@voicenter-team/nuxt-llms-generator 0.1.1 → 0.1.3

package/README.md

@@ -63,9 +63,9 @@ flowchart TB
         ENV2[🌐 Site 2<br/>SITE_ENV=partner]
         ENV3[🌐 Site 3<br/>SITE_ENV=staging]
 
-        CACHE1[💾 .llms-cache/main/]
-        CACHE2[💾 .llms-cache/partner/]
-        CACHE3[💾 .llms-cache/staging/]
+        CACHE1[💾 .llms-templates/main/]
+        CACHE2[💾 .llms-templates/partner/]
+        CACHE3[💾 .llms-templates/staging/]
 
         OUT1[📂 .output/llms/main/]
         OUT2[📂 .output/llms/partner/]
@@ -140,9 +140,9 @@ flowchart TB
 The system automatically adapts to different environments using the `SITE_ENV` variable:
 
 ```bash
-SITE_ENV=main     → UmbracoData-main.json     → .llms-cache/main/     → .output/llms/main/
-SITE_ENV=partner  → UmbracoData-partner.json  → .llms-cache/partner/  → .output/llms/partner/
-SITE_ENV=staging  → UmbracoData-staging.json  → .llms-cache/staging/  → .output/llms/staging/
+SITE_ENV=main     → UmbracoData-main.json     → .llms-templates/main/     → .output/llms/main/
+SITE_ENV=partner  → UmbracoData-partner.json  → .llms-templates/partner/  → .output/llms/partner/
+SITE_ENV=staging  → UmbracoData-staging.json  → .llms-templates/staging/  → .output/llms/staging/
 ```
 
 Each environment maintains its own isolated cache and output, preventing conflicts while sharing the same codebase and configuration logic.
@@ -231,9 +231,8 @@ npm run build
 |----------------------|----------|----------|------------------|--------------------------------------------|
 | `anthropicApiKey`    | `string` | ✅        | -                | Your Claude API key from Anthropic         |
 | `umbracoDataPath`    | `string` | ✅        | -                | Path to your UmbracoData.json file         |
-| `templatesOutputDir` | `string` | ❌        | `./templates`    | Directory for generated Mustache templates |
+| `templatesDir`       | `string` | ❌        | `./.llms-templates` | Directory for templates and cache files     |
 | `finalOutputDir`     | `string` | ❌        | `./.output/llms` | Output directory for final documentation   |
-| `cacheDir`           | `string` | ❌        | `./.llms-cache`  | Template cache directory                   |
 
 ### Generation Options
 
@@ -272,9 +271,8 @@ export default defineNuxtConfig({
 
     // Environment-specific paths
     umbracoDataPath: `./public/UmbracoData-${siteEnv}.json`,
-    cacheDir: `./.llms-cache/${siteEnv}`,
+    templatesDir: `./.llms-templates/${siteEnv}`,
     finalOutputDir: `./.output/llms/${siteEnv}`,
-    templatesOutputDir: `./templates/${siteEnv}`,
 
     // Shared settings
     maxConcurrent: 5,
@@ -300,10 +298,10 @@ export default defineNuxtConfig({
 
 ```
 project/
-├── .llms-cache/
-│   ├── main/        # Main site cache
-│   ├── partner/     # Partner site cache
-│   └── staging/     # Staging site cache
+├── .llms-templates/
+│   ├── main/        # Main site templates & cache
+│   ├── partner/     # Partner site templates & cache
+│   └── staging/     # Staging site templates & cache
 ├── .output/
 │   └── llms/
 │       ├── main/    # Main site docs
@@ -569,9 +567,8 @@ interface LLMSConfig {
   umbracoDataPath: string;
 
   // Optional with defaults
-  templatesOutputDir?: string;        // './templates'
+  templatesDir?: string;              // './.llms-templates'
   finalOutputDir?: string;            // './.output/llms'
-  cacheDir?: string;                  // './.llms-cache'
   anthropicModel?: string;            // 'claude-3-5-sonnet-20241022'
   maxConcurrent?: number;             // 5
   enableLLMSFullTxt?: boolean;        // true

package/dist/chunks/llms-files-generator.mjs

@@ -1,10 +1,10 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { existsSync, mkdirSync, readFileSync, writeFileSync, unlinkSync } from 'fs';
 import { join, dirname, basename } from 'path';
 import Mustache from 'mustache';
 import Anthropic from '@anthropic-ai/sdk';
 import { createHash } from 'crypto';
 import { JSONPath } from 'jsonpath-plus';
-import { T as TemplateError, E as ErrorCode, w as withErrorHandling } from '../shared/nuxt-llms-generator.3c82efc0.mjs';
+import { T as TemplateError, E as ErrorCode, w as withErrorHandling } from '../shared/nuxt-llms-generator.ab079b9f.mjs';
 import { NodeHtmlMarkdown } from 'node-html-markdown';
 import '@nuxt/kit';
 import 'zod';
@@ -367,12 +367,14 @@ class PromptAnalyzer {
 
 class LLMSCache {
   cacheFilePath;
+  templatesDir;
   cache = {};
-  constructor(cacheDir) {
-    if (!existsSync(cacheDir)) {
-      mkdirSync(cacheDir, { recursive: true });
+  constructor(templatesDir) {
+    this.templatesDir = templatesDir;
+    if (!existsSync(templatesDir)) {
+      mkdirSync(templatesDir, { recursive: true });
     }
-    this.cacheFilePath = join(cacheDir, "templates.json");
+    this.cacheFilePath = join(templatesDir, "cache.json");
     this.loadCache();
   }
   loadCache() {
@@ -404,39 +406,95 @@ class LLMSCache {
     const cached = this.cache[pageId];
     if (!cached)
       return null;
+    const templatePath = join(this.templatesDir, `${pageId}.mustache`);
+    let template = "";
+    try {
+      if (existsSync(templatePath)) {
+        template = readFileSync(templatePath, "utf-8");
+      } else {
+        console.warn(`Template file not found: ${templatePath}`);
+        return null;
+      }
+    } catch (error) {
+      console.warn(`Failed to read template file for ${pageId}:`, error);
+      return null;
+    }
     return {
       pageId,
-      templatePath: "",
-      template: cached.template,
-      hash: cached.hash,
+      templatePath,
+      template,
+      structureHash: cached.structureHash,
+      metadata: {
+        url: cached.pageId,
+        // PageContentHash.pageId contains the URL
+        templateAlias: cached.templateAlias,
+        jpath: cached.jpath,
+        generatedAt: new Date(cached.lastUpdated)
+      }
+    };
+  }
+  getTemplateMetadata(pageId) {
+    const cached = this.cache[pageId];
+    if (!cached)
+      return null;
+    return {
+      structureHash: cached.structureHash,
       metadata: {
-        url: cached.metadata.pageId,
-        // Contains the URL from setTemplate
-        templateAlias: cached.metadata.templateAlias,
-        jpath: cached.metadata.jpath,
-        generatedAt: new Date(cached.metadata.lastUpdated)
+        url: cached.pageId,
+        // PageContentHash.pageId contains the URL
+        templateAlias: cached.templateAlias,
+        jpath: cached.jpath,
+        generatedAt: new Date(cached.lastUpdated)
       }
     };
   }
+  getTemplatePath(pageId) {
+    return join(this.templatesDir, `${pageId}.mustache`);
+  }
   getTemplateHash(pageId) {
-    return this.cache[pageId]?.hash || null;
+    return this.cache[pageId]?.structureHash || null;
   }
-  setTemplate(pageId, template, hash, metadata) {
+  setTemplate(pageId, template, structureHash, jpath, templateAlias, url) {
+    const templatePath = join(this.templatesDir, `${pageId}.mustache`);
+    try {
+      writeFileSync(templatePath, template, "utf-8");
+    } catch (error) {
+      console.error(`Failed to save template file for ${pageId}:`, error);
+      return;
+    }
     this.cache[pageId] = {
-      hash,
-      template,
-      metadata: {
-        ...metadata,
-        lastUpdated: /* @__PURE__ */ new Date()
-      }
+      pageId: url,
+      // Store URL in pageId field for backwards compatibility
+      jpath,
+      templateAlias,
+      structureHash,
+      lastUpdated: /* @__PURE__ */ new Date()
     };
     this.saveCache();
   }
   removeTemplate(pageId) {
+    const templatePath = join(this.templatesDir, `${pageId}.mustache`);
+    try {
+      if (existsSync(templatePath)) {
+        unlinkSync(templatePath);
+      }
+    } catch (error) {
+      console.warn(`Failed to remove template file for ${pageId}:`, error);
+    }
     delete this.cache[pageId];
     this.saveCache();
   }
   clearCache() {
+    for (const pageId of Object.keys(this.cache)) {
+      const templatePath = join(this.templatesDir, `${pageId}.mustache`);
+      try {
+        if (existsSync(templatePath)) {
+          unlinkSync(templatePath);
+        }
+      } catch (error) {
+        console.warn(`Failed to remove template file for ${pageId}:`, error);
+      }
+    }
     this.cache = {};
     this.saveCache();
   }
@@ -446,9 +504,9 @@ class LLMSCache {
     let oldestTemplate = null;
     let newestTemplate = null;
     for (const template of templates) {
-      const alias = template.metadata.templateAlias;
+      const alias = template.templateAlias;
       templatesByAlias[alias] = (templatesByAlias[alias] || 0) + 1;
-      const date = new Date(template.metadata.lastUpdated);
+      const date = new Date(template.lastUpdated);
       if (!oldestTemplate || date < oldestTemplate) {
         oldestTemplate = date;
       }
@@ -467,9 +525,9 @@ class LLMSCache {
     const cutoffDate = new Date(Date.now() - maxAgeMs);
     let removedCount = 0;
     for (const [pageId, template] of Object.entries(this.cache)) {
-      const templateDate = new Date(template.metadata.lastUpdated);
+      const templateDate = new Date(template.lastUpdated);
       if (templateDate < cutoffDate) {
-        delete this.cache[pageId];
+        this.removeTemplate(pageId);
         removedCount++;
       }
     }
@@ -557,7 +615,9 @@ function excludeChildrenFromContent(content) {
   return cleanContent;
 }
 function generatePageId(urlItem) {
-  return `${urlItem.TemplateAlias}_${urlItem.nodeID}`;
+  const templateAlias = urlItem.TemplateAlias || "UnknownTemplate";
+  const nodeID = urlItem.nodeID || "UnknownNode";
+  return `${templateAlias}_${nodeID}`;
 }
 function estimateContentTokens(content) {
   try {
@@ -651,8 +711,8 @@ function getPageVisibilityStats(umbracoData) {
 
 class TemplateCleanup {
   cache;
-  constructor(cacheDir) {
-    this.cache = new LLMSCache(cacheDir);
+  constructor(templatesDir) {
+    this.cache = new LLMSCache(templatesDir);
   }
   /**
   * Find templates that no longer have corresponding pages
@@ -1077,11 +1137,7 @@ class TemplateGenerator {
     this.config = config;
     this.anthropicClient = new AnthropicClient(config);
     this.promptAnalyzer = new PromptAnalyzer();
-    if (config.cacheDir) {
-      this.cache = new LLMSCache(config.cacheDir);
-    } else {
-      this.cache = new LLMSCache(".llms-cache");
-    }
+    this.cache = new LLMSCache(config.templatesDir || "./.llms-templates");
     this.ensureOutputDirectories();
   }
   async generateTemplate(pageContent, urlItem) {
@@ -1100,7 +1156,7 @@ class TemplateGenerator {
   async generateAllTemplates(umbracoData) {
     const templates = [];
     const maxConcurrent = this.config.maxConcurrent || 5;
-    await performAutomaticCleanup(umbracoData, this.config.cacheDir || "./cache", {
+    await performAutomaticCleanup(umbracoData, this.config.templatesDir || "./.llms-templates", {
       enableAutoCleanup: this.config.enableAutoCleanup ?? true,
       cleanupOrphaned: this.config.cleanupOrphaned ?? true,
       cleanupHidden: this.config.cleanupHidden ?? true,
@@ -1189,7 +1245,7 @@ class TemplateGenerator {
     const response = await this.anthropicClient.generateTemplate(request);
     const currentHash = generatePageStructureHash(pageContent, { excludeChildren: true });
     const templatePath = join(
-      this.config.templatesOutputDir,
+      this.config.templatesDir,
       `${pageId}.mustache`
     );
     this.saveTemplate(templatePath, response.template);
@@ -1197,7 +1253,7 @@ class TemplateGenerator {
       pageId,
       templatePath,
       template: response.template,
-      hash: currentHash,
+      structureHash: currentHash,
       metadata: {
         url: urlItem.url,
         templateAlias: urlItem.TemplateAlias,
@@ -1205,13 +1261,7 @@ class TemplateGenerator {
         generatedAt: /* @__PURE__ */ new Date()
       }
     };
-    this.cache.setTemplate(pageId, response.template, currentHash, {
-      pageId: urlItem.url,
-      // Store URL as pageId for cache lookup
-      jpath: urlItem.Jpath,
-      templateAlias: urlItem.TemplateAlias,
-      structureHash: currentHash
-    });
+    this.cache.setTemplate(pageId, response.template, currentHash, urlItem.Jpath, urlItem.TemplateAlias, urlItem.url);
     return generatedTemplate;
   }
   async renderTemplate(template, data) {
@@ -1251,8 +1301,8 @@ class TemplateGenerator {
     writeFileSync(templatePath, content, "utf-8");
   }
   ensureOutputDirectories() {
-    if (!existsSync(this.config.templatesOutputDir)) {
-      mkdirSync(this.config.templatesOutputDir, { recursive: true });
+    if (!existsSync(this.config.templatesDir || "./.llms-templates")) {
+      mkdirSync(this.config.templatesDir || "./.llms-templates", { recursive: true });
     }
     if (this.config.finalOutputDir && !existsSync(this.config.finalOutputDir)) {
       mkdirSync(this.config.finalOutputDir, { recursive: true });
@@ -1275,6 +1325,7 @@ class TemplateGenerator {
 class LLMSFilesGenerator {
   config;
   templateGenerator;
+  nhm = new NodeHtmlMarkdown();
   constructor(config) {
     this.config = config;
     this.templateGenerator = new TemplateGenerator(config);
@@ -1391,7 +1442,6 @@ class LLMSFilesGenerator {
 
 `;
     }
-    content += "This document contains all website content in a single file for comprehensive AI analysis.\n\n";
     content += "---\n\n";
     for (const mdFile of mdFiles) {
       const urlItem = umbracoData.urlList.find((item) => item.url === mdFile.url);
@@ -1468,11 +1518,22 @@ class LLMSFilesGenerator {
   }
   extractSiteTitle(umbracoData) {
     const siteData = umbracoData.SiteData;
-    return siteData?.pageTitle || siteData?.mainHeaderBlockTitle || "Website Documentation";
+    const rawTitle = siteData?.pageTitle || siteData?.mainHeaderBlockTitle || "Website Documentation";
+    return this.cleanHtmlContent(rawTitle);
   }
   extractSiteDescription(umbracoData) {
     const siteData = umbracoData.SiteData;
-    return siteData?.pageDescription || siteData?.ogDescription || null;
+    const rawDescription = siteData?.pageDescription || siteData?.ogDescription || null;
+    return rawDescription ? this.cleanHtmlContent(rawDescription) : null;
+  }
+  cleanHtmlContent(content) {
+    if (!this.config.enableHtmlToMarkdown) {
+      return content;
+    }
+    if (/<[^>]+>/.test(content)) {
+      return this.nhm.translate(content).trim();
+    }
+    return content;
   }
   extractPageTitle(umbracoData, urlItem) {
     if (!urlItem)
@@ -1505,7 +1566,14 @@ class LLMSFilesGenerator {
     return names[category] || category.charAt(0).toUpperCase() + category.slice(1);
   }
   sanitizeUrlForFilename(url) {
-    return url.replace(/^\//, "").replace(/\/$/, "").replace(/\//g, "-").replace(/[^a-zA-Z0-9\-_]/g, "").replace(/^$/, "index");
+    let filename = url.replace(/^\//, "").replace(/\/$/, "").replace(/\//g, "-").replace(/[^a-zA-Z0-9\-_.]/g, "").replace(/--+/g, "-").replace(/^-+|-+$/g, "");
+    if (!filename || filename === "") {
+      filename = "index";
+    }
+    if (filename.startsWith("-") || filename.startsWith(".")) {
+      filename = "page-" + filename.replace(/^[-.]/, "");
+    }
+    return filename;
   }
   getRelativeFilePath(fullPath) {
     const filename = fullPath.split("/").pop() || "";

package/dist/module.d.mts

@@ -3,9 +3,8 @@ import * as _nuxt_schema from '@nuxt/schema';
 interface LLMSConfig {
     anthropicApiKey: string;
     umbracoDataPath: string;
-    templatesOutputDir: string;
+    templatesDir: string;
     finalOutputDir: string;
-    cacheDir?: string;
     anthropicModel?: string;
     maxConcurrent?: number;
     enableLLMSFullTxt?: boolean;
@@ -55,17 +54,13 @@ interface PageContentHash {
     lastUpdated: Date;
 }
 interface TemplateCache {
-    [pageId: string]: {
-        hash: string;
-        template: string;
-        metadata: PageContentHash;
-    };
+    [pageId: string]: PageContentHash;
 }
 interface GeneratedTemplate {
     pageId: string;
     templatePath: string;
     template: string;
-    hash: string;
+    structureHash: string;
     metadata: {
         url: string;
         templateAlias: string;

package/dist/module.d.ts

@@ -3,9 +3,8 @@ import * as _nuxt_schema from '@nuxt/schema';
 interface LLMSConfig {
     anthropicApiKey: string;
     umbracoDataPath: string;
-    templatesOutputDir: string;
+    templatesDir: string;
     finalOutputDir: string;
-    cacheDir?: string;
     anthropicModel?: string;
     maxConcurrent?: number;
     enableLLMSFullTxt?: boolean;
@@ -55,17 +54,13 @@ interface PageContentHash {
     lastUpdated: Date;
 }
 interface TemplateCache {
-    [pageId: string]: {
-        hash: string;
-        template: string;
-        metadata: PageContentHash;
-    };
+    [pageId: string]: PageContentHash;
 }
 interface GeneratedTemplate {
     pageId: string;
     templatePath: string;
     template: string;
-    hash: string;
+    structureHash: string;
     metadata: {
         url: string;
         templateAlias: string;

package/dist/module.json

@@ -4,5 +4,5 @@
   "compatibility": {
     "nuxt": "^3.0.0"
   },
-  "version": "0.1.1"
+  "version": "0.1.3"
 }

package/dist/module.mjs

@@ -1,4 +1,4 @@
-export { l as default } from './shared/nuxt-llms-generator.3c82efc0.mjs';
+export { l as default } from './shared/nuxt-llms-generator.ab079b9f.mjs';
 import '@nuxt/kit';
 import 'fs';
 import 'path';

package/dist/shared/{nuxt-llms-generator.3c82efc0.mjs → nuxt-llms-generator.ab079b9f.mjs}

@@ -10,9 +10,8 @@ const anthropicApiKey = z.string().min(10, "API key must be at least 10 characte
 const LLMSConfigSchema = z.object({
   anthropicApiKey,
   umbracoDataPath: existingPath,
-  templatesOutputDir: z.string().min(1, "Templates output directory is required"),
+  templatesDir: z.string().min(1, "Templates directory is required"),
   finalOutputDir: z.string().optional().default("public"),
-  cacheDir: z.string().optional().default(".llms-cache"),
   anthropicModel: z.string().optional(),
   maxConcurrent: z.number().int().min(1, "maxConcurrent must be at least 1").max(10, "maxConcurrent should not exceed 10 to avoid rate limits").optional().default(3),
   enableLLMSFullTxt: z.boolean().optional().default(true),
@@ -207,7 +206,7 @@ const DEFAULT_OPTIONS = {
   maxConcurrent: 5,
   enableLLMSFullTxt: true,
   enableIndividualMd: true,
-  cacheDir: ".llms-cache",
+  templatesDir: ".llms-templates",
   finalOutputDir: "public",
   enableAutoCleanup: true,
   cleanupOrphaned: true,
@@ -240,23 +239,23 @@ const llmsModule = defineNuxtModule({
       logger.warn("No Umbraco data path provided. LLMS Generator will be disabled.");
       return;
     }
-    if (!options.templatesOutputDir) {
-      logger.warn("No templates output directory provided. LLMS Generator will be disabled.");
+    if (!options.templatesDir) {
+      logger.warn("No templates directory provided. LLMS Generator will be disabled.");
       return;
     }
     const configForValidation = {
       anthropicApiKey: options.anthropicApiKey,
       umbracoDataPath: resolve(nuxt.options.rootDir, options.umbracoDataPath),
-      templatesOutputDir: resolve(nuxt.options.rootDir, options.templatesOutputDir),
+      templatesDir: resolve(nuxt.options.rootDir, options.templatesDir),
       finalOutputDir: resolve(nuxt.options.rootDir, options.finalOutputDir ?? "public"),
-      cacheDir: options.cacheDir ? resolve(nuxt.options.rootDir, options.cacheDir) : void 0,
       anthropicModel: options.anthropicModel || DEFAULT_OPTIONS.anthropicModel,
       maxConcurrent: options.maxConcurrent || DEFAULT_OPTIONS.maxConcurrent,
       enableLLMSFullTxt: options.enableLLMSFullTxt ?? DEFAULT_OPTIONS.enableLLMSFullTxt,
       enableIndividualMd: options.enableIndividualMd ?? DEFAULT_OPTIONS.enableIndividualMd,
       enableAutoCleanup: options.enableAutoCleanup ?? DEFAULT_OPTIONS.enableAutoCleanup,
       cleanupOrphaned: options.cleanupOrphaned ?? DEFAULT_OPTIONS.cleanupOrphaned,
-      cleanupHidden: options.cleanupHidden ?? DEFAULT_OPTIONS.cleanupHidden
+      cleanupHidden: options.cleanupHidden ?? DEFAULT_OPTIONS.cleanupHidden,
+      enableHtmlToMarkdown: options.enableHtmlToMarkdown ?? DEFAULT_OPTIONS.enableHtmlToMarkdown
     };
     let moduleOptions;
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voicenter-team/nuxt-llms-generator",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Nuxt 3 module for automatically generating AI-optimized documentation files (llms.txt, llms-full.txt, and individual .md files) from Umbraco CMS data using Anthropic's Claude API.",
   "repository": "your-org/my-module",
   "license": "MIT",