@voicenter-team/nuxt-llms-generator 0.1.0 → 0.1.2

package/README.md

@@ -63,9 +63,9 @@ flowchart TB
         ENV2[🌐 Site 2<br/>SITE_ENV=partner]
         ENV3[🌐 Site 3<br/>SITE_ENV=staging]
 
-        CACHE1[💾 .llms-cache/main/]
-        CACHE2[💾 .llms-cache/partner/]
-        CACHE3[💾 .llms-cache/staging/]
+        CACHE1[💾 .llms-templates/main/]
+        CACHE2[💾 .llms-templates/partner/]
+        CACHE3[💾 .llms-templates/staging/]
 
         OUT1[📂 .output/llms/main/]
         OUT2[📂 .output/llms/partner/]
@@ -140,9 +140,9 @@ flowchart TB
 The system automatically adapts to different environments using the `SITE_ENV` variable:
 
 ```bash
-SITE_ENV=main     → UmbracoData-main.json     → .llms-cache/main/     → .output/llms/main/
-SITE_ENV=partner  → UmbracoData-partner.json  → .llms-cache/partner/  → .output/llms/partner/
-SITE_ENV=staging  → UmbracoData-staging.json  → .llms-cache/staging/  → .output/llms/staging/
+SITE_ENV=main     → UmbracoData-main.json     → .llms-templates/main/     → .output/llms/main/
+SITE_ENV=partner  → UmbracoData-partner.json  → .llms-templates/partner/  → .output/llms/partner/
+SITE_ENV=staging  → UmbracoData-staging.json  → .llms-templates/staging/  → .output/llms/staging/
 ```
 
 Each environment maintains its own isolated cache and output, preventing conflicts while sharing the same codebase and configuration logic.
@@ -231,18 +231,18 @@ npm run build
 |----------------------|----------|----------|------------------|--------------------------------------------|
 | `anthropicApiKey`    | `string` | ✅        | -                | Your Claude API key from Anthropic         |
 | `umbracoDataPath`    | `string` | ✅        | -                | Path to your UmbracoData.json file         |
-| `templatesOutputDir` | `string` | ❌        | `./templates`    | Directory for generated Mustache templates |
+| `templatesDir`       | `string` | ❌        | `./.llms-templates` | Directory for templates and cache files     |
 | `finalOutputDir`     | `string` | ❌        | `./.output/llms` | Output directory for final documentation   |
-| `cacheDir`           | `string` | ❌        | `./.llms-cache`  | Template cache directory                   |
 
 ### Generation Options
 
-| Option               | Type      | Default                      | Description                            |
-|----------------------|-----------|------------------------------|----------------------------------------|
-| `enableIndividualMd` | `boolean` | `true`                       | Generate individual .md files per page |
-| `enableLLMSFullTxt`  | `boolean` | `true`                       | Generate combined llms-full.txt file   |
-| `maxConcurrent`      | `number`  | `5`                          | Maximum concurrent API requests        |
-| `anthropicModel`     | `string`  | `claude-3-5-sonnet-20241022` | Claude model to use                    |
+| Option                 | Type      | Default                      | Description                              |
+|------------------------|-----------|------------------------------|------------------------------------------|
+| `enableIndividualMd`   | `boolean` | `true`                       | Generate individual .md files per page  |
+| `enableLLMSFullTxt`    | `boolean` | `true`                       | Generate combined llms-full.txt file    |
+| `enableHtmlToMarkdown` | `boolean` | `true`                       | Convert HTML content to markdown using [node-html-markdown](https://www.npmjs.com/package/node-html-markdown) |
+| `maxConcurrent`        | `number`  | `5`                          | Maximum concurrent API requests          |
+| `anthropicModel`       | `string`  | `claude-3-5-sonnet-20241022` | Claude model to use                      |
 
 ### Cleanup Options
 
@@ -271,9 +271,8 @@ export default defineNuxtConfig({
 
     // Environment-specific paths
     umbracoDataPath: `./public/UmbracoData-${siteEnv}.json`,
-    cacheDir: `./.llms-cache/${siteEnv}`,
+    templatesDir: `./.llms-templates/${siteEnv}`,
     finalOutputDir: `./.output/llms/${siteEnv}`,
-    templatesOutputDir: `./templates/${siteEnv}`,
 
     // Shared settings
     maxConcurrent: 5,
@@ -299,10 +298,10 @@ export default defineNuxtConfig({
 
 ```
 project/
-├── .llms-cache/
-│   ├── main/        # Main site cache
-│   ├── partner/     # Partner site cache
-│   └── staging/     # Staging site cache
+├── .llms-templates/
+│   ├── main/        # Main site templates & cache
+│   ├── partner/     # Partner site templates & cache
+│   └── staging/     # Staging site templates & cache
 ├── .output/
 │   └── llms/
 │       ├── main/    # Main site docs
@@ -518,7 +517,16 @@ ls -la public/UmbracoData.json
    {
      enableAutoCleanup: true,
      cleanupOrphaned: true,
-     cleanupHidden: true
+     cleanupHidden: true,
+     enableHtmlToMarkdown: true  // Clean HTML from CMS content
+   }
+   ```
+
+4. **HTML Content Processing**:
+   ```typescript
+   {
+     enableHtmlToMarkdown: true,  // Convert <p>, <h1>, etc. to clean markdown
+     enableHtmlToMarkdown: false  // Keep HTML as-is (if AI already generates clean content)
    }
    ```
 
@@ -559,13 +567,13 @@ interface LLMSConfig {
   umbracoDataPath: string;
 
   // Optional with defaults
-  templatesOutputDir?: string;        // './templates'
+  templatesDir?: string;              // './.llms-templates'
   finalOutputDir?: string;            // './.output/llms'
-  cacheDir?: string;                  // './.llms-cache'
   anthropicModel?: string;            // 'claude-3-5-sonnet-20241022'
   maxConcurrent?: number;             // 5
   enableLLMSFullTxt?: boolean;        // true
   enableIndividualMd?: boolean;       // true
+  enableHtmlToMarkdown?: boolean;     // true
   enableAutoCleanup?: boolean;        // true
   cleanupOrphaned?: boolean;          // true
   cleanupHidden?: boolean;            // true

package/dist/chunks/llms-files-generator.mjs

@@ -1,10 +1,11 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
-import { join, dirname } from 'path';
+import { existsSync, mkdirSync, readFileSync, writeFileSync, unlinkSync } from 'fs';
+import { join, dirname, basename } from 'path';
 import Mustache from 'mustache';
 import Anthropic from '@anthropic-ai/sdk';
 import { createHash } from 'crypto';
 import { JSONPath } from 'jsonpath-plus';
-import { T as TemplateError, E as ErrorCode, w as withErrorHandling } from '../shared/nuxt-llms-generator.dc009f50.mjs';
+import { T as TemplateError, E as ErrorCode, w as withErrorHandling } from '../shared/nuxt-llms-generator.ab079b9f.mjs';
+import { NodeHtmlMarkdown } from 'node-html-markdown';
 import '@nuxt/kit';
 import 'zod';
 
@@ -366,12 +367,14 @@ class PromptAnalyzer {
 
 class LLMSCache {
   cacheFilePath;
+  templatesDir;
   cache = {};
-  constructor(cacheDir) {
-    if (!existsSync(cacheDir)) {
-      mkdirSync(cacheDir, { recursive: true });
+  constructor(templatesDir) {
+    this.templatesDir = templatesDir;
+    if (!existsSync(templatesDir)) {
+      mkdirSync(templatesDir, { recursive: true });
     }
-    this.cacheFilePath = join(cacheDir, "templates.json");
+    this.cacheFilePath = join(templatesDir, "cache.json");
     this.loadCache();
   }
   loadCache() {
@@ -403,39 +406,95 @@ class LLMSCache {
     const cached = this.cache[pageId];
     if (!cached)
       return null;
+    const templatePath = join(this.templatesDir, `${pageId}.mustache`);
+    let template = "";
+    try {
+      if (existsSync(templatePath)) {
+        template = readFileSync(templatePath, "utf-8");
+      } else {
+        console.warn(`Template file not found: ${templatePath}`);
+        return null;
+      }
+    } catch (error) {
+      console.warn(`Failed to read template file for ${pageId}:`, error);
+      return null;
+    }
     return {
       pageId,
-      templatePath: "",
-      template: cached.template,
-      hash: cached.hash,
+      templatePath,
+      template,
+      structureHash: cached.structureHash,
+      metadata: {
+        url: cached.pageId,
+        // PageContentHash.pageId contains the URL
+        templateAlias: cached.templateAlias,
+        jpath: cached.jpath,
+        generatedAt: new Date(cached.lastUpdated)
+      }
+    };
+  }
+  getTemplateMetadata(pageId) {
+    const cached = this.cache[pageId];
+    if (!cached)
+      return null;
+    return {
+      structureHash: cached.structureHash,
       metadata: {
-        url: cached.metadata.pageId,
-        // Contains the URL from setTemplate
-        templateAlias: cached.metadata.templateAlias,
-        jpath: cached.metadata.jpath,
-        generatedAt: new Date(cached.metadata.lastUpdated)
+        url: cached.pageId,
+        // PageContentHash.pageId contains the URL
+        templateAlias: cached.templateAlias,
+        jpath: cached.jpath,
+        generatedAt: new Date(cached.lastUpdated)
       }
     };
   }
+  getTemplatePath(pageId) {
+    return join(this.templatesDir, `${pageId}.mustache`);
+  }
   getTemplateHash(pageId) {
-    return this.cache[pageId]?.hash || null;
+    return this.cache[pageId]?.structureHash || null;
   }
-  setTemplate(pageId, template, hash, metadata) {
+  setTemplate(pageId, template, structureHash, jpath, templateAlias, url) {
+    const templatePath = join(this.templatesDir, `${pageId}.mustache`);
+    try {
+      writeFileSync(templatePath, template, "utf-8");
+    } catch (error) {
+      console.error(`Failed to save template file for ${pageId}:`, error);
+      return;
+    }
     this.cache[pageId] = {
-      hash,
-      template,
-      metadata: {
-        ...metadata,
-        lastUpdated: /* @__PURE__ */ new Date()
-      }
+      pageId: url,
+      // Store URL in pageId field for backwards compatibility
+      jpath,
+      templateAlias,
+      structureHash,
+      lastUpdated: /* @__PURE__ */ new Date()
     };
     this.saveCache();
   }
   removeTemplate(pageId) {
+    const templatePath = join(this.templatesDir, `${pageId}.mustache`);
+    try {
+      if (existsSync(templatePath)) {
+        unlinkSync(templatePath);
+      }
+    } catch (error) {
+      console.warn(`Failed to remove template file for ${pageId}:`, error);
+    }
     delete this.cache[pageId];
     this.saveCache();
   }
   clearCache() {
+    for (const pageId of Object.keys(this.cache)) {
+      const templatePath = join(this.templatesDir, `${pageId}.mustache`);
+      try {
+        if (existsSync(templatePath)) {
+          unlinkSync(templatePath);
+        }
+      } catch (error) {
+        console.warn(`Failed to remove template file for ${pageId}:`, error);
+      }
+    }
     this.cache = {};
     this.saveCache();
   }
@@ -445,9 +504,9 @@ class LLMSCache {
     let oldestTemplate = null;
     let newestTemplate = null;
     for (const template of templates) {
-      const alias = template.metadata.templateAlias;
+      const alias = template.templateAlias;
       templatesByAlias[alias] = (templatesByAlias[alias] || 0) + 1;
-      const date = new Date(template.metadata.lastUpdated);
+      const date = new Date(template.lastUpdated);
       if (!oldestTemplate || date < oldestTemplate) {
         oldestTemplate = date;
       }
@@ -466,9 +525,9 @@ class LLMSCache {
     const cutoffDate = new Date(Date.now() - maxAgeMs);
     let removedCount = 0;
     for (const [pageId, template] of Object.entries(this.cache)) {
-      const templateDate = new Date(template.metadata.lastUpdated);
+      const templateDate = new Date(template.lastUpdated);
       if (templateDate < cutoffDate) {
-        delete this.cache[pageId];
+        this.removeTemplate(pageId);
         removedCount++;
       }
     }
@@ -650,8 +709,8 @@ function getPageVisibilityStats(umbracoData) {
 
 class TemplateCleanup {
   cache;
-  constructor(cacheDir) {
-    this.cache = new LLMSCache(cacheDir);
+  constructor(templatesDir) {
+    this.cache = new LLMSCache(templatesDir);
   }
   /**
   * Find templates that no longer have corresponding pages
@@ -1067,15 +1126,16 @@ class TemplateGenerator {
   promptAnalyzer;
   cache;
   config;
+  nhm = new NodeHtmlMarkdown(
+    {},
+    void 0,
+    void 0
+  );
   constructor(config) {
     this.config = config;
     this.anthropicClient = new AnthropicClient(config);
     this.promptAnalyzer = new PromptAnalyzer();
-    if (config.cacheDir) {
-      this.cache = new LLMSCache(config.cacheDir);
-    } else {
-      this.cache = new LLMSCache(".llms-cache");
-    }
+    this.cache = new LLMSCache(config.templatesDir || "./.llms-templates");
     this.ensureOutputDirectories();
   }
   async generateTemplate(pageContent, urlItem) {
@@ -1094,7 +1154,7 @@ class TemplateGenerator {
   async generateAllTemplates(umbracoData) {
     const templates = [];
     const maxConcurrent = this.config.maxConcurrent || 5;
-    await performAutomaticCleanup(umbracoData, this.config.cacheDir || "./cache", {
+    await performAutomaticCleanup(umbracoData, this.config.templatesDir || "./.llms-templates", {
       enableAutoCleanup: this.config.enableAutoCleanup ?? true,
       cleanupOrphaned: this.config.cleanupOrphaned ?? true,
       cleanupHidden: this.config.cleanupHidden ?? true,
@@ -1183,7 +1243,7 @@ class TemplateGenerator {
     const response = await this.anthropicClient.generateTemplate(request);
     const currentHash = generatePageStructureHash(pageContent, { excludeChildren: true });
     const templatePath = join(
-      this.config.templatesOutputDir,
+      this.config.templatesDir,
       `${pageId}.mustache`
     );
     this.saveTemplate(templatePath, response.template);
@@ -1191,7 +1251,7 @@ class TemplateGenerator {
       pageId,
       templatePath,
       template: response.template,
-      hash: currentHash,
+      structureHash: currentHash,
       metadata: {
         url: urlItem.url,
         templateAlias: urlItem.TemplateAlias,
@@ -1199,19 +1259,17 @@ class TemplateGenerator {
         generatedAt: /* @__PURE__ */ new Date()
       }
     };
-    this.cache.setTemplate(pageId, response.template, currentHash, {
-      pageId: urlItem.url,
-      // Store URL as pageId for cache lookup
-      jpath: urlItem.Jpath,
-      templateAlias: urlItem.TemplateAlias,
-      structureHash: currentHash
-    });
+    this.cache.setTemplate(pageId, response.template, currentHash, urlItem.Jpath, urlItem.TemplateAlias, urlItem.url);
     return generatedTemplate;
   }
   async renderTemplate(template, data) {
     return withErrorHandling(async () => {
       const validatedTemplate = await templateValidationPipeline.validateAndFix(template);
-      return Mustache.render(validatedTemplate, data);
+      const renderedContent = Mustache.render(validatedTemplate, data);
+      if (this.config.enableHtmlToMarkdown) {
+        return this.nhm.translate(renderedContent);
+      }
+      return renderedContent;
     }, {
       template: template.substring(0, 200) + "...",
       dataKeys: Object.keys(data)
@@ -1241,8 +1299,8 @@ class TemplateGenerator {
     writeFileSync(templatePath, content, "utf-8");
   }
   ensureOutputDirectories() {
-    if (!existsSync(this.config.templatesOutputDir)) {
-      mkdirSync(this.config.templatesOutputDir, { recursive: true });
+    if (!existsSync(this.config.templatesDir || "./.llms-templates")) {
+      mkdirSync(this.config.templatesDir || "./.llms-templates", { recursive: true });
     }
     if (this.config.finalOutputDir && !existsSync(this.config.finalOutputDir)) {
       mkdirSync(this.config.finalOutputDir, { recursive: true });
@@ -1502,8 +1560,8 @@ class LLMSFilesGenerator {
     return filename;
   }
   getLLMSFilePath(fullPath) {
-    const filename = fullPath.split("/").pop() || "";
-    return `llms/${filename}`;
+    const filename = basename(fullPath);
+    return `/llms/${filename}`;
   }
   getOutputDir() {
     return this.config.finalOutputDir || "dist";

package/dist/module.d.mts

@@ -3,9 +3,8 @@ import * as _nuxt_schema from '@nuxt/schema';
 interface LLMSConfig {
     anthropicApiKey: string;
     umbracoDataPath: string;
-    templatesOutputDir: string;
+    templatesDir: string;
     finalOutputDir: string;
-    cacheDir?: string;
     anthropicModel?: string;
     maxConcurrent?: number;
     enableLLMSFullTxt?: boolean;
@@ -13,6 +12,7 @@ interface LLMSConfig {
     enableAutoCleanup?: boolean;
     cleanupOrphaned?: boolean;
     cleanupHidden?: boolean;
+    enableHtmlToMarkdown?: boolean;
 }
 interface UmbracoUrlItem {
     nodeID: number;
@@ -54,17 +54,13 @@ interface PageContentHash {
     lastUpdated: Date;
 }
 interface TemplateCache {
-    [pageId: string]: {
-        hash: string;
-        template: string;
-        metadata: PageContentHash;
-    };
+    [pageId: string]: PageContentHash;
 }
 interface GeneratedTemplate {
     pageId: string;
     templatePath: string;
     template: string;
-    hash: string;
+    structureHash: string;
     metadata: {
         url: string;
         templateAlias: string;

package/dist/module.d.ts

@@ -3,9 +3,8 @@ import * as _nuxt_schema from '@nuxt/schema';
 interface LLMSConfig {
     anthropicApiKey: string;
     umbracoDataPath: string;
-    templatesOutputDir: string;
+    templatesDir: string;
     finalOutputDir: string;
-    cacheDir?: string;
     anthropicModel?: string;
     maxConcurrent?: number;
     enableLLMSFullTxt?: boolean;
@@ -13,6 +12,7 @@ interface LLMSConfig {
     enableAutoCleanup?: boolean;
     cleanupOrphaned?: boolean;
     cleanupHidden?: boolean;
+    enableHtmlToMarkdown?: boolean;
 }
 interface UmbracoUrlItem {
     nodeID: number;
@@ -54,17 +54,13 @@ interface PageContentHash {
     lastUpdated: Date;
 }
 interface TemplateCache {
-    [pageId: string]: {
-        hash: string;
-        template: string;
-        metadata: PageContentHash;
-    };
+    [pageId: string]: PageContentHash;
 }
 interface GeneratedTemplate {
     pageId: string;
     templatePath: string;
     template: string;
-    hash: string;
+    structureHash: string;
     metadata: {
         url: string;
         templateAlias: string;

package/dist/module.json

@@ -4,5 +4,5 @@
   "compatibility": {
     "nuxt": "^3.0.0"
   },
-  "version": "0.1.0"
+  "version": "0.1.2"
 }

package/dist/module.mjs

@@ -1,4 +1,4 @@
-export { l as default } from './shared/nuxt-llms-generator.dc009f50.mjs';
+export { l as default } from './shared/nuxt-llms-generator.ab079b9f.mjs';
 import '@nuxt/kit';
 import 'fs';
 import 'path';

package/dist/shared/{nuxt-llms-generator.dc009f50.mjs → nuxt-llms-generator.ab079b9f.mjs}

@@ -10,16 +10,16 @@ const anthropicApiKey = z.string().min(10, "API key must be at least 10 characte
 const LLMSConfigSchema = z.object({
   anthropicApiKey,
   umbracoDataPath: existingPath,
-  templatesOutputDir: z.string().min(1, "Templates output directory is required"),
+  templatesDir: z.string().min(1, "Templates directory is required"),
   finalOutputDir: z.string().optional().default("public"),
-  cacheDir: z.string().optional().default(".llms-cache"),
   anthropicModel: z.string().optional(),
   maxConcurrent: z.number().int().min(1, "maxConcurrent must be at least 1").max(10, "maxConcurrent should not exceed 10 to avoid rate limits").optional().default(3),
   enableLLMSFullTxt: z.boolean().optional().default(true),
   enableIndividualMd: z.boolean().optional().default(true),
   enableAutoCleanup: z.boolean().optional().default(true),
   cleanupOrphaned: z.boolean().optional().default(true),
-  cleanupHidden: z.boolean().optional().default(true)
+  cleanupHidden: z.boolean().optional().default(true),
+  enableHtmlToMarkdown: z.boolean().optional().default(true)
 }).strict();
 class SchemaValidator {
   static validateConfig(config) {
@@ -206,11 +206,12 @@ const DEFAULT_OPTIONS = {
   maxConcurrent: 5,
   enableLLMSFullTxt: true,
   enableIndividualMd: true,
-  cacheDir: ".llms-cache",
+  templatesDir: ".llms-templates",
   finalOutputDir: "public",
   enableAutoCleanup: true,
   cleanupOrphaned: true,
-  cleanupHidden: true
+  cleanupHidden: true,
+  enableHtmlToMarkdown: true
 };
 const llmsModule = defineNuxtModule({
   meta: {
@@ -238,23 +239,23 @@ const llmsModule = defineNuxtModule({
       logger.warn("No Umbraco data path provided. LLMS Generator will be disabled.");
       return;
     }
-    if (!options.templatesOutputDir) {
-      logger.warn("No templates output directory provided. LLMS Generator will be disabled.");
+    if (!options.templatesDir) {
+      logger.warn("No templates directory provided. LLMS Generator will be disabled.");
       return;
     }
     const configForValidation = {
       anthropicApiKey: options.anthropicApiKey,
       umbracoDataPath: resolve(nuxt.options.rootDir, options.umbracoDataPath),
-      templatesOutputDir: resolve(nuxt.options.rootDir, options.templatesOutputDir),
+      templatesDir: resolve(nuxt.options.rootDir, options.templatesDir),
       finalOutputDir: resolve(nuxt.options.rootDir, options.finalOutputDir ?? "public"),
-      cacheDir: options.cacheDir ? resolve(nuxt.options.rootDir, options.cacheDir) : void 0,
       anthropicModel: options.anthropicModel || DEFAULT_OPTIONS.anthropicModel,
       maxConcurrent: options.maxConcurrent || DEFAULT_OPTIONS.maxConcurrent,
       enableLLMSFullTxt: options.enableLLMSFullTxt ?? DEFAULT_OPTIONS.enableLLMSFullTxt,
       enableIndividualMd: options.enableIndividualMd ?? DEFAULT_OPTIONS.enableIndividualMd,
       enableAutoCleanup: options.enableAutoCleanup ?? DEFAULT_OPTIONS.enableAutoCleanup,
       cleanupOrphaned: options.cleanupOrphaned ?? DEFAULT_OPTIONS.cleanupOrphaned,
-      cleanupHidden: options.cleanupHidden ?? DEFAULT_OPTIONS.cleanupHidden
+      cleanupHidden: options.cleanupHidden ?? DEFAULT_OPTIONS.cleanupHidden,
+      enableHtmlToMarkdown: options.enableHtmlToMarkdown ?? DEFAULT_OPTIONS.enableHtmlToMarkdown
     };
     let moduleOptions;
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voicenter-team/nuxt-llms-generator",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Nuxt 3 module for automatically generating AI-optimized documentation files (llms.txt, llms-full.txt, and individual .md files) from Umbraco CMS data using Anthropic's Claude API.",
   "repository": "your-org/my-module",
   "license": "MIT",
@@ -42,8 +42,11 @@
     "@anthropic-ai/sdk": "^0.30.0",
     "@nuxt/kit": "^3.11.2",
     "@voicenter-team/eslint-config-ts": "^1.0.22",
+    "i": "^0.3.7",
     "jsonpath-plus": "^8.0.0",
     "mustache": "^4.2.0",
+    "node-html-markdown": "^1.3.0",
+    "npm": "^11.6.0",
     "zod": "^4.1.9"
   },
   "devDependencies": {