@voicenter-team/nuxt-llms-generator 0.1.12 → 0.1.13

package/dist/chunks/llms-files-generator.mjs

@@ -269,6 +269,17 @@ users[2]{id,name,role}:
 
 **CRITICAL:** Use the EXACT property names shown in TOON \`{braces}\` for your Mustache bindings.
 
+---
+## \u{1F6AB} Content Exclusion Rules
+When generating the template, **DO NOT** create sections for these types of data even if they appear in TOON:
+- **Image/media properties**: URLs to images, avatars, thumbnails, icons, or any media files
+- **UI-only labels**: Search placeholders, filter menu text, "Show More"/"Show Less", pagination labels
+- **SEO/meta fields**: ogTitle, ogDescription, ogImage, canonical URLs, changefreq, priority, sitemap fields
+- **Legal boilerplate**: Copyright text, "All rights reserved"
+- **Navigation chrome**: Breadcrumbs, menu items, footer links \u2014 unless they ARE the primary page content
+- **System identifiers**: Internal IDs, GUIDs, sort orders, node paths, template aliases
+- **Empty/null values**: Skip any property that holds no meaningful value
+**Focus ONLY on**: titles, descriptions, features, pricing, reviews, specifications, contact info, and other business-relevant content.
 ---
 
 ## \u{1F3AF} TRUE PURPOSE: Help LLMs Answer Questions Efficiently
@@ -980,6 +991,11 @@ function getValueType(value) {
   return typeof value;
 }
 
+const PLACEHOLDER_PATTERNS = [
+  "lorem ipsum",
+  "dolor sit amet",
+  "consectetuer adipi"
+];
 function shouldGenerateTemplate(umbracoData, urlItem) {
   try {
     const pageContent = extractPageContent(umbracoData, urlItem.Jpath);
@@ -992,6 +1008,16 @@ function shouldGenerateTemplate(umbracoData, urlItem) {
       console.log(`Page ${urlItem.url} is hidden (hidePage: ${hidePage}), skipping template generation`);
       return false;
     }
+    const title = pageContent.pageTitle ?? pageContent.pageTittle ?? pageContent.ogTitle ?? pageContent.headerBlockTitle;
+    if (!title || title === "undefined" || title === "null") {
+      console.log(`Page ${urlItem.url} has no valid title, skipping template generation`);
+      return false;
+    }
+    const bodyText = JSON.stringify(pageContent).toLowerCase();
+    if (PLACEHOLDER_PATTERNS.some((p) => bodyText.includes(p))) {
+      console.log(`Page ${urlItem.url} contains placeholder text, skipping template generation`);
+      return false;
+    }
     return true;
   } catch (error) {
     console.error(`Error checking visibility for ${urlItem.url}:`, error);
@@ -1207,6 +1233,19 @@ async function performAutomaticCleanup(umbracoData, cacheDir, options = {}) {
   return stats;
 }
 
+function sanitizeRenderedMarkdown(markdown) {
+  let output = markdown;
+  output = output.replace(/!\[.*?]\(.*?\)/g, "");
+  output = output.replace(/^(#{1,6})\s+\d+,\s*/gm, "$1 ");
+  output = output.replace(/&#x2F;/g, "/").replace(/&#39;/g, "'").replace(/&#x27;/g, "'").replace(/&quot;/g, '"').replace(/&amp;/g, "&").replace(/&#x3D;/g, "=").replace(/&#x60;/g, "`").replace(/&lt;/g, "<").replace(/&gt;/g, ">");
+  output = output.replace(/^- .+?:\s*$/gm, "");
+  output = output.replace(/\[הרחבה]\([^)]*\)/g, "");
+  output = output.replace(/(?<!:)\/{2,}/g, "/");
+  output = output.replace(/^(#{2,6})\s+.+\n(\s*\n)+(?=#{1,6}\s|$)/gm, "");
+  output = output.replace(/\n{3,}/g, "\n\n");
+  return output.trim();
+}
+
 class TemplateGenerator {
   anthropicClient;
   promptAnalyzer;
@@ -1320,7 +1359,6 @@ class TemplateGenerator {
     if (tokensBeforeTruncation > tokensAfterTruncation) {
       console.warn(`Page ${pageId} content truncated: ${tokensBeforeTruncation} -> ${tokensAfterTruncation} tokens`);
     }
-    this.promptAnalyzer.analyzeContent(truncatedContent, urlItem);
     const request = {
       pageContent: truncatedContent,
       templateAlias: urlItem.TemplateAlias,
@@ -1351,7 +1389,14 @@ class TemplateGenerator {
   }
   async renderTemplate(template, data) {
     return withErrorHandling(async () => {
-      return Mustache.render(template, data);
+      const originalEscape = Mustache.escape;
+      Mustache.escape = (text) => text;
+      try {
+        const rendered = Mustache.render(template, data);
+        return sanitizeRenderedMarkdown(rendered);
+      } finally {
+        Mustache.escape = originalEscape;
+      }
     }, {
       template: template.substring(0, 200) + "...",
       dataKeys: Object.keys(data)
@@ -1480,7 +1525,8 @@ class LLMSFilesGenerator {
     content += `This website contains comprehensive information about ${siteTitle.toLowerCase()}. The content is organized into the following sections:
 
 `;
-    const pagesByCategory = this.groupPagesByCategory(mdFiles);
+    const deduplicatedFiles = this.deduplicateByUrl(mdFiles);
+    const pagesByCategory = this.groupPagesByCategory(deduplicatedFiles);
     for (const [category, pages] of Object.entries(pagesByCategory)) {
       if (pages.length === 0)
         continue;
@@ -1523,7 +1569,8 @@ class LLMSFilesGenerator {
 `;
     }
     content += "---\n\n";
-    for (const mdFile of mdFiles) {
+    const deduplicatedFiles = this.deduplicateByUrl(mdFiles);
+    for (const mdFile of deduplicatedFiles) {
       const urlItem = this.umbracoData.urlList.find((item) => item.url === mdFile.url);
       if (!urlItem)
         continue;
@@ -1564,6 +1611,15 @@ class LLMSFilesGenerator {
   *   /marketplace           -> category "marketplace"
   *   /                      -> category "main"
   */
+  deduplicateByUrl(mdFiles) {
+    const seen = /* @__PURE__ */ new Set();
+    return mdFiles.filter((file) => {
+      if (seen.has(file.url))
+        return false;
+      seen.add(file.url);
+      return true;
+    });
+  }
   groupPagesByCategory(mdFiles) {
     const categories = {};
     for (const mdFile of mdFiles) {
@@ -1609,8 +1665,7 @@ class LLMSFilesGenerator {
   }
   extractSiteTitle() {
     const siteData = this.umbracoData.SiteData;
-    const rawTitle = siteData?.pageTitle || siteData?.mainHeaderBlockTitle || "Website Documentation";
-    return rawTitle;
+    return siteData?.pageTitle || siteData?.mainHeaderBlockTitle || "Website Documentation";
   }
   extractSiteDescription() {
     const siteData = this.umbracoData.SiteData;
@@ -1633,11 +1688,11 @@ class LLMSFilesGenerator {
     const pageContent = extractPageContent(this.umbracoData, urlItem.Jpath);
     if (!pageContent)
       return `${urlItem.TemplateAlias} page`;
-    const desc = pageContent.pageDescription || pageContent.description || pageContent.headerBlockSubtitle;
-    if (desc && typeof desc === "string") {
-      return desc;
+    const desc = pageContent.pageDescription || pageContent.description || pageContent.headerBlockSubtitle || pageContent.ogDescription;
+    if (desc && typeof desc === "string" && desc.trim().length > 0) {
+      return desc.trim();
     }
-    return `Information about ${urlItem.url}`;
+    return `${urlItem.TemplateAlias} page`;
   }
   sanitizeUrlForFilename(url) {
     if (!url || url === "/")

package/dist/module.json

@@ -4,5 +4,5 @@
   "compatibility": {
     "nuxt": "^3.0.0"
   },
-  "version": "0.1.12"
+  "version": "0.1.13"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voicenter-team/nuxt-llms-generator",
-  "version": "0.1.12",
+  "version": "0.1.13",
   "description": "Nuxt 3 module for automatically generating AI-optimized documentation files (llms.txt, llms-full.txt, and individual .md files) from Umbraco CMS data using Anthropic's Claude API.",
   "repository": "https://github.com/VoicenterTeam/nuxt-llms-generator",
   "license": "MIT",