agentsite-kit 1.0.0

package/dist/chunk-AYLZUDQP.js

@@ -0,0 +1,281 @@
+import {
+  extractContent,
+  getLlmConfig,
+  llmExtractTags,
+  loadConfig,
+  loadPlugins,
+  log,
+  runHook,
+  sha256,
+  spinner
+} from "./chunk-YWUDTSOR.js";
+
+// src/commands/generate.ts
+import { readFileSync as readFileSync2, writeFileSync, existsSync as existsSync2, mkdirSync } from "fs";
+
+// src/generator/llms-txt.ts
+function generateLlmsTxt(scanResult, siteName, siteDescription) {
+  const lines = [];
+  lines.push(`# ${siteName}`);
+  lines.push(`> ${siteDescription}`);
+  lines.push("");
+  const grouped = /* @__PURE__ */ new Map();
+  for (const page of scanResult.pages) {
+    const group = grouped.get(page.type) ?? [];
+    group.push(page);
+    grouped.set(page.type, group);
+  }
+  const sectionOrder = ["docs", "faq", "product", "blog", "pricing", "about", "contact", "homepage", "unknown"];
+  const sectionLabels = {
+    docs: "Docs",
+    faq: "FAQ",
+    product: "Products",
+    blog: "Articles",
+    pricing: "Pricing",
+    about: "About",
+    contact: "Contact",
+    homepage: "Home",
+    unknown: "Other"
+  };
+  for (const type of sectionOrder) {
+    const pages = grouped.get(type);
+    if (!pages?.length) continue;
+    lines.push(`## ${sectionLabels[type] ?? type}`);
+    for (const page of pages) {
+      const summary = page.summary ? `: ${page.summary}` : "";
+      lines.push(`- [${page.title || page.url}](${page.url})${summary}`);
+    }
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+
+// src/generator/agent-sitemap.ts
+function generateAgentSitemap(scanResult) {
+  const entries = scanResult.pages.map((p) => ({
+    url: p.url,
+    title: p.title,
+    type: p.type,
+    summary: p.summary,
+    lastModified: p.lastModified,
+    wordCount: p.wordCount
+  }));
+  return {
+    version: "1.0",
+    generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    siteUrl: scanResult.siteUrl,
+    totalPages: entries.length,
+    pages: entries
+  };
+}
+
+// src/generator/agent-index.ts
+function generateAgentIndex(scanResult, siteName, siteDescription) {
+  const typeGroups = {};
+  for (const p of scanResult.pages) {
+    typeGroups[p.type] = (typeGroups[p.type] ?? 0) + 1;
+  }
+  const endpoints = {
+    search: "/api/search?q={query}",
+    pages: "/api/pages/{id}",
+    faq: "/api/faq",
+    products: "/api/products",
+    docs: "/api/docs",
+    articles: "/api/articles",
+    pricing: "/api/pricing",
+    changelog: "/api/changelog",
+    stats: "/api/stats",
+    config: "/api/config"
+  };
+  return {
+    version: "1.0",
+    site: {
+      name: siteName,
+      description: siteDescription,
+      url: scanResult.siteUrl
+    },
+    stats: {
+      totalPages: scanResult.totalPages,
+      scannedAt: scanResult.scannedAt,
+      pageTypes: typeGroups
+    },
+    endpoints,
+    files: {
+      "llms.txt": "LLM-friendly site overview",
+      "agent-sitemap.json": "Machine-readable sitemap with metadata",
+      "data/docs.json": "Documentation entries",
+      "data/faq.json": "FAQ entries",
+      "data/articles.json": "Blog/article entries",
+      "data/products.json": "Product entries",
+      "data/pricing.json": "Pricing/plan entries",
+      "data/changelog.json": "Changelog/release entries"
+    }
+  };
+}
+
+// src/generator/structured-export.ts
+import { readFileSync, existsSync } from "fs";
+function loadCachedHtml(outDir, url) {
+  const path = `${outDir}/cache/pages/${sha256(url)}.html`;
+  if (!existsSync(path)) return null;
+  return readFileSync(path, "utf-8");
+}
+async function tryExtractTags(llm, title, bodyText) {
+  if (!llm) return [];
+  try {
+    return await llmExtractTags(llm, title, bodyText);
+  } catch {
+    return [];
+  }
+}
+async function generateStructuredExports(scanResult, outDir, config) {
+  const docs = [];
+  const faq = [];
+  const products = [];
+  const articles = [];
+  const pricing = [];
+  const changelog = [];
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const llm = getLlmConfig(config);
+  for (const page of scanResult.pages) {
+    const html = loadCachedHtml(outDir, page.url);
+    if (!html) continue;
+    const content = extractContent(html, page.url);
+    switch (page.type) {
+      case "docs": {
+        const tags = await tryExtractTags(llm, page.title, content.bodyText);
+        docs.push({
+          title: page.title,
+          url: page.url,
+          section: page.headings[0] ?? "",
+          summary: page.summary,
+          tags,
+          updated_at: page.lastModified ?? now
+        });
+        break;
+      }
+      case "faq":
+        if (content.faqItems.length > 0) {
+          for (const item of content.faqItems) {
+            faq.push({
+              question: item.question,
+              answer: item.answer,
+              category: page.headings[0] ?? "General",
+              url: page.url,
+              updated_at: page.lastModified ?? now
+            });
+          }
+        } else {
+          faq.push({
+            question: page.title,
+            answer: page.summary,
+            category: "General",
+            url: page.url,
+            updated_at: page.lastModified ?? now
+          });
+        }
+        break;
+      case "product":
+        products.push({
+          product_name: page.title,
+          description: page.summary,
+          features: content.features.slice(0, 10),
+          pricing: "",
+          url: page.url,
+          updated_at: page.lastModified ?? now
+        });
+        break;
+      case "blog": {
+        const tags = await tryExtractTags(llm, page.title, content.bodyText);
+        articles.push({
+          title: page.title,
+          summary: page.summary,
+          published_at: page.lastModified ?? now,
+          updated_at: page.lastModified ?? now,
+          tags,
+          url: page.url
+        });
+        break;
+      }
+      case "pricing":
+        pricing.push({
+          plan_name: page.title,
+          price: content.bodyText.match(/\$\d+(?:\.\d{2})?/)?.[0] ?? "",
+          features: content.features.slice(0, 10),
+          url: page.url,
+          updated_at: page.lastModified ?? now
+        });
+        break;
+      case "changelog":
+        changelog.push({
+          version: page.version ?? page.title,
+          date: page.publishedAt ?? page.lastModified ?? now,
+          changes: content.headings.slice(1),
+          url: page.url
+        });
+        break;
+    }
+  }
+  return { docs, faq, products, articles, pricing, changelog };
+}
+
+// src/commands/generate.ts
+async function runGenerate(configOverride) {
+  const config = configOverride ?? loadConfig();
+  const outDir = config.output.dir;
+  const scanPath = `${outDir}/scan-result.json`;
+  if (!existsSync2(scanPath)) {
+    log.error("No scan result found. Run `agentsite scan` first.");
+    throw new Error("No scan result found");
+  }
+  const scanResult = JSON.parse(readFileSync2(scanPath, "utf-8"));
+  mkdirSync(`${outDir}/data`, { recursive: true });
+  const plugins = await loadPlugins(config.plugins ?? []);
+  await runHook(plugins, "beforeGenerate", scanResult);
+  const sp = spinner("Generating files...");
+  let count = 0;
+  const llmsTxt = generateLlmsTxt(scanResult, config.site.name, config.site.description);
+  writeFileSync(`${outDir}/llms.txt`, llmsTxt, "utf-8");
+  count++;
+  const sitemap = generateAgentSitemap(scanResult);
+  writeFileSync(`${outDir}/agent-sitemap.json`, JSON.stringify(sitemap, null, 2), "utf-8");
+  count++;
+  const index = generateAgentIndex(scanResult, config.site.name, config.site.description);
+  writeFileSync(`${outDir}/agent-index.json`, JSON.stringify(index, null, 2), "utf-8");
+  count++;
+  const { docs, faq, products, articles, pricing, changelog } = await generateStructuredExports(scanResult, outDir, config);
+  writeFileSync(`${outDir}/data/docs.json`, JSON.stringify(docs, null, 2), "utf-8");
+  writeFileSync(`${outDir}/data/faq.json`, JSON.stringify(faq, null, 2), "utf-8");
+  writeFileSync(`${outDir}/data/products.json`, JSON.stringify(products, null, 2), "utf-8");
+  writeFileSync(`${outDir}/data/articles.json`, JSON.stringify(articles, null, 2), "utf-8");
+  writeFileSync(`${outDir}/data/pricing.json`, JSON.stringify(pricing, null, 2), "utf-8");
+  writeFileSync(`${outDir}/data/changelog.json`, JSON.stringify(changelog, null, 2), "utf-8");
+  count += 6;
+  await runHook(plugins, "afterGenerate", outDir);
+  sp.succeed(`Generated ${count} files`);
+  log.info(`Output directory: ${outDir}/`);
+  log.info(`  llms.txt (${llmsTxt.length} bytes)`);
+  log.info(`  agent-sitemap.json (${scanResult.totalPages} pages)`);
+  log.info(`  agent-index.json`);
+  log.info(`  data/docs.json (${docs.length} entries)`);
+  log.info(`  data/faq.json (${faq.length} entries)`);
+  log.info(`  data/products.json (${products.length} entries)`);
+  log.info(`  data/articles.json (${articles.length} entries)`);
+  log.info(`  data/pricing.json (${pricing.length} entries)`);
+  log.info(`  data/changelog.json (${changelog.length} entries)`);
+}
+function registerGenerateCommand(program) {
+  program.command("generate").description("Generate Agent-friendly files from scan results").action(async () => {
+    await runGenerate();
+  });
+}
+
+export {
+  generateLlmsTxt,
+  generateAgentSitemap,
+  generateAgentIndex,
+  generateStructuredExports,
+  runGenerate,
+  registerGenerateCommand
+};
+//# sourceMappingURL=chunk-AYLZUDQP.js.map

package/dist/chunk-AYLZUDQP.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/commands/generate.ts","../src/generator/llms-txt.ts","../src/generator/agent-sitemap.ts","../src/generator/agent-index.ts","../src/generator/structured-export.ts"],"sourcesContent":["import { Command } from 'commander';\nimport { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';\nimport { loadConfig } from '../config/loader.js';\nimport { generateLlmsTxt } from '../generator/llms-txt.js';\nimport { generateAgentSitemap } from '../generator/agent-sitemap.js';\nimport { generateAgentIndex } from '../generator/agent-index.js';\nimport { generateStructuredExports } from '../generator/structured-export.js';\nimport { loadPlugins, runHook } from '../plugins/loader.js';\nimport { log, spinner } from '../utils/logger.js';\nimport type { ScanResult } from '../types/page.js';\n\nexport async function runGenerate(configOverride?: ReturnType<typeof loadConfig>): Promise<void> {\n  const config = configOverride ?? loadConfig();\n  const outDir = config.output.dir;\n  const scanPath = `${outDir}/scan-result.json`;\n\n  if (!existsSync(scanPath)) {\n    log.error('No scan result found. Run `agentsite scan` first.');\n    throw new Error('No scan result found');\n  }\n\n  const scanResult: ScanResult = JSON.parse(readFileSync(scanPath, 'utf-8'));\n  mkdirSync(`${outDir}/data`, { recursive: true });\n\n  // Load plugins\n  const plugins = await loadPlugins(config.plugins ?? []);\n  await runHook(plugins, 'beforeGenerate', scanResult);\n\n  const sp = spinner('Generating files...');\n  let count = 0;\n\n  // llms.txt\n  const llmsTxt = generateLlmsTxt(scanResult, config.site.name, config.site.description);\n  writeFileSync(`${outDir}/llms.txt`, llmsTxt, 'utf-8');\n  count++;\n\n  // agent-sitemap.json\n  const sitemap = generateAgentSitemap(scanResult);\n  writeFileSync(`${outDir}/agent-sitemap.json`, JSON.stringify(sitemap, null, 2), 'utf-8');\n  count++;\n\n  // agent-index.json\n  const index = generateAgentIndex(scanResult, config.site.name, config.site.description);\n  writeFileSync(`${outDir}/agent-index.json`, JSON.stringify(index, null, 2), 'utf-8');\n  count++;\n\n  // Structured exports\n  const { docs, faq, products, articles, pricing, changelog } = await generateStructuredExports(scanResult, outDir, config);\n  writeFileSync(`${outDir}/data/docs.json`, JSON.stringify(docs, null, 2), 'utf-8');\n  writeFileSync(`${outDir}/data/faq.json`, JSON.stringify(faq, null, 2), 'utf-8');\n  writeFileSync(`${outDir}/data/products.json`, JSON.stringify(products, null, 2), 'utf-8');\n  writeFileSync(`${outDir}/data/articles.json`, JSON.stringify(articles, null, 2), 'utf-8');\n  writeFileSync(`${outDir}/data/pricing.json`, JSON.stringify(pricing, null, 2), 'utf-8');\n  writeFileSync(`${outDir}/data/changelog.json`, JSON.stringify(changelog, null, 2), 'utf-8');\n  count += 6;\n\n  // Run afterGenerate plugin hooks\n  await runHook(plugins, 'afterGenerate', outDir);\n\n  sp.succeed(`Generated ${count} files`);\n\n  log.info(`Output directory: ${outDir}/`);\n  log.info(`  llms.txt (${llmsTxt.length} bytes)`);\n  log.info(`  agent-sitemap.json (${scanResult.totalPages} pages)`);\n  log.info(`  agent-index.json`);\n  log.info(`  data/docs.json (${docs.length} entries)`);\n  log.info(`  data/faq.json (${faq.length} entries)`);\n  log.info(`  data/products.json (${products.length} entries)`);\n  log.info(`  data/articles.json (${articles.length} entries)`);\n  log.info(`  data/pricing.json (${pricing.length} entries)`);\n  log.info(`  data/changelog.json (${changelog.length} entries)`);\n}\n\nexport function registerGenerateCommand(program: Command) {\n  program\n    .command('generate')\n    .description('Generate Agent-friendly files from scan results')\n    .action(async () => {\n      await runGenerate();\n    });\n}\n","import type { ScanResult } from '../types/page.js';\n\nexport function generateLlmsTxt(scanResult: ScanResult, siteName: string, siteDescription: string): string {\n  const lines: string[] = [];\n  lines.push(`# ${siteName}`);\n  lines.push(`> ${siteDescription}`);\n  lines.push('');\n\n  const grouped = new Map<string, typeof scanResult.pages>();\n  for (const page of scanResult.pages) {\n    const group = grouped.get(page.type) ?? [];\n    group.push(page);\n    grouped.set(page.type, group);\n  }\n\n  const sectionOrder = ['docs', 'faq', 'product', 'blog', 'pricing', 'about', 'contact', 'homepage', 'unknown'];\n  const sectionLabels: Record<string, string> = {\n    docs: 'Docs',\n    faq: 'FAQ',\n    product: 'Products',\n    blog: 'Articles',\n    pricing: 'Pricing',\n    about: 'About',\n    contact: 'Contact',\n    homepage: 'Home',\n    unknown: 'Other',\n  };\n\n  for (const type of sectionOrder) {\n    const pages = grouped.get(type);\n    if (!pages?.length) continue;\n\n    lines.push(`## ${sectionLabels[type] ?? type}`);\n    for (const page of pages) {\n      const summary = page.summary ? `: ${page.summary}` : '';\n      lines.push(`- [${page.title || page.url}](${page.url})${summary}`);\n    }\n    lines.push('');\n  }\n\n  return lines.join('\\n');\n}\n","import type { ScanResult } from '../types/page.js';\n\ninterface AgentSitemapEntry {\n  url: string;\n  title: string;\n  type: string;\n  summary: string;\n  lastModified?: string;\n  wordCount: number;\n}\n\nexport function generateAgentSitemap(scanResult: ScanResult): object {\n  const entries: AgentSitemapEntry[] = scanResult.pages.map((p) => ({\n    url: p.url,\n    title: p.title,\n    type: p.type,\n    summary: p.summary,\n    lastModified: p.lastModified,\n    wordCount: p.wordCount,\n  }));\n\n  return {\n    version: '1.0',\n    generatedAt: new Date().toISOString(),\n    siteUrl: scanResult.siteUrl,\n    totalPages: entries.length,\n    pages: entries,\n  };\n}\n","import type { ScanResult } from '../types/page.js';\n\nexport function generateAgentIndex(scanResult: ScanResult, siteName: string, siteDescription: string): object {\n  const typeGroups: Record<string, number> = {};\n  for (const p of scanResult.pages) {\n    typeGroups[p.type] = (typeGroups[p.type] ?? 0) + 1;\n  }\n\n  const endpoints: Record<string, string> = {\n    search: '/api/search?q={query}',\n    pages: '/api/pages/{id}',\n    faq: '/api/faq',\n    products: '/api/products',\n    docs: '/api/docs',\n    articles: '/api/articles',\n    pricing: '/api/pricing',\n    changelog: '/api/changelog',\n    stats: '/api/stats',\n    config: '/api/config',\n  };\n\n  return {\n    version: '1.0',\n    site: {\n      name: siteName,\n      description: siteDescription,\n      url: scanResult.siteUrl,\n    },\n    stats: {\n      totalPages: scanResult.totalPages,\n      scannedAt: scanResult.scannedAt,\n      pageTypes: typeGroups,\n    },\n    endpoints,\n    files: {\n      'llms.txt': 'LLM-friendly site overview',\n      'agent-sitemap.json': 'Machine-readable sitemap with metadata',\n      'data/docs.json': 'Documentation entries',\n      'data/faq.json': 'FAQ entries',\n      'data/articles.json': 'Blog/article entries',\n      'data/products.json': 'Product entries',\n      'data/pricing.json': 'Pricing/plan entries',\n      'data/changelog.json': 'Changelog/release entries',\n    },\n  };\n}\n","import { readFileSync, existsSync } from 'node:fs';\nimport { sha256 } from '../utils/hash.js';\nimport { extractContent } from '../scanner/content-extractor.js';\nimport { getLlmConfig, type LlmConfig } from '../llm/client.js';\nimport { llmExtractTags } from '../llm/summarizer.js';\nimport type { ScanResult } from '../types/page.js';\nimport type { DocEntry, FaqEntry, ProductEntry, ArticleEntry, PricingEntry, ChangelogEntry } from '../types/content.js';\nimport type { AgentSiteConfig } from '../types/config.js';\n\nfunction loadCachedHtml(outDir: string, url: string): string | null {\n  const path = `${outDir}/cache/pages/${sha256(url)}.html`;\n  if (!existsSync(path)) return null;\n  return readFileSync(path, 'utf-8');\n}\n\nasync function tryExtractTags(llm: LlmConfig | null, title: string, bodyText: string): Promise<string[]> {\n  if (!llm) return [];\n  try {\n    return await llmExtractTags(llm, title, bodyText);\n  } catch {\n    return [];\n  }\n}\n\nexport async function generateStructuredExports(scanResult: ScanResult, outDir: string, config?: AgentSiteConfig) {\n  const docs: DocEntry[] = [];\n  const faq: FaqEntry[] = [];\n  const products: ProductEntry[] = [];\n  const articles: ArticleEntry[] = [];\n  const pricing: PricingEntry[] = [];\n  const changelog: ChangelogEntry[] = [];\n  const now = new Date().toISOString();\n  const llm = getLlmConfig(config);\n\n  for (const page of scanResult.pages) {\n    const html = loadCachedHtml(outDir, page.url);\n    if (!html) continue;\n\n    const content = extractContent(html, page.url);\n\n    switch (page.type) {\n      case 'docs': {\n        const tags = await tryExtractTags(llm, page.title, content.bodyText);\n        docs.push({\n          title: page.title,\n          url: page.url,\n          section: page.headings[0] ?? '',\n          summary: page.summary,\n          tags,\n          updated_at: page.lastModified ?? now,\n        });\n        break;\n      }\n\n      case 'faq':\n        if (content.faqItems.length > 0) {\n          for (const item of content.faqItems) {\n            faq.push({\n              question: item.question,\n              answer: item.answer,\n              category: page.headings[0] ?? 'General',\n              url: page.url,\n              updated_at: page.lastModified ?? now,\n            });\n          }\n        } else {\n          faq.push({\n            question: page.title,\n            answer: page.summary,\n            category: 'General',\n            url: page.url,\n            updated_at: page.lastModified ?? now,\n          });\n        }\n        break;\n\n      case 'product':\n        products.push({\n          product_name: page.title,\n          description: page.summary,\n          features: content.features.slice(0, 10),\n          pricing: '',\n          url: page.url,\n          updated_at: page.lastModified ?? now,\n        });\n        break;\n\n      case 'blog': {\n        const tags = await tryExtractTags(llm, page.title, content.bodyText);\n        articles.push({\n          title: page.title,\n          summary: page.summary,\n          published_at: page.lastModified ?? now,\n          updated_at: page.lastModified ?? now,\n          tags,\n          url: page.url,\n        });\n        break;\n      }\n\n      case 'pricing':\n        pricing.push({\n          plan_name: page.title,\n          price: content.bodyText.match(/\\$\\d+(?:\\.\\d{2})?/)?.[0] ?? '',\n          features: content.features.slice(0, 10),\n          url: page.url,\n          updated_at: page.lastModified ?? now,\n        });\n        break;\n\n      case 'changelog':\n        changelog.push({\n          version: page.version ?? page.title,\n          date: page.publishedAt ?? page.lastModified ?? now,\n          changes: content.headings.slice(1),\n          url: page.url,\n        });\n        break;\n    }\n  }\n\n  return { docs, faq, products, articles, pricing, changelog };\n}\n"],"mappings":";;;;;;;;;;;;;AACA,SAAS,gBAAAA,eAAc,eAAe,cAAAC,aAAY,iBAAiB;;;ACC5D,SAAS,gBAAgB,YAAwB,UAAkB,iBAAiC;AACzG,QAAM,QAAkB,CAAC;AACzB,QAAM,KAAK,KAAK,QAAQ,EAAE;AAC1B,QAAM,KAAK,KAAK,eAAe,EAAE;AACjC,QAAM,KAAK,EAAE;AAEb,QAAM,UAAU,oBAAI,IAAqC;AACzD,aAAW,QAAQ,WAAW,OAAO;AACnC,UAAM,QAAQ,QAAQ,IAAI,KAAK,IAAI,KAAK,CAAC;AACzC,UAAM,KAAK,IAAI;AACf,YAAQ,IAAI,KAAK,MAAM,KAAK;AAAA,EAC9B;AAEA,QAAM,eAAe,CAAC,QAAQ,OAAO,WAAW,QAAQ,WAAW,SAAS,WAAW,YAAY,SAAS;AAC5G,QAAM,gBAAwC;AAAA,IAC5C,MAAM;AAAA,IACN,KAAK;AAAA,IACL,SAAS;AAAA,IACT,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO;AAAA,IACP,SAAS;AAAA,IACT,UAAU;AAAA,IACV,SAAS;AAAA,EACX;AAEA,aAAW,QAAQ,cAAc;AAC/B,UAAM,QAAQ,QAAQ,IAAI,IAAI;AAC9B,QAAI,CAAC,OAAO,OAAQ;AAEpB,UAAM,KAAK,MAAM,cAAc,IAAI,KAAK,IAAI,EAAE;AAC9C,eAAW,QAAQ,OAAO;AACxB,YAAM,UAAU,KAAK,UAAU,KAAK,KAAK,OAAO,KAAK;AACrD,YAAM,KAAK,MAAM,KAAK,SAAS,KAAK,GAAG,KAAK,KAAK,GAAG,IAAI,OAAO,EAAE;AAAA,IACnE;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;;;AC9BO,SAAS,qBAAqB,YAAgC;AACnE,QAAM,UAA+B,WAAW,MAAM,IAAI,CAAC,OAAO;AAAA,IAChE,KAAK,EAAE;AAAA,IACP,OAAO,EAAE;AAAA,IACT,MAAM,EAAE;AAAA,IACR,SAAS,EAAE;AAAA,IACX,cAAc,EAAE;AAAA,IAChB,WAAW,EAAE;AAAA,EACf,EAAE;AAEF,SAAO;AAAA,IACL,SAAS;AAAA,IACT,cAAa,oBAAI,KAAK,GAAE,YAAY;AAAA,IACpC,SAAS,WAAW;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,OAAO;AAAA,EACT;AACF;;;AC1BO,SAAS,mBAAmB,YAAwB,UAAkB,iBAAiC;AAC5G,QAAM,aAAqC,CAAC;AAC5C,aAAW,KAAK,WAAW,OAAO;AAChC,eAAW,EAAE,IAAI,KAAK,WAAW,EAAE,IAAI,KAAK,KAAK;AAAA,EACnD;AAEA,QAAM,YAAoC;AAAA,IACxC,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,KAAK;AAAA,IACL,UAAU;AAAA,IACV,MAAM;AAAA,IACN,UAAU;AAAA,IACV,SAAS;AAAA,IACT,WAAW;AAAA,IACX,OAAO;AAAA,IACP,QAAQ;AAAA,EACV;AAEA,SAAO;AAAA,IACL,SAAS;AAAA,IACT,MAAM;AAAA,MACJ,MAAM;AAAA,MACN,aAAa;AAAA,MACb,KAAK,WAAW;AAAA,IAClB;AAAA,IACA,OAAO;AAAA,MACL,YAAY,WAAW;AAAA,MACvB,WAAW,WAAW;AAAA,MACtB,WAAW;AAAA,IACb;AAAA,IACA;AAAA,IACA,OAAO;AAAA,MACL,YAAY;AAAA,MACZ,sBAAsB;AAAA,MACtB,kBAAkB;AAAA,MAClB,iBAAiB;AAAA,MACjB,sBAAsB;AAAA,MACtB,sBAAsB;AAAA,MACtB,qBAAqB;AAAA,MACrB,uBAAuB;AAAA,IACzB;AAAA,EACF;AACF;;;AC7CA,SAAS,cAAc,kBAAkB;AASzC,SAAS,eAAe,QAAgB,KAA4B;AAClE,QAAM,OAAO,GAAG,MAAM,gBAAgB,OAAO,GAAG,CAAC;AACjD,MAAI,CAAC,WAAW,IAAI,EAAG,QAAO;AAC9B,SAAO,aAAa,MAAM,OAAO;AACnC;AAEA,eAAe,eAAe,KAAuB,OAAe,UAAqC;AACvG,MAAI,CAAC,IAAK,QAAO,CAAC;AAClB,MAAI;AACF,WAAO,MAAM,eAAe,KAAK,OAAO,QAAQ;AAAA,EAClD,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;AAEA,eAAsB,0BAA0B,YAAwB,QAAgB,QAA0B;AAChH,QAAM,OAAmB,CAAC;AAC1B,QAAM,MAAkB,CAAC;AACzB,QAAM,WAA2B,CAAC;AAClC,QAAM,WAA2B,CAAC;AAClC,QAAM,UAA0B,CAAC;AACjC,QAAM,YAA8B,CAAC;AACrC,QAAM,OAAM,oBAAI,KAAK,GAAE,YAAY;AACnC,QAAM,MAAM,aAAa,MAAM;AAE/B,aAAW,QAAQ,WAAW,OAAO;AACnC,UAAM,OAAO,eAAe,QAAQ,KAAK,GAAG;AAC5C,QAAI,CAAC,KAAM;AAEX,UAAM,UAAU,eAAe,MAAM,KAAK,GAAG;AAE7C,YAAQ,KAAK,MAAM;AAAA,MACjB,KAAK,QAAQ;AACX,cAAM,OAAO,MAAM,eAAe,KAAK,KAAK,OAAO,QAAQ,QAAQ;AACnE,aAAK,KAAK;AAAA,UACR,OAAO,KAAK;AAAA,UACZ,KAAK,KAAK;AAAA,UACV,SAAS,KAAK,SAAS,CAAC,KAAK;AAAA,UAC7B,SAAS,KAAK;AAAA,UACd;AAAA,UACA,YAAY,KAAK,gBAAgB;AAAA,QACnC,CAAC;AACD;AAAA,MACF;AAAA,MAEA,KAAK;AACH,YAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,qBAAW,QAAQ,QAAQ,UAAU;AACnC,gBAAI,KAAK;AAAA,cACP,UAAU,KAAK;AAAA,cACf,QAAQ,KAAK;AAAA,cACb,UAAU,KAAK,SAAS,CAAC,KAAK;AAAA,cAC9B,KAAK,KAAK;AAAA,cACV,YAAY,KAAK,gBAAgB;AAAA,YACnC,CAAC;AAAA,UACH;AAAA,QACF,OAAO;AACL,cAAI,KAAK;AAAA,YACP,UAAU,KAAK;AAAA,YACf,QAAQ,KAAK;AAAA,YACb,UAAU;AAAA,YACV,KAAK,KAAK;AAAA,YACV,YAAY,KAAK,gBAAgB;AAAA,UACnC,CAAC;AAAA,QACH;AACA;AAAA,MAEF,KAAK;AACH,iBAAS,KAAK;AAAA,UACZ,cAAc,KAAK;AAAA,UACnB,aAAa,KAAK;AAAA,UAClB,UAAU,QAAQ,SAAS,MAAM,GAAG,EAAE;AAAA,UACtC,SAAS;AAAA,UACT,KAAK,KAAK;AAAA,UACV,YAAY,KAAK,gBAAgB;AAAA,QACnC,CAAC;AACD;AAAA,MAEF,KAAK,QAAQ;AACX,cAAM,OAAO,MAAM,eAAe,KAAK,KAAK,OAAO,QAAQ,QAAQ;AACnE,iBAAS,KAAK;AAAA,UACZ,OAAO,KAAK;AAAA,UACZ,SAAS,KAAK;AAAA,UACd,cAAc,KAAK,gBAAgB;AAAA,UACnC,YAAY,KAAK,gBAAgB;AAAA,UACjC;AAAA,UACA,KAAK,KAAK;AAAA,QACZ,CAAC;AACD;AAAA,MACF;AAAA,MAEA,KAAK;AACH,gBAAQ,KAAK;AAAA,UACX,WAAW,KAAK;AAAA,UAChB,OAAO,QAAQ,SAAS,MAAM,mBAAmB,IAAI,CAAC,KAAK;AAAA,UAC3D,UAAU,QAAQ,SAAS,MAAM,GAAG,EAAE;AAAA,UACtC,KAAK,KAAK;AAAA,UACV,YAAY,KAAK,gBAAgB;AAAA,QACnC,CAAC;AACD;AAAA,MAEF,KAAK;AACH,kBAAU,KAAK;AAAA,UACb,SAAS,KAAK,WAAW,KAAK;AAAA,UAC9B,MAAM,KAAK,eAAe,KAAK,gBAAgB;AAAA,UAC/C,SAAS,QAAQ,SAAS,MAAM,CAAC;AAAA,UACjC,KAAK,KAAK;AAAA,QACZ,CAAC;AACD;AAAA,IACJ;AAAA,EACF;AAEA,SAAO,EAAE,MAAM,KAAK,UAAU,UAAU,SAAS,UAAU;AAC7D;;;AJ/GA,eAAsB,YAAY,gBAA+D;AAC/F,QAAM,SAAS,kBAAkB,WAAW;AAC5C,QAAM,SAAS,OAAO,OAAO;AAC7B,QAAM,WAAW,GAAG,MAAM;AAE1B,MAAI,CAACC,YAAW,QAAQ,GAAG;AACzB,QAAI,MAAM,mDAAmD;AAC7D,UAAM,IAAI,MAAM,sBAAsB;AAAA,EACxC;AAEA,QAAM,aAAyB,KAAK,MAAMC,cAAa,UAAU,OAAO,CAAC;AACzE,YAAU,GAAG,MAAM,SAAS,EAAE,WAAW,KAAK,CAAC;AAG/C,QAAM,UAAU,MAAM,YAAY,OAAO,WAAW,CAAC,CAAC;AACtD,QAAM,QAAQ,SAAS,kBAAkB,UAAU;AAEnD,QAAM,KAAK,QAAQ,qBAAqB;AACxC,MAAI,QAAQ;AAGZ,QAAM,UAAU,gBAAgB,YAAY,OAAO,KAAK,MAAM,OAAO,KAAK,WAAW;AACrF,gBAAc,GAAG,MAAM,aAAa,SAAS,OAAO;AACpD;AAGA,QAAM,UAAU,qBAAqB,UAAU;AAC/C,gBAAc,GAAG,MAAM,uBAAuB,KAAK,UAAU,SAAS,MAAM,CAAC,GAAG,OAAO;AACvF;AAGA,QAAM,QAAQ,mBAAmB,YAAY,OAAO,KAAK,MAAM,OAAO,KAAK,WAAW;AACtF,gBAAc,GAAG,MAAM,qBAAqB,KAAK,UAAU,OAAO,MAAM,CAAC,GAAG,OAAO;AACnF;AAGA,QAAM,EAAE,MAAM,KAAK,UAAU,UAAU,SAAS,UAAU,IAAI,MAAM,0BAA0B,YAAY,QAAQ,MAAM;AACxH,gBAAc,GAAG,MAAM,mBAAmB,KAAK,UAAU,MAAM,MAAM,CAAC,GAAG,OAAO;AAChF,gBAAc,GAAG,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,CAAC,GAAG,OAAO;AAC9E,gBAAc,GAAG,MAAM,uBAAuB,KAAK,UAAU,UAAU,MAAM,CAAC,GAAG,OAAO;AACxF,gBAAc,GAAG,MAAM,uBAAuB,KAAK,UAAU,UAAU,MAAM,CAAC,GAAG,OAAO;AACxF,gBAAc,GAAG,MAAM,sBAAsB,KAAK,UAAU,SAAS,MAAM,CAAC,GAAG,OAAO;AACtF,gBAAc,GAAG,MAAM,wBAAwB,KAAK,UAAU,WAAW,MAAM,CAAC,GAAG,OAAO;AAC1F,WAAS;AAGT,QAAM,QAAQ,SAAS,iBAAiB,MAAM;AAE9C,KAAG,QAAQ,aAAa,KAAK,QAAQ;AAErC,MAAI,KAAK,qBAAqB,MAAM,GAAG;AACvC,MAAI,KAAK,eAAe,QAAQ,MAAM,SAAS;AAC/C,MAAI,KAAK,yBAAyB,WAAW,UAAU,SAAS;AAChE,MAAI,KAAK,oBAAoB;AAC7B,MAAI,KAAK,qBAAqB,KAAK,MAAM,WAAW;AACpD,MAAI,KAAK,oBAAoB,IAAI,MAAM,WAAW;AAClD,MAAI,KAAK,yBAAyB,SAAS,MAAM,WAAW;AAC5D,MAAI,KAAK,yBAAyB,SAAS,MAAM,WAAW;AAC5D,MAAI,KAAK,wBAAwB,QAAQ,MAAM,WAAW;AAC1D,MAAI,KAAK,0BAA0B,UAAU,MAAM,WAAW;AAChE;AAEO,SAAS,wBAAwB,SAAkB;AACxD,UACG,QAAQ,UAAU,EAClB,YAAY,iDAAiD,EAC7D,OAAO,YAAY;AAClB,UAAM,YAAY;AAAA,EACpB,CAAC;AACL;","names":["readFileSync","existsSync","existsSync","readFileSync"]}

package/dist/chunk-YWR5EH3F.js

@@ -0,0 +1,339 @@
+import {
+  chatCompletion,
+  extractContent,
+  getLlmConfig,
+  llmSummarize,
+  loadConfig,
+  loadPlugins,
+  log,
+  runAfterScan,
+  runHook,
+  sha256,
+  spinner
+} from "./chunk-YWUDTSOR.js";
+
+// src/commands/scan.ts
+import { writeFileSync as writeFileSync2, mkdirSync as mkdirSync2 } from "fs";
+
+// src/scanner/crawler.ts
+import axios from "axios";
+import * as cheerio from "cheerio";
+import { writeFileSync, mkdirSync } from "fs";
+import pLimit from "p-limit";
+
+// src/utils/url.ts
+function normalizeUrl(raw) {
+  let url = raw.trim();
+  if (!/^https?:\/\//i.test(url)) url = "https://" + url;
+  return url.replace(/\/+$/, "");
+}
+function isSameOrigin(base, target) {
+  try {
+    return new URL(base).origin === new URL(target).origin;
+  } catch {
+    return false;
+  }
+}
+function urlToId(url) {
+  return Buffer.from(url).toString("base64url");
+}
+
+// src/scanner/crawler.ts
+async function crawlSite(config, seedUrls, onPage) {
+  const { maxPages, concurrency, delayMs } = config.scan;
+  const baseUrl = config.site.url;
+  const visited = /* @__PURE__ */ new Set();
+  const queue = [...seedUrls, baseUrl];
+  const results = [];
+  const limit = pLimit(concurrency);
+  const cacheDir = `${config.output.dir}/cache/pages`;
+  mkdirSync(cacheDir, { recursive: true });
+  const delay = (ms) => new Promise((r) => setTimeout(r, ms));
+  while (queue.length > 0 && visited.size < maxPages) {
+    const batch = queue.splice(0, concurrency).filter((u) => !visited.has(u));
+    if (batch.length === 0) continue;
+    const tasks = batch.map(
+      (url) => limit(async () => {
+        if (visited.has(url) || visited.size >= maxPages) return null;
+        visited.add(url);
+        try {
+          const res = await axios.get(url, {
+            timeout: 15e3,
+            headers: { "User-Agent": "AgentSite-Kit/0.1" },
+            maxRedirects: 3
+          });
+          const html = res.data;
+          onPage?.(url, visited.size);
+          const filename = sha256(url);
+          writeFileSync(`${cacheDir}/${filename}.html`, html, "utf-8");
+          const $ = cheerio.load(html);
+          $("a[href]").each((_, el) => {
+            try {
+              const href = $(el).attr("href");
+              if (!href) return;
+              const resolved = new URL(href, url).href.split("#")[0].split("?")[0];
+              if (isSameOrigin(baseUrl, resolved) && !visited.has(resolved)) {
+                queue.push(resolved);
+              }
+            } catch {
+            }
+          });
+          if (delayMs > 0) await delay(delayMs);
+          return { url, html, status: res.status };
+        } catch {
+          return null;
+        }
+      })
+    );
+    const settled = await Promise.all(tasks);
+    for (const r of settled) {
+      if (r) results.push(r);
+    }
+  }
+  return results;
+}
+
+// src/scanner/sitemap-parser.ts
+import axios2 from "axios";
+import { XMLParser } from "fast-xml-parser";
+async function parseSitemap(siteUrl) {
+  const urls = [];
+  const sitemapUrl = `${siteUrl}/sitemap.xml`;
+  try {
+    const res = await axios2.get(sitemapUrl, {
+      timeout: 1e4,
+      headers: { "User-Agent": "AgentSite-Kit/0.1" }
+    });
+    const parser = new XMLParser();
+    const parsed = parser.parse(res.data);
+    if (parsed.sitemapindex?.sitemap) {
+      const sitemaps = Array.isArray(parsed.sitemapindex.sitemap) ? parsed.sitemapindex.sitemap : [parsed.sitemapindex.sitemap];
+      for (const sm of sitemaps) {
+        if (sm.loc) {
+          const childUrls = await parseSitemapFromUrl(sm.loc);
+          urls.push(...childUrls);
+        }
+      }
+    }
+    if (parsed.urlset?.url) {
+      const entries = Array.isArray(parsed.urlset.url) ? parsed.urlset.url : [parsed.urlset.url];
+      for (const entry of entries) {
+        if (entry.loc) urls.push(entry.loc);
+      }
+    }
+  } catch {
+  }
+  return urls;
+}
+async function parseSitemapFromUrl(url) {
+  try {
+    const res = await axios2.get(url, {
+      timeout: 1e4,
+      headers: { "User-Agent": "AgentSite-Kit/0.1" }
+    });
+    const parser = new XMLParser();
+    const parsed = parser.parse(res.data);
+    if (parsed.urlset?.url) {
+      const entries = Array.isArray(parsed.urlset.url) ? parsed.urlset.url : [parsed.urlset.url];
+      return entries.map((e) => e.loc).filter(Boolean);
+    }
+  } catch {
+  }
+  return [];
+}
+
+// src/scanner/page-classifier.ts
+var URL_PATTERNS = [
+  [/\/(docs?|documentation|guide|manual|reference|api-docs)(\/|$)/i, "docs", 3],
+  [/\/(faq|frequently-asked|help\/faq)(\/|$)/i, "faq", 3],
+  [/\/(blog|posts?|articles?|news)(\/|$)/i, "blog", 3],
+  [/\/(products?|features?|solutions?)(\/|$)/i, "product", 2],
+  [/\/(pricing|plans?|packages?)(\/|$)/i, "pricing", 3],
+  [/\/(about|about-us|team|company)(\/|$)/i, "about", 3],
+  [/\/(contact|contact-us|support)(\/|$)/i, "contact", 3],
+  [/\/(changelog|changes|release-notes|releases|updates|whats-new)(\/|$)/i, "changelog", 3]
+];
+var TITLE_KEYWORDS = [
+  [/\b(documentation|docs|guide|reference|api)\b/i, "docs", 2],
+  [/\b(faq|frequently asked|common questions)\b/i, "faq", 2],
+  [/\b(blog|article|post)\b/i, "blog", 2],
+  [/\b(pricing|plans|packages)\b/i, "pricing", 2],
+  [/\b(product|feature|solution)\b/i, "product", 1],
+  [/\b(about|team|company|who we are)\b/i, "about", 2],
+  [/\b(contact|get in touch|support)\b/i, "contact", 2],
+  [/\b(changelog|release notes|what's new|updates|releases)\b/i, "changelog", 2]
+];
+function classifyPage(input) {
+  const { url, title, bodyText } = input;
+  const scores = /* @__PURE__ */ new Map();
+  const add = (type, weight) => {
+    scores.set(type, (scores.get(type) ?? 0) + weight);
+  };
+  try {
+    const parsed = new URL(url);
+    if (parsed.pathname === "/" || parsed.pathname === "") {
+      return "homepage";
+    }
+  } catch {
+  }
+  for (const [pattern, type, weight] of URL_PATTERNS) {
+    if (pattern.test(url)) add(type, weight);
+  }
+  for (const [pattern, type, weight] of TITLE_KEYWORDS) {
+    if (pattern.test(title)) add(type, weight);
+  }
+  const qaPairs = (bodyText.match(/\?[\s\n]/g) || []).length;
+  if (qaPairs >= 3) add("faq", 2);
+  if (/\$\d+|\d+\/mo|per month|free tier/i.test(bodyText)) add("pricing", 2);
+  let best = "unknown";
+  let bestScore = 1;
+  for (const [type, score] of scores) {
+    if (score > bestScore) {
+      best = type;
+      bestScore = score;
+    }
+  }
+  return best;
+}
+
+// src/llm/classifier.ts
+var VALID_TYPES = ["homepage", "docs", "faq", "blog", "product", "pricing", "about", "contact", "unknown"];
+async function llmClassifyPage(config, url, title, bodyText) {
+  const truncated = bodyText.slice(0, 2e3);
+  const response = await chatCompletion(config, [
+    {
+      role: "system",
+      content: `You are a web page classifier. Classify the page into exactly one of these types: ${VALID_TYPES.join(", ")}. Reply with only the type name, nothing else.`
+    },
+    {
+      role: "user",
+      content: `URL: ${url}
+Title: ${title}
+
+Content:
+${truncated}`
+    }
+  ], { temperature: 0.1, maxTokens: 32 });
+  const cleaned = response.toLowerCase().trim();
+  if (VALID_TYPES.includes(cleaned)) return cleaned;
+  return "unknown";
+}
+
+// src/commands/scan.ts
+async function runScan(configOverride) {
+  const config = configOverride ?? loadConfig();
+  const outDir = config.output.dir;
+  mkdirSync2(`${outDir}/cache/pages`, { recursive: true });
+  mkdirSync2(`${outDir}/data`, { recursive: true });
+  const plugins = await loadPlugins(config.plugins ?? []);
+  await runHook(plugins, "beforeScan", config);
+  const llmConfig = getLlmConfig(config);
+  if (llmConfig) {
+    log.info("LLM enabled \u2014 using AI-assisted classification & summarization");
+  }
+  const sp = spinner("Fetching sitemap...");
+  const sitemapUrls = await parseSitemap(config.site.url);
+  if (sitemapUrls.length > 0) {
+    sp.succeed(`Found ${sitemapUrls.length} URLs in sitemap`);
+  } else {
+    sp.info("No sitemap found, will discover pages by crawling");
+  }
+  const crawlSp = spinner("Crawling site...");
+  const crawled = await crawlSite(config, sitemapUrls, (url, i) => {
+    crawlSp.text = `Crawling (${i}/${config.scan.maxPages})... ${url}`;
+  });
+  crawlSp.succeed(`Crawled ${crawled.length} pages`);
+  const extractSp = spinner("Extracting content...");
+  let pages = [];
+  for (let idx = 0; idx < crawled.length; idx++) {
+    const { url, html } = crawled[idx];
+    const content = extractContent(html, url);
+    extractSp.text = `Processing (${idx + 1}/${crawled.length})... ${url}`;
+    let pageType;
+    if (llmConfig) {
+      try {
+        pageType = await llmClassifyPage(llmConfig, url, content.title, content.bodyText);
+      } catch {
+        pageType = classifyPage({
+          url,
+          title: content.title,
+          metaOgType: content.metaOgType,
+          headings: content.headings,
+          bodyText: content.bodyText
+        });
+      }
+    } else {
+      pageType = classifyPage({
+        url,
+        title: content.title,
+        metaOgType: content.metaOgType,
+        headings: content.headings,
+        bodyText: content.bodyText
+      });
+    }
+    let summary = content.summary;
+    if (llmConfig) {
+      try {
+        summary = await llmSummarize(llmConfig, content.title, content.bodyText);
+      } catch {
+      }
+    }
+    pages.push({
+      url,
+      title: content.title,
+      type: pageType,
+      contentHash: sha256(content.bodyText),
+      summary,
+      headings: content.headings,
+      lastModified: content.lastModified,
+      scannedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      wordCount: content.wordCount,
+      tags: content.tags,
+      version: content.version,
+      author: content.author,
+      publishedAt: content.publishedAt,
+      updatedAt: content.lastModified
+    });
+  }
+  extractSp.succeed(`Extracted content from ${pages.length} pages`);
+  pages = await runAfterScan(plugins, pages);
+  const result = {
+    siteUrl: config.site.url,
+    scannedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    totalPages: pages.length,
+    pages
+  };
+  writeFileSync2(`${outDir}/scan-result.json`, JSON.stringify(result, null, 2), "utf-8");
+  log.success(`Scan result saved to ${outDir}/scan-result.json`);
+  const typeCounts = /* @__PURE__ */ new Map();
+  for (const p of pages) {
+    typeCounts.set(p.type, (typeCounts.get(p.type) ?? 0) + 1);
+  }
+  log.info("Page types:");
+  for (const [type, count] of typeCounts) {
+    console.log(`  ${type}: ${count}`);
+  }
+  return result;
+}
+function registerScanCommand(program) {
+  program.command("scan").description("Scan your website and classify pages").option("--no-llm", "Disable LLM-assisted classification and summarization").action(async (opts) => {
+    if (!opts.llm) {
+      const config = loadConfig();
+      config.llm = void 0;
+      await runScan(config);
+    } else {
+      await runScan();
+    }
+  });
+}
+
+export {
+  normalizeUrl,
+  urlToId,
+  crawlSite,
+  parseSitemap,
+  classifyPage,
+  runScan,
+  registerScanCommand
+};
+//# sourceMappingURL=chunk-YWR5EH3F.js.map