nuxt-ai-ready 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +1 -0
  2. package/dist/module.d.mts +12 -1
  3. package/dist/module.json +1 -1
  4. package/dist/module.mjs +226 -352
  5. package/dist/runtime/index.d.ts +5 -7
  6. package/dist/runtime/index.js +14 -3
  7. package/dist/runtime/llms-txt-format.d.ts +8 -0
  8. package/dist/runtime/llms-txt-format.js +32 -0
  9. package/dist/runtime/llms-txt-utils.d.ts +2 -5
  10. package/dist/runtime/llms-txt-utils.js +31 -88
  11. package/dist/runtime/server/db/index.d.ts +3 -7
  12. package/dist/runtime/server/db/index.js +14 -31
  13. package/dist/runtime/server/db/queries.d.ts +106 -26
  14. package/dist/runtime/server/db/queries.js +330 -33
  15. package/dist/runtime/server/db/schema-sql.d.ts +3 -0
  16. package/dist/runtime/server/db/schema-sql.js +71 -0
  17. package/dist/runtime/server/db/shared.d.ts +88 -0
  18. package/dist/runtime/server/db/shared.js +117 -0
  19. package/dist/runtime/server/mcp/resources/pages.js +11 -3
  20. package/dist/runtime/server/mcp/tools/list-pages.js +33 -7
  21. package/dist/runtime/server/mcp/tools/search-pages.js +3 -3
  22. package/dist/runtime/server/middleware/markdown.js +2 -3
  23. package/dist/runtime/server/middleware/markdown.prerender.js +6 -5
  24. package/dist/runtime/server/plugins/db-restore.js +10 -8
  25. package/dist/runtime/server/plugins/sitemap-seeder.js +44 -0
  26. package/dist/runtime/server/routes/__ai-ready/indexnow.post.d.ts +2 -0
  27. package/dist/runtime/server/routes/__ai-ready/indexnow.post.js +18 -0
  28. package/dist/runtime/server/routes/__ai-ready/poll.post.d.ts +8 -0
  29. package/dist/runtime/server/routes/__ai-ready/poll.post.js +25 -0
  30. package/dist/runtime/server/routes/__ai-ready/prune.post.d.ts +14 -0
  31. package/dist/runtime/server/routes/__ai-ready/prune.post.js +21 -0
  32. package/dist/runtime/server/routes/__ai-ready/status.get.d.ts +2 -0
  33. package/dist/runtime/server/routes/__ai-ready/status.get.js +28 -0
  34. package/dist/runtime/server/routes/__ai-ready-debug.get.js +13 -14
  35. package/dist/runtime/server/routes/indexnow-key.get.d.ts +6 -0
  36. package/dist/runtime/server/routes/indexnow-key.get.js +10 -0
  37. package/dist/runtime/server/routes/llms-full.txt.get.d.ts +1 -1
  38. package/dist/runtime/server/routes/llms-full.txt.get.js +34 -3
  39. package/dist/runtime/server/tasks/ai-ready-index.d.ts +11 -0
  40. package/dist/runtime/server/tasks/ai-ready-index.js +39 -0
  41. package/dist/runtime/server/utils/batchIndex.d.ts +26 -0
  42. package/dist/runtime/server/utils/batchIndex.js +53 -0
  43. package/dist/runtime/server/utils/indexPage.d.ts +8 -2
  44. package/dist/runtime/server/utils/indexPage.js +38 -23
  45. package/dist/runtime/server/utils/indexnow.d.ts +27 -0
  46. package/dist/runtime/server/utils/indexnow.js +66 -0
  47. package/dist/runtime/server/utils/keywords.d.ts +0 -4
  48. package/dist/runtime/server/utils/keywords.js +0 -3
  49. package/dist/runtime/server/utils/llms-full.d.ts +11 -0
  50. package/dist/runtime/server/utils/llms-full.js +39 -0
  51. package/dist/runtime/server/utils/sitemap.js +3 -3
  52. package/dist/runtime/server/utils.d.ts +12 -10
  53. package/dist/runtime/server/utils.js +63 -94
  54. package/dist/runtime/types.d.ts +64 -40
  55. package/package.json +28 -29
  56. package/dist/runtime/mcp.d.ts +0 -32
  57. package/dist/runtime/mcp.js +0 -5
  58. package/dist/runtime/server/db/dump.d.ts +0 -29
  59. package/dist/runtime/server/db/dump.js +0 -29
  60. package/dist/runtime/server/db/schema.d.ts +0 -8
  61. package/dist/runtime/server/db/schema.js +0 -124
  62. package/dist/runtime/server/plugins/page-indexer.js +0 -68
  63. package/dist/runtime/server/tsconfig.json +0 -3
  64. package/dist/runtime/server/utils/pageData.d.ts +0 -28
  65. package/dist/runtime/server/utils/pageData.js +0 -43
  66. package/mcp.d.ts +0 -4
  67. /package/dist/runtime/{nuxt → app}/plugins/md-hints.prerender.d.ts +0 -0
  68. /package/dist/runtime/{nuxt → app}/plugins/md-hints.prerender.js +0 -0
  69. /package/dist/runtime/server/plugins/{page-indexer.d.ts → sitemap-seeder.d.ts} +0 -0
@@ -1,35 +1,39 @@
1
1
  import { useNitroApp, useRuntimeConfig } from "nitropack/runtime";
2
- import { useDatabase } from "../db/index.js";
3
- import { getPage, isPageFresh, upsertPage } from "../db/queries.js";
2
+ import { getPageHash, isPageFresh, queryPages, upsertPage } from "../db/queries.js";
3
+ import { computeContentHash } from "../db/shared.js";
4
4
  import { logger } from "../logger.js";
5
- import { convertHtmlToMarkdownMeta } from "../utils.js";
6
- import { extractKeywords, stripMarkdown } from "./keywords.js";
7
- export async function indexPage(route, html, options = {}) {
5
+ import { convertHtmlToMarkdown } from "../utils.js";
6
+ import { extractKeywords } from "./keywords.js";
7
+ export const INDEXING_HEADER = "x-ai-ready-indexing";
8
+ export async function indexPage(route, html, options = {}, event) {
8
9
  const config = useRuntimeConfig()["nuxt-ai-ready"];
9
- const db = await useDatabase();
10
- const ttl = options.ttl ?? config.ttl ?? 0;
11
- if (!options.force && await isPageFresh(db, route, ttl)) {
10
+ const ttl = options.ttl ?? config.runtimeSync.ttl;
11
+ if (!options.force && await isPageFresh(event, route, ttl)) {
12
12
  logger.debug(`[indexPage] Skipping ${route} - still fresh`);
13
13
  return { success: true, skipped: true, isUpdate: true };
14
14
  }
15
- const existing = await getPage(db, route);
15
+ const existing = await queryPages(event, { route });
16
16
  const isUpdate = !!existing;
17
17
  const isError = html.includes("__NUXT_ERROR__") || html.includes("nuxt-error-page");
18
- const result = await convertHtmlToMarkdownMeta(html, route, config.mdreamOptions);
18
+ const result = await convertHtmlToMarkdown(html, route, config.mdreamOptions, { extractUpdatedAt: true });
19
19
  const updatedAt = result.updatedAt || (/* @__PURE__ */ new Date()).toISOString();
20
20
  const headings = JSON.stringify(result.headings);
21
- const keywords = extractKeywords(stripMarkdown(result.markdown), result.metaKeywords);
22
- await upsertPage(db, {
21
+ const keywords = extractKeywords(result.textContent, result.metaKeywords);
22
+ const contentHash = await computeContentHash(result.markdown);
23
+ const existingHash = await getPageHash(event, route);
24
+ const contentChanged = existingHash !== contentHash;
25
+ await upsertPage(event, {
23
26
  route,
24
27
  title: result.title,
25
28
  description: result.description,
26
29
  markdown: result.markdown,
27
30
  headings,
28
31
  keywords,
32
+ contentHash,
29
33
  updatedAt,
30
34
  isError
31
35
  });
32
- logger.debug(`[indexPage] Indexed: ${route} "${result.title}"`);
36
+ logger.debug(`[indexPage] Indexed: ${route} "${result.title}" (changed=${contentChanged})`);
33
37
  if (!options.skipHook) {
34
38
  const nitro = useNitroApp();
35
39
  const hookContext = {
@@ -40,13 +44,15 @@ export async function indexPage(route, html, options = {}) {
40
44
  keywords,
41
45
  markdown: result.markdown,
42
46
  updatedAt,
43
- isUpdate
47
+ isUpdate,
48
+ contentChanged
44
49
  };
45
50
  await nitro.hooks.callHook("ai-ready:page:indexed", hookContext);
46
51
  }
47
52
  return {
48
53
  success: true,
49
54
  isUpdate,
55
+ contentChanged,
50
56
  data: {
51
57
  route,
52
58
  title: result.title,
@@ -54,23 +60,32 @@ export async function indexPage(route, html, options = {}) {
54
60
  headings,
55
61
  keywords,
56
62
  markdown: result.markdown,
63
+ contentHash,
57
64
  updatedAt
58
65
  }
59
66
  };
60
67
  }
61
68
  export async function indexPageByRoute(route, event, options = {}) {
62
- const fetchFn = event?.fetch || globalThis.$fetch;
63
- const html = await fetchFn(route, {
64
- headers: { "x-ai-ready-internal": "1" }
69
+ const html = await event.$fetch(route, {
70
+ headers: { [INDEXING_HEADER]: "1" }
65
71
  }).catch((err) => {
66
- logger.error(`[indexPageByRoute] Failed to fetch ${route}:`, err);
72
+ logger.warn(`[indexPageByRoute] Failed to fetch ${route}:`, err.message);
67
73
  return null;
68
74
  });
69
- if (!html) {
75
+ if (!html || typeof html !== "string") {
76
+ if (options.markFailedAsError) {
77
+ await upsertPage(event, {
78
+ route,
79
+ title: "",
80
+ description: "",
81
+ markdown: "",
82
+ headings: "[]",
83
+ keywords: [],
84
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
85
+ isError: true
86
+ });
87
+ }
70
88
  return { success: false, error: `Failed to fetch HTML for ${route}` };
71
89
  }
72
- if (typeof html !== "string") {
73
- return { success: false, error: `Response for ${route} is not HTML` };
74
- }
75
- return indexPage(route, html, options);
90
+ return indexPage(route, html, options, event);
76
91
  }
@@ -0,0 +1,27 @@
1
+ import type { H3Event } from 'h3';
2
+ export interface IndexNowResult {
3
+ success: boolean;
4
+ submitted: number;
5
+ remaining: number;
6
+ error?: string;
7
+ }
8
+ export interface IndexNowConfig {
9
+ enabled?: boolean;
10
+ key?: string;
11
+ host?: string;
12
+ }
13
+ /**
14
+ * Submit URLs to IndexNow API
15
+ */
16
+ export declare function submitToIndexNow(routes: string[], config: {
17
+ key: string;
18
+ host?: string;
19
+ }, siteUrl: string): Promise<{
20
+ success: boolean;
21
+ error?: string;
22
+ }>;
23
+ /**
24
+ * Submit pending pages to IndexNow
25
+ * Queries DB for pages needing sync, submits, marks as synced
26
+ */
27
+ export declare function syncToIndexNow(event: H3Event, limit?: number): Promise<IndexNowResult>;
@@ -0,0 +1,66 @@
1
+ import { useSiteConfig } from "#imports";
2
+ import { useRuntimeConfig } from "nitropack/runtime";
3
+ import {
4
+ countPagesNeedingIndexNowSync,
5
+ getPagesNeedingIndexNowSync,
6
+ markIndexNowSynced,
7
+ updateIndexNowStats
8
+ } from "../db/queries.js";
9
+ import { logger } from "../logger.js";
10
+ export async function submitToIndexNow(routes, config, siteUrl) {
11
+ if (!siteUrl) {
12
+ return { success: false, error: "Site URL not configured" };
13
+ }
14
+ const host = config.host || "api.indexnow.org";
15
+ const endpoint = `https://${host}/indexnow`;
16
+ const urlList = routes.map(
17
+ (route) => route.startsWith("http") ? route : `${siteUrl}${route}`
18
+ );
19
+ const body = {
20
+ host: new URL(siteUrl).host,
21
+ key: config.key,
22
+ keyLocation: `${siteUrl}/${config.key}.txt`,
23
+ urlList
24
+ };
25
+ logger.debug(`[indexnow] Submitting ${urlList.length} URLs to ${endpoint}`);
26
+ const response = await $fetch(endpoint, {
27
+ method: "POST",
28
+ headers: { "Content-Type": "application/json" },
29
+ body
30
+ }).catch((err) => ({ error: err.message }));
31
+ if (response && typeof response === "object" && "error" in response) {
32
+ logger.warn(`[indexnow] Submission failed: ${response.error}`);
33
+ return { success: false, error: response.error };
34
+ }
35
+ logger.debug(`[indexnow] Successfully submitted ${urlList.length} URLs`);
36
+ return { success: true };
37
+ }
38
+ export async function syncToIndexNow(event, limit = 100) {
39
+ const config = useRuntimeConfig(event)["nuxt-ai-ready"];
40
+ const siteConfig = useSiteConfig();
41
+ if (!config.indexNow?.enabled || !config.indexNow?.key) {
42
+ return { success: false, submitted: 0, remaining: 0, error: "IndexNow not configured" };
43
+ }
44
+ if (!siteConfig.url) {
45
+ return { success: false, submitted: 0, remaining: 0, error: "Site URL not configured" };
46
+ }
47
+ const pages = await getPagesNeedingIndexNowSync(event, limit);
48
+ if (pages.length === 0) {
49
+ return { success: true, submitted: 0, remaining: 0 };
50
+ }
51
+ const routes = pages.map((p) => p.route);
52
+ const result = await submitToIndexNow(routes, config.indexNow, siteConfig.url);
53
+ if (result.success) {
54
+ await markIndexNowSynced(event, routes);
55
+ await updateIndexNowStats(event, routes.length);
56
+ } else {
57
+ await updateIndexNowStats(event, 0, result.error);
58
+ }
59
+ const remaining = await countPagesNeedingIndexNowSync(event);
60
+ return {
61
+ success: result.success,
62
+ submitted: result.success ? routes.length : 0,
63
+ remaining,
64
+ error: result.error
65
+ };
66
+ }
@@ -2,7 +2,3 @@
2
2
  * Extract keywords from text using word frequency
3
3
  */
4
4
  export declare function extractKeywords(text: string, metaKeywords?: string, max?: number): string[];
5
- /**
6
- * Strip markdown formatting to get plain text
7
- */
8
- export declare function stripMarkdown(markdown: string): string;
@@ -277,6 +277,3 @@ export function extractKeywords(text, metaKeywords, max = 10) {
277
277
  freq.set(word, (freq.get(word) || 0) + 1);
278
278
  return Array.from(freq.entries()).sort((a, b) => b[1] - a[1]).slice(0, max).map(([word]) => word);
279
279
  }
280
- export function stripMarkdown(markdown) {
281
- return markdown.replace(/```[\s\S]*?```/g, "").replace(/`[^`]+`/g, "").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/!\[[^\]]*\]\([^)]+\)/g, "").replace(/^#{1,6}\s+/gm, "").replace(/[*_]{1,2}([^*_]+)[*_]{1,2}/g, "$1").replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim();
282
- }
@@ -0,0 +1,11 @@
1
+ /**
2
+ * Utilities for formatting llms-full.txt content
3
+ */
4
+ import type { LlmsTxtConfig } from '../../types.js';
5
+ export declare function formatPageForLlmsFullTxt(route: string, title: string, description: string, markdown: string, siteUrl?: string): string;
6
+ export interface SiteInfo {
7
+ name?: string;
8
+ url?: string;
9
+ description?: string;
10
+ }
11
+ export declare function buildLlmsFullTxtHeader(siteInfo?: SiteInfo, llmsTxtConfig?: LlmsTxtConfig): string;
@@ -0,0 +1,39 @@
1
+ import { normalizeLlmsTxtConfig } from "../../llms-txt-format.js";
2
+ export function formatPageForLlmsFullTxt(route, title, description, markdown, siteUrl) {
3
+ const canonicalUrl = siteUrl ? `${siteUrl.replace(/\/$/, "")}${route}` : route;
4
+ const heading = title && title !== route ? `### ${title}` : `### ${route}`;
5
+ const content = markdown.replace(/^---\n[\s\S]*?\n---\n*/, "").replace(/^(#{1,6}) ([^\n]+)$/gm, (_, hashes, text) => `h${hashes.length}. ${text}`);
6
+ const parts = [heading, ""];
7
+ parts.push(`Source: ${canonicalUrl}`);
8
+ if (description)
9
+ parts.push(`Description: ${description}`);
10
+ parts.push("");
11
+ if (content.trim()) {
12
+ parts.push(content.trim());
13
+ parts.push("");
14
+ }
15
+ parts.push("---");
16
+ parts.push("");
17
+ return `${parts.join("\n")}
18
+ `;
19
+ }
20
+ export function buildLlmsFullTxtHeader(siteInfo, llmsTxtConfig) {
21
+ const parts = [];
22
+ parts.push(`# ${siteInfo?.name || siteInfo?.url || "Site"}`);
23
+ if (siteInfo?.description)
24
+ parts.push(`
25
+ > ${siteInfo.description}`);
26
+ if (siteInfo?.url)
27
+ parts.push(`
28
+ Canonical Origin: ${siteInfo.url}`);
29
+ parts.push("");
30
+ if (llmsTxtConfig) {
31
+ const normalizedContent = normalizeLlmsTxtConfig(llmsTxtConfig);
32
+ if (normalizedContent) {
33
+ parts.push(normalizedContent);
34
+ parts.push("");
35
+ }
36
+ }
37
+ parts.push("## Pages\n\n");
38
+ return parts.join("\n");
39
+ }
@@ -3,16 +3,16 @@ import { logger } from "../logger.js";
3
3
  export async function fetchSitemapUrls(event) {
4
4
  const sitemapRes = await event.$fetch("/sitemap.xml", { responseType: "text" }).catch(() => null);
5
5
  if (!sitemapRes) {
6
- logger.warn("Sitemap not found at /sitemap.xml - llms.txt will have no pages listed");
6
+ logger.warn("Sitemap not found at /sitemap.xml - ensure @nuxtjs/sitemap is installed and configured");
7
7
  return [];
8
8
  }
9
9
  const result = await parseSitemapXml(sitemapRes).catch((e) => {
10
- logger.warn("Failed to parse sitemap.xml:", e);
10
+ logger.warn(`Failed to parse sitemap.xml: ${e instanceof Error ? e.message : e}`);
11
11
  return { urls: [] };
12
12
  });
13
13
  const urls = result?.urls || [];
14
14
  if (urls.length === 0) {
15
- logger.warn("Sitemap is empty - llms.txt will have no pages listed");
15
+ logger.warn("Sitemap is empty - add routes to sitemap or configure sitemap.sources in nuxt.config");
16
16
  }
17
17
  return urls.map((entry) => {
18
18
  if (typeof entry === "string")
@@ -1,23 +1,25 @@
1
1
  import type { H3Event } from 'h3';
2
2
  import type { ModulePublicRuntimeConfig } from '../../module.js';
3
- export declare function normalizeWhitespace(text: string): string;
4
3
  export declare function getMarkdownRenderInfo(event: H3Event, explicitOnly?: boolean): {
5
4
  path: string;
6
5
  isExplicit: boolean;
7
6
  } | null;
8
- export declare function clientPrefersMarkdown(event: H3Event): boolean;
9
- export declare function convertHtmlToMarkdown(html: string, url: string, config: ModulePublicRuntimeConfig, route: string, event: H3Event): Promise<{
10
- markdown: string;
11
- title: string;
12
- description: string;
13
- headings: Record<string, string>[];
14
- metaKeywords: string;
15
- }>;
16
- export declare function convertHtmlToMarkdownMeta(html: string, url: string, mdreamOptions: ModulePublicRuntimeConfig['mdreamOptions']): Promise<{
7
+ interface ConvertHtmlOptions {
8
+ /** Extract updatedAt from meta tags */
9
+ extractUpdatedAt?: boolean;
10
+ /** Call runtime hooks (ai-ready:mdreamConfig, ai-ready:markdown) */
11
+ hooks?: {
12
+ route: string;
13
+ event: H3Event;
14
+ };
15
+ }
16
+ export declare function convertHtmlToMarkdown(html: string, url: string, mdreamOptions: ModulePublicRuntimeConfig['mdreamOptions'], opts?: ConvertHtmlOptions): Promise<{
17
17
  updatedAt?: string | undefined;
18
18
  markdown: string;
19
19
  title: string;
20
20
  description: string;
21
21
  headings: Record<string, string>[];
22
22
  metaKeywords: string;
23
+ textContent: string;
23
24
  }>;
25
+ export {};
@@ -4,9 +4,44 @@ import { htmlToMarkdown } from "mdream";
4
4
  import { extractionPlugin } from "mdream/plugins";
5
5
  import { withMinimalPreset } from "mdream/preset/minimal";
6
6
  import { useNitroApp } from "nitropack/runtime";
7
- export function normalizeWhitespace(text) {
7
+ function normalizeWhitespace(text) {
8
8
  return text.replace(/\u00A0/g, " ");
9
9
  }
10
+ function buildMdreamOptions(url, mdreamOptions, meta, extractUpdatedAt = false) {
11
+ const extractPlugin = extractionPlugin({
12
+ "title": (el) => {
13
+ meta.title = el.textContent;
14
+ },
15
+ 'meta[name="description"]': (el) => {
16
+ meta.description = el.attributes.content || "";
17
+ },
18
+ 'meta[name="keywords"]': (el) => {
19
+ meta.metaKeywords = el.attributes.content || "";
20
+ },
21
+ "h1, h2, h3, h4, h5, h6": (el) => {
22
+ const text = el.textContent?.trim();
23
+ if (text)
24
+ meta.headings.push({ [el.name.toLowerCase()]: text });
25
+ },
26
+ "p, li, td, th, blockquote, figcaption": (el) => {
27
+ const text = el.textContent?.trim();
28
+ if (text)
29
+ meta.textContent.push(text);
30
+ },
31
+ ...extractUpdatedAt && {
32
+ 'meta[property="article:modified_time"], meta[name="last-modified"], meta[name="updated"], meta[property="og:updated_time"], meta[name="lastmod"]': (el) => {
33
+ if (!meta.updatedAt && el.attributes.content)
34
+ meta.updatedAt = el.attributes.content;
35
+ }
36
+ }
37
+ });
38
+ let options = { origin: url, ...mdreamOptions };
39
+ if (mdreamOptions?.preset === "minimal") {
40
+ options = withMinimalPreset(options);
41
+ }
42
+ options.plugins = [extractPlugin, ...options.plugins || []];
43
+ return options;
44
+ }
10
45
  export function getMarkdownRenderInfo(event, explicitOnly = false) {
11
46
  const originalPath = event.path;
12
47
  if (originalPath.startsWith("/api") || originalPath.startsWith("/_") || originalPath.startsWith("/@")) {
@@ -32,7 +67,7 @@ export function getMarkdownRenderInfo(event, explicitOnly = false) {
32
67
  }
33
68
  return { path, isExplicit };
34
69
  }
35
- export function clientPrefersMarkdown(event) {
70
+ function clientPrefersMarkdown(event) {
36
71
  const accept = getHeader(event, "accept") || "";
37
72
  const secFetchDest = getHeader(event, "sec-fetch-dest") || "";
38
73
  if (secFetchDest === "document") {
@@ -50,100 +85,34 @@ export function clientPrefersMarkdown(event) {
50
85
  }
51
86
  return false;
52
87
  }
53
- export async function convertHtmlToMarkdown(html, url, config, route, event) {
54
- const nitroApp = useNitroApp();
55
- let title = "";
56
- let description = "";
57
- let metaKeywords = "";
58
- const headings = [];
59
- const extractPlugin = extractionPlugin({
60
- title(el) {
61
- title = el.textContent;
62
- },
63
- 'meta[name="description"]': (el) => {
64
- description = el.attributes.content || "";
65
- },
66
- 'meta[name="keywords"]': (el) => {
67
- metaKeywords = el.attributes.content || "";
68
- },
69
- "h1, h2, h3, h4, h5, h6": (el) => {
70
- const text = el.textContent?.trim();
71
- const level = el.name.toLowerCase();
72
- if (text)
73
- headings.push({ [level]: text });
74
- }
75
- });
76
- let options = {
77
- origin: url,
78
- ...config.mdreamOptions
79
- };
80
- if (config.mdreamOptions?.preset === "minimal") {
81
- options = withMinimalPreset(options);
82
- options.plugins = [extractPlugin, ...options.plugins || []];
83
- } else {
84
- options.plugins = [extractPlugin, ...options.plugins || []];
85
- }
86
- await nitroApp.hooks.callHook("ai-ready:mdreamConfig", options);
87
- let markdown = htmlToMarkdown(html, options);
88
- const context = {
89
- html,
90
- markdown,
91
- route,
92
- title,
93
- description,
94
- isPrerender: false,
95
- event
96
- };
97
- await nitroApp.hooks.callHook("ai-ready:markdown", context);
98
- markdown = normalizeWhitespace(context.markdown);
99
- return { markdown, title: normalizeWhitespace(title), description: normalizeWhitespace(description), headings, metaKeywords };
100
- }
101
- export async function convertHtmlToMarkdownMeta(html, url, mdreamOptions) {
102
- let title = "";
103
- let description = "";
104
- let metaKeywords = "";
105
- let updatedAt;
106
- const headings = [];
107
- const extractPlugin = extractionPlugin({
108
- title(el) {
109
- title = el.textContent;
110
- },
111
- 'meta[name="description"]': (el) => {
112
- description = el.attributes.content || "";
113
- },
114
- 'meta[name="keywords"]': (el) => {
115
- metaKeywords = el.attributes.content || "";
116
- },
117
- 'meta[property="article:modified_time"], meta[name="last-modified"], meta[name="updated"], meta[property="og:updated_time"], meta[name="lastmod"]': (el) => {
118
- if (!updatedAt && el.attributes.content) {
119
- updatedAt = el.attributes.content;
120
- }
121
- },
122
- "h1, h2, h3, h4, h5, h6": (el) => {
123
- const text = el.textContent?.trim();
124
- const level = el.name.toLowerCase();
125
- if (text)
126
- headings.push({ [level]: text });
127
- }
128
- });
129
- let options = {
130
- origin: url,
131
- ...mdreamOptions
132
- };
133
- if (mdreamOptions?.preset === "minimal") {
134
- options = withMinimalPreset(options);
135
- options.plugins = [extractPlugin, ...options.plugins || []];
88
+ export async function convertHtmlToMarkdown(html, url, mdreamOptions, opts = {}) {
89
+ const meta = { title: "", description: "", metaKeywords: "", headings: [], textContent: [] };
90
+ const options = buildMdreamOptions(url, mdreamOptions, meta, opts.extractUpdatedAt);
91
+ let markdown;
92
+ if (opts.hooks) {
93
+ const nitroApp = useNitroApp();
94
+ await nitroApp.hooks.callHook("ai-ready:mdreamConfig", options);
95
+ const context = {
96
+ html,
97
+ markdown: htmlToMarkdown(html, options),
98
+ route: opts.hooks.route,
99
+ title: meta.title,
100
+ description: meta.description,
101
+ isPrerender: false,
102
+ event: opts.hooks.event
103
+ };
104
+ await nitroApp.hooks.callHook("ai-ready:markdown", context);
105
+ markdown = context.markdown;
136
106
  } else {
137
- options.plugins = [extractPlugin, ...options.plugins || []];
107
+ markdown = htmlToMarkdown(html, options);
138
108
  }
139
- const rawMarkdown = htmlToMarkdown(html, options);
140
- const markdown = normalizeWhitespace(rawMarkdown);
141
109
  return {
142
- markdown,
143
- title: normalizeWhitespace(title),
144
- description: normalizeWhitespace(description),
145
- headings,
146
- metaKeywords,
147
- ...updatedAt && { updatedAt }
110
+ markdown: normalizeWhitespace(markdown),
111
+ title: normalizeWhitespace(meta.title),
112
+ description: normalizeWhitespace(meta.description),
113
+ headings: meta.headings,
114
+ metaKeywords: meta.metaKeywords,
115
+ textContent: meta.textContent.join(" "),
116
+ ...meta.updatedAt && { updatedAt: meta.updatedAt }
148
117
  };
149
118
  }
@@ -74,13 +74,6 @@ export interface ModuleOptions {
74
74
  * @default 600 (10 minutes)
75
75
  */
76
76
  cacheMaxAgeSeconds?: number;
77
- /**
78
- * TTL for re-indexing pages in seconds
79
- * Pages are automatically indexed on visit; this controls how often to refresh
80
- * 0 = never re-index (rely on prerendered data)
81
- * @default 0
82
- */
83
- ttl?: number;
84
77
  /**
85
78
  * Database configuration for page storage
86
79
  * Uses db0 for cross-platform SQLite support
@@ -110,6 +103,68 @@ export interface ModuleOptions {
110
103
  */
111
104
  authToken?: string;
112
105
  };
106
+ /**
107
+ * Runtime sync configuration (opt-in for dynamic content sites)
108
+ * When enabled, pages are re-indexed at runtime from sitemap
109
+ * @default disabled - prerendered data is used
110
+ */
111
+ runtimeSync?: {
112
+ /**
113
+ * Enable runtime sync
114
+ * When false (default), runtime uses prerendered data only
115
+ * @default false
116
+ */
117
+ enabled?: boolean;
118
+ /**
119
+ * TTL for refresh in seconds (sitemap + page re-indexing)
120
+ * Controls how often to refresh sitemap routes and re-index stale pages
121
+ * @default 3600 (1 hour)
122
+ */
123
+ ttl?: number;
124
+ /**
125
+ * Pages to index per batch
126
+ * @default 20
127
+ */
128
+ batchSize?: number;
129
+ /**
130
+ * Cron expression for scheduled indexing (e.g. every 5 minutes)
131
+ * When set, enables automatic background indexing via Nitro task
132
+ */
133
+ cron?: string;
134
+ /**
135
+ * Secret token for authenticating poll endpoint
136
+ * When set, requires ?secret=<token> query param
137
+ */
138
+ secret?: string;
139
+ /**
140
+ * TTL for pruning stale routes in seconds
141
+ * Routes not seen in sitemap for longer than this are deleted
142
+ * 0 = never prune (default)
143
+ * @default 0
144
+ */
145
+ pruneTtl?: number;
146
+ };
147
+ /**
148
+ * IndexNow configuration for instant search engine notifications
149
+ * Submits changed URLs to Bing, Yandex, Naver, Seznam via IndexNow protocol
150
+ */
151
+ indexNow?: {
152
+ /**
153
+ * Enable IndexNow submissions
154
+ * @default false
155
+ */
156
+ enabled?: boolean;
157
+ /**
158
+ * Your IndexNow API key
159
+ * Get one from https://www.bing.com/indexnow
160
+ */
161
+ key?: string;
162
+ /**
163
+ * IndexNow endpoint host
164
+ * @default 'api.indexnow.org'
165
+ */
166
+ host?: string;
167
+ };
113
168
  }
114
169
  /**
115
170
  * Page-level entry for discovery and metadata queries
@@ -199,28 +254,6 @@ export interface LlmsTxtConfig {
199
254
  /** Notes section (always appears at the end) */
200
255
  notes?: string | string[];
201
256
  }
202
- /**
203
- * Hook context for page markdown processing (Nuxt build-time hook)
204
- *
205
- * Called once per page during prerender when markdown content has changed.
206
- * Allows integrations to process page content (e.g., for embedding generation).
207
- *
208
- * @example Process page markdown
209
- * nuxt.hooks.hook('ai-ready:page:markdown', async (context) => {
210
- * console.log(`Processing ${context.route}: ${context.title}`)
211
- * // Generate embeddings, update search index, etc.
212
- * })
213
- */
214
- export interface PageMarkdownContext {
215
- /** The route being processed (e.g., '/about') */
216
- route: string;
217
- /** The markdown content */
218
- markdown: string;
219
- /** Page title extracted from HTML */
220
- title: string;
221
- /** Page description from meta tags */
222
- description: string;
223
- }
224
257
  /**
225
258
  * Context for runtime page indexing hook
226
259
  */
@@ -241,15 +274,6 @@ export interface PageIndexedContext {
241
274
  updatedAt: string;
242
275
  /** Whether this is a new page or an update */
243
276
  isUpdate: boolean;
244
- }
245
- declare module 'nitropack/types' {
246
- interface NitroRuntimeHooks {
247
- 'ai-ready:markdown': (context: MarkdownContext) => void | Promise<void>;
248
- 'ai-ready:mdreamConfig': (config: import('mdream').HTMLToMarkdownOptions) => void | Promise<void>;
249
- /**
250
- * Called when a page is indexed at runtime
251
- * Use this to sync with external systems (embeddings, search, etc)
252
- */
253
- 'ai-ready:page:indexed': (context: PageIndexedContext) => void | Promise<void>;
254
- }
277
+ /** Whether the content hash changed (triggers IndexNow sync) */
278
+ contentChanged: boolean;
255
279
  }