nuxt-ai-ready 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/module.d.mts +12 -1
- package/dist/module.json +1 -1
- package/dist/module.mjs +226 -352
- package/dist/runtime/index.d.ts +5 -7
- package/dist/runtime/index.js +14 -3
- package/dist/runtime/llms-txt-format.d.ts +8 -0
- package/dist/runtime/llms-txt-format.js +32 -0
- package/dist/runtime/llms-txt-utils.d.ts +2 -5
- package/dist/runtime/llms-txt-utils.js +31 -88
- package/dist/runtime/server/db/index.d.ts +3 -7
- package/dist/runtime/server/db/index.js +14 -31
- package/dist/runtime/server/db/queries.d.ts +106 -26
- package/dist/runtime/server/db/queries.js +330 -33
- package/dist/runtime/server/db/schema-sql.d.ts +3 -0
- package/dist/runtime/server/db/schema-sql.js +71 -0
- package/dist/runtime/server/db/shared.d.ts +88 -0
- package/dist/runtime/server/db/shared.js +117 -0
- package/dist/runtime/server/mcp/resources/pages.js +11 -3
- package/dist/runtime/server/mcp/tools/list-pages.js +33 -7
- package/dist/runtime/server/mcp/tools/search-pages.js +3 -3
- package/dist/runtime/server/middleware/markdown.js +2 -3
- package/dist/runtime/server/middleware/markdown.prerender.js +6 -5
- package/dist/runtime/server/plugins/db-restore.js +10 -8
- package/dist/runtime/server/plugins/sitemap-seeder.js +44 -0
- package/dist/runtime/server/routes/__ai-ready/indexnow.post.d.ts +2 -0
- package/dist/runtime/server/routes/__ai-ready/indexnow.post.js +18 -0
- package/dist/runtime/server/routes/__ai-ready/poll.post.d.ts +8 -0
- package/dist/runtime/server/routes/__ai-ready/poll.post.js +25 -0
- package/dist/runtime/server/routes/__ai-ready/prune.post.d.ts +14 -0
- package/dist/runtime/server/routes/__ai-ready/prune.post.js +21 -0
- package/dist/runtime/server/routes/__ai-ready/status.get.d.ts +2 -0
- package/dist/runtime/server/routes/__ai-ready/status.get.js +28 -0
- package/dist/runtime/server/routes/__ai-ready-debug.get.js +13 -14
- package/dist/runtime/server/routes/indexnow-key.get.d.ts +6 -0
- package/dist/runtime/server/routes/indexnow-key.get.js +10 -0
- package/dist/runtime/server/routes/llms-full.txt.get.d.ts +1 -1
- package/dist/runtime/server/routes/llms-full.txt.get.js +34 -3
- package/dist/runtime/server/tasks/ai-ready-index.d.ts +11 -0
- package/dist/runtime/server/tasks/ai-ready-index.js +39 -0
- package/dist/runtime/server/utils/batchIndex.d.ts +26 -0
- package/dist/runtime/server/utils/batchIndex.js +53 -0
- package/dist/runtime/server/utils/indexPage.d.ts +8 -2
- package/dist/runtime/server/utils/indexPage.js +38 -23
- package/dist/runtime/server/utils/indexnow.d.ts +27 -0
- package/dist/runtime/server/utils/indexnow.js +66 -0
- package/dist/runtime/server/utils/keywords.d.ts +0 -4
- package/dist/runtime/server/utils/keywords.js +0 -3
- package/dist/runtime/server/utils/llms-full.d.ts +11 -0
- package/dist/runtime/server/utils/llms-full.js +39 -0
- package/dist/runtime/server/utils/sitemap.js +3 -3
- package/dist/runtime/server/utils.d.ts +12 -10
- package/dist/runtime/server/utils.js +63 -94
- package/dist/runtime/types.d.ts +64 -40
- package/package.json +28 -29
- package/dist/runtime/mcp.d.ts +0 -32
- package/dist/runtime/mcp.js +0 -5
- package/dist/runtime/server/db/dump.d.ts +0 -29
- package/dist/runtime/server/db/dump.js +0 -29
- package/dist/runtime/server/db/schema.d.ts +0 -8
- package/dist/runtime/server/db/schema.js +0 -124
- package/dist/runtime/server/plugins/page-indexer.js +0 -68
- package/dist/runtime/server/tsconfig.json +0 -3
- package/dist/runtime/server/utils/pageData.d.ts +0 -28
- package/dist/runtime/server/utils/pageData.js +0 -43
- package/mcp.d.ts +0 -4
- /package/dist/runtime/{nuxt → app}/plugins/md-hints.prerender.d.ts +0 -0
- /package/dist/runtime/{nuxt → app}/plugins/md-hints.prerender.js +0 -0
- /package/dist/runtime/server/plugins/{page-indexer.d.ts → sitemap-seeder.d.ts} +0 -0
|
@@ -1,35 +1,39 @@
|
|
|
1
1
|
import { useNitroApp, useRuntimeConfig } from "nitropack/runtime";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { getPageHash, isPageFresh, queryPages, upsertPage } from "../db/queries.js";
|
|
3
|
+
import { computeContentHash } from "../db/shared.js";
|
|
4
4
|
import { logger } from "../logger.js";
|
|
5
|
-
import {
|
|
6
|
-
import { extractKeywords
|
|
7
|
-
export
|
|
5
|
+
import { convertHtmlToMarkdown } from "../utils.js";
|
|
6
|
+
import { extractKeywords } from "./keywords.js";
|
|
7
|
+
export const INDEXING_HEADER = "x-ai-ready-indexing";
|
|
8
|
+
export async function indexPage(route, html, options = {}, event) {
|
|
8
9
|
const config = useRuntimeConfig()["nuxt-ai-ready"];
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
if (!options.force && await isPageFresh(db, route, ttl)) {
|
|
10
|
+
const ttl = options.ttl ?? config.runtimeSync.ttl;
|
|
11
|
+
if (!options.force && await isPageFresh(event, route, ttl)) {
|
|
12
12
|
logger.debug(`[indexPage] Skipping ${route} - still fresh`);
|
|
13
13
|
return { success: true, skipped: true, isUpdate: true };
|
|
14
14
|
}
|
|
15
|
-
const existing = await
|
|
15
|
+
const existing = await queryPages(event, { route });
|
|
16
16
|
const isUpdate = !!existing;
|
|
17
17
|
const isError = html.includes("__NUXT_ERROR__") || html.includes("nuxt-error-page");
|
|
18
|
-
const result = await
|
|
18
|
+
const result = await convertHtmlToMarkdown(html, route, config.mdreamOptions, { extractUpdatedAt: true });
|
|
19
19
|
const updatedAt = result.updatedAt || (/* @__PURE__ */ new Date()).toISOString();
|
|
20
20
|
const headings = JSON.stringify(result.headings);
|
|
21
|
-
const keywords = extractKeywords(
|
|
22
|
-
await
|
|
21
|
+
const keywords = extractKeywords(result.textContent, result.metaKeywords);
|
|
22
|
+
const contentHash = await computeContentHash(result.markdown);
|
|
23
|
+
const existingHash = await getPageHash(event, route);
|
|
24
|
+
const contentChanged = existingHash !== contentHash;
|
|
25
|
+
await upsertPage(event, {
|
|
23
26
|
route,
|
|
24
27
|
title: result.title,
|
|
25
28
|
description: result.description,
|
|
26
29
|
markdown: result.markdown,
|
|
27
30
|
headings,
|
|
28
31
|
keywords,
|
|
32
|
+
contentHash,
|
|
29
33
|
updatedAt,
|
|
30
34
|
isError
|
|
31
35
|
});
|
|
32
|
-
logger.debug(`[indexPage] Indexed: ${route} "${result.title}"`);
|
|
36
|
+
logger.debug(`[indexPage] Indexed: ${route} "${result.title}" (changed=${contentChanged})`);
|
|
33
37
|
if (!options.skipHook) {
|
|
34
38
|
const nitro = useNitroApp();
|
|
35
39
|
const hookContext = {
|
|
@@ -40,13 +44,15 @@ export async function indexPage(route, html, options = {}) {
|
|
|
40
44
|
keywords,
|
|
41
45
|
markdown: result.markdown,
|
|
42
46
|
updatedAt,
|
|
43
|
-
isUpdate
|
|
47
|
+
isUpdate,
|
|
48
|
+
contentChanged
|
|
44
49
|
};
|
|
45
50
|
await nitro.hooks.callHook("ai-ready:page:indexed", hookContext);
|
|
46
51
|
}
|
|
47
52
|
return {
|
|
48
53
|
success: true,
|
|
49
54
|
isUpdate,
|
|
55
|
+
contentChanged,
|
|
50
56
|
data: {
|
|
51
57
|
route,
|
|
52
58
|
title: result.title,
|
|
@@ -54,23 +60,32 @@ export async function indexPage(route, html, options = {}) {
|
|
|
54
60
|
headings,
|
|
55
61
|
keywords,
|
|
56
62
|
markdown: result.markdown,
|
|
63
|
+
contentHash,
|
|
57
64
|
updatedAt
|
|
58
65
|
}
|
|
59
66
|
};
|
|
60
67
|
}
|
|
61
68
|
export async function indexPageByRoute(route, event, options = {}) {
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
headers: { "x-ai-ready-internal": "1" }
|
|
69
|
+
const html = await event.$fetch(route, {
|
|
70
|
+
headers: { [INDEXING_HEADER]: "1" }
|
|
65
71
|
}).catch((err) => {
|
|
66
|
-
logger.
|
|
72
|
+
logger.warn(`[indexPageByRoute] Failed to fetch ${route}:`, err.message);
|
|
67
73
|
return null;
|
|
68
74
|
});
|
|
69
|
-
if (!html) {
|
|
75
|
+
if (!html || typeof html !== "string") {
|
|
76
|
+
if (options.markFailedAsError) {
|
|
77
|
+
await upsertPage(event, {
|
|
78
|
+
route,
|
|
79
|
+
title: "",
|
|
80
|
+
description: "",
|
|
81
|
+
markdown: "",
|
|
82
|
+
headings: "[]",
|
|
83
|
+
keywords: [],
|
|
84
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
85
|
+
isError: true
|
|
86
|
+
});
|
|
87
|
+
}
|
|
70
88
|
return { success: false, error: `Failed to fetch HTML for ${route}` };
|
|
71
89
|
}
|
|
72
|
-
|
|
73
|
-
return { success: false, error: `Response for ${route} is not HTML` };
|
|
74
|
-
}
|
|
75
|
-
return indexPage(route, html, options);
|
|
90
|
+
return indexPage(route, html, options, event);
|
|
76
91
|
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { H3Event } from 'h3';
|
|
2
|
+
export interface IndexNowResult {
|
|
3
|
+
success: boolean;
|
|
4
|
+
submitted: number;
|
|
5
|
+
remaining: number;
|
|
6
|
+
error?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface IndexNowConfig {
|
|
9
|
+
enabled?: boolean;
|
|
10
|
+
key?: string;
|
|
11
|
+
host?: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Submit URLs to IndexNow API
|
|
15
|
+
*/
|
|
16
|
+
export declare function submitToIndexNow(routes: string[], config: {
|
|
17
|
+
key: string;
|
|
18
|
+
host?: string;
|
|
19
|
+
}, siteUrl: string): Promise<{
|
|
20
|
+
success: boolean;
|
|
21
|
+
error?: string;
|
|
22
|
+
}>;
|
|
23
|
+
/**
|
|
24
|
+
* Submit pending pages to IndexNow
|
|
25
|
+
* Queries DB for pages needing sync, submits, marks as synced
|
|
26
|
+
*/
|
|
27
|
+
export declare function syncToIndexNow(event: H3Event, limit?: number): Promise<IndexNowResult>;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { useSiteConfig } from "#imports";
|
|
2
|
+
import { useRuntimeConfig } from "nitropack/runtime";
|
|
3
|
+
import {
|
|
4
|
+
countPagesNeedingIndexNowSync,
|
|
5
|
+
getPagesNeedingIndexNowSync,
|
|
6
|
+
markIndexNowSynced,
|
|
7
|
+
updateIndexNowStats
|
|
8
|
+
} from "../db/queries.js";
|
|
9
|
+
import { logger } from "../logger.js";
|
|
10
|
+
export async function submitToIndexNow(routes, config, siteUrl) {
|
|
11
|
+
if (!siteUrl) {
|
|
12
|
+
return { success: false, error: "Site URL not configured" };
|
|
13
|
+
}
|
|
14
|
+
const host = config.host || "api.indexnow.org";
|
|
15
|
+
const endpoint = `https://${host}/indexnow`;
|
|
16
|
+
const urlList = routes.map(
|
|
17
|
+
(route) => route.startsWith("http") ? route : `${siteUrl}${route}`
|
|
18
|
+
);
|
|
19
|
+
const body = {
|
|
20
|
+
host: new URL(siteUrl).host,
|
|
21
|
+
key: config.key,
|
|
22
|
+
keyLocation: `${siteUrl}/${config.key}.txt`,
|
|
23
|
+
urlList
|
|
24
|
+
};
|
|
25
|
+
logger.debug(`[indexnow] Submitting ${urlList.length} URLs to ${endpoint}`);
|
|
26
|
+
const response = await $fetch(endpoint, {
|
|
27
|
+
method: "POST",
|
|
28
|
+
headers: { "Content-Type": "application/json" },
|
|
29
|
+
body
|
|
30
|
+
}).catch((err) => ({ error: err.message }));
|
|
31
|
+
if (response && typeof response === "object" && "error" in response) {
|
|
32
|
+
logger.warn(`[indexnow] Submission failed: ${response.error}`);
|
|
33
|
+
return { success: false, error: response.error };
|
|
34
|
+
}
|
|
35
|
+
logger.debug(`[indexnow] Successfully submitted ${urlList.length} URLs`);
|
|
36
|
+
return { success: true };
|
|
37
|
+
}
|
|
38
|
+
export async function syncToIndexNow(event, limit = 100) {
|
|
39
|
+
const config = useRuntimeConfig(event)["nuxt-ai-ready"];
|
|
40
|
+
const siteConfig = useSiteConfig();
|
|
41
|
+
if (!config.indexNow?.enabled || !config.indexNow?.key) {
|
|
42
|
+
return { success: false, submitted: 0, remaining: 0, error: "IndexNow not configured" };
|
|
43
|
+
}
|
|
44
|
+
if (!siteConfig.url) {
|
|
45
|
+
return { success: false, submitted: 0, remaining: 0, error: "Site URL not configured" };
|
|
46
|
+
}
|
|
47
|
+
const pages = await getPagesNeedingIndexNowSync(event, limit);
|
|
48
|
+
if (pages.length === 0) {
|
|
49
|
+
return { success: true, submitted: 0, remaining: 0 };
|
|
50
|
+
}
|
|
51
|
+
const routes = pages.map((p) => p.route);
|
|
52
|
+
const result = await submitToIndexNow(routes, config.indexNow, siteConfig.url);
|
|
53
|
+
if (result.success) {
|
|
54
|
+
await markIndexNowSynced(event, routes);
|
|
55
|
+
await updateIndexNowStats(event, routes.length);
|
|
56
|
+
} else {
|
|
57
|
+
await updateIndexNowStats(event, 0, result.error);
|
|
58
|
+
}
|
|
59
|
+
const remaining = await countPagesNeedingIndexNowSync(event);
|
|
60
|
+
return {
|
|
61
|
+
success: result.success,
|
|
62
|
+
submitted: result.success ? routes.length : 0,
|
|
63
|
+
remaining,
|
|
64
|
+
error: result.error
|
|
65
|
+
};
|
|
66
|
+
}
|
|
@@ -2,7 +2,3 @@
|
|
|
2
2
|
* Extract keywords from text using word frequency
|
|
3
3
|
*/
|
|
4
4
|
export declare function extractKeywords(text: string, metaKeywords?: string, max?: number): string[];
|
|
5
|
-
/**
|
|
6
|
-
* Strip markdown formatting to get plain text
|
|
7
|
-
*/
|
|
8
|
-
export declare function stripMarkdown(markdown: string): string;
|
|
@@ -277,6 +277,3 @@ export function extractKeywords(text, metaKeywords, max = 10) {
|
|
|
277
277
|
freq.set(word, (freq.get(word) || 0) + 1);
|
|
278
278
|
return Array.from(freq.entries()).sort((a, b) => b[1] - a[1]).slice(0, max).map(([word]) => word);
|
|
279
279
|
}
|
|
280
|
-
export function stripMarkdown(markdown) {
|
|
281
|
-
return markdown.replace(/```[\s\S]*?```/g, "").replace(/`[^`]+`/g, "").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/!\[[^\]]*\]\([^)]+\)/g, "").replace(/^#{1,6}\s+/gm, "").replace(/[*_]{1,2}([^*_]+)[*_]{1,2}/g, "$1").replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim();
|
|
282
|
-
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Utilities for formatting llms-full.txt content
|
|
3
|
+
*/
|
|
4
|
+
import type { LlmsTxtConfig } from '../../types.js';
|
|
5
|
+
export declare function formatPageForLlmsFullTxt(route: string, title: string, description: string, markdown: string, siteUrl?: string): string;
|
|
6
|
+
export interface SiteInfo {
|
|
7
|
+
name?: string;
|
|
8
|
+
url?: string;
|
|
9
|
+
description?: string;
|
|
10
|
+
}
|
|
11
|
+
export declare function buildLlmsFullTxtHeader(siteInfo?: SiteInfo, llmsTxtConfig?: LlmsTxtConfig): string;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { normalizeLlmsTxtConfig } from "../../llms-txt-format.js";
|
|
2
|
+
export function formatPageForLlmsFullTxt(route, title, description, markdown, siteUrl) {
|
|
3
|
+
const canonicalUrl = siteUrl ? `${siteUrl.replace(/\/$/, "")}${route}` : route;
|
|
4
|
+
const heading = title && title !== route ? `### ${title}` : `### ${route}`;
|
|
5
|
+
const content = markdown.replace(/^---\n[\s\S]*?\n---\n*/, "").replace(/^(#{1,6}) ([^\n]+)$/gm, (_, hashes, text) => `h${hashes.length}. ${text}`);
|
|
6
|
+
const parts = [heading, ""];
|
|
7
|
+
parts.push(`Source: ${canonicalUrl}`);
|
|
8
|
+
if (description)
|
|
9
|
+
parts.push(`Description: ${description}`);
|
|
10
|
+
parts.push("");
|
|
11
|
+
if (content.trim()) {
|
|
12
|
+
parts.push(content.trim());
|
|
13
|
+
parts.push("");
|
|
14
|
+
}
|
|
15
|
+
parts.push("---");
|
|
16
|
+
parts.push("");
|
|
17
|
+
return `${parts.join("\n")}
|
|
18
|
+
`;
|
|
19
|
+
}
|
|
20
|
+
export function buildLlmsFullTxtHeader(siteInfo, llmsTxtConfig) {
|
|
21
|
+
const parts = [];
|
|
22
|
+
parts.push(`# ${siteInfo?.name || siteInfo?.url || "Site"}`);
|
|
23
|
+
if (siteInfo?.description)
|
|
24
|
+
parts.push(`
|
|
25
|
+
> ${siteInfo.description}`);
|
|
26
|
+
if (siteInfo?.url)
|
|
27
|
+
parts.push(`
|
|
28
|
+
Canonical Origin: ${siteInfo.url}`);
|
|
29
|
+
parts.push("");
|
|
30
|
+
if (llmsTxtConfig) {
|
|
31
|
+
const normalizedContent = normalizeLlmsTxtConfig(llmsTxtConfig);
|
|
32
|
+
if (normalizedContent) {
|
|
33
|
+
parts.push(normalizedContent);
|
|
34
|
+
parts.push("");
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
parts.push("## Pages\n\n");
|
|
38
|
+
return parts.join("\n");
|
|
39
|
+
}
|
|
@@ -3,16 +3,16 @@ import { logger } from "../logger.js";
|
|
|
3
3
|
export async function fetchSitemapUrls(event) {
|
|
4
4
|
const sitemapRes = await event.$fetch("/sitemap.xml", { responseType: "text" }).catch(() => null);
|
|
5
5
|
if (!sitemapRes) {
|
|
6
|
-
logger.warn("Sitemap not found at /sitemap.xml -
|
|
6
|
+
logger.warn("Sitemap not found at /sitemap.xml - ensure @nuxtjs/sitemap is installed and configured");
|
|
7
7
|
return [];
|
|
8
8
|
}
|
|
9
9
|
const result = await parseSitemapXml(sitemapRes).catch((e) => {
|
|
10
|
-
logger.warn(
|
|
10
|
+
logger.warn(`Failed to parse sitemap.xml: ${e instanceof Error ? e.message : e}`);
|
|
11
11
|
return { urls: [] };
|
|
12
12
|
});
|
|
13
13
|
const urls = result?.urls || [];
|
|
14
14
|
if (urls.length === 0) {
|
|
15
|
-
logger.warn("Sitemap is empty -
|
|
15
|
+
logger.warn("Sitemap is empty - add routes to sitemap or configure sitemap.sources in nuxt.config");
|
|
16
16
|
}
|
|
17
17
|
return urls.map((entry) => {
|
|
18
18
|
if (typeof entry === "string")
|
|
@@ -1,23 +1,25 @@
|
|
|
1
1
|
import type { H3Event } from 'h3';
|
|
2
2
|
import type { ModulePublicRuntimeConfig } from '../../module.js';
|
|
3
|
-
export declare function normalizeWhitespace(text: string): string;
|
|
4
3
|
export declare function getMarkdownRenderInfo(event: H3Event, explicitOnly?: boolean): {
|
|
5
4
|
path: string;
|
|
6
5
|
isExplicit: boolean;
|
|
7
6
|
} | null;
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
}
|
|
16
|
-
|
|
7
|
+
interface ConvertHtmlOptions {
|
|
8
|
+
/** Extract updatedAt from meta tags */
|
|
9
|
+
extractUpdatedAt?: boolean;
|
|
10
|
+
/** Call runtime hooks (ai-ready:mdreamConfig, ai-ready:markdown) */
|
|
11
|
+
hooks?: {
|
|
12
|
+
route: string;
|
|
13
|
+
event: H3Event;
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
export declare function convertHtmlToMarkdown(html: string, url: string, mdreamOptions: ModulePublicRuntimeConfig['mdreamOptions'], opts?: ConvertHtmlOptions): Promise<{
|
|
17
17
|
updatedAt?: string | undefined;
|
|
18
18
|
markdown: string;
|
|
19
19
|
title: string;
|
|
20
20
|
description: string;
|
|
21
21
|
headings: Record<string, string>[];
|
|
22
22
|
metaKeywords: string;
|
|
23
|
+
textContent: string;
|
|
23
24
|
}>;
|
|
25
|
+
export {};
|
|
@@ -4,9 +4,44 @@ import { htmlToMarkdown } from "mdream";
|
|
|
4
4
|
import { extractionPlugin } from "mdream/plugins";
|
|
5
5
|
import { withMinimalPreset } from "mdream/preset/minimal";
|
|
6
6
|
import { useNitroApp } from "nitropack/runtime";
|
|
7
|
-
|
|
7
|
+
function normalizeWhitespace(text) {
|
|
8
8
|
return text.replace(/\u00A0/g, " ");
|
|
9
9
|
}
|
|
10
|
+
function buildMdreamOptions(url, mdreamOptions, meta, extractUpdatedAt = false) {
|
|
11
|
+
const extractPlugin = extractionPlugin({
|
|
12
|
+
"title": (el) => {
|
|
13
|
+
meta.title = el.textContent;
|
|
14
|
+
},
|
|
15
|
+
'meta[name="description"]': (el) => {
|
|
16
|
+
meta.description = el.attributes.content || "";
|
|
17
|
+
},
|
|
18
|
+
'meta[name="keywords"]': (el) => {
|
|
19
|
+
meta.metaKeywords = el.attributes.content || "";
|
|
20
|
+
},
|
|
21
|
+
"h1, h2, h3, h4, h5, h6": (el) => {
|
|
22
|
+
const text = el.textContent?.trim();
|
|
23
|
+
if (text)
|
|
24
|
+
meta.headings.push({ [el.name.toLowerCase()]: text });
|
|
25
|
+
},
|
|
26
|
+
"p, li, td, th, blockquote, figcaption": (el) => {
|
|
27
|
+
const text = el.textContent?.trim();
|
|
28
|
+
if (text)
|
|
29
|
+
meta.textContent.push(text);
|
|
30
|
+
},
|
|
31
|
+
...extractUpdatedAt && {
|
|
32
|
+
'meta[property="article:modified_time"], meta[name="last-modified"], meta[name="updated"], meta[property="og:updated_time"], meta[name="lastmod"]': (el) => {
|
|
33
|
+
if (!meta.updatedAt && el.attributes.content)
|
|
34
|
+
meta.updatedAt = el.attributes.content;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
let options = { origin: url, ...mdreamOptions };
|
|
39
|
+
if (mdreamOptions?.preset === "minimal") {
|
|
40
|
+
options = withMinimalPreset(options);
|
|
41
|
+
}
|
|
42
|
+
options.plugins = [extractPlugin, ...options.plugins || []];
|
|
43
|
+
return options;
|
|
44
|
+
}
|
|
10
45
|
export function getMarkdownRenderInfo(event, explicitOnly = false) {
|
|
11
46
|
const originalPath = event.path;
|
|
12
47
|
if (originalPath.startsWith("/api") || originalPath.startsWith("/_") || originalPath.startsWith("/@")) {
|
|
@@ -32,7 +67,7 @@ export function getMarkdownRenderInfo(event, explicitOnly = false) {
|
|
|
32
67
|
}
|
|
33
68
|
return { path, isExplicit };
|
|
34
69
|
}
|
|
35
|
-
|
|
70
|
+
function clientPrefersMarkdown(event) {
|
|
36
71
|
const accept = getHeader(event, "accept") || "";
|
|
37
72
|
const secFetchDest = getHeader(event, "sec-fetch-dest") || "";
|
|
38
73
|
if (secFetchDest === "document") {
|
|
@@ -50,100 +85,34 @@ export function clientPrefersMarkdown(event) {
|
|
|
50
85
|
}
|
|
51
86
|
return false;
|
|
52
87
|
}
|
|
53
|
-
export async function convertHtmlToMarkdown(html, url,
|
|
54
|
-
const
|
|
55
|
-
|
|
56
|
-
let
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
const level = el.name.toLowerCase();
|
|
72
|
-
if (text)
|
|
73
|
-
headings.push({ [level]: text });
|
|
74
|
-
}
|
|
75
|
-
});
|
|
76
|
-
let options = {
|
|
77
|
-
origin: url,
|
|
78
|
-
...config.mdreamOptions
|
|
79
|
-
};
|
|
80
|
-
if (config.mdreamOptions?.preset === "minimal") {
|
|
81
|
-
options = withMinimalPreset(options);
|
|
82
|
-
options.plugins = [extractPlugin, ...options.plugins || []];
|
|
83
|
-
} else {
|
|
84
|
-
options.plugins = [extractPlugin, ...options.plugins || []];
|
|
85
|
-
}
|
|
86
|
-
await nitroApp.hooks.callHook("ai-ready:mdreamConfig", options);
|
|
87
|
-
let markdown = htmlToMarkdown(html, options);
|
|
88
|
-
const context = {
|
|
89
|
-
html,
|
|
90
|
-
markdown,
|
|
91
|
-
route,
|
|
92
|
-
title,
|
|
93
|
-
description,
|
|
94
|
-
isPrerender: false,
|
|
95
|
-
event
|
|
96
|
-
};
|
|
97
|
-
await nitroApp.hooks.callHook("ai-ready:markdown", context);
|
|
98
|
-
markdown = normalizeWhitespace(context.markdown);
|
|
99
|
-
return { markdown, title: normalizeWhitespace(title), description: normalizeWhitespace(description), headings, metaKeywords };
|
|
100
|
-
}
|
|
101
|
-
export async function convertHtmlToMarkdownMeta(html, url, mdreamOptions) {
|
|
102
|
-
let title = "";
|
|
103
|
-
let description = "";
|
|
104
|
-
let metaKeywords = "";
|
|
105
|
-
let updatedAt;
|
|
106
|
-
const headings = [];
|
|
107
|
-
const extractPlugin = extractionPlugin({
|
|
108
|
-
title(el) {
|
|
109
|
-
title = el.textContent;
|
|
110
|
-
},
|
|
111
|
-
'meta[name="description"]': (el) => {
|
|
112
|
-
description = el.attributes.content || "";
|
|
113
|
-
},
|
|
114
|
-
'meta[name="keywords"]': (el) => {
|
|
115
|
-
metaKeywords = el.attributes.content || "";
|
|
116
|
-
},
|
|
117
|
-
'meta[property="article:modified_time"], meta[name="last-modified"], meta[name="updated"], meta[property="og:updated_time"], meta[name="lastmod"]': (el) => {
|
|
118
|
-
if (!updatedAt && el.attributes.content) {
|
|
119
|
-
updatedAt = el.attributes.content;
|
|
120
|
-
}
|
|
121
|
-
},
|
|
122
|
-
"h1, h2, h3, h4, h5, h6": (el) => {
|
|
123
|
-
const text = el.textContent?.trim();
|
|
124
|
-
const level = el.name.toLowerCase();
|
|
125
|
-
if (text)
|
|
126
|
-
headings.push({ [level]: text });
|
|
127
|
-
}
|
|
128
|
-
});
|
|
129
|
-
let options = {
|
|
130
|
-
origin: url,
|
|
131
|
-
...mdreamOptions
|
|
132
|
-
};
|
|
133
|
-
if (mdreamOptions?.preset === "minimal") {
|
|
134
|
-
options = withMinimalPreset(options);
|
|
135
|
-
options.plugins = [extractPlugin, ...options.plugins || []];
|
|
88
|
+
export async function convertHtmlToMarkdown(html, url, mdreamOptions, opts = {}) {
|
|
89
|
+
const meta = { title: "", description: "", metaKeywords: "", headings: [], textContent: [] };
|
|
90
|
+
const options = buildMdreamOptions(url, mdreamOptions, meta, opts.extractUpdatedAt);
|
|
91
|
+
let markdown;
|
|
92
|
+
if (opts.hooks) {
|
|
93
|
+
const nitroApp = useNitroApp();
|
|
94
|
+
await nitroApp.hooks.callHook("ai-ready:mdreamConfig", options);
|
|
95
|
+
const context = {
|
|
96
|
+
html,
|
|
97
|
+
markdown: htmlToMarkdown(html, options),
|
|
98
|
+
route: opts.hooks.route,
|
|
99
|
+
title: meta.title,
|
|
100
|
+
description: meta.description,
|
|
101
|
+
isPrerender: false,
|
|
102
|
+
event: opts.hooks.event
|
|
103
|
+
};
|
|
104
|
+
await nitroApp.hooks.callHook("ai-ready:markdown", context);
|
|
105
|
+
markdown = context.markdown;
|
|
136
106
|
} else {
|
|
137
|
-
|
|
107
|
+
markdown = htmlToMarkdown(html, options);
|
|
138
108
|
}
|
|
139
|
-
const rawMarkdown = htmlToMarkdown(html, options);
|
|
140
|
-
const markdown = normalizeWhitespace(rawMarkdown);
|
|
141
109
|
return {
|
|
142
|
-
markdown,
|
|
143
|
-
title: normalizeWhitespace(title),
|
|
144
|
-
description: normalizeWhitespace(description),
|
|
145
|
-
headings,
|
|
146
|
-
metaKeywords,
|
|
147
|
-
|
|
110
|
+
markdown: normalizeWhitespace(markdown),
|
|
111
|
+
title: normalizeWhitespace(meta.title),
|
|
112
|
+
description: normalizeWhitespace(meta.description),
|
|
113
|
+
headings: meta.headings,
|
|
114
|
+
metaKeywords: meta.metaKeywords,
|
|
115
|
+
textContent: meta.textContent.join(" "),
|
|
116
|
+
...meta.updatedAt && { updatedAt: meta.updatedAt }
|
|
148
117
|
};
|
|
149
118
|
}
|
package/dist/runtime/types.d.ts
CHANGED
|
@@ -74,13 +74,6 @@ export interface ModuleOptions {
|
|
|
74
74
|
* @default 600 (10 minutes)
|
|
75
75
|
*/
|
|
76
76
|
cacheMaxAgeSeconds?: number;
|
|
77
|
-
/**
|
|
78
|
-
* TTL for re-indexing pages in seconds
|
|
79
|
-
* Pages are automatically indexed on visit; this controls how often to refresh
|
|
80
|
-
* 0 = never re-index (rely on prerendered data)
|
|
81
|
-
* @default 0
|
|
82
|
-
*/
|
|
83
|
-
ttl?: number;
|
|
84
77
|
/**
|
|
85
78
|
* Database configuration for page storage
|
|
86
79
|
* Uses db0 for cross-platform SQLite support
|
|
@@ -110,6 +103,68 @@ export interface ModuleOptions {
|
|
|
110
103
|
*/
|
|
111
104
|
authToken?: string;
|
|
112
105
|
};
|
|
106
|
+
/**
|
|
107
|
+
* Runtime sync configuration (opt-in for dynamic content sites)
|
|
108
|
+
* When enabled, pages are re-indexed at runtime from sitemap
|
|
109
|
+
* @default disabled - prerendered data is used
|
|
110
|
+
*/
|
|
111
|
+
runtimeSync?: {
|
|
112
|
+
/**
|
|
113
|
+
* Enable runtime sync
|
|
114
|
+
* When false (default), runtime uses prerendered data only
|
|
115
|
+
* @default false
|
|
116
|
+
*/
|
|
117
|
+
enabled?: boolean;
|
|
118
|
+
/**
|
|
119
|
+
* TTL for refresh in seconds (sitemap + page re-indexing)
|
|
120
|
+
* Controls how often to refresh sitemap routes and re-index stale pages
|
|
121
|
+
* @default 3600 (1 hour)
|
|
122
|
+
*/
|
|
123
|
+
ttl?: number;
|
|
124
|
+
/**
|
|
125
|
+
* Pages to index per batch
|
|
126
|
+
* @default 20
|
|
127
|
+
*/
|
|
128
|
+
batchSize?: number;
|
|
129
|
+
/**
|
|
130
|
+
* Cron expression for scheduled indexing (e.g. every 5 minutes)
|
|
131
|
+
* When set, enables automatic background indexing via Nitro task
|
|
132
|
+
*/
|
|
133
|
+
cron?: string;
|
|
134
|
+
/**
|
|
135
|
+
* Secret token for authenticating poll endpoint
|
|
136
|
+
* When set, requires ?secret=<token> query param
|
|
137
|
+
*/
|
|
138
|
+
secret?: string;
|
|
139
|
+
/**
|
|
140
|
+
* TTL for pruning stale routes in seconds
|
|
141
|
+
* Routes not seen in sitemap for longer than this are deleted
|
|
142
|
+
* 0 = never prune (default)
|
|
143
|
+
* @default 0
|
|
144
|
+
*/
|
|
145
|
+
pruneTtl?: number;
|
|
146
|
+
};
|
|
147
|
+
/**
|
|
148
|
+
* IndexNow configuration for instant search engine notifications
|
|
149
|
+
* Submits changed URLs to Bing, Yandex, Naver, Seznam via IndexNow protocol
|
|
150
|
+
*/
|
|
151
|
+
indexNow?: {
|
|
152
|
+
/**
|
|
153
|
+
* Enable IndexNow submissions
|
|
154
|
+
* @default false
|
|
155
|
+
*/
|
|
156
|
+
enabled?: boolean;
|
|
157
|
+
/**
|
|
158
|
+
* Your IndexNow API key
|
|
159
|
+
* Get one from https://www.bing.com/indexnow
|
|
160
|
+
*/
|
|
161
|
+
key?: string;
|
|
162
|
+
/**
|
|
163
|
+
* IndexNow endpoint host
|
|
164
|
+
* @default 'api.indexnow.org'
|
|
165
|
+
*/
|
|
166
|
+
host?: string;
|
|
167
|
+
};
|
|
113
168
|
}
|
|
114
169
|
/**
|
|
115
170
|
* Page-level entry for discovery and metadata queries
|
|
@@ -199,28 +254,6 @@ export interface LlmsTxtConfig {
|
|
|
199
254
|
/** Notes section (always appears at the end) */
|
|
200
255
|
notes?: string | string[];
|
|
201
256
|
}
|
|
202
|
-
/**
|
|
203
|
-
* Hook context for page markdown processing (Nuxt build-time hook)
|
|
204
|
-
*
|
|
205
|
-
* Called once per page during prerender when markdown content has changed.
|
|
206
|
-
* Allows integrations to process page content (e.g., for embedding generation).
|
|
207
|
-
*
|
|
208
|
-
* @example Process page markdown
|
|
209
|
-
* nuxt.hooks.hook('ai-ready:page:markdown', async (context) => {
|
|
210
|
-
* console.log(`Processing ${context.route}: ${context.title}`)
|
|
211
|
-
* // Generate embeddings, update search index, etc.
|
|
212
|
-
* })
|
|
213
|
-
*/
|
|
214
|
-
export interface PageMarkdownContext {
|
|
215
|
-
/** The route being processed (e.g., '/about') */
|
|
216
|
-
route: string;
|
|
217
|
-
/** The markdown content */
|
|
218
|
-
markdown: string;
|
|
219
|
-
/** Page title extracted from HTML */
|
|
220
|
-
title: string;
|
|
221
|
-
/** Page description from meta tags */
|
|
222
|
-
description: string;
|
|
223
|
-
}
|
|
224
257
|
/**
|
|
225
258
|
* Context for runtime page indexing hook
|
|
226
259
|
*/
|
|
@@ -241,15 +274,6 @@ export interface PageIndexedContext {
|
|
|
241
274
|
updatedAt: string;
|
|
242
275
|
/** Whether this is a new page or an update */
|
|
243
276
|
isUpdate: boolean;
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
interface NitroRuntimeHooks {
|
|
247
|
-
'ai-ready:markdown': (context: MarkdownContext) => void | Promise<void>;
|
|
248
|
-
'ai-ready:mdreamConfig': (config: import('mdream').HTMLToMarkdownOptions) => void | Promise<void>;
|
|
249
|
-
/**
|
|
250
|
-
* Called when a page is indexed at runtime
|
|
251
|
-
* Use this to sync with external systems (embeddings, search, etc)
|
|
252
|
-
*/
|
|
253
|
-
'ai-ready:page:indexed': (context: PageIndexedContext) => void | Promise<void>;
|
|
254
|
-
}
|
|
277
|
+
/** Whether the content hash changed (triggers IndexNow sync) */
|
|
278
|
+
contentChanged: boolean;
|
|
255
279
|
}
|