nuxt-ai-ready 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/module.d.mts +9 -0
- package/dist/module.json +1 -1
- package/dist/module.mjs +178 -22
- package/dist/runtime/index.d.ts +9 -0
- package/dist/runtime/index.js +4 -0
- package/dist/runtime/mcp.d.ts +32 -0
- package/dist/runtime/mcp.js +5 -0
- package/dist/runtime/server/db/dump.d.ts +29 -0
- package/dist/runtime/server/db/dump.js +29 -0
- package/dist/runtime/server/db/index.d.ts +12 -0
- package/dist/runtime/server/db/index.js +56 -0
- package/dist/runtime/server/db/queries.d.ts +84 -0
- package/dist/runtime/server/db/queries.js +79 -0
- package/dist/runtime/server/db/schema.d.ts +8 -0
- package/dist/runtime/server/db/schema.js +121 -0
- package/dist/runtime/server/mcp/tools/search-pages.js +19 -0
- package/dist/runtime/server/middleware/markdown.prerender.js +11 -2
- package/dist/runtime/server/plugins/db-restore.d.ts +2 -0
- package/dist/runtime/server/plugins/db-restore.js +24 -0
- package/dist/runtime/server/plugins/page-indexer.d.ts +2 -0
- package/dist/runtime/server/plugins/page-indexer.js +68 -0
- package/dist/runtime/server/utils/indexPage.d.ts +38 -0
- package/dist/runtime/server/utils/indexPage.js +76 -0
- package/dist/runtime/server/utils/keywords.d.ts +8 -0
- package/dist/runtime/server/utils/keywords.js +282 -0
- package/dist/runtime/server/utils/pageData.d.ts +4 -2
- package/dist/runtime/server/utils/pageData.js +19 -41
- package/dist/runtime/server/utils.d.ts +4 -2
- package/dist/runtime/server/utils.js +11 -2
- package/dist/runtime/types.d.ts +64 -0
- package/mcp.d.ts +4 -0
- package/package.json +29 -19
- package/dist/runtime/server/mcp/tools/search-pages-fuzzy.js +0 -25
package/README.md
CHANGED
|
@@ -20,7 +20,8 @@ Nuxt AI Ready implements both. It converts your pages to markdown, generates llm
|
|
|
20
20
|
- 🚀 **On-Demand Markdown**: Any route available as `.md` (e.g., `/about` → `/about.md`), automatically served to AI crawlers
|
|
21
21
|
- 📡 **Content Signals**: Configure AI training/search/input permissions via [Nuxt Robots](https://nuxtseo.com/robots)
|
|
22
22
|
- 🌐 **Sitemap Integration**: Index AI-allowed pages via [Nuxt Sitemap](https://nuxtseo.com/sitemap)
|
|
23
|
-
- ⚡ **MCP Server**: `list_pages` and `
|
|
23
|
+
- ⚡ **MCP Server**: `list_pages` and `search_pages` tools with FTS5 full-text search
|
|
24
|
+
- 🗄️ **Runtime Indexing**: Index pages on-demand without prerendering, with SQLite/D1/LibSQL support
|
|
24
25
|
- 🧠 **[RAG Ready](https://nuxtseo.com/ai-ready/advanced/rag-example)**: Markdown output optimized for vectorizing and semantic search
|
|
25
26
|
|
|
26
27
|
## Installation
|
package/dist/module.d.mts
CHANGED
|
@@ -7,6 +7,7 @@ interface ParsedMarkdownResult {
|
|
|
7
7
|
title: string;
|
|
8
8
|
description: string;
|
|
9
9
|
headings: Array<Record<string, string>>;
|
|
10
|
+
keywords?: string[];
|
|
10
11
|
updatedAt?: string;
|
|
11
12
|
}
|
|
12
13
|
|
|
@@ -31,6 +32,14 @@ interface ModulePublicRuntimeConfig {
|
|
|
31
32
|
version: string;
|
|
32
33
|
mdreamOptions: ModuleOptions['mdreamOptions'];
|
|
33
34
|
markdownCacheHeaders: Required<NonNullable<ModuleOptions['markdownCacheHeaders']>>;
|
|
35
|
+
ttl: number;
|
|
36
|
+
database: {
|
|
37
|
+
type: 'sqlite' | 'd1' | 'libsql';
|
|
38
|
+
filename?: string;
|
|
39
|
+
bindingName?: string;
|
|
40
|
+
url?: string;
|
|
41
|
+
authToken?: string;
|
|
42
|
+
};
|
|
34
43
|
}
|
|
35
44
|
declare const _default: _nuxt_schema.NuxtModule<ModuleOptions, ModuleOptions, false>;
|
|
36
45
|
|
package/dist/module.json
CHANGED
package/dist/module.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { appendFile, mkdir, writeFile, readFile, stat, access } from 'node:fs/promises';
|
|
2
|
-
import { join, dirname } from 'node:path';
|
|
2
|
+
import { join as join$1, dirname } from 'node:path';
|
|
3
3
|
import { useLogger, useNuxt, defineNuxtModule, createResolver, addTypeTemplate, hasNuxtModule, addServerHandler, addPlugin } from '@nuxt/kit';
|
|
4
4
|
import defu from 'defu';
|
|
5
5
|
import { useSiteConfig, installNuxtSiteConfig, withSiteUrl } from 'nuxt-site-config/kit';
|
|
@@ -10,6 +10,7 @@ import { isTest, isCI } from 'std-env';
|
|
|
10
10
|
import { parseSitemapXml } from '@nuxtjs/sitemap/utils';
|
|
11
11
|
import { colorize } from 'consola/utils';
|
|
12
12
|
import { withBase } from 'ufo';
|
|
13
|
+
import { join, isAbsolute } from 'pathe';
|
|
13
14
|
|
|
14
15
|
const logger = useLogger("nuxt-ai-ready");
|
|
15
16
|
|
|
@@ -64,6 +65,45 @@ function hookNuxtSeoProLicense() {
|
|
|
64
65
|
}
|
|
65
66
|
}
|
|
66
67
|
|
|
68
|
+
async function resolveDatabaseAdapter(type, _opts) {
|
|
69
|
+
const connectors = {
|
|
70
|
+
d1: "db0/connectors/cloudflare-d1",
|
|
71
|
+
libsql: "db0/connectors/libsql/node"
|
|
72
|
+
};
|
|
73
|
+
if (type !== "sqlite" && connectors[type]) {
|
|
74
|
+
return connectors[type];
|
|
75
|
+
}
|
|
76
|
+
if (process.versions.bun) {
|
|
77
|
+
return "db0/connectors/bun-sqlite";
|
|
78
|
+
}
|
|
79
|
+
const nodeVersion = Number.parseInt(process.versions.node?.split(".")[0] || "0");
|
|
80
|
+
if (nodeVersion >= 22) {
|
|
81
|
+
return "db0/connectors/node-sqlite";
|
|
82
|
+
}
|
|
83
|
+
return "db0/connectors/better-sqlite3";
|
|
84
|
+
}
|
|
85
|
+
function refineDatabaseConfig(config, rootDir) {
|
|
86
|
+
const type = config.type || "sqlite";
|
|
87
|
+
if (type === "sqlite") {
|
|
88
|
+
const filename = config.filename || ".data/ai-ready/pages.db";
|
|
89
|
+
return {
|
|
90
|
+
type: "sqlite",
|
|
91
|
+
filename: isAbsolute(filename) ? filename : join(rootDir, filename)
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
if (type === "d1") {
|
|
95
|
+
return {
|
|
96
|
+
type: "d1",
|
|
97
|
+
bindingName: config.bindingName || "AI_READY_DB"
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
type: "libsql",
|
|
102
|
+
url: config.url,
|
|
103
|
+
authToken: config.authToken
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
67
107
|
function normalizeLink(link) {
|
|
68
108
|
const parts = [];
|
|
69
109
|
parts.push(`- [${link.title}](${link.href})`);
|
|
@@ -178,7 +218,7 @@ function formatPageForLlmsFullTxt(route, title, description, markdown, siteUrl)
|
|
|
178
218
|
`;
|
|
179
219
|
}
|
|
180
220
|
async function processMarkdownRoute(state, nuxt, route, parsed, lastmod, options) {
|
|
181
|
-
const { markdown, title, description, headings, updatedAt: metaUpdatedAt } = parsed;
|
|
221
|
+
const { markdown, title, description, headings, keywords, updatedAt: metaUpdatedAt } = parsed;
|
|
182
222
|
let updatedAt = (lastmod instanceof Date ? lastmod.toISOString() : lastmod) || (/* @__PURE__ */ new Date()).toISOString();
|
|
183
223
|
if (metaUpdatedAt) {
|
|
184
224
|
const parsedDate = new Date(metaUpdatedAt);
|
|
@@ -192,6 +232,7 @@ async function processMarkdownRoute(state, nuxt, route, parsed, lastmod, options
|
|
|
192
232
|
title,
|
|
193
233
|
description,
|
|
194
234
|
headings: flattenHeadings(headings),
|
|
235
|
+
keywords: keywords || [],
|
|
195
236
|
updatedAt,
|
|
196
237
|
markdown
|
|
197
238
|
};
|
|
@@ -273,6 +314,92 @@ function detectSitemapPrerender(sitemapName = "sitemap.xml") {
|
|
|
273
314
|
usePrerenderHook: shouldHookIntoPrerender && !prerenderSitemap
|
|
274
315
|
};
|
|
275
316
|
}
|
|
317
|
+
async function createDatabaseDump(entries, errorRoutes, dbConfig) {
|
|
318
|
+
const Database = (await import('better-sqlite3')).default;
|
|
319
|
+
const dbPath = dbConfig.filename || ".data/ai-ready/pages.db";
|
|
320
|
+
await mkdir(dirname(dbPath), { recursive: true });
|
|
321
|
+
const db = new Database(dbPath);
|
|
322
|
+
db.exec(`
|
|
323
|
+
CREATE TABLE IF NOT EXISTS pages (
|
|
324
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
325
|
+
route TEXT UNIQUE NOT NULL,
|
|
326
|
+
route_key TEXT UNIQUE NOT NULL,
|
|
327
|
+
title TEXT NOT NULL DEFAULT '',
|
|
328
|
+
description TEXT NOT NULL DEFAULT '',
|
|
329
|
+
markdown TEXT NOT NULL DEFAULT '',
|
|
330
|
+
headings TEXT NOT NULL DEFAULT '[]',
|
|
331
|
+
keywords TEXT NOT NULL DEFAULT '[]',
|
|
332
|
+
updated_at TEXT NOT NULL,
|
|
333
|
+
indexed_at INTEGER NOT NULL,
|
|
334
|
+
is_error INTEGER NOT NULL DEFAULT 0
|
|
335
|
+
);
|
|
336
|
+
CREATE INDEX IF NOT EXISTS idx_pages_route ON pages(route);
|
|
337
|
+
CREATE INDEX IF NOT EXISTS idx_pages_is_error ON pages(is_error);
|
|
338
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(
|
|
339
|
+
route, title, description, markdown, headings, keywords,
|
|
340
|
+
content=pages, content_rowid=id
|
|
341
|
+
);
|
|
342
|
+
CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN
|
|
343
|
+
INSERT INTO pages_fts(rowid, route, title, description, markdown, headings, keywords)
|
|
344
|
+
VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);
|
|
345
|
+
END;
|
|
346
|
+
CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN
|
|
347
|
+
INSERT INTO pages_fts(pages_fts, rowid, route, title, description, markdown, headings, keywords)
|
|
348
|
+
VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);
|
|
349
|
+
END;
|
|
350
|
+
CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN
|
|
351
|
+
INSERT INTO pages_fts(pages_fts, rowid, route, title, description, markdown, headings, keywords)
|
|
352
|
+
VALUES('delete', old.id, old.route, old.title, old.description, old.markdown, old.headings, old.keywords);
|
|
353
|
+
INSERT INTO pages_fts(rowid, route, title, description, markdown, headings, keywords)
|
|
354
|
+
VALUES (new.id, new.route, new.title, new.description, new.markdown, new.headings, new.keywords);
|
|
355
|
+
END;
|
|
356
|
+
`);
|
|
357
|
+
const insertStmt = db.prepare(`
|
|
358
|
+
INSERT OR REPLACE INTO pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error)
|
|
359
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
360
|
+
`);
|
|
361
|
+
const normalizeRouteKey = (route) => route.replace(/^\//, "").replace(/\//g, ":") || "index";
|
|
362
|
+
const now = Date.now();
|
|
363
|
+
for (const entry of entries) {
|
|
364
|
+
insertStmt.run(
|
|
365
|
+
entry.route,
|
|
366
|
+
normalizeRouteKey(entry.route),
|
|
367
|
+
entry.title,
|
|
368
|
+
entry.description,
|
|
369
|
+
entry.markdown,
|
|
370
|
+
entry.headings,
|
|
371
|
+
JSON.stringify(entry.keywords),
|
|
372
|
+
entry.updatedAt,
|
|
373
|
+
now,
|
|
374
|
+
0
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
for (const route of errorRoutes) {
|
|
378
|
+
insertStmt.run(
|
|
379
|
+
route,
|
|
380
|
+
normalizeRouteKey(route),
|
|
381
|
+
"",
|
|
382
|
+
"",
|
|
383
|
+
"",
|
|
384
|
+
"[]",
|
|
385
|
+
"[]",
|
|
386
|
+
(/* @__PURE__ */ new Date()).toISOString(),
|
|
387
|
+
now,
|
|
388
|
+
1
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
const rows = db.prepare(`
|
|
392
|
+
SELECT route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error
|
|
393
|
+
FROM pages
|
|
394
|
+
`).all();
|
|
395
|
+
db.close();
|
|
396
|
+
const json = JSON.stringify(rows);
|
|
397
|
+
const encoder = new TextEncoder();
|
|
398
|
+
const stream = new Blob([encoder.encode(json)]).stream();
|
|
399
|
+
const compressed = stream.pipeThrough(new CompressionStream("gzip"));
|
|
400
|
+
const buffer = await new Response(compressed).arrayBuffer();
|
|
401
|
+
return Buffer.from(buffer).toString("base64");
|
|
402
|
+
}
|
|
276
403
|
async function prerenderRoute(nitro, route) {
|
|
277
404
|
const start = Date.now();
|
|
278
405
|
const encodedRoute = encodeURI(route);
|
|
@@ -282,7 +409,7 @@ async function prerenderRoute(nitro, route) {
|
|
|
282
409
|
retry: nitro.options.prerender.retry,
|
|
283
410
|
retryDelay: nitro.options.prerender.retryDelay
|
|
284
411
|
});
|
|
285
|
-
const filePath = join(nitro.options.output.publicDir, route);
|
|
412
|
+
const filePath = join$1(nitro.options.output.publicDir, route);
|
|
286
413
|
await mkdir(dirname(filePath), { recursive: true });
|
|
287
414
|
const data = res._data;
|
|
288
415
|
if (data === void 0)
|
|
@@ -298,8 +425,9 @@ async function prerenderRoute(nitro, route) {
|
|
|
298
425
|
}
|
|
299
426
|
function setupPrerenderHandler(pageDataPath, siteInfo, llmsTxtConfig) {
|
|
300
427
|
const nuxt = useNuxt();
|
|
428
|
+
const dbConfig = refineDatabaseConfig({}, nuxt.options.rootDir);
|
|
301
429
|
nuxt.hooks.hook("nitro:init", async (nitro) => {
|
|
302
|
-
const llmsFullTxtPath = join(nitro.options.output.publicDir, "llms-full.txt");
|
|
430
|
+
const llmsFullTxtPath = join$1(nitro.options.output.publicDir, "llms-full.txt");
|
|
303
431
|
const state = createCrawlerState(pageDataPath, llmsFullTxtPath, siteInfo, llmsTxtConfig);
|
|
304
432
|
let initPromise = null;
|
|
305
433
|
nitro.hooks.hook("prerender:generate", async (route) => {
|
|
@@ -319,7 +447,7 @@ function setupPrerenderHandler(pageDataPath, siteInfo, llmsTxtConfig) {
|
|
|
319
447
|
initPromise = initCrawler(state);
|
|
320
448
|
await initPromise;
|
|
321
449
|
const parsed = JSON.parse(route.contents || "{}");
|
|
322
|
-
const { markdown, title, description, headings, updatedAt: metaUpdatedAt } = parsed;
|
|
450
|
+
const { markdown, title, description, headings, keywords, updatedAt: metaUpdatedAt } = parsed;
|
|
323
451
|
let updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
324
452
|
if (metaUpdatedAt) {
|
|
325
453
|
const parsedDate = new Date(metaUpdatedAt);
|
|
@@ -339,6 +467,7 @@ function setupPrerenderHandler(pageDataPath, siteInfo, llmsTxtConfig) {
|
|
|
339
467
|
title,
|
|
340
468
|
description,
|
|
341
469
|
headings: flattenHeadings(headings),
|
|
470
|
+
keywords: keywords || [],
|
|
342
471
|
updatedAt,
|
|
343
472
|
markdown
|
|
344
473
|
};
|
|
@@ -361,24 +490,32 @@ function setupPrerenderHandler(pageDataPath, siteInfo, llmsTxtConfig) {
|
|
|
361
490
|
}
|
|
362
491
|
logger.debug(`Wrote ${state.errorRoutes.size} error routes to page data`);
|
|
363
492
|
}
|
|
493
|
+
const publicDataDir = join$1(nitro.options.output.publicDir, "__ai-ready");
|
|
494
|
+
await mkdir(publicDataDir, { recursive: true });
|
|
364
495
|
if (state.pageDataPath) {
|
|
365
496
|
const jsonlContent = await readFile(state.pageDataPath, "utf-8").catch(() => "");
|
|
366
497
|
if (jsonlContent) {
|
|
367
498
|
const entries = jsonlContent.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
368
|
-
const pages = entries.filter((e) => !e._error)
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
499
|
+
const pages = entries.filter((e) => !e._error);
|
|
500
|
+
const errorRoutesList = entries.filter((e) => e._error).map((e) => e.route);
|
|
501
|
+
const jsonContent = JSON.stringify({
|
|
502
|
+
pages: pages.map((p) => ({
|
|
503
|
+
route: p.route,
|
|
504
|
+
title: p.title,
|
|
505
|
+
description: p.description,
|
|
506
|
+
headings: p.headings,
|
|
507
|
+
keywords: p.keywords || [],
|
|
508
|
+
updatedAt: p.updatedAt
|
|
509
|
+
})),
|
|
510
|
+
errorRoutes: errorRoutesList
|
|
511
|
+
});
|
|
512
|
+
const publicJsonPath = join$1(publicDataDir, "pages.json");
|
|
380
513
|
await writeFile(publicJsonPath, jsonContent, "utf-8");
|
|
381
514
|
logger.debug(`Wrote ${pages.length} pages to __ai-ready/pages.json`);
|
|
515
|
+
const dumpData = await createDatabaseDump(pages, errorRoutesList, dbConfig);
|
|
516
|
+
const dumpPath = join$1(publicDataDir, "pages.dump");
|
|
517
|
+
await writeFile(dumpPath, dumpData, "utf-8");
|
|
518
|
+
logger.debug(`Created database dump at __ai-ready/pages.dump (${(dumpData.length / 1024).toFixed(1)}kb compressed)`);
|
|
382
519
|
}
|
|
383
520
|
}
|
|
384
521
|
const llmsStats = await prerenderRoute(nitro, "/llms.txt");
|
|
@@ -469,6 +606,11 @@ const module$1 = defineNuxtModule({
|
|
|
469
606
|
hookNuxtSeoProLicense();
|
|
470
607
|
nuxt.options.nitro.alias = nuxt.options.nitro.alias || {};
|
|
471
608
|
nuxt.options.alias["#ai-ready"] = resolve("./runtime");
|
|
609
|
+
({ resolver: createResolver(import.meta.url) });
|
|
610
|
+
const dbType = config.database?.type || "sqlite";
|
|
611
|
+
const adapterPath = await resolveDatabaseAdapter(dbType);
|
|
612
|
+
nuxt.options.alias["#ai-ready/adapter"] = adapterPath;
|
|
613
|
+
nuxt.options.nitro.alias["#ai-ready/adapter"] = adapterPath;
|
|
472
614
|
if (!nuxt.options.mcp?.name) {
|
|
473
615
|
nuxt.options.mcp = nuxt.options.mcp || {};
|
|
474
616
|
nuxt.options.mcp.name = useSiteConfig().name;
|
|
@@ -489,13 +631,14 @@ const module$1 = defineNuxtModule({
|
|
|
489
631
|
addTypeTemplate({
|
|
490
632
|
filename: "types/nuxt-ai-ready.d.ts",
|
|
491
633
|
getContents: () => `// Generated by nuxt-ai-ready
|
|
492
|
-
import type { MarkdownContext } from 'nuxt-ai-ready'
|
|
634
|
+
import type { MarkdownContext, PageIndexedContext } from 'nuxt-ai-ready'
|
|
493
635
|
import type { HTMLToMarkdownOptions } from 'mdream'
|
|
494
636
|
|
|
495
637
|
declare module 'nitropack/types' {
|
|
496
638
|
interface NitroRuntimeHooks {
|
|
497
639
|
'ai-ready:markdown': (context: MarkdownContext) => void | Promise<void>
|
|
498
640
|
'ai-ready:mdreamConfig': (config: HTMLToMarkdownOptions) => void | Promise<void>
|
|
641
|
+
'ai-ready:page:indexed': (context: PageIndexedContext) => void | Promise<void>
|
|
499
642
|
}
|
|
500
643
|
}
|
|
501
644
|
|
|
@@ -506,6 +649,7 @@ declare module '#ai-ready-virtual/read-page-data.mjs' {
|
|
|
506
649
|
title: string
|
|
507
650
|
description: string
|
|
508
651
|
headings: string
|
|
652
|
+
keywords: string[]
|
|
509
653
|
updatedAt: string
|
|
510
654
|
markdown: string
|
|
511
655
|
}>
|
|
@@ -517,6 +661,12 @@ declare module '#ai-ready-virtual/page-data.mjs' {
|
|
|
517
661
|
export const pages: never[]
|
|
518
662
|
}
|
|
519
663
|
|
|
664
|
+
declare module '#ai-ready/adapter' {
|
|
665
|
+
import type { Connector } from 'db0'
|
|
666
|
+
const connector: (config: unknown) => Connector
|
|
667
|
+
export default connector
|
|
668
|
+
}
|
|
669
|
+
|
|
520
670
|
export {}
|
|
521
671
|
`
|
|
522
672
|
}, { nitro: true });
|
|
@@ -570,8 +720,8 @@ export {}
|
|
|
570
720
|
await nuxt.callHook("ai-ready:llms-txt", llmsTxtPayload);
|
|
571
721
|
mergedLlmsTxt.sections = llmsTxtPayload.sections;
|
|
572
722
|
mergedLlmsTxt.notes = llmsTxtPayload.notes.length > 0 ? llmsTxtPayload.notes : void 0;
|
|
573
|
-
const prerenderCacheDir = join(nuxt.options.rootDir, "node_modules/.cache/nuxt-seo/ai-ready/routes");
|
|
574
|
-
const pageDataPath = join(nuxt.options.buildDir, ".data/ai-ready/page-data.jsonl");
|
|
723
|
+
const prerenderCacheDir = join$1(nuxt.options.rootDir, "node_modules/.cache/nuxt-seo/ai-ready/routes");
|
|
724
|
+
const pageDataPath = join$1(nuxt.options.buildDir, ".data/ai-ready/page-data.jsonl");
|
|
575
725
|
nuxt.hooks.hook("nitro:config", (nitroConfig) => {
|
|
576
726
|
nitroConfig.experimental = nitroConfig.experimental || {};
|
|
577
727
|
nitroConfig.experimental.asyncContext = true;
|
|
@@ -594,6 +744,7 @@ export async function readPageDataFromFilesystem() {
|
|
|
594
744
|
nitroConfig.virtual["#ai-ready-virtual/page-data.mjs"] = `export const pages = []
|
|
595
745
|
export const errorRoutes = []`;
|
|
596
746
|
});
|
|
747
|
+
const database = refineDatabaseConfig(config.database || {}, nuxt.options.rootDir);
|
|
597
748
|
nuxt.options.runtimeConfig["nuxt-ai-ready"] = {
|
|
598
749
|
version: version || "0.0.0",
|
|
599
750
|
debug: config.debug || false,
|
|
@@ -604,8 +755,13 @@ export const errorRoutes = []`;
|
|
|
604
755
|
}),
|
|
605
756
|
llmsTxt: mergedLlmsTxt,
|
|
606
757
|
cacheMaxAgeSeconds: config.cacheMaxAgeSeconds ?? 600,
|
|
607
|
-
prerenderCacheDir
|
|
758
|
+
prerenderCacheDir,
|
|
759
|
+
ttl: config.ttl ?? 0,
|
|
760
|
+
database
|
|
608
761
|
};
|
|
762
|
+
nuxt.options.nitro.plugins = nuxt.options.nitro.plugins || [];
|
|
763
|
+
nuxt.options.nitro.plugins.push(resolve("./runtime/server/plugins/db-restore"));
|
|
764
|
+
nuxt.options.nitro.plugins.push(resolve("./runtime/server/plugins/page-indexer"));
|
|
609
765
|
addServerHandler({
|
|
610
766
|
middleware: true,
|
|
611
767
|
handler: resolve("./runtime/server/middleware/markdown.prerender")
|
|
@@ -649,7 +805,7 @@ export const errorRoutes = []`;
|
|
|
649
805
|
nuxt.options.nitro.routeRules["/llms-full.txt"] = { headers: { "Content-Type": "text/plain; charset=utf-8" } };
|
|
650
806
|
nuxt.hooks.hook("nitro:build:before", (nitro) => {
|
|
651
807
|
nitro.hooks.hook("compiled", async () => {
|
|
652
|
-
const headersPath = join(nitro.options.output.publicDir, "_headers");
|
|
808
|
+
const headersPath = join$1(nitro.options.output.publicDir, "_headers");
|
|
653
809
|
const exists = await access(headersPath).then(() => true).catch(() => false);
|
|
654
810
|
if (exists) {
|
|
655
811
|
await appendFile(headersPath, `
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export { indexPage, indexPageByRoute } from './server/utils/indexPage.js';
|
|
2
|
+
export type { IndexPageOptions, IndexPageResult } from './server/utils/indexPage.js';
|
|
3
|
+
export { getPages, getPagesList, getErrorRoutes } from './server/utils/pageData.js';
|
|
4
|
+
export type { PageEntry, PageData, PageListItem } from './server/utils/pageData.js';
|
|
5
|
+
export { useDatabase } from './server/db/index.js';
|
|
6
|
+
export type { DatabaseAdapter } from './server/db/schema.js';
|
|
7
|
+
export { searchPages, getAllPages, getPage, getPageWithMarkdown, upsertPage, getPageCount } from './server/db/queries.js';
|
|
8
|
+
export type { SearchResult, PageRow } from './server/db/queries.js';
|
|
9
|
+
export type { MarkdownContext, PageIndexedContext, PageMarkdownContext, } from './types.js';
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { indexPage, indexPageByRoute } from "./server/utils/indexPage.js";
|
|
2
|
+
export { getPages, getPagesList, getErrorRoutes } from "./server/utils/pageData.js";
|
|
3
|
+
export { useDatabase } from "./server/db/index.js";
|
|
4
|
+
export { searchPages, getAllPages, getPage, getPageWithMarkdown, upsertPage, getPageCount } from "./server/db/queries.js";
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export declare const tools: readonly [{
|
|
2
|
+
name: string;
|
|
3
|
+
description: string;
|
|
4
|
+
inputSchema: {};
|
|
5
|
+
cache: "1h";
|
|
6
|
+
handler(): Promise<{
|
|
7
|
+
content: {
|
|
8
|
+
type: "text";
|
|
9
|
+
text: string;
|
|
10
|
+
}[];
|
|
11
|
+
}>;
|
|
12
|
+
}, import("@nuxtjs/mcp-toolkit").McpToolDefinition<Readonly<{
|
|
13
|
+
[k: string]: import("zod/v4/core").$ZodType<unknown, unknown, import("zod/v4/core").$ZodTypeInternals<unknown, unknown>>;
|
|
14
|
+
}>, Readonly<{
|
|
15
|
+
[k: string]: import("zod/v4/core").$ZodType<unknown, unknown, import("zod/v4/core").$ZodTypeInternals<unknown, unknown>>;
|
|
16
|
+
}>>];
|
|
17
|
+
export declare const resources: readonly [{
|
|
18
|
+
uri: string;
|
|
19
|
+
name: string;
|
|
20
|
+
description: string;
|
|
21
|
+
metadata: {
|
|
22
|
+
mimeType: string;
|
|
23
|
+
};
|
|
24
|
+
cache: "1h";
|
|
25
|
+
handler(uri: URL): Promise<{
|
|
26
|
+
contents: {
|
|
27
|
+
uri: string;
|
|
28
|
+
mimeType: string;
|
|
29
|
+
text: string;
|
|
30
|
+
}[];
|
|
31
|
+
}>;
|
|
32
|
+
}];
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { DatabaseAdapter } from './schema.js';
|
|
2
|
+
export interface DumpRow {
|
|
3
|
+
route: string;
|
|
4
|
+
route_key: string;
|
|
5
|
+
title: string;
|
|
6
|
+
description: string;
|
|
7
|
+
markdown: string;
|
|
8
|
+
headings: string;
|
|
9
|
+
keywords: string;
|
|
10
|
+
updated_at: string;
|
|
11
|
+
indexed_at: number;
|
|
12
|
+
is_error: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Export all pages as JSON for dump
|
|
16
|
+
*/
|
|
17
|
+
export declare function exportDump(db: DatabaseAdapter): Promise<DumpRow[]>;
|
|
18
|
+
/**
|
|
19
|
+
* Compress dump data to base64 gzip
|
|
20
|
+
*/
|
|
21
|
+
export declare function compressDump(data: DumpRow[]): Promise<string>;
|
|
22
|
+
/**
|
|
23
|
+
* Decompress dump from base64 gzip
|
|
24
|
+
*/
|
|
25
|
+
export declare function decompressDump(base64: string): Promise<DumpRow[]>;
|
|
26
|
+
/**
|
|
27
|
+
* Import dump into database
|
|
28
|
+
*/
|
|
29
|
+
export declare function importDump(db: DatabaseAdapter, rows: DumpRow[]): Promise<void>;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export async function exportDump(db) {
|
|
2
|
+
return db.all(`
|
|
3
|
+
SELECT route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error
|
|
4
|
+
FROM pages
|
|
5
|
+
`);
|
|
6
|
+
}
|
|
7
|
+
export async function compressDump(data) {
|
|
8
|
+
const json = JSON.stringify(data);
|
|
9
|
+
const encoder = new TextEncoder();
|
|
10
|
+
const stream = new Blob([encoder.encode(json)]).stream();
|
|
11
|
+
const compressed = stream.pipeThrough(new CompressionStream("gzip"));
|
|
12
|
+
const buffer = await new Response(compressed).arrayBuffer();
|
|
13
|
+
return Buffer.from(buffer).toString("base64");
|
|
14
|
+
}
|
|
15
|
+
export async function decompressDump(base64) {
|
|
16
|
+
const buffer = Buffer.from(base64, "base64");
|
|
17
|
+
const stream = new Blob([buffer]).stream();
|
|
18
|
+
const decompressed = stream.pipeThrough(new DecompressionStream("gzip"));
|
|
19
|
+
const text = await new Response(decompressed).text();
|
|
20
|
+
return JSON.parse(text);
|
|
21
|
+
}
|
|
22
|
+
export async function importDump(db, rows) {
|
|
23
|
+
for (const row of rows) {
|
|
24
|
+
await db.exec(`
|
|
25
|
+
INSERT OR REPLACE INTO pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error)
|
|
26
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
27
|
+
`, [row.route, row.route_key, row.title, row.description, row.markdown, row.headings, row.keywords, row.updated_at, row.indexed_at, row.is_error]);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { H3Event } from 'h3';
|
|
2
|
+
import type { DatabaseAdapter } from './schema.js';
|
|
3
|
+
/**
|
|
4
|
+
* Get the database adapter instance
|
|
5
|
+
* Initializes the database on first call
|
|
6
|
+
*/
|
|
7
|
+
export declare function useDatabase(event?: H3Event): Promise<DatabaseAdapter>;
|
|
8
|
+
/**
|
|
9
|
+
* Reset database connection (for testing)
|
|
10
|
+
*/
|
|
11
|
+
export declare function _resetDatabase(): void;
|
|
12
|
+
export type { DatabaseAdapter } from './schema.js';
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { useRuntimeConfig } from "nitropack/runtime";
|
|
2
|
+
import { mkdir } from "node:fs/promises";
|
|
3
|
+
import { dirname } from "pathe";
|
|
4
|
+
import { initSchema } from "./schema.js";
|
|
5
|
+
import adapter from "#ai-ready/adapter";
|
|
6
|
+
let _db = null;
|
|
7
|
+
let _initPromise = null;
|
|
8
|
+
export function useDatabase(event) {
|
|
9
|
+
if (!_initPromise) {
|
|
10
|
+
_initPromise = initDatabase(event);
|
|
11
|
+
}
|
|
12
|
+
return _initPromise;
|
|
13
|
+
}
|
|
14
|
+
async function initDatabase(event) {
|
|
15
|
+
const config = useRuntimeConfig()["nuxt-ai-ready"];
|
|
16
|
+
if (config.database.type === "d1") {
|
|
17
|
+
const binding = event?.context?.cloudflare?.env?.[config.database.bindingName || "AI_READY_DB"];
|
|
18
|
+
if (!binding) {
|
|
19
|
+
throw new Error(`D1 binding '${config.database.bindingName || "AI_READY_DB"}' not found in event context`);
|
|
20
|
+
}
|
|
21
|
+
_db = adapter({ binding });
|
|
22
|
+
} else if (config.database.type === "libsql") {
|
|
23
|
+
_db = adapter({
|
|
24
|
+
url: config.database.url,
|
|
25
|
+
authToken: config.database.authToken
|
|
26
|
+
});
|
|
27
|
+
} else {
|
|
28
|
+
const dbPath = config.database.filename || ".data/ai-ready/pages.db";
|
|
29
|
+
await mkdir(dirname(dbPath), { recursive: true });
|
|
30
|
+
_db = adapter({ path: dbPath });
|
|
31
|
+
}
|
|
32
|
+
if (!_db) {
|
|
33
|
+
throw new Error("Failed to initialize database connector");
|
|
34
|
+
}
|
|
35
|
+
const dbAdapter = createAdapter(_db);
|
|
36
|
+
await initSchema(dbAdapter);
|
|
37
|
+
return dbAdapter;
|
|
38
|
+
}
|
|
39
|
+
function createAdapter(db) {
|
|
40
|
+
return {
|
|
41
|
+
all: async (sql, params = []) => {
|
|
42
|
+
const result = await db.prepare(sql).all(...params);
|
|
43
|
+
return result || [];
|
|
44
|
+
},
|
|
45
|
+
first: async (sql, params = []) => {
|
|
46
|
+
return db.prepare(sql).get(...params);
|
|
47
|
+
},
|
|
48
|
+
exec: async (sql, params = []) => {
|
|
49
|
+
await db.prepare(sql).run(...params);
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
export function _resetDatabase() {
|
|
54
|
+
_db = null;
|
|
55
|
+
_initPromise = null;
|
|
56
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import type { DatabaseAdapter } from './schema.js';
|
|
2
|
+
export interface PageRow {
|
|
3
|
+
id: number;
|
|
4
|
+
route: string;
|
|
5
|
+
route_key: string;
|
|
6
|
+
title: string;
|
|
7
|
+
description: string;
|
|
8
|
+
markdown: string;
|
|
9
|
+
headings: string;
|
|
10
|
+
keywords: string;
|
|
11
|
+
updated_at: string;
|
|
12
|
+
indexed_at: number;
|
|
13
|
+
is_error: number;
|
|
14
|
+
}
|
|
15
|
+
export interface PageEntry {
|
|
16
|
+
route: string;
|
|
17
|
+
title: string;
|
|
18
|
+
description: string;
|
|
19
|
+
headings: string;
|
|
20
|
+
keywords: string[];
|
|
21
|
+
updatedAt: string;
|
|
22
|
+
}
|
|
23
|
+
export interface PageData extends PageEntry {
|
|
24
|
+
markdown: string;
|
|
25
|
+
}
|
|
26
|
+
export interface SearchResult {
|
|
27
|
+
route: string;
|
|
28
|
+
title: string;
|
|
29
|
+
description: string;
|
|
30
|
+
score: number;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Get all non-error pages
|
|
34
|
+
*/
|
|
35
|
+
export declare function getAllPages(db: DatabaseAdapter): Promise<PageEntry[]>;
|
|
36
|
+
/**
|
|
37
|
+
* Get a single page by route
|
|
38
|
+
*/
|
|
39
|
+
export declare function getPage(db: DatabaseAdapter, route: string): Promise<PageEntry | undefined>;
|
|
40
|
+
/**
|
|
41
|
+
* Get a page with markdown content
|
|
42
|
+
*/
|
|
43
|
+
export declare function getPageWithMarkdown(db: DatabaseAdapter, route: string): Promise<PageData | undefined>;
|
|
44
|
+
/**
|
|
45
|
+
* Full-text search using FTS5
|
|
46
|
+
* @param query Search query string
|
|
47
|
+
* @param opts Search options
|
|
48
|
+
*/
|
|
49
|
+
export declare function searchPages(db: DatabaseAdapter, query: string, opts?: {
|
|
50
|
+
limit?: number;
|
|
51
|
+
}): Promise<SearchResult[]>;
|
|
52
|
+
/**
|
|
53
|
+
* Insert or update a page
|
|
54
|
+
*/
|
|
55
|
+
export declare function upsertPage(db: DatabaseAdapter, page: {
|
|
56
|
+
route: string;
|
|
57
|
+
title: string;
|
|
58
|
+
description: string;
|
|
59
|
+
markdown: string;
|
|
60
|
+
headings: string;
|
|
61
|
+
keywords: string[];
|
|
62
|
+
updatedAt: string;
|
|
63
|
+
isError?: boolean;
|
|
64
|
+
}): Promise<void>;
|
|
65
|
+
/**
|
|
66
|
+
* Get all error routes
|
|
67
|
+
*/
|
|
68
|
+
export declare function getErrorRoutes(db: DatabaseAdapter): Promise<string[]>;
|
|
69
|
+
/**
|
|
70
|
+
* Check if a page is fresh (within TTL)
|
|
71
|
+
*/
|
|
72
|
+
export declare function isPageFresh(db: DatabaseAdapter, route: string, ttlSeconds: number): Promise<boolean>;
|
|
73
|
+
/**
|
|
74
|
+
* Delete a page by route
|
|
75
|
+
*/
|
|
76
|
+
export declare function deletePage(db: DatabaseAdapter, route: string): Promise<void>;
|
|
77
|
+
/**
|
|
78
|
+
* Get page count
|
|
79
|
+
*/
|
|
80
|
+
export declare function getPageCount(db: DatabaseAdapter): Promise<number>;
|
|
81
|
+
/**
|
|
82
|
+
* Check if database has any pages
|
|
83
|
+
*/
|
|
84
|
+
export declare function hasPages(db: DatabaseAdapter): Promise<boolean>;
|