nuxt-ai-ready 0.6.2 → 0.7.0

package/dist/runtime/index.d.ts

@@ -1,9 +1,7 @@
+export { useDatabase } from './server/db/index.js';
+export type { DatabaseAdapter } from './server/db/index.js';
+export { countPages, getSitemapSeededAt, getStaleRoutes, isPageFresh, pruneStaleRoutes, queryPages, searchPages, seedRoutes, setSitemapSeededAt, streamPages, upsertPage, } from './server/db/queries.js';
+export type { CountPagesOptions, PageData, PageEntry, PageRow, QueryPagesOptions, SearchPagesOptions, SearchResult, StreamPagesOptions, UpsertPageInput, } from './server/db/queries.js';
 export { indexPage, indexPageByRoute } from './server/utils/indexPage.js';
 export type { IndexPageOptions, IndexPageResult } from './server/utils/indexPage.js';
-export { getPages, getPagesList, getErrorRoutes } from './server/utils/pageData.js';
-export type { PageEntry, PageData, PageListItem } from './server/utils/pageData.js';
-export { useDatabase } from './server/db/index.js';
-export type { DatabaseAdapter } from './server/db/schema.js';
-export { searchPages, getAllPages, getPage, getPageWithMarkdown, upsertPage, getPageCount } from './server/db/queries.js';
-export type { SearchResult, PageRow } from './server/db/queries.js';
-export type { MarkdownContext, PageIndexedContext, PageMarkdownContext, } from './types.js';
+export type { MarkdownContext, PageIndexedContext, } from './types.js';

package/dist/runtime/index.js

@@ -1,4 +1,15 @@
-export { indexPage, indexPageByRoute } from "./server/utils/indexPage.js";
-export { getPages, getPagesList, getErrorRoutes } from "./server/utils/pageData.js";
 export { useDatabase } from "./server/db/index.js";
-export { searchPages, getAllPages, getPage, getPageWithMarkdown, upsertPage, getPageCount } from "./server/db/queries.js";
+export {
+  countPages,
+  getSitemapSeededAt,
+  getStaleRoutes,
+  isPageFresh,
+  pruneStaleRoutes,
+  queryPages,
+  searchPages,
+  seedRoutes,
+  setSitemapSeededAt,
+  streamPages,
+  upsertPage
+} from "./server/db/queries.js";
+export { indexPage, indexPageByRoute } from "./server/utils/indexPage.js";

package/dist/runtime/llms-txt-format.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Pure formatting functions for llms.txt - no runtime dependencies
+ */
+import type { LlmsTxtConfig } from './types.js';
+/**
+ * Normalize llms.txt structured configuration to markdown string
+ */
+export declare function normalizeLlmsTxtConfig(config: LlmsTxtConfig): string;

package/dist/runtime/llms-txt-format.js

@@ -0,0 +1,32 @@
+function normalizeLink(link) {
+  const parts = [];
+  parts.push(`- [${link.title}](${link.href})`);
+  if (link.description)
+    parts.push(`  ${link.description}`);
+  return parts.join("\n");
+}
+function normalizeSection(section) {
+  const parts = [];
+  parts.push(`## ${section.title}`);
+  parts.push("");
+  if (section.description) {
+    const descriptions = Array.isArray(section.description) ? section.description : [section.description];
+    parts.push(...descriptions);
+    parts.push("");
+  }
+  if (section.links?.length)
+    parts.push(...section.links.map(normalizeLink));
+  return parts.join("\n");
+}
+export function normalizeLlmsTxtConfig(config) {
+  const parts = [];
+  if (config.sections?.length)
+    parts.push(...config.sections.map(normalizeSection));
+  if (config.notes) {
+    parts.push("## Notes");
+    parts.push("");
+    const notes = Array.isArray(config.notes) ? config.notes : [config.notes];
+    parts.push(...notes);
+  }
+  return parts.join("\n\n");
+}

package/dist/runtime/llms-txt-utils.d.ts

@@ -1,7 +1,4 @@
 import type { H3Event } from 'h3';
-import type { LlmsTxtConfig } from './types.js';
-/**
- * Normalize llms.txt structured configuration to markdown string
- */
-export declare function normalizeLlmsTxtConfig(config: LlmsTxtConfig): string;
+import { normalizeLlmsTxtConfig } from './llms-txt-format.js';
+export { normalizeLlmsTxtConfig };
 export declare function buildLlmsTxt(event: H3Event): Promise<string>;

package/dist/runtime/llms-txt-utils.js

@@ -1,7 +1,9 @@
 import { getSiteConfig } from "#site-config/server/composables/getSiteConfig";
 import { useRuntimeConfig } from "nitropack/runtime";
-import { getErrorRoutes, getPages } from "./server/utils/pageData.js";
+import { normalizeLlmsTxtConfig } from "./llms-txt-format.js";
+import { queryPages } from "./server/db/queries.js";
 import { fetchSitemapUrls } from "./server/utils/sitemap.js";
+export { normalizeLlmsTxtConfig };
 function getGroupPrefix(url, depth) {
   const segments = url.split("/").filter(Boolean);
   if (segments.length === 0)
@@ -13,14 +15,12 @@ function getGroupPrefix(url, depth) {
 function getPathSegments(pathname) {
   return pathname.split("/").filter((s) => Boolean(s));
 }
-function sortPagesByPath(pages) {
+function analyzePageGroups(pages) {
   const twoSegmentCount = /* @__PURE__ */ new Map();
+  const segmentHasNested = /* @__PURE__ */ new Map();
   for (const page of pages) {
     const prefix = getGroupPrefix(page.pathname, 2);
     twoSegmentCount.set(prefix, (twoSegmentCount.get(prefix) || 0) + 1);
-  }
-  const segmentHasNested = /* @__PURE__ */ new Map();
-  for (const page of pages) {
     const segments = getPathSegments(page.pathname);
     const firstSegment = segments[0] || "";
     if (!segmentHasNested.has(firstSegment))
@@ -28,25 +28,25 @@ function sortPagesByPath(pages) {
     if (segments.length > 1)
       segmentHasNested.set(firstSegment, true);
   }
+  return { twoSegmentCount, segmentHasNested };
+}
+function getPageGroupKey(pathname, { twoSegmentCount, segmentHasNested }) {
+  const segments = getPathSegments(pathname);
+  const firstSegment = segments[0] || "";
+  const twoSegPrefix = getGroupPrefix(pathname, 2);
+  const twoSegCount = twoSegmentCount.get(twoSegPrefix) || 0;
+  let groupKey = twoSegCount > 1 ? twoSegPrefix : `/${firstSegment}`;
+  if (segments.length <= 1 && !segmentHasNested.get(firstSegment))
+    groupKey = "";
+  return groupKey;
+}
+function sortPagesByPath(pages) {
+  const analysis = analyzePageGroups(pages);
   return pages.sort((a, b) => {
     const segmentsA = getPathSegments(a.pathname);
     const segmentsB = getPathSegments(b.pathname);
-    const firstSegmentA = segmentsA[0] || "";
-    const firstSegmentB = segmentsB[0] || "";
-    const twoSegPrefixA = getGroupPrefix(a.pathname, 2);
-    const twoSegPrefixB = getGroupPrefix(b.pathname, 2);
-    const twoSegCountA = twoSegmentCount.get(twoSegPrefixA) || 0;
-    const twoSegCountB = twoSegmentCount.get(twoSegPrefixB) || 0;
-    let groupKeyA = twoSegCountA > 1 ? twoSegPrefixA : `/${firstSegmentA}`;
-    let groupKeyB = twoSegCountB > 1 ? twoSegPrefixB : `/${firstSegmentB}`;
-    const isRootLevelA = segmentsA.length <= 1;
-    const isRootLevelB = segmentsB.length <= 1;
-    const hasNestedA = segmentHasNested.get(firstSegmentA);
-    const hasNestedB = segmentHasNested.get(firstSegmentB);
-    if (isRootLevelA && !hasNestedA)
-      groupKeyA = "";
-    if (isRootLevelB && !hasNestedB)
-      groupKeyB = "";
+    const groupKeyA = getPageGroupKey(a.pathname, analysis);
+    const groupKeyB = getPageGroupKey(b.pathname, analysis);
     if (groupKeyA === "" && groupKeyB !== "")
       return -1;
     if (groupKeyA !== "" && groupKeyB === "")
@@ -66,39 +66,16 @@ function sortPagesByPath(pages) {
     return segmentsA.length - segmentsB.length;
   });
 }
-function getPageGroupKey(pathname, twoSegmentCount, segmentHasNested) {
-  const segments = getPathSegments(pathname);
-  const firstSegment = segments[0] || "";
-  const twoSegPrefix = getGroupPrefix(pathname, 2);
-  const twoSegCount = twoSegmentCount.get(twoSegPrefix) || 0;
-  let groupKey = twoSegCount > 1 ? twoSegPrefix : `/${firstSegment}`;
-  const isRootLevel = segments.length <= 1;
-  const hasNested = segmentHasNested.get(firstSegment);
-  if (isRootLevel && !hasNested)
-    groupKey = "";
-  return groupKey;
-}
 function formatPagesWithGroups(pages) {
   if (pages.length === 0)
     return [];
-  const twoSegmentCount = /* @__PURE__ */ new Map();
-  const segmentHasNested = /* @__PURE__ */ new Map();
-  for (const page of pages) {
-    const prefix = getGroupPrefix(page.pathname, 2);
-    twoSegmentCount.set(prefix, (twoSegmentCount.get(prefix) || 0) + 1);
-    const segments = getPathSegments(page.pathname);
-    const firstSegment = segments[0] || "";
-    if (!segmentHasNested.has(firstSegment))
-      segmentHasNested.set(firstSegment, false);
-    if (segments.length > 1)
-      segmentHasNested.set(firstSegment, true);
-  }
+  const analysis = analyzePageGroups(pages);
   const lines = [];
   let currentGroup = "";
   let segmentGroupIndex = 0;
   let urlsInCurrentGroup = 0;
   for (const page of pages) {
-    const groupKey = getPageGroupKey(page.pathname, twoSegmentCount, segmentHasNested);
+    const groupKey = getPageGroupKey(page.pathname, analysis);
     if (groupKey !== currentGroup) {
       if (urlsInCurrentGroup > 0) {
         const shouldAddBlankLine = segmentGroupIndex === 0 || segmentGroupIndex >= 1 && segmentGroupIndex <= 2 && urlsInCurrentGroup > 1;
@@ -118,41 +95,6 @@ function formatPagesWithGroups(pages) {
   }
   return lines;
 }
-function normalizeLink(link) {
-  const parts = [];
-  parts.push(`- [${link.title}](${link.href})`);
-  if (link.description) {
-    parts.push(`  ${link.description}`);
-  }
-  return parts.join("\n");
-}
-function normalizeSection(section) {
-  const parts = [];
-  parts.push(`## ${section.title}`);
-  parts.push("");
-  if (section.description) {
-    const descriptions = Array.isArray(section.description) ? section.description : [section.description];
-    parts.push(...descriptions);
-    parts.push("");
-  }
-  if (section.links?.length) {
-    parts.push(...section.links.map(normalizeLink));
-  }
-  return parts.join("\n");
-}
-export function normalizeLlmsTxtConfig(config) {
-  const parts = [];
-  if (config.sections?.length) {
-    parts.push(...config.sections.map(normalizeSection));
-  }
-  if (config.notes) {
-    parts.push("## Notes");
-    parts.push("");
-    const notes = Array.isArray(config.notes) ? config.notes : [config.notes];
-    parts.push(...notes);
-  }
-  return parts.join("\n\n");
-}
 export async function buildLlmsTxt(event) {
   const runtimeConfig = useRuntimeConfig(event);
   const aiReadyConfig = runtimeConfig["nuxt-ai-ready"];
@@ -185,20 +127,21 @@ Canonical Origin: ${siteConfig.url}`);
     parts.push(normalizedContent);
     parts.push("");
   }
-  const pages = await getPages();
+  const pages = await queryPages(event);
   const urls = await fetchSitemapUrls(event);
-  const errorRoutes = await getErrorRoutes();
-  const devModeHint = import.meta.dev && pages.size === 0 ? " (dev mode - run `nuxi generate` for page titles)" : "";
+  const errorRoutes = await queryPages(event, { where: { hasError: true } });
+  const errorSet = new Set(errorRoutes.map((e) => e.route));
+  const devModeHint = import.meta.dev && pages.length === 0 ? " (dev mode - run `nuxi generate` for page titles)" : "";
   const prerendered = [];
   const seenPaths = /* @__PURE__ */ new Set();
-  for (const [pathname, page] of pages) {
-    prerendered.push({ pathname, title: page.title, description: page.description });
-    seenPaths.add(pathname);
+  for (const page of pages) {
+    prerendered.push({ pathname: page.route, title: page.title, description: page.description });
+    seenPaths.add(page.route);
   }
   const other = [];
   for (const url of urls) {
     const pathname = url.loc.startsWith("http") ? new URL(url.loc).pathname : url.loc;
-    if (!seenPaths.has(pathname) && !errorRoutes.has(pathname)) {
+    if (!seenPaths.has(pathname) && !errorSet.has(pathname)) {
       other.push({ pathname });
       seenPaths.add(pathname);
     }

package/dist/runtime/server/db/index.d.ts

@@ -1,12 +1,8 @@
 import type { H3Event } from 'h3';
-import type { DatabaseAdapter } from './schema.js';
+import type { DatabaseAdapter } from './shared.js';
 /**
  * Get the database adapter instance
- * Initializes the database on first call
+ * Cached on nitro context for request lifecycle
  */
 export declare function useDatabase(event?: H3Event): Promise<DatabaseAdapter>;
-/**
- * Reset database connection (for testing)
- */
-export declare function _resetDatabase(): void;
-export type { DatabaseAdapter } from './schema.js';
+export type { DatabaseAdapter } from './shared.js';

package/dist/runtime/server/db/index.js

@@ -1,56 +1,39 @@
-import { useRuntimeConfig } from "nitropack/runtime";
 import { mkdir } from "node:fs/promises";
-import { dirname } from "pathe";
-import { initSchema } from "./schema.js";
 import adapter from "#ai-ready/adapter";
-let _db = null;
-let _initPromise = null;
+import { useNitroApp, useRuntimeConfig } from "nitropack/runtime";
+import { dirname } from "pathe";
+import { createAdapter, initSchema } from "./shared.js";
+const DB_KEY = "_aiReadyDb";
 export function useDatabase(event) {
-  if (!_initPromise) {
-    _initPromise = initDatabase(event);
+  const nitro = useNitroApp();
+  if (!nitro[DB_KEY]) {
+    nitro[DB_KEY] = initDatabase(event);
   }
-  return _initPromise;
+  return nitro[DB_KEY];
 }
 async function initDatabase(event) {
   const config = useRuntimeConfig()["nuxt-ai-ready"];
+  let connector;
   if (config.database.type === "d1") {
     const binding = event?.context?.cloudflare?.env?.[config.database.bindingName || "AI_READY_DB"];
     if (!binding) {
       throw new Error(`D1 binding '${config.database.bindingName || "AI_READY_DB"}' not found in event context`);
     }
-    _db = adapter({ binding });
+    connector = adapter({ binding });
   } else if (config.database.type === "libsql") {
-    _db = adapter({
+    connector = adapter({
       url: config.database.url,
       authToken: config.database.authToken
     });
   } else {
     const dbPath = config.database.filename || ".data/ai-ready/pages.db";
     await mkdir(dirname(dbPath), { recursive: true });
-    _db = adapter({ path: dbPath });
+    connector = adapter({ path: dbPath });
   }
-  if (!_db) {
+  if (!connector) {
     throw new Error("Failed to initialize database connector");
   }
-  const dbAdapter = createAdapter(_db);
+  const dbAdapter = createAdapter(connector);
   await initSchema(dbAdapter);
   return dbAdapter;
 }
-function createAdapter(db) {
-  return {
-    all: async (sql, params = []) => {
-      const result = await db.prepare(sql).all(...params);
-      return result || [];
-    },
-    first: async (sql, params = []) => {
-      return db.prepare(sql).get(...params);
-    },
-    exec: async (sql, params = []) => {
-      await db.prepare(sql).run(...params);
-    }
-  };
-}
-export function _resetDatabase() {
-  _db = null;
-  _initPromise = null;
-}

package/dist/runtime/server/db/queries.d.ts

@@ -1,4 +1,4 @@
-import type { DatabaseAdapter } from './schema.js';
+import type { H3Event } from 'h3';
 export interface PageRow {
     id: number;
     route: string;
@@ -8,9 +8,14 @@ export interface PageRow {
     markdown: string;
     headings: string;
     keywords: string;
+    content_hash: string | null;
     updated_at: string;
     indexed_at: number;
     is_error: number;
+    indexed: number;
+    source: 'prerender' | 'runtime';
+    last_seen_at: number | null;
+    indexnow_synced_at: number | null;
 }
 export interface PageEntry {
     route: string;
@@ -29,56 +34,131 @@ export interface SearchResult {
     description: string;
     score: number;
 }
+export interface QueryPagesOptions {
+    route?: string;
+    includeMarkdown?: boolean;
+    where?: {
+        pending?: boolean;
+        hasError?: boolean;
+        source?: 'prerender' | 'runtime';
+    };
+    limit?: number;
+    offset?: number;
+}
 /**
- * Get all non-error pages
- */
-export declare function getAllPages(db: DatabaseAdapter): Promise<PageEntry[]>;
-/**
- * Get a single page by route
+ * Unified page query function
+ * @param event - H3Event (optional, used for db context)
+ * @param options - Query options
  */
-export declare function getPage(db: DatabaseAdapter, route: string): Promise<PageEntry | undefined>;
+export declare function queryPages(event?: H3Event, options?: QueryPagesOptions): Promise<PageEntry[] | PageData[]>;
+export declare function queryPages(event: H3Event | undefined, options: QueryPagesOptions & {
+    route: string;
+}): Promise<PageEntry | PageData | undefined>;
+export declare function queryPages(event: H3Event | undefined, options: QueryPagesOptions & {
+    route: string;
+    includeMarkdown: true;
+}): Promise<PageData | undefined>;
+export declare function queryPages(event: H3Event | undefined, options: QueryPagesOptions & {
+    includeMarkdown: true;
+}): Promise<PageData[]>;
+export interface StreamPagesOptions {
+    batchSize?: number;
+}
 /**
- * Get a page with markdown content
+ * Stream pages using cursor-based pagination
+ * Yields pages one batch at a time to avoid loading all into memory
  */
-export declare function getPageWithMarkdown(db: DatabaseAdapter, route: string): Promise<PageData | undefined>;
+export declare function streamPages(event?: H3Event, options?: StreamPagesOptions): AsyncGenerator<PageData, void, unknown>;
+export interface CountPagesOptions {
+    where?: {
+        pending?: boolean;
+        hasError?: boolean;
+        source?: 'prerender' | 'runtime';
+    };
+}
 /**
- * Full-text search using FTS5
- * @param query Search query string
- * @param opts Search options
+ * Count pages matching criteria
  */
-export declare function searchPages(db: DatabaseAdapter, query: string, opts?: {
+export declare function countPages(event?: H3Event, options?: CountPagesOptions): Promise<number>;
+export interface SearchPagesOptions {
     limit?: number;
-}): Promise<SearchResult[]>;
+}
 /**
- * Insert or update a page
+ * Full-text search using FTS5
+ * Note: FTS is only available at runtime, not during prerender
  */
-export declare function upsertPage(db: DatabaseAdapter, page: {
+export declare function searchPages(event: H3Event | undefined, query: string, options?: SearchPagesOptions): Promise<SearchResult[]>;
+export interface UpsertPageInput {
     route: string;
     title: string;
     description: string;
     markdown: string;
     headings: string;
     keywords: string[];
+    contentHash?: string;
     updatedAt: string;
     isError?: boolean;
-}): Promise<void>;
+    source?: 'prerender' | 'runtime';
+}
 /**
- * Get all error routes
+ * Insert or update a page
  */
-export declare function getErrorRoutes(db: DatabaseAdapter): Promise<string[]>;
+export declare function upsertPage(event: H3Event | undefined, page: UpsertPageInput): Promise<void>;
 /**
  * Check if a page is fresh (within TTL)
  */
-export declare function isPageFresh(db: DatabaseAdapter, route: string, ttlSeconds: number): Promise<boolean>;
+export declare function isPageFresh(event: H3Event | undefined, route: string, ttlSeconds: number): Promise<boolean>;
+/**
+ * Get existing content hash for a page (for change detection)
+ * @internal
+ */
+export declare function getPageHash(event: H3Event | undefined, route: string): Promise<string | null>;
 /**
- * Delete a page by route
+ * Seed routes from sitemap (insert with indexed=0 if not exists)
  */
-export declare function deletePage(db: DatabaseAdapter, route: string): Promise<void>;
+export declare function seedRoutes(event: H3Event | undefined, routes: string[]): Promise<number>;
 /**
- * Get page count
+ * Get sitemap seeded timestamp from _ai_ready_info
  */
-export declare function getPageCount(db: DatabaseAdapter): Promise<number>;
+export declare function getSitemapSeededAt(event: H3Event | undefined): Promise<number | undefined>;
+/**
+ * Set sitemap seeded timestamp
+ */
+export declare function setSitemapSeededAt(event: H3Event | undefined, timestamp: number): Promise<void>;
+/**
+ * Prune routes not seen in sitemap for longer than threshold
+ * Only prunes routes with source='runtime' (never prerendered pages)
+ */
+export declare function pruneStaleRoutes(event: H3Event | undefined, staleThresholdSeconds: number): Promise<number>;
+/**
+ * Get stale routes that would be pruned (for preview)
+ */
+export declare function getStaleRoutes(event: H3Event | undefined, staleThresholdSeconds: number): Promise<string[]>;
+/**
+ * Get pages needing IndexNow sync (content changed since last sync)
+ */
+export declare function getPagesNeedingIndexNowSync(event: H3Event | undefined, limit?: number): Promise<{
+    route: string;
+}[]>;
+/**
+ * Count pages needing IndexNow sync
+ */
+export declare function countPagesNeedingIndexNowSync(event: H3Event | undefined): Promise<number>;
+/**
+ * Mark pages as synced to IndexNow
+ */
+export declare function markIndexNowSynced(event: H3Event | undefined, routes: string[]): Promise<void>;
+/**
+ * Update IndexNow stats after submission
+ * Uses atomic SQL to handle concurrent updates safely
+ */
+export declare function updateIndexNowStats(event: H3Event | undefined, submitted: number, error?: string): Promise<void>;
+export interface IndexNowStats {
+    totalSubmitted: number;
+    lastSubmittedAt: number | null;
+    lastError: string | null;
+}
 /**
- * Check if database has any pages
+ * Get IndexNow stats
  */
-export declare function hasPages(db: DatabaseAdapter): Promise<boolean>;
+export declare function getIndexNowStats(event: H3Event | undefined): Promise<IndexNowStats>;