nuxt-ai-ready 2.0.1 → 2.2.0

package/dist/runtime/server/db/drizzle/queries.js

@@ -1,10 +1,11 @@
-import { and, count, desc, eq, gt, isNotNull, isNull, like, lt, or, sql } from "drizzle-orm";
+import { and, count, desc, eq, gt, isNotNull, isNull, lt, or, sql } from "drizzle-orm";
 import { cronRuns, info, pages, sitemaps } from "#ai-ready-virtual/db-schema.mjs";
 import { useRuntimeConfig } from "#nuxtseo/nitro";
 import { parseSitemapCrawlState, serializeSitemapCrawlState } from "../../utils/sitemap-crawl-state.js";
 import { resolveFtsTokenizer as validateFtsTokenizer } from "../schema-sql.js";
+import { LIKE_ESCAPE, likeSubstring, maxRowsPerInsert } from "../shared.js";
 import { useDrizzle } from "./client.js";
-import { useRawDb } from "./raw.js";
+import { getRawExecutor, useRawDb } from "./raw.js";
 function resolveFtsTokenizer(event) {
   const cfg = useRuntimeConfig(event);
   return validateFtsTokenizer(cfg["nuxt-ai-ready"]?.ftsTokenizer);
@@ -127,7 +128,8 @@ export async function getPageLastmods(event) {
 export async function searchPages(event, query, options) {
   const client = await useDrizzle(event);
   const limitNum = options?.limit || 20;
-  const searchTerm = `%${query}%`;
+  const searchTerm = likeSubstring(query);
+  const matches = (col) => sql`${col} LIKE ${searchTerm} ESCAPE ${LIKE_ESCAPE}`;
   if (client.dialect === "sqlite") {
     try {
       const ftsResult = await client.db.all(sql`
@@ -155,10 +157,10 @@ export async function searchPages(event, query, options) {
     and(
       eq(pages.isError, 0),
       or(
-        like(pages.title, searchTerm),
-        like(pages.description, searchTerm),
-        like(pages.markdown, searchTerm),
-        like(pages.headings, searchTerm)
+        matches(pages.title),
+        matches(pages.description),
+        matches(pages.markdown),
+        matches(pages.headings)
       )
     )
   ).limit(limitNum);
@@ -172,6 +174,8 @@ export async function countPages(event, options) {
   }
   if (options?.errors !== void 0) {
     conditions.push(eq(pages.isError, options.errors ? 1 : 0));
+  } else {
+    conditions.push(eq(pages.isError, 0));
   }
   let query = client.db.select({ count: count() }).from(pages);
   if (conditions.length > 0) {
@@ -180,13 +184,12 @@ export async function countPages(event, options) {
   const result = await query;
   return Number(result[0]?.count || 0);
 }
-export async function deletePage(event, route) {
-  const client = await useDrizzle(event);
-  await client.db.delete(pages).where(eq(pages.route, route));
-}
 export async function getPendingPages(event, limit = 10) {
   const client = await useDrizzle(event);
-  return client.db.select({ route: pages.route }).from(pages).where(eq(pages.indexed, 0)).limit(limit);
+  return client.db.select({ route: pages.route }).from(pages).where(and(
+    eq(pages.indexed, 0),
+    eq(pages.isError, 0)
+  )).orderBy(pages.route).limit(limit);
 }
 export async function markPageIndexed(event, route) {
   const client = await useDrizzle(event);
@@ -202,7 +205,7 @@ export async function markRoutesPending(event, routes) {
     const batch = routes.slice(i, i + D1_MAX_IN_ROUTES);
     const placeholders = batch.map(() => "?").join(",");
     stmts.push({
-      sql: `UPDATE ai_ready_pages SET indexed = 0 WHERE route IN (${placeholders})`,
+      sql: `UPDATE ai_ready_pages SET indexed = 0, is_error = 0 WHERE route IN (${placeholders})`,
       params: batch
     });
   }
@@ -229,16 +232,23 @@ export async function deleteInfoValue(event, key) {
   const client = await useDrizzle(event);
   await client.db.delete(info).where(eq(info.id, key));
 }
-const SCHEMA_VERSION = "v2.3.0-drizzle";
+const SQLITE_SCHEMA_VERSION = "v2.3.0-drizzle";
+const LEGACY_POSTGRES_INTEGER_SCHEMA_VERSION = "v2.3.0-drizzle";
+const POSTGRES_SCHEMA_VERSION = "v2.3.0-drizzle-postgres-bigint";
 export async function initSchema(event) {
   const client = await useDrizzle(event);
   const tokenizer = resolveFtsTokenizer(event);
+  const schemaVersion = client.dialect === "postgres" ? POSTGRES_SCHEMA_VERSION : SQLITE_SCHEMA_VERSION;
   const currentVersion = await getSchemaVersion(client);
   const currentTokenizer = client.dialect === "postgres" ? null : await getStoredTokenizer(client);
-  const versionMatches = currentVersion === SCHEMA_VERSION;
+  const versionMatches = currentVersion === schemaVersion;
   const tokenizerMatches = client.dialect === "postgres" || !currentTokenizer || currentTokenizer === tokenizer;
   if (versionMatches && tokenizerMatches)
     return;
+  if (client.dialect === "postgres" && currentVersion === LEGACY_POSTGRES_INTEGER_SCHEMA_VERSION) {
+    await migratePostgresIntegerTimestamps(client);
+    return;
+  }
   if (currentVersion) {
     if (client.dialect === "postgres") {
       await dropPostgresTables(client);
@@ -251,9 +261,9 @@ export async function initSchema(event) {
   } else {
     await createSQLiteTables(client, tokenizer);
   }
-  await client.db.insert(info).values({ id: "schema", version: SCHEMA_VERSION }).onConflictDoUpdate({
+  await client.db.insert(info).values({ id: "schema", version: schemaVersion }).onConflictDoUpdate({
     target: info.id,
-    set: { version: SCHEMA_VERSION }
+    set: { version: schemaVersion }
   });
   if (client.dialect !== "postgres") {
     await client.db.insert(info).values({ id: "fts_tokenizer", value: tokenizer }).onConflictDoUpdate({
@@ -264,23 +274,38 @@ export async function initSchema(event) {
 }
 async function getSchemaVersion(client) {
   try {
-    const result = await client.db.all(
-      sql`SELECT version FROM _ai_ready_info WHERE id = 'schema'`
-    );
-    return result?.[0]?.version || null;
-  } catch {
-    return null;
+    const [row] = await client.db.select({ version: info.version }).from(info).where(eq(info.id, "schema")).limit(1);
+    return row?.version || null;
+  } catch (error) {
+    if (isMissingInfoTableError(error))
+      return null;
+    throw error;
   }
 }
 async function getStoredTokenizer(client) {
   try {
-    const result = await client.db.all(
-      sql`SELECT value FROM _ai_ready_info WHERE id = 'fts_tokenizer'`
-    );
-    return result?.[0]?.value || null;
-  } catch {
-    return null;
+    const [row] = await client.db.select({ value: info.value }).from(info).where(eq(info.id, "fts_tokenizer")).limit(1);
+    return row?.value || null;
+  } catch (error) {
+    if (isMissingInfoTableError(error))
+      return null;
+    throw error;
+  }
+}
+function isMissingInfoTableError(error) {
+  let current = error;
+  const seen = /* @__PURE__ */ new Set();
+  while (current && typeof current === "object" && !seen.has(current)) {
+    seen.add(current);
+    const candidate = current;
+    if (candidate.code === "42P01")
+      return true;
+    if (typeof candidate.message === "string" && candidate.message.includes("no such table") && candidate.message.includes("_ai_ready_info")) {
+      return true;
+    }
+    current = candidate.cause;
   }
+  return false;
 }
 const SQLITE_DROP_STATEMENTS = [
   sql`DROP TABLE IF EXISTS ai_ready_pages_fts`,
@@ -305,6 +330,19 @@ async function dropPostgresTables(client) {
   for (const stmt of POSTGRES_DROP_STATEMENTS)
     await client.db.execute(stmt);
 }
+async function migratePostgresIntegerTimestamps(client) {
+  await getRawExecutor(client).batch([
+    { sql: "ALTER TABLE ai_ready_pages ALTER COLUMN indexed_at TYPE BIGINT USING indexed_at::BIGINT" },
+    { sql: "ALTER TABLE ai_ready_pages ALTER COLUMN last_seen_at TYPE BIGINT USING last_seen_at::BIGINT" },
+    { sql: "ALTER TABLE ai_ready_cron_runs ALTER COLUMN started_at TYPE BIGINT USING started_at::BIGINT" },
+    { sql: "ALTER TABLE ai_ready_cron_runs ALTER COLUMN finished_at TYPE BIGINT USING finished_at::BIGINT" },
+    { sql: "ALTER TABLE ai_ready_sitemaps ALTER COLUMN last_crawled_at TYPE BIGINT USING last_crawled_at::BIGINT" },
+    {
+      sql: "UPDATE _ai_ready_info SET version = ? WHERE id = 'schema'",
+      params: [POSTGRES_SCHEMA_VERSION]
+    }
+  ]);
+}
 async function createSQLiteTables(client, ftsTokenizer) {
   const statements = [
     sql`CREATE TABLE IF NOT EXISTS ai_ready_pages (
@@ -404,11 +442,11 @@ async function createPostgresTables(client) {
       keywords TEXT NOT NULL DEFAULT '[]',
       content_hash TEXT,
       updated_at TEXT NOT NULL,
-      indexed_at INTEGER NOT NULL,
+      indexed_at BIGINT NOT NULL,
       is_error INTEGER NOT NULL DEFAULT 0,
       indexed INTEGER NOT NULL DEFAULT 0,
       source TEXT NOT NULL DEFAULT 'prerender',
-      last_seen_at INTEGER,
+      last_seen_at BIGINT,
       locale TEXT NOT NULL DEFAULT ''
     )`,
     sql`CREATE TABLE IF NOT EXISTS _ai_ready_info (
@@ -420,8 +458,8 @@ async function createPostgresTables(client) {
     )`,
     sql`CREATE TABLE IF NOT EXISTS ai_ready_cron_runs (
       id SERIAL PRIMARY KEY,
-      started_at INTEGER NOT NULL,
-      finished_at INTEGER,
+      started_at BIGINT NOT NULL,
+      finished_at BIGINT,
       duration_ms INTEGER,
       pages_indexed INTEGER DEFAULT 0,
       pages_remaining INTEGER DEFAULT 0,
@@ -431,7 +469,7 @@ async function createPostgresTables(client) {
     sql`CREATE TABLE IF NOT EXISTS ai_ready_sitemaps (
       name TEXT PRIMARY KEY,
       route TEXT NOT NULL,
-      last_crawled_at INTEGER,
+      last_crawled_at BIGINT,
       url_count INTEGER DEFAULT 0,
       error_count INTEGER DEFAULT 0,
       last_error TEXT,
@@ -640,21 +678,24 @@ export async function seedRoutes(event, routes) {
     const route = normalizeRoute(raw);
     byRoute.set(route, normalizeRouteKey(route));
   }
-  const ROWS_PER_INSERT = 20;
+  const rowsPerInsert = maxRowsPerInsert(4);
   const db = await useRawDb(event);
   const now = (/* @__PURE__ */ new Date()).toISOString();
   const nowMs = Date.now();
   const entries = [...byRoute.entries()];
   const stmts = [];
-  for (let i = 0; i < entries.length; i += ROWS_PER_INSERT) {
-    const batch = entries.slice(i, i + ROWS_PER_INSERT);
+  for (let i = 0; i < entries.length; i += rowsPerInsert) {
+    const batch = entries.slice(i, i + rowsPerInsert);
     const valuesSql = batch.map(() => `(?, ?, '', '', '', '[]', '[]', ?, 0, 0, 0, 'runtime', ?)`).join(", ");
     const params = batch.flatMap(([route, routeKey]) => [route, routeKey, now, nowMs]);
     stmts.push({
       sql: `
         INSERT INTO ai_ready_pages (route, route_key, title, description, markdown, headings, keywords, updated_at, indexed_at, is_error, indexed, source, last_seen_at)
         VALUES ${valuesSql}
-        ON CONFLICT(route) DO UPDATE SET last_seen_at = excluded.last_seen_at
+        ON CONFLICT(route) DO UPDATE SET
+          last_seen_at = excluded.last_seen_at,
+          is_error = 0,
+          indexed = CASE WHEN ai_ready_pages.is_error = 1 THEN 0 ELSE ai_ready_pages.indexed END
       `,
       params
     });

package/dist/runtime/server/db/drizzle/raw.d.ts

@@ -7,7 +7,7 @@ import type { DrizzleDatabase } from './client.js';
 /**
  * Register the underlying driver for raw SQL access
  */
-export declare function registerDriver(db: DrizzleDatabase['db'], type: 'better-sqlite3' | 'node-sqlite' | 'libsql' | 'neon' | 'd1', driver: unknown): void;
+export declare function registerDriver(db: DrizzleDatabase['db'], type: 'better-sqlite3' | 'node-sqlite' | 'libsql' | 'neon' | 'postgres' | 'd1', driver: unknown): void;
 /**
  * Get raw SQL executor for a Drizzle client
  */
@@ -33,4 +33,4 @@ export declare function useRawDb(event?: H3Event): Promise<RawExecutor>;
 /**
  * Close underlying database driver connection
  */
-export declare function closeDriver(db: DrizzleDatabase['db']): void;
+export declare function closeDriver(db: DrizzleDatabase['db']): Promise<void>;

package/dist/runtime/server/db/drizzle/raw.js

@@ -5,6 +5,10 @@ export function registerDriver(db, type, driver) {
 }
 const RE_PARAM_PLACEHOLDER = /\?/g;
 const MAX_BATCH_STATEMENTS = 100;
+function toPostgresQuery(query) {
+  let index = 0;
+  return query.replace(RE_PARAM_PLACEHOLDER, () => `$${++index}`);
+}
 export function getRawExecutor(client) {
   const cached = driverCache.get(client.db);
   if (!cached) {
@@ -35,11 +39,13 @@ export function getRawExecutor(client) {
         }
         case "neon": {
           const sqlFn = driver;
-          let idx = 0;
-          const pgQuery = query.replace(RE_PARAM_PLACEHOLDER, () => `$${++idx}`);
-          const result = await sqlFn.query(pgQuery, params);
+          const result = await sqlFn.query(toPostgresQuery(query), params);
           return result.rows || result;
         }
+        case "postgres": {
+          const sqlClient = driver;
+          return await sqlClient.unsafe(toPostgresQuery(query), params);
+        }
       }
     },
     async first(query, params = []) {
@@ -70,9 +76,12 @@ export function getRawExecutor(client) {
         }
         case "neon": {
           const sqlFn = driver;
-          let idx = 0;
-          const pgQuery = query.replace(RE_PARAM_PLACEHOLDER, () => `$${++idx}`);
-          await sqlFn.query(pgQuery, params);
+          await sqlFn.query(toPostgresQuery(query), params);
+          break;
+        }
+        case "postgres": {
+          const sqlClient = driver;
+          await sqlClient.unsafe(toPostgresQuery(query), params);
           break;
         }
       }
@@ -127,12 +136,17 @@ export function getRawExecutor(client) {
           const sqlFn = driver;
           for (let i = 0; i < queries.length; i += MAX_BATCH_STATEMENTS) {
             const chunk = queries.slice(i, i + MAX_BATCH_STATEMENTS);
-            const pgQueries = chunk.map((q) => {
-              let idx = 0;
-              const pgQuery = q.sql.replace(RE_PARAM_PLACEHOLDER, () => `$${++idx}`);
-              return sqlFn.query(pgQuery, q.params || []);
-            });
-            await sqlFn.transaction(pgQueries);
+            await sqlFn.transaction((tx) => chunk.map((q) => tx.query(toPostgresQuery(q.sql), q.params || [])));
+          }
+          break;
+        }
+        case "postgres": {
+          const sqlClient = driver;
+          for (let i = 0; i < queries.length; i += MAX_BATCH_STATEMENTS) {
+            const chunk = queries.slice(i, i + MAX_BATCH_STATEMENTS);
+            await sqlClient.begin((transaction) => Promise.all(chunk.map(
+              (q) => transaction.unsafe(toPostgresQuery(q.sql), q.params || [])
+            )));
           }
           break;
         }
@@ -144,7 +158,7 @@ export async function useRawDb(event) {
   const client = await useDrizzle(event);
   return getRawExecutor(client);
 }
-export function closeDriver(db) {
+export async function closeDriver(db) {
   const cached = driverCache.get(db);
   if (!cached)
     return;
@@ -165,6 +179,11 @@ export function closeDriver(db) {
       client.close?.();
       break;
     }
+    case "postgres": {
+      const sqlClient = driver;
+      await sqlClient.end();
+      break;
+    }
   }
   driverCache.delete(db);
 }

package/dist/runtime/server/db/queries.d.ts

@@ -48,6 +48,7 @@ export interface QueryPagesOptions {
     limit?: number;
     offset?: number;
 }
+type DatabaseNumber = number | string | bigint;
 /**
  * Get lastmod (updatedAt) for all indexed pages
  * Returns a Map for O(1) lookup when enriching sitemaps
@@ -92,7 +93,7 @@ export interface SearchPagesOptions {
     limit?: number;
 }
 /**
- * Full-text search using FTS5
+ * Full-text search using FTS5 or PostgreSQL ILIKE
  * Note: FTS is only available at runtime, not during prerender
  */
 export declare function searchPages(event: H3Event | undefined, query: string, options?: SearchPagesOptions): Promise<SearchResult[]>;
@@ -139,15 +140,8 @@ export declare function getPageHash(event: H3Event | undefined, route: string):
 export declare function seedRoutes(event: H3Event | undefined, routes: Array<string | {
     route: string;
     locale?: string;
+    url?: string;
 }>): Promise<number>;
-/**
- * Get sitemap seeded timestamp from _ai_ready_info
- */
-export declare function getSitemapSeededAt(event: H3Event | undefined): Promise<number | undefined>;
-/**
- * Set sitemap seeded timestamp
- */
-export declare function setSitemapSeededAt(event: H3Event | undefined, timestamp: number): Promise<void>;
 /**
  * Prune routes not seen in sitemap for longer than threshold
  * Only prunes routes with source='runtime' (never prerendered pages)
@@ -159,11 +153,11 @@ export declare function pruneStaleRoutes(event: H3Event | undefined, staleThresh
 export declare function getStaleRoutes(event: H3Event | undefined, staleThresholdSeconds: number): Promise<string[]>;
 export interface CronRunRow {
     id: number;
-    started_at: number;
-    finished_at: number | null;
-    duration_ms: number | null;
-    pages_indexed: number;
-    pages_remaining: number;
+    started_at: DatabaseNumber;
+    finished_at: DatabaseNumber | null;
+    duration_ms: DatabaseNumber | null;
+    pages_indexed: DatabaseNumber;
+    pages_remaining: DatabaseNumber;
     errors: string;
     status: 'running' | 'success' | 'partial' | 'error';
 }
@@ -193,10 +187,6 @@ export declare function completeCronRun(event: H3Event | undefined, runId: numbe
  * Get recent cron runs
  */
 export declare function getRecentCronRuns(event: H3Event | undefined, limit?: number): Promise<CronRun[]>;
-/**
- * Clean up old cron runs (keep last N)
- */
-export declare function cleanupOldCronRuns(event: H3Event | undefined, keepCount?: number): Promise<number>;
 /**
  * Clean up cron runs older than specified age
  */
@@ -213,15 +203,27 @@ export interface CronFastPathStatus {
  * Reduces 6+ sequential DB calls to 1
  */
 export declare function getCronFastPathStatus(event: H3Event | undefined, sitemapIntervalMinutes?: number): Promise<CronFastPathStatus | null>;
-/**
- * Try to acquire cron lock. Returns true if acquired, false if another run is active.
- * Uses atomic INSERT OR REPLACE with conditional check to prevent race conditions.
- */
-export declare function tryAcquireCronLock(event: H3Event | undefined): Promise<boolean>;
-/**
- * Release cron lock
- */
-export declare function releaseCronLock(event: H3Event | undefined): Promise<void>;
+export type CronLockAcquire = {
+    _tag: 'acquired';
+    token: string;
+} | {
+    _tag: 'held';
+};
+/**
+ * Try to acquire cron lock. Returns the ownership token on success, `held`
+ * when a live lock belongs to another run.
+ *
+ * The value carries a random token plus expiry in one row, so the
+ * compare-expired-then-set upsert is atomic and ownership is unique even for
+ * two isolates that start in the same millisecond. A legacy value that
+ * predates tokens parses as no expiry, so it is treated as expired.
+ */
+export declare function tryAcquireCronLock(event: H3Event | undefined): Promise<CronLockAcquire>;
+/**
+ * Release cron lock. Only deletes the row when its token is ours, so a run
+ * that outlived the lock TTL cannot delete the new owner's lock.
+ */
+export declare function releaseCronLock(event: H3Event | undefined, token: string): Promise<void>;
 export interface CronLockStatus {
     held: boolean;
     since: number | null;
@@ -303,3 +305,4 @@ export declare function getRecentlyIndexedPages(event: H3Event | undefined, limi
  * Count pages indexed in a time window
  */
 export declare function countRecentlyIndexed(event: H3Event | undefined, sinceMs: number): Promise<number>;
+export {};