@lunora/d1 1.0.0-alpha.25 → 1.0.0-alpha.26

package/dist/index.d.mts

@@ -52,11 +52,21 @@ interface ExportGlobalArgs {
   tables?: ReadonlyArray<string>;
 }
 /**
-* Yield rows from every requested `.global()` table in batches. Uses
-* `LIMIT ?/OFFSET ?` because D1 globals don't have a stable keyset abstraction
-* here (the writer's `findMany` does, but at the cost of routing through the
-* full validator pipeline; for a snapshot stream a plain offset scan is
-* sufficient and predictable).
+* Yield rows from every requested `.global()` table in batches.
+*
+* Keyset-paginates on the physical primary key (`id` — every `.global()` table
+* carries it, see `frameworkColumnDdl`): `WHERE "id" > ? ORDER BY "id" LIMIT ?`,
+* carrying the last id forward. A plain `LIMIT/OFFSET` scan would be wrong here —
+* SQLite gives no ordering guarantee for an unordered SELECT and each page is a
+* separate query, so pages could overlap or skip rows (and any concurrent
+* insert/delete would shift offsets and silently drop/duplicate rows in the
+* snapshot). Keyset paging is deterministic under concurrent writes and avoids
+* O(n^2) OFFSET scans on large tables.
+*
+* Tables are provisioned first (idempotent `CREATE … IF NOT EXISTS`): `.global()`
+* tables are created lazily on first write, so a fresh deployment — or any table
+* never written — would otherwise abort the stream with a raw `no such table`
+* instead of exporting it as empty.
 */
 declare const exportGlobalRows: (exec: SqlCtxExec, schema: SchemaLike, args: ExportGlobalArgs) => AsyncGenerator<ExportRow, void, undefined>;
 interface ImportGlobalArgs {
@@ -90,7 +100,6 @@ declare const importGlobalRows: (writer: DatabaseWriterLike, schema: SchemaLike,
 */
 interface D1DatabaseLike {
   batch?: (statements: D1PreparedStatementLike[]) => Promise<unknown[]>;
-  exec?: (sql: string) => Promise<unknown>;
   prepare: (sql: string) => D1PreparedStatementLike;
   withSession: (bookmark?: string) => D1SessionLike;
 }

package/dist/index.d.ts

@@ -52,11 +52,21 @@ interface ExportGlobalArgs {
   tables?: ReadonlyArray<string>;
 }
 /**
-* Yield rows from every requested `.global()` table in batches. Uses
-* `LIMIT ?/OFFSET ?` because D1 globals don't have a stable keyset abstraction
-* here (the writer's `findMany` does, but at the cost of routing through the
-* full validator pipeline; for a snapshot stream a plain offset scan is
-* sufficient and predictable).
+* Yield rows from every requested `.global()` table in batches.
+*
+* Keyset-paginates on the physical primary key (`id` — every `.global()` table
+* carries it, see `frameworkColumnDdl`): `WHERE "id" > ? ORDER BY "id" LIMIT ?`,
+* carrying the last id forward. A plain `LIMIT/OFFSET` scan would be wrong here —
+* SQLite gives no ordering guarantee for an unordered SELECT and each page is a
+* separate query, so pages could overlap or skip rows (and any concurrent
+* insert/delete would shift offsets and silently drop/duplicate rows in the
+* snapshot). Keyset paging is deterministic under concurrent writes and avoids
+* O(n^2) OFFSET scans on large tables.
+*
+* Tables are provisioned first (idempotent `CREATE … IF NOT EXISTS`): `.global()`
+* tables are created lazily on first write, so a fresh deployment — or any table
+* never written — would otherwise abort the stream with a raw `no such table`
+* instead of exporting it as empty.
 */
 declare const exportGlobalRows: (exec: SqlCtxExec, schema: SchemaLike, args: ExportGlobalArgs) => AsyncGenerator<ExportRow, void, undefined>;
 interface ImportGlobalArgs {
@@ -90,7 +100,6 @@ declare const importGlobalRows: (writer: DatabaseWriterLike, schema: SchemaLike,
 */
 interface D1DatabaseLike {
   batch?: (statements: D1PreparedStatementLike[]) => Promise<unknown[]>;
-  exec?: (sql: string) => Promise<unknown>;
   prepare: (sql: string) => D1PreparedStatementLike;
   withSession: (bookmark?: string) => D1SessionLike;
 }

package/dist/index.mjs

@@ -1,7 +1,7 @@
-export { exportGlobalRows, importGlobalRows, selectGlobalTables } from './packem_shared/exportGlobalRows-BgfNSwCG.mjs';
-export { D1Client, D1Session } from './packem_shared/D1Client-DA3flo1o.mjs';
+export { exportGlobalRows, importGlobalRows, selectGlobalTables } from './packem_shared/exportGlobalRows-q0RgCLPM.mjs';
+export { D1Client, D1Session } from './packem_shared/D1Client-DSj8p7c4.mjs';
 export { createD1CtxDb, readD1CdcChanges, runD1AggregateMigrations, runD1CdcMigration, runD1GlobalTableMigrations, runD1RankMigrations, runD1SearchMigrations, trimD1CdcChanges } from './packem_shared/createD1CtxDb-BMR8J0dT.mjs';
-export { facetGlobalColumn, listGlobalTables, readGlobalTablePage } from './packem_shared/facetGlobalColumn-DxrhOr12.mjs';
-export { MigrationRunner } from './packem_shared/MigrationRunner-BQx8HAGh.mjs';
+export { facetGlobalColumn, listGlobalTables, readGlobalTablePage } from './packem_shared/facetGlobalColumn-DnglIe4P.mjs';
+export { MigrationRunner } from './packem_shared/MigrationRunner-A-o1LTKj.mjs';
 export { default as sqliteDialect } from './packem_shared/sqliteDialect-DqYnHPuu.mjs';
 export { createSqlCtxDb } from '@lunora/sql-store';

package/dist/packem_shared/{D1Client-DA3flo1o.mjs → D1Client-DSj8p7c4.mjs}

@@ -1,6 +1,28 @@
 import { drizzle } from 'drizzle-orm/d1';
 
+const evictOldestEntry = (map, capacity) => {
+  if (map.size < capacity) {
+    return;
+  }
+  const oldest = map.keys().next().value;
+  if (oldest !== void 0) {
+    map.delete(oldest);
+  }
+};
+
 const STMT_CACHE_CAPACITY = 256;
+const prepareCached = (cache, prepare, sql) => {
+  const cached = cache.get(sql);
+  if (cached) {
+    cache.delete(sql);
+    cache.set(sql, cached);
+    return cached;
+  }
+  const stmt = prepare(sql);
+  evictOldestEntry(cache, STMT_CACHE_CAPACITY);
+  cache.set(sql, stmt);
+  return stmt;
+};
 const D1_FIRST_UNCONSTRAINED = "first-unconstrained";
 class D1Session {
   session;
@@ -10,21 +32,7 @@ class D1Session {
     this.session = session;
   }
   prepare(sql) {
-    const cached = this.stmtCache.get(sql);
-    if (cached) {
-      this.stmtCache.delete(sql);
-      this.stmtCache.set(sql, cached);
-      return cached;
-    }
-    const stmt = this.session.prepare(sql);
-    if (this.stmtCache.size >= STMT_CACHE_CAPACITY) {
-      const oldest = this.stmtCache.keys().next().value;
-      if (oldest !== void 0) {
-        this.stmtCache.delete(oldest);
-      }
-    }
-    this.stmtCache.set(sql, stmt);
-    return stmt;
+    return prepareCached(this.stmtCache, (text) => this.session.prepare(text), sql);
   }
   // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-parameters -- T types the result rows for the caller and is forwarded to the prepared statement.
   async run(sql, ...binds) {
@@ -84,21 +92,7 @@ class D1Client {
    * even when the previous caller already called `.bind(...).run()`.
    */
   prepare(sql) {
-    const cached = this.stmtCache.get(sql);
-    if (cached) {
-      this.stmtCache.delete(sql);
-      this.stmtCache.set(sql, cached);
-      return cached;
-    }
-    const stmt = this.db.prepare(sql);
-    if (this.stmtCache.size >= STMT_CACHE_CAPACITY) {
-      const oldest = this.stmtCache.keys().next().value;
-      if (oldest !== void 0) {
-        this.stmtCache.delete(oldest);
-      }
-    }
-    this.stmtCache.set(sql, stmt);
-    return stmt;
+    return prepareCached(this.stmtCache, (text) => this.db.prepare(text), sql);
   }
   /**
    * Drizzle handle over the bare `env.DB` binding. Used for typed queries

package/dist/packem_shared/{MigrationRunner-BQx8HAGh.mjs → MigrationRunner-A-o1LTKj.mjs}

@@ -1,19 +1,19 @@
 import { LunoraError } from '@lunora/errors';
 import { sql } from 'drizzle-orm';
-import { D1Client } from './D1Client-DA3flo1o.mjs';
+import { D1Client } from './D1Client-DSj8p7c4.mjs';
 
 const TRACKING_TABLE_NAME = "__drizzle_migrations";
-const TRACKING_TABLE_DDL = `CREATE TABLE IF NOT EXISTS ${TRACKING_TABLE_NAME} (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT NOT NULL, created_at NUMERIC)`;
+const TRACKING_TABLE_DDL = `CREATE TABLE IF NOT EXISTS ${TRACKING_TABLE_NAME} (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT NOT NULL UNIQUE, created_at NUMERIC)`;
 const WHITESPACE_RE = /\s/u;
-const TRAILING_SEMICOLON_RE = /;\s*$/u;
 const SHA256_HEX_RE = /^[0-9a-f]{64}$/u;
+const TRACKING_HASH_UNIQUE_RE = new RegExp(String.raw`UNIQUE constraint failed:\s*${TRACKING_TABLE_NAME}\.hash`, "iu");
 const assertSingleStatement = (migration) => {
   const text = migration.sql;
   let inSingle = false;
   let inDouble = false;
   let inLineComment = false;
   let inBlockComment = false;
-  let seenStatement = false;
+  let terminatorIndex;
   for (let index = 0; index < text.length; index += 1) {
     const character = text[index];
     const next = text[index + 1];
@@ -50,14 +50,6 @@ const assertSingleStatement = (migration) => {
       }
       continue;
     }
-    if (character === "'") {
-      inSingle = true;
-      continue;
-    }
-    if (character === '"') {
-      inDouble = true;
-      continue;
-    }
     if (character === "-" && next === "-") {
       inLineComment = true;
       index += 1;
@@ -68,17 +60,28 @@ const assertSingleStatement = (migration) => {
       index += 1;
       continue;
     }
-    if (character === ";") {
-      seenStatement = true;
+    if (terminatorIndex !== void 0) {
+      if (character !== void 0 && !WHITESPACE_RE.test(character)) {
+        throw new LunoraError(
+          "INTERNAL",
+          `Migration "${migration.name}" (v${String(migration.version)}) contains more than one SQL statement. Split it into separate migrations — batch() runs them atomically.`
+        );
+      }
       continue;
     }
-    if (seenStatement && character !== void 0 && !WHITESPACE_RE.test(character)) {
-      throw new LunoraError(
-        "INTERNAL",
-        `Migration "${migration.name}" (v${String(migration.version)}) contains more than one SQL statement. Split it into separate migrations — batch() runs them atomically.`
-      );
+    if (character === "'") {
+      inSingle = true;
+      continue;
+    }
+    if (character === '"') {
+      inDouble = true;
+      continue;
+    }
+    if (character === ";") {
+      terminatorIndex = index;
     }
   }
+  return terminatorIndex;
 };
 const hashMigration = async (text) => {
   const bytes = new TextEncoder().encode(text);
@@ -112,20 +115,32 @@ class MigrationRunner {
         skipped.push({ name: migration.name, version: migration.version });
         continue;
       }
-      await this.applyOne(migration, hash);
-      applied.push({ name: migration.name, version: migration.version });
+      const didApply = await this.applyOne(migration, hash);
+      if (didApply) {
+        applied.push({ name: migration.name, version: migration.version });
+      } else {
+        skipped.push({ name: migration.name, version: migration.version });
+      }
     }
     return { applied, skipped };
   }
   async applyOne(migration, hash) {
-    assertSingleStatement(migration);
-    const statementText = migration.sql.replace(TRAILING_SEMICOLON_RE, "").trim();
+    const terminatorIndex = assertSingleStatement(migration);
+    const statementText = (terminatorIndex === void 0 ? migration.sql : migration.sql.slice(0, terminatorIndex)).trim();
     if (!SHA256_HEX_RE.test(hash)) {
       throw new LunoraError("INTERNAL", `migration "${migration.name}" produced a non-hex hash; refusing to inline into SQL`);
     }
     const trackingInsertSql = `INSERT INTO ${TRACKING_TABLE_NAME} (hash, created_at) VALUES ('${hash}', ${String(Date.now())})`;
     const items = [this.client.drizzle.run(sql.raw(statementText)), this.client.drizzle.run(sql.raw(trackingInsertSql))];
-    await this.client.batch(items);
+    try {
+      await this.client.batch(items);
+    } catch (error) {
+      if (TRACKING_HASH_UNIQUE_RE.test(error instanceof Error ? error.message : String(error))) {
+        return false;
+      }
+      throw error;
+    }
+    return true;
   }
   assertUniqueVersions() {
     const seen = /* @__PURE__ */ new Set();

package/dist/packem_shared/{exportGlobalRows-BgfNSwCG.mjs → exportGlobalRows-q0RgCLPM.mjs}

@@ -1,4 +1,5 @@
 import { toErrorBody } from '@lunora/errors';
+import { runD1GlobalTableMigrations } from './createD1CtxDb-BMR8J0dT.mjs';
 import { quoteIdentifier } from './quoteIdentifier-B-ZeSe1V.mjs';
 import { decodeGlobalRow } from '@lunora/sql-store';
 
@@ -20,16 +21,21 @@ const decodeRow = (schema, table, row) => {
 const exportGlobalRows = async function* (exec, schema, args) {
   const tables = selectGlobalTables(schema, args.tables);
   const batchSize = args.batchSize ?? DEFAULT_BATCH_SIZE;
+  await runD1GlobalTableMigrations(exec, schema);
   for (const table of tables) {
-    let offset = 0;
+    const quoted = quoteIdentifier(table);
+    let lastId;
     let hasMore = true;
     while (hasMore) {
-      const rows = await exec.all(`SELECT * FROM ${quoteIdentifier(table)} LIMIT ? OFFSET ?`, [batchSize, offset]);
+      const rows = lastId === void 0 ? await exec.all(`SELECT * FROM ${quoted} ORDER BY "id" LIMIT ?`, [batchSize]) : await exec.all(`SELECT * FROM ${quoted} WHERE "id" > ? ORDER BY "id" LIMIT ?`, [lastId, batchSize]);
       for (const row of rows) {
         yield { doc: decodeRow(schema, table, row), table };
       }
+      const last = rows.at(-1);
+      if (last !== void 0) {
+        lastId = String(last["id"]);
+      }
       hasMore = rows.length === batchSize;
-      offset += rows.length;
     }
   }
 };

package/dist/packem_shared/{facetGlobalColumn-DxrhOr12.mjs → facetGlobalColumn-DnglIe4P.mjs}

@@ -1,5 +1,6 @@
 import { LunoraError } from '@lunora/errors';
 import { runD1GlobalTableMigrations } from './createD1CtxDb-BMR8J0dT.mjs';
+import { quoteIdentifier } from './quoteIdentifier-B-ZeSe1V.mjs';
 import { decodeGlobalRow } from '@lunora/sql-store';
 
 const ensureGlobalTables = (exec, schema) => runD1GlobalTableMigrations(exec, schema);
@@ -7,7 +8,6 @@ const DEFAULT_PAGE_SIZE = 50;
 const MAX_PAGE_SIZE = 500;
 const DEFAULT_FACET_LIMIT = 30;
 const MAX_FACET_LIMIT = 200;
-const quoteIdentifier = (name) => `"${name.replaceAll('"', '""')}"`;
 const clamp = (value, min, max) => Math.min(Math.max(value, min), max);
 const INTERNAL_TABLE = /^sqlite_|^_cf_|^d1_|^__cdc|__agg_|__rank_|__fts_/u;
 const isInternalTable = (name) => INTERNAL_TABLE.test(name);
@@ -31,6 +31,9 @@ const buildEqPredicate = (schema, table, displayColumns, filters) => {
     if (!displayColumns.includes(filter.column)) {
       throw new LunoraError("UNKNOWN_COLUMN", `unknown column: ${filter.column}`, { status: 404 });
     }
+    if (schema.tables[table] === void 0 && SENSITIVE_COLUMN.test(filter.column)) {
+      throw new LunoraError("FORBIDDEN", `cannot filter on a redacted column: ${filter.column}`, { status: 403 });
+    }
     const quoted = quoteIdentifier(physicalColumnName(schema, table, filter.column));
     if (filter.value === null || filter.value === void 0) {
       clauses.push(`${quoted} IS NULL`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lunora/d1",
-  "version": "1.0.0-alpha.25",
+  "version": "1.0.0-alpha.26",
   "description": "D1 adapter for Lunora .global() tables, wrapping the Sessions API for read-your-writes",
   "keywords": [
     "cloudflare",
@@ -50,9 +50,9 @@
     "access": "public"
   },
   "dependencies": {
-    "@lunora/do": "1.0.0-alpha.26",
-    "@lunora/errors": "1.0.0-alpha.3",
-    "@lunora/sql-store": "1.0.0-alpha.26",
+    "@lunora/do": "1.0.0-alpha.27",
+    "@lunora/errors": "1.0.0-alpha.4",
+    "@lunora/sql-store": "1.0.0-alpha.27",
     "drizzle-orm": "^0.45.2"
   },
   "engines": {