@lunora/sql-store 1.0.0-alpha.8 → 1.0.0-alpha.80

package/dist/index.d.mts

@@ -1,121 +1,150 @@
-import { ServerDefaultContextLike, DatabaseWriterLike, SchedulerLike, SchemaLike, TableDefinitionLike, ValidatorLike } from '@lunora/do';
-import { SqlDialect, SqlRunResult } from "./dialect.mjs";
+import { TableDefinitionLike, SchemaLike, ServerDefaultContextLike, DatabaseWriterLike, CrossShardReadArgs, QueryPage, SchedulerLike, ValidatorLike } from '@lunora/shard-engine';
+import { SqlRunResult, SqlDialect } from "./dialect.mjs";
 export type { SqlExec } from "./dialect.mjs";
 import 'drizzle-orm';
 /**
-* Async SQL surface the D1 ORM needs: `all` for reads, `run` for writes.
-* Satisfied by a `D1Session`/`D1Client` in production and a `node:sqlite`
-* adapter in tests, so the query logic runs against a real SQLite engine.
-*/
+ * Async SQL surface the D1 ORM needs: `all` for reads, `run` for writes.
+ * Satisfied by a `D1Session`/`D1Client` in production and a `node:sqlite`
+ * adapter in tests, so the query logic runs against a real SQLite engine.
+ */
 interface SqlCtxExec {
   all: (sql: string, parameters: ReadonlyArray<unknown>) => Promise<Record<string, unknown>[]>;
+  batch?: (statements: ReadonlyArray<{
+    params: ReadonlyArray<unknown>;
+    sql: string;
+  }>) => Promise<void>;
   run: (sql: string, parameters: ReadonlyArray<unknown>) => Promise<SqlRunResult | void>;
 }
+/**
+ * Decode a SELECTed row back into a document: `id` → `_id`, `_creationTime`
+ * preserved, and every column run through the shared {@link sqliteDecode} so the
+ * stored form is reversed back into its JS shape. Exported so the data-browser
+ * (`introspect.ts`) and admin export/import paths share the exact same decode.
+ *
+ * The decode is engine-agnostic: every backend stores SQLite-shaped values
+ * (boolean → 1/0, JSON → text, bigint → decimal string), and `sqliteDecode` is
+ * robust to a driver returning either the stored string OR a natively-parsed
+ * value (e.g. mysql2 returns JSON columns pre-parsed) — so the same decoder is
+ * correct on SQLite, Postgres and MySQL.
+ */
+declare const decodeGlobalRow: (definition: TableDefinitionLike, row: Record<string, unknown>) => Record<string, unknown>;
+/**
+ * Provision the search companions, then index one bounded page of the rows that
+ * predate each index — unless it is declared `staged: true`, which leaves the
+ * whole backfill to {@link backfillSqlSearchIndexes}.
+ *
+ * Idempotent (`CREATE … IF NOT EXISTS` throughout, and the backfill resumes
+ * from recorded progress).
+ */
+declare const runSqlSearchMigrations: (exec: SqlCtxExec, schema: SchemaLike, dialect: SqlDialect) => Promise<void>;
+/**
+ * Run every declared search index — including the `staged: true` ones the
+ * migration pass skips — through to completion. The entry point a host calls
+ * out-of-band after deploying a search index over a table too large to index a
+ * page at a time.
+ *
+ * Idempotent and resumable: an index recorded as complete is skipped, and an
+ * interrupted run picks up from its cursor.
+ */
+declare const backfillSqlSearchIndexes: (exec: SqlCtxExec, schema: SchemaLike, dialect: SqlDialect) => Promise<void>;
 interface SqlCtxDbOptions {
   /**
-  * Resolved request auth handed to `.serverDefault(fn)` column factories so
-  * server-trusted columns (owner/tenant ids) stamp from the verified caller,
-  * never the client. The generated worker passes the per-request identity;
-  * absent it, server-trusted columns stamp the anonymous slice (`userId: null`).
-  */
+   * Resolved request auth handed to `.serverDefault(fn)` column factories so
+   * server-trusted columns (owner/tenant ids) stamp from the verified caller,
+   * never the client. The generated worker passes the per-request identity;
+   * absent it, server-trusted columns stamp the anonymous slice (`userId: null`).
+   */
   auth?: ServerDefaultContextLike["auth"];
   /**
-  * Opt into change-data-capture: when `true`, every committed write appends a
-  * post-image to the `__cdc_log` table (created lazily alongside the other
-  * companion tables). Backs CDC streaming export for `.global()` tables — the
-  * log is for export/CDC consumers, NOT point-in-time recovery: D1's PITR is
-  * the platform's own Time Travel (`wrangler d1 time-travel restore`), an
-  * atomic restore, not a changelog replay. Leave undefined for zero-cost
-  * legacy behaviour.
-  */
+   * Opt into change-data-capture: when `true`, every committed write appends a
+   * post-image to the `__cdc_log` table (created lazily alongside the other
+   * companion tables). Backs CDC streaming export for `.global()` tables — the
+   * log is for export/CDC consumers, NOT point-in-time recovery: D1's PITR is
+   * the platform's own Time Travel (`wrangler d1 time-travel restore`), an
+   * atomic restore, not a changelog replay. Leave undefined for zero-cost
+   * legacy behaviour.
+   */
   cdc?: boolean;
   clock?: () => number;
   /**
-  * Cross-shard counter for **reverse cross-backend relations** — the `_count`
-  * mirror of the `crossShardReader` option below.
-  */
+   * Cross-shard counter for **reverse cross-backend relations** — the `_count`
+   * mirror of the `crossShardReader` option below.
+   */
   crossShardCounter?: DatabaseWriterLike["count"];
   /**
-  * Optional cross-shard reader for **reverse cross-backend relations**: a
-  * `.global()` (D1) parent loading a shard-local (`.shardBy()`/root) child.
-  * Such a child's rows are partitioned across every shard DO, so the local D1
-  * writer can't resolve it. When provided, the relation loader routes the
-  * child's read through this (the host wires it to the Query Coordinator's
-  * RLS-correct `fanOut`, with identity forwarded so each shard applies its own
-  * RLS). Absent it, loading such a relation throws a clear "not supported"
-  * error (legacy behaviour). The forward direction (shard-local parent →
-  * global child) and same-backend relations never touch this.
-  */
-  crossShardReader?: DatabaseWriterLike["findMany"];
+   * Optional cross-shard reader for **reverse cross-backend relations**: a
+   * `.global()` (D1) parent loading a shard-local (`.shardBy()`/root) child.
+   * Such a child's rows are partitioned across every shard DO, so the local D1
+   * writer can't resolve it. When provided, the relation loader routes the
+   * child's read through this (the host wires it to the Query Coordinator's
+   * RLS-correct `fanOut`, with identity forwarded so each shard applies its own
+   * RLS). Absent it, loading such a relation throws a clear "not supported"
+   * error (legacy behaviour). The forward direction (shard-local parent →
+   * global child) and same-backend relations never touch this.
+   *
+   * Takes {@link CrossShardReadArgs}, not `QueryArgs`: the hop is JSON, so the
+   * RLS filters are handed over as data (see that type's docblock).
+   */
+  crossShardReader?: (table: string, args: CrossShardReadArgs) => Promise<QueryPage>;
   /**
-  * The SQL dialect that shapes every statement (identifier quoting, value
-  * encode/decode, column types, upserts, RETURNING vs affected-rows).
-  * `@lunora/d1` passes its `sqliteDialect`; the PlanetScale/Hyperdrive backend
-  * passes its Postgres/MySQL dialect. Required — the core is engine-blind.
-  */
+   * The SQL dialect that shapes every statement (identifier quoting, value
+   * encode/decode, column types, upserts, RETURNING vs affected-rows).
+   * `@lunora/d1` passes its `sqliteDialect`; the PlanetScale/Hyperdrive backend
+   * passes its Postgres/MySQL dialect. Required — the core is engine-blind.
+   */
   dialect: SqlDialect;
   exec: SqlCtxExec;
   idGenerator?: () => string;
   /**
-  * Ceiling on the number of child join keys a relation-crossing `where`
-  * predicate may materialize before the semijoin pre-resolver fails closed
-  * (`relation predicate … exceeding the N-key limit`). D1 has no EXISTS
-  * push-down, so an overflow here can only fail closed — never truncate the
-  * `IN (...)` and silently mis-match. Defaults to the pre-resolver's shared
-  * key cap when omitted.
-  */
+   * Ceiling on the number of child join keys a relation-crossing `where`
+   * predicate may materialize before the semijoin pre-resolver fails closed
+   * (`relation predicate … exceeding the N-key limit`). D1 has no EXISTS
+   * push-down, so an overflow here can only fail closed — never truncate the
+   * `IN (...)` and silently mis-match. Defaults to the pre-resolver's shared
+   * key cap when omitted.
+   */
   maxRelationKeys?: number;
   /**
-  * Scheduler exposed to global-table trigger handlers as `ctx.scheduler`.
-  * Absent it, `ctx.scheduler` is a stub that throws on use — pass one when
-  * triggers on `.global()` tables need to enqueue follow-up work.
-  */
+   * Scheduler exposed to global-table trigger handlers as `ctx.scheduler`.
+   * Absent it, `ctx.scheduler` is a stub that throws on use — pass one when
+   * triggers on `.global()` tables need to enqueue follow-up work.
+   */
   scheduler?: SchedulerLike;
   schema: SchemaLike;
 }
 /**
-* Decode a SELECTed row back into a document: `id` → `_id`, `_creationTime`
-* preserved, and every column run through the shared {@link sqliteDecode} so the
-* stored form is reversed back into its JS shape. Exported so the data-browser
-* (`introspect.ts`) and admin export/import paths share the exact same decode.
-*
-* The decode is engine-agnostic: every backend stores SQLite-shaped values
-* (boolean → 1/0, JSON → text, bigint → decimal string), and `sqliteDecode` is
-* robust to a driver returning either the stored string OR a natively-parsed
-* value (e.g. mysql2 returns JSON columns pre-parsed) — so the same decoder is
-* correct on SQLite, Postgres and MySQL.
-*/
-declare const decodeGlobalRow: (definition: TableDefinitionLike, row: Record<string, unknown>) => Record<string, unknown>;
+ * Auto-provision every `.global()` table from the schema: `CREATE TABLE IF NOT
+ * EXISTS` with the physical `id`/`_creationTime` columns plus a typed column per
+ * declared field, then its secondary and `.unique()` indexes. This is the D1
+ * twin of `@lunora/do`'s `runShardMigrations` (which self-creates shard-local
+ * tables) — it makes the schema the single source of truth for global tables
+ * too, so a fresh database serves them without a hand-applied migration. The
+ * column set and dialect match exactly what this module reads and writes
+ * (`columnRef`, `serializeColumnValue`, `decodeGlobalRow`).
+ *
+ * Idempotent (`CREATE TABLE/INDEX IF NOT EXISTS`); additive only — it never
+ * drops or retypes an existing column, so destructive schema changes still need
+ * an explicit migration.
+ */
 declare const runSqlGlobalTableMigrations: (exec: SqlCtxExec, schema: SchemaLike, dialect: SqlDialect) => Promise<void>;
 /**
-* Materialize the `__agg_&lt;index>` companion tables for every declared
-* `aggregateIndex` on a global table. Global tables in Lunora ship their own
-* DDL — counter tables are opt-in so production hosts can decide where they
-* live. Tests and dev hosts can call this once after their schema migration to
-* unlock O(1) counts.
-*
-* Idempotent (`CREATE TABLE IF NOT EXISTS`).
-*/
+ * Materialize the `__agg_<index>` companion tables for every declared
+ * `aggregateIndex` on a global table. Global tables in Lunora ship their own
+ * DDL — counter tables are opt-in so production hosts can decide where they
+ * live. Tests and dev hosts can call this once after their schema migration to
+ * unlock O(1) counts.
+ *
+ * Idempotent (`CREATE TABLE IF NOT EXISTS`).
+ */
 declare const runSqlAggregateMigrations: (exec: SqlCtxExec, schema: SchemaLike, dialect: SqlDialect) => Promise<void>;
 /**
-* Materialize the `__rank_&lt;index>` companion tables for every declared
-* `rankIndex` on a global table. Mirrors `runSqlAggregateMigrations` — same
-* opt-in pattern so production hosts decide whether to spend the DDL.
-*
-* Idempotent (`CREATE TABLE IF NOT EXISTS` + `createIndexIfNotExists`).
-*/
+ * Materialize the `__rank_<index>` companion tables for every declared
+ * `rankIndex` on a global table. Mirrors `runSqlAggregateMigrations` — same
+ * opt-in pattern so production hosts decide whether to spend the DDL.
+ *
+ * Idempotent (`CREATE TABLE IF NOT EXISTS` + `createIndexIfNotExists`).
+ */
 declare const runSqlRankMigrations: (exec: SqlCtxExec, schema: SchemaLike, dialect: SqlDialect) => Promise<void>;
-/**
-* Materialize the `__fts_&lt;index>` FTS5 shadow tables for every declared
-* `.searchIndex()` on a global table. Mirrors `runSqlAggregateMigrations` — same
-* opt-in pattern so production hosts decide whether to spend the DDL. Only runs
-* on engines that ship FTS5 (D1 does; the `node:sqlite` test runner doesn't,
-* where `.search()` transparently falls back to a scan). `__text__` holds the
-* indexed field; `__id__` (UNINDEXED) joins back to the row.
-*
-* Idempotent (`CREATE VIRTUAL TABLE IF NOT EXISTS`).
-*/
-declare const runSqlSearchMigrations: (exec: SqlCtxExec, schema: SchemaLike, dialect: SqlDialect) => Promise<void>;
 /** One change-data-capture entry: a committed mutation, in monotonic `seq` order. Mirrors the DO twin. */
 interface CdcChange {
   /** Post-image document for insert/update; absent for delete (the `id` identifies the removed row). */
@@ -131,9 +160,9 @@ interface CdcChange {
 /** Create the `__cdc_log` table. Idempotent; only run when CDC is enabled. */
 declare const runSqlCdcMigration: (exec: SqlCtxExec, dialect: SqlDialect) => Promise<void>;
 /**
-* Read changelog entries newer than `sinceSeq` in commit order, up to `limit`
-* (clamped to [1, 10000]); plus the cursor to resume from.
-*/
+ * Read changelog entries newer than `sinceSeq` in commit order, up to `limit`
+ * (clamped to [1, 10000]); plus the cursor to resume from.
+ */
 declare const readSqlCdcChanges: (exec: SqlCtxExec, options: {
   limit?: number;
   sinceSeq?: number;
@@ -144,6 +173,10 @@ declare const readSqlCdcChanges: (exec: SqlCtxExec, options: {
 /** Drop changelog entries at or below a checkpointed `throughSeq` (retention). */
 declare const trimSqlCdcChanges: (exec: SqlCtxExec, throughSeq: number, dialect: SqlDialect) => Promise<void>;
 declare const createSqlCtxDb: (options: SqlCtxDbOptions) => DatabaseWriterLike;
+/** Reserved table holding one backfill-progress row per search companion. */
+declare const SEARCH_STATE_TABLE = "__lunora_search_state";
+/** Create the progress table. Idempotent; runs alongside the companion DDL. */
+declare const migrateSearchState: (exec: SqlCtxExec, dialect: SqlDialect) => Promise<void>;
 /** Map a JS value onto its SQLite storage form — SQLite has no boolean, so true/false → 1/0. */
 declare const sqliteEncode: (value: unknown) => unknown;
 /** Parse `raw` as JSON, returning `raw` unchanged when it is not valid JSON. */
@@ -151,24 +184,69 @@ declare const tryJsonParse: (raw: string) => unknown;
 /** Decode a `bigint` column: a decimal string back into a `BigInt`, else verbatim. */
 declare const decodeBigint: (raw: unknown) => unknown;
 /**
-* Resolve the *effective* storage kind of a column validator. Encoding keys off
-* the runtime value's JS type, so a `v.optional(inner)` column stores its
-* present value exactly as `inner` would. The validator's own `kind` is
-* `"optional"`, which hides that — unwrap to the inner validator's kind so the
-* decode reverses the real storage form. The inner validator is stashed on
-* `_meta.inner` by `@lunora/values`' `createValidator`.
-*/
+ * Resolve the *effective* storage kind of a column validator: `v.optional(inner)`
+ * unwrapped to `inner`'s kind, since encoding keys off the runtime value's JS
+ * type and the validator's own `kind` of `"optional"` hides that.
+ *
+ * A thin alias over `shared/effective-kind`, which is where the rule lives so
+ * the DO row store applies the identical one — it reads the same validators and
+ * has the same failure mode, and two copies drifted the last time.
+ */
 declare const effectiveColumnKind: (validator: ValidatorLike) => string | undefined;
 /**
-* Inverse of {@link sqliteEncode}: map a SQLite storage value back onto its JS
-* form, driven by the field's effective validator `kind`:
-*
-* - `boolean`: 1/0 → true/false (SQLite has no boolean type).
-* - `bigint`: decimal string → `BigInt`.
-* - `object`/`array`/`record`: JSON string → parsed value.
-* - `union`/`any`: parsed back only when the stored string is a JSON non-scalar
-*   (a scalar union member round-trips through SQLite's native column type).
-* - everything else (string/number/date/timestamp/id/literal): verbatim.
-*/
+ * Inverse of {@link sqliteEncode}: map a SQLite storage value back onto its JS
+ * form, driven by the field's effective validator `kind`:
+ *
+ * - `boolean`: 1/0 → true/false (SQLite has no boolean type).
+ * - `bigint`: decimal string → `BigInt`.
+ * - `bytes`: normalizes any driver return shape to a genuine `ArrayBuffer` — a
+ *   view (`Uint8Array`/`Buffer`/…) is sliced to its own byte window, a plain
+ *   `ArrayBuffer` passes through. Required because `v.bytes()` validates
+ *   `value instanceof ArrayBuffer` and different backends return different BLOB
+ *   shapes (workerd D1 `ArrayBuffer`, node:sqlite `Uint8Array`, pg/mysql2 `Buffer`).
+ * - `object`/`array`/`record`: JSON string → parsed value.
+ * - `union`/`any`/`from`: parsed back only when the stored string is a JSON
+ *   non-scalar (a scalar member round-trips through SQLite's native column type).
+ *   `from` belongs to THIS group, not to `object`/`array`/`record`: an external
+ *   Standard Schema can describe a string just as easily as an object, and
+ *   {@link sqliteEncode} keys off the runtime JS type — so a `v.from(z.string())`
+ *   column holding `"123"` is stored verbatim, and unconditional parsing would
+ *   read it back as the NUMBER 123.
+ *   CAVEAT: a union/any/from member is stored verbatim by {@link sqliteEncode}, so
+ *   a legitimate *string* value that itself looks like JSON (`'{"a":1}'`, `'[1,2]'`)
+ *   is ambiguous on read and decodes back to the parsed object/array, not the
+ *   original string. This is inherent to sharing one TEXT column between a string
+ *   and an object member; disambiguating would require a breaking storage-format
+ *   change (tagging encoded non-scalars), so it is documented rather than fixed.
+ * - everything else (string/number/date/timestamp/id/literal): verbatim.
+ */
 declare const sqliteDecode: (raw: unknown, kind: string | undefined) => unknown;
-export { type SqlCtxDbOptions, type SqlCtxExec, type SqlDialect, type SqlRunResult, createSqlCtxDb, decodeBigint, decodeGlobalRow, effectiveColumnKind, readSqlCdcChanges, runSqlAggregateMigrations, runSqlCdcMigration, runSqlGlobalTableMigrations, runSqlRankMigrations, runSqlSearchMigrations, sqliteDecode, sqliteEncode, trimSqlCdcChanges, tryJsonParse };
+export { SEARCH_STATE_TABLE,
+/**
+ * `@lunora/sql-store` — the internal, dialect-parameterized SQL store core
+ * shared by Lunora's `.global()` table backends.
+ *
+ * One ORM implementation (`createSqlCtxDb`) drives any SQL engine through a
+ * `SqlDialect`: SQLite via `@lunora/d1`, and Postgres/MySQL (PlanetScale, Neon,
+ * any Hyperdrive-reachable database) via `@lunora/hyperdrive`. Reactivity is
+ * unaffected — the writer is injected as `globalDb` into `createShardCtxDb`,
+ * whose `broadcast` hook drives live queries regardless of engine.
+ *
+ * This package is internal: consumers depend on `@lunora/d1` or
+ * `@lunora/hyperdrive`, which assemble the concrete dialect and wrap the core.
+ */
+type SqlCtxDbOptions,
+/**
+ * `@lunora/sql-store` — the internal, dialect-parameterized SQL store core
+ * shared by Lunora's `.global()` table backends.
+ *
+ * One ORM implementation (`createSqlCtxDb`) drives any SQL engine through a
+ * `SqlDialect`: SQLite via `@lunora/d1`, and Postgres/MySQL (PlanetScale, Neon,
+ * any Hyperdrive-reachable database) via `@lunora/hyperdrive`. Reactivity is
+ * unaffected — the writer is injected as `globalDb` into `createShardCtxDb`,
+ * whose `broadcast` hook drives live queries regardless of engine.
+ *
+ * This package is internal: consumers depend on `@lunora/d1` or
+ * `@lunora/hyperdrive`, which assemble the concrete dialect and wrap the core.
+ */
+type SqlCtxExec, type SqlDialect, type SqlRunResult, backfillSqlSearchIndexes, createSqlCtxDb, decodeBigint, decodeGlobalRow, effectiveColumnKind, migrateSearchState, readSqlCdcChanges, runSqlAggregateMigrations, runSqlCdcMigration, runSqlGlobalTableMigrations, runSqlRankMigrations, runSqlSearchMigrations, sqliteDecode, sqliteEncode, trimSqlCdcChanges, tryJsonParse };