@sixb/sqlite 0.1.15 → 0.1.17

package/src/migrations.ts

@@ -121,6 +121,11 @@ import projectionRunSupersessionSql from "./migrations/051-projection-run-supers
 import compactSourceStorageSql from "./migrations/052-compact-source-storage.sql" with {
   type: "text",
 }
+import replacementPlansSql from "./migrations/053-replacement-plans.sql" with { type: "text" }
+import slimOntologyOutboxSql from "./migrations/054-slim-ontology-outbox.sql" with { type: "text" }
+import commitTouchesSql from "./migrations/055-commit-touches.sql" with { type: "text" }
+import audioTranscriptionSql from "./migrations/056-audio-transcription.sql" with { type: "text" }
+import rerankingModelKindSql from "./migrations/057-reranking-model-kind.sql" with { type: "text" }
 
 const MIGRATIONS_TABLE_SQL = `
   CREATE TABLE IF NOT EXISTS sixb_migrations (
@@ -246,6 +251,26 @@ export const sqliteStorageMigrations = defineMigrations({
       },
       { checksum: checksum(compactSourceStorageSql) }
     ),
+    sqliteSql("053-replacement-plans", replacementPlansSql),
+    sqliteStep(
+      "054-slim-ontology-outbox",
+      (db) => {
+        // SQLite has no procedural block to refuse the conversion in SQL: check before it.
+        const mismatched = unrebuildableOutboxEvents(db)
+        if (mismatched > 0) {
+          throw new Error(
+            `[SixbSqliteStorage] ${mismatched} outbox events would not rebuild from their commit.`
+          )
+        }
+        db.run(slimOntologyOutboxSql)
+      },
+      { checksum: checksum(slimOntologyOutboxSql) }
+    ),
+    sqliteSql("055-commit-touches", commitTouchesSql),
+    sqliteSql("056-audio-transcription", audioTranscriptionSql),
+    sqliteSql("057-reranking-model-kind", rerankingModelKindSql),
+    sqliteSql("058-file-download-grants", fileDownloadGrantsSql),
+    sqliteSql("059-drained-source-versions", drainedSourceVersionsSql),
   ],
 })
 
@@ -416,6 +441,92 @@ function sourceStorageCounts(db: Database): {
     .get() as { readonly versions: number; readonly roots: number; readonly rows: number }
 }
 
+/**
+ * Stored events that would not rebuild exactly from their commit and execution. Who asked and what
+ * wrote are derived from the execution exactly as 047 derived the commit's copies; both sides were
+ * written as canonical JSON, so equal JSON compares as equal text.
+ */
+function unrebuildableOutboxEvents(db: Database): number {
+  const row = db
+    .query(
+      `SELECT COUNT(*) AS mismatched
+       FROM ontology_outbox AS outbox
+       LEFT JOIN ontology_commits AS commits
+         ON commits.project_id = outbox.project_id AND commits.id = outbox.commit_id
+       LEFT JOIN executions
+         ON executions.project_id = commits.project_id AND executions.id = commits.execution_id
+       WHERE executions.id IS NULL
+         OR outbox.envelope ->> '$.id' IS NOT outbox.id
+         OR outbox.envelope ->> '$.commitOrdinal' IS NOT outbox.commit_ordinal
+         OR outbox.envelope ->> '$.commitId' IS NOT commits.id
+         OR outbox.envelope ->> '$.projectId' IS NOT commits.project_id
+         OR outbox.envelope ->> '$.schemaVersion' IS NOT 1
+         OR outbox.envelope ->> '$.occurredAt' IS NOT commits.committed_at
+         OR outbox.envelope -> '$.origin' IS NOT json(commits.origin)
+         OR outbox.envelope ->> '$.correlationId' IS NOT executions.correlation_id
+         OR outbox.envelope -> '$.requestedBy' IS NOT CASE
+           WHEN executions.requested_by_user_id IS NOT NULL
+             THEN json_object('id', executions.requested_by_user_id, 'type', 'user')
+           WHEN executions.requested_by_service_account_id IS NOT NULL
+             THEN json_object('id', executions.requested_by_service_account_id, 'type', 'serviceAccount')
+         END
+         OR outbox.envelope -> '$.executor' IS NOT CASE executions.executor_kind
+           WHEN 'request' THEN json_object('requestId', executions.executor_id, 'type', 'request')
+           WHEN 'agent' THEN json_object('runId', executions.executor_id, 'type', 'agent')
+           WHEN 'kernel' THEN json_object(
+             'operation',
+             CASE executions.authority_kernel_operation
+               WHEN 'ontology.recover'
+                 THEN json_object('recoveryId', executions.executor_id, 'type', 'ontology.recover')
+               ELSE json_object('indexingId', executions.executor_id, 'type', 'ontology.indexVectors')
+             END,
+             'type', 'kernel'
+           )
+           ELSE json_object(
+             'id', executions.authority_primitive_id,
+             'kind', executions.executor_kind,
+             'runId', executions.executor_id,
+             'type', 'primitive'
+           )
+         END
+         OR outbox.envelope ->> '$.topic' IS NOT CASE substr(outbox.envelope ->> '$.type', 1, 4)
+           WHEN 'obje' THEN 'objects'
+           WHEN 'link' THEN 'links'
+           WHEN 'tele' THEN 'telemetry'
+         END
+         OR outbox.envelope ->> '$.partitionKey' IS NOT CASE outbox.envelope ->> '$.topic'
+           WHEN 'objects' THEN (outbox.envelope ->> '$.payload.objectTypeId') || ':' ||
+             (outbox.envelope ->> '$.payload.primaryId')
+           WHEN 'links' THEN (outbox.envelope ->> '$.payload.sourceTypeId') || ':' ||
+             (outbox.envelope ->> '$.payload.sourceId') || ':' ||
+             (outbox.envelope ->> '$.payload.linkId')
+           WHEN 'telemetry' THEN (outbox.envelope ->> '$.payload.objectTypeId') || ':' ||
+             (outbox.envelope ->> '$.payload.objectId') || ':' ||
+             (outbox.envelope ->> '$.payload.propertyId')
+         END
+         OR (
+           outbox.envelope ->> '$.type' IN ('object.created', 'link.created')
+           AND (
+             (SELECT COUNT(*) FROM json_each(outbox.envelope, '$.payload.propertyChanges'))
+               IS NOT (SELECT COUNT(*) FROM json_each(outbox.envelope, '$.payload.properties'))
+             OR EXISTS (
+               SELECT 1
+               FROM json_each(outbox.envelope, '$.payload.propertyChanges') AS change
+               LEFT JOIN json_each(outbox.envelope, '$.payload.properties') AS property
+                 ON property.key = change.key
+               WHERE property.key IS NULL
+                 OR change.value ->> '$.operation' IS NOT 'created'
+                 OR json_type(change.value, '$.after') IS NOT property.type
+                 OR change.value ->> '$.after' IS NOT property.value
+                 OR (SELECT COUNT(*) FROM json_each(change.value)) IS NOT 2
+             )
+           )
+         )`
+    )
+    .get() as { readonly mismatched: number }
+  return row.mismatched
+}
+
 function checksum(value: string): string {
   return createHash("sha256").update(value).digest("hex")
 }
@@ -482,3 +593,7 @@ import connectorOptionalPkceSql from "./migrations/039-connector-optional-pkce.s
   type: "text",
 }
 import fileUploadSessionsSql from "./migrations/048-file-upload-sessions.sql" with { type: "text" }
+import fileDownloadGrantsSql from "./migrations/058-file-download-grants.sql" with { type: "text" }
+import drainedSourceVersionsSql from "./migrations/059-drained-source-versions.sql" with {
+  type: "text",
+}

package/src/objects/README.md

@@ -0,0 +1,19 @@
+# Object reads
+
+`read-scope.ts` derives visible objects, links and property grants from the selected roots and
+paths. The selected reader holds its traversal-budget probe and terminal statements on one WAL
+snapshot. SQL must filter by those relations before ranking or limiting results.
+
+SQLite flattens CTEs. Expose identity columns from the grant/visible relation and put authorized
+parents before indexed lookups with `CROSS JOIN`; exposing stored identities can cause a full
+grant scan per traversed edge. The query-plan regression test checks those lookups.
+
+Vector `list()` materializes at most the candidate budget plus one identity/vector pair per
+eligible object. Its count controls distance evaluation and travels with the ranked rows, so
+admission, total and ranking share one scan. Returned objects are hydrated after top-k; candidate
+filters and traversals may still evaluate masked properties while deciding eligibility.
+Overflow fails before exposing rows; it never silently ranks a truncated candidate set.
+
+An outer zero limit can suppress the count row; that case uses the separate bounded probe.
+Count, existence and facet terminals also retain that probe. Totals count the top-k set before
+the outer limit, and all candidates must grant every profile source property.

package/src/objects/query-compiler.ts

@@ -24,7 +24,14 @@ export interface SqliteObjectQueryPageRow {
 }
 
 export interface CompiledObjectQuery {
-  vectorProbe?: { sql: string; args: SqliteValue[]; limit: number }
+  vectorProbe?: {
+    sql: string
+    args: SqliteValue[]
+    limit: number
+    /** Maximum returned rows and total before the outermost limit, respectively. */
+    rowLimit: number
+    totalLimit: number
+  }
   sql: string
   args: SqliteValue[]
   totalSql: string
@@ -98,7 +105,7 @@ export function compileObjectQuery(
   if (hasVectorProfile(query) && !isVectorProfileQuery(query)) {
     throw new ObjectQueryExecutionError(
       "unsupported_vector_composition",
-      "Vector search supports one profile with filters before ranking and limit/project after."
+      "Vector search supports one profile with filters, traversals or sets before ranking and limit/project after."
     )
   }
   const source = options.source ?? DEFAULT_OBJECT_QUERY_SOURCE
@@ -139,6 +146,11 @@ function compileObjectQueryInternal(
   ctx: CompileContext
 ): CompiledObjectQuery {
   switch (query.kind) {
+    case "rerank":
+      throw new ObjectQueryExecutionError(
+        "reranking_requires_runtime",
+        "Reranking must execute in the model runtime, not storage."
+      )
     case "start":
       return compileStart(projectId, query, ctx)
     case "refs":
@@ -231,9 +243,31 @@ function compileVector(
     { kind: "column", column: "_vector_score", direction: "desc" },
     ...identityOrderFields().map((field) => ({ ...field, binary: true })),
   ])
-  const sql = `SELECT * FROM (SELECT input.*, max(-1, min(1, 1 - vec_distance_cosine(vectors.embedding, ?))) AS _vector_score
-    ${candidates}) AS ranked ORDER BY ${order.sql} LIMIT ?`
-  const args = [encodeSqliteVector(values), ...candidateArgs, query.k]
+  const rankedOrder = compileOrder(order.fields, "ranked")
+  // Materialize only bounded, authorized candidates once. CASE prevents distance evaluation
+  // above the envelope; the reader rejects the count before exposing any rows or scores.
+  const sql = `SELECT stored.project_id, stored.object_type_id, stored.primary_id,
+    stored.properties, stored.properties AS _cursor_properties,
+    stored.created_at, stored.updated_at, stored.version, stored.last_commit_id,
+    ranked._vector_candidate_count, ranked._vector_score
+    FROM (
+    WITH _sixb_vector_candidates AS MATERIALIZED (
+      SELECT input.project_id, input.object_type_id, input.primary_id,
+        vectors.embedding AS _vector_embedding ${candidates} LIMIT ?
+    ), _sixb_vector_budget AS (
+      SELECT COUNT(*) AS _vector_candidate_count FROM _sixb_vector_candidates
+    )
+    SELECT input.project_id, input.object_type_id, input.primary_id, budget._vector_candidate_count,
+      CASE WHEN budget._vector_candidate_count <= ?
+        THEN max(-1, min(1, 1 - vec_distance_cosine(input._vector_embedding, ?)))
+      END AS _vector_score
+    FROM _sixb_vector_candidates AS input CROSS JOIN _sixb_vector_budget AS budget
+    ORDER BY ${order.sql} LIMIT ?
+  ) AS ranked CROSS JOIN ${ctx.source.objectsTable} AS stored
+    ON stored.project_id = ranked.project_id AND stored.object_type_id = ranked.object_type_id
+      AND stored.primary_id = ranked.primary_id
+  ORDER BY ${rankedOrder.sql}`
+  const args = [...candidateArgs, limit + 1, limit, encodeSqliteVector(values), query.k]
   return {
     sql,
     args,
@@ -245,6 +279,8 @@ function compileVector(
       sql: `SELECT COUNT(*) AS total FROM (SELECT 1 ${candidates} LIMIT ?) AS candidates`,
       args: [...candidateArgs, limit + 1],
       limit,
+      rowLimit: query.k,
+      totalLimit: query.k,
     },
     hasMore: () => false,
     trimRows: identityRows,
@@ -451,7 +487,13 @@ function compileLimit(
       FROM (${input.sql}) AS input
     `,
     totalArgs: input.args,
-    vectorProbe: input.vectorProbe,
+    vectorProbe: input.vectorProbe
+      ? {
+          ...input.vectorProbe,
+          rowLimit: Math.min(input.vectorProbe.rowLimit, rowLimit),
+          totalLimit: input.vectorProbe.rowLimit,
+        }
+      : undefined,
     order: input.order,
     hasMore: (rowCount, total) =>
       total === undefined ? ctx.probeLimit && rowCount > limit : limit < total,
@@ -852,7 +894,7 @@ function compileProject(
         input.created_at,
         input.updated_at,
         input.version,
-        input.last_commit_id${input.vectorProbe ? ", input._vector_score" : ""}
+        input.last_commit_id${input.vectorProbe ? ", input._vector_score, input._vector_candidate_count" : ""}
       FROM (${input.sql}) AS input
       ORDER BY ${inputOrder.sql}
     `,

package/src/objects/read-scope.ts

@@ -22,6 +22,8 @@ export interface SqliteSelectedObjectReadSource extends SqliteObjectQuerySource
  * CROSS JOIN keeps authorized parents/grants before indexed edge/object lookups, even when
  * statistics would otherwise make SQLite rescan all grants for every stored object.
  * Property grants are grouped only after path-sensitive reachability has been resolved.
+ * Expose identities from grants/visible, so flattened joins can seek those relations by key
+ * before loading stored values. Exposing stored identities makes traversal rescan grants per edge.
  */
 export function compileSqliteSelectedObjectReadSource(
   projectId: string,
@@ -105,7 +107,7 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
       _sixb_scope_root_facts(project_id, root_id, node_id, object_type_id, primary_id) AS (
         SELECT anchor.project_id, root.root_id, root.node_id, root.object_type_id, root.primary_id
         FROM _sixb_scope_roots AS root
-        JOIN objects AS anchor
+        CROSS JOIN objects AS anchor
           ON anchor.project_id = ?
          AND anchor.object_type_id = root.object_type_id
          AND anchor.primary_id = root.primary_id
@@ -187,7 +189,7 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
         GROUP BY reachable.project_id, reachable.object_type_id, reachable.primary_id
       ),
       _sixb_scope_objects AS (
-        SELECT stored.project_id, stored.object_type_id, stored.primary_id,
+        SELECT grants.project_id, grants.object_type_id, grants.primary_id,
           COALESCE((
             SELECT json_group_object(property.key, ${sqliteJsonEachValue("property")})
             FROM json_each(stored.properties) AS property
@@ -277,12 +279,12 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
       ),
       _sixb_scope_links AS (
         SELECT
-          stored.project_id,
-          stored.source_type_id,
-          stored.source_id,
-          stored.link_id,
-          stored.target_type_id,
-          stored.target_id,
+          visible.project_id,
+          visible.source_type_id,
+          visible.source_id,
+          visible.link_id,
+          visible.target_type_id,
+          visible.target_id,
           CASE
             WHEN stored.properties IS NULL THEN NULL
             ELSE NULLIF(

package/src/objects/reader.ts

@@ -54,23 +54,23 @@ export class SqliteObjectReader {
     ensureSqliteVectorSearch(this.db)
     const probe = compiled.vectorProbe
     const row = this.db.query<{ total: number }, SqliteValue[]>(probe.sql).get(...probe.args)
-    if (Number(row?.total ?? 0) > probe.limit) {
-      throw new ObjectQueryExecutionError(
-        "vector_work_limit_exceeded",
-        `Exact vector search supports at most ${probe.limit} eligible vectors at this dimension. Narrow the query filters.`
-      )
-    }
+    assertVectorCandidateCount(row?.total, probe.limit)
   }
 
   queryObjects(params: QueryObjectsInput): QueryObjectsResult {
-    this.assertVectorSearch(params)
     const source = this.source
     const compiled = compileObjectQuery(params.projectId, params.query, {
       includeTotal: params.includeTotal,
       source,
     })
-    const total = params.includeTotal === false ? undefined : readTotal(this.db, compiled)
+    if (compiled.vectorProbe) ensureSqliteVectorSearch(this.db)
+    const ordinaryTotal =
+      params.includeTotal === false || compiled.vectorProbe
+        ? undefined
+        : readTotal(this.db, compiled)
     const rawRows = this.db.query(compiled.sql).all(...compiled.args) as ObjectQueryDatabaseRow[]
+    const vectorTotal = this.vectorResultTotal(compiled, rawRows)
+    const total = params.includeTotal === false ? undefined : (vectorTotal ?? ordinaryTotal)
     const rows = compiled.trimRows(rawRows) as readonly ObjectQueryDatabaseRow[]
     const hasMore =
       total === undefined && compiled.hasMoreProbe
@@ -87,6 +87,26 @@ export class SqliteObjectReader {
     }
   }
 
+  private vectorResultTotal(
+    compiled: CompiledObjectQuery,
+    rows: readonly ObjectQueryDatabaseRow[]
+  ): number | undefined {
+    const probe = compiled.vectorProbe
+    if (!probe) return undefined
+
+    // A zero outer limit can hide the count row. Keep admission and total semantics for that
+    // uncommon case, without rescanning candidates for ordinary vector queries.
+    let count = rows[0]?._vector_candidate_count
+    if (rows.length === 0) {
+      count =
+        probe.rowLimit === 0
+          ? this.db.query<{ total: number }, SqliteValue[]>(probe.sql).get(...probe.args)?.total
+          : 0
+    }
+    assertVectorCandidateCount(count, probe.limit)
+    return Math.min(count, probe.totalLimit)
+  }
+
   countObjects(params: CountObjectsInput): CountObjectsResult {
     this.assertVectorSearch(params)
     const source = this.source
@@ -116,6 +136,7 @@ export class SqliteObjectReader {
       source,
     })
     return {
+      total: readTotal(this.db, compiled),
       facets: params.facets.map((facet) => ({
         propertyId: facet.propertyId,
         buckets: readFacetBuckets(this.db, compiled, facet),
@@ -519,6 +540,21 @@ function sqliteFacetValue(valueType: string | null, value: unknown): unknown {
   return value
 }
 
+function assertVectorCandidateCount(count: unknown, limit: number): asserts count is number {
+  if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0) {
+    throw new ObjectQueryExecutionError(
+      "invalid_vector_candidate_count",
+      "Storage did not return a valid vector candidate count."
+    )
+  }
+  if (count > limit) {
+    throw new ObjectQueryExecutionError(
+      "vector_work_limit_exceeded",
+      `Exact vector search supports at most ${limit} eligible vectors at this dimension. Narrow the query filters.`
+    )
+  }
+}
+
 function existsProbeQuery(query: ObjectQuery): ObjectQuery {
   return { kind: "limit", limit: 1, input: stripOuterRowShape(query) }
 }

package/src/objects/rows.ts

@@ -107,6 +107,7 @@ export interface DatabaseRow {
 }
 
 export interface ObjectQueryDatabaseRow extends DatabaseRow {
+  _vector_candidate_count?: number
   _vector_score?: number
   _cursor_properties?: string
   /** `json_object(linkId, value, ...)` serialized text from an `expand` pushdown; absent otherwise. */

package/src/objects/storage.ts

@@ -161,9 +161,7 @@ export class SqliteObjectStorage implements ObjectStorage {
   }
 
   async facetObjects(params: FacetObjectsInput): Promise<FacetObjectsResult> {
-    if (hasVectorProfile(params.query))
-      return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params))
-    return this.reader.facetObjects(params)
+    return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params))
   }
 
   async getByPrimaryId(params: {
@@ -252,23 +250,19 @@ export class SqliteObjectStorage implements ObjectStorage {
     limit: number
   }): Promise<{ objects: readonly ObjectRow[]; nextPrimaryId?: string }> {
     assertReconciliationPageLimit(params.limit)
+    // A cursor written as `? IS NULL OR …` cannot seek, so every page would rescan from the start.
+    const cursor = params.afterPrimaryId === undefined ? [] : [params.afterPrimaryId]
     const rows = this.db
       .query(
         `
           SELECT * FROM objects
           WHERE project_id = ? AND object_type_id = ?
-            AND (? IS NULL OR primary_id > ?)
+            ${cursor.length > 0 ? "AND primary_id > ?" : ""}
           ORDER BY primary_id ASC
           LIMIT ?
         `
       )
-      .all(
-        params.projectId,
-        params.objectTypeId,
-        params.afterPrimaryId ?? null,
-        params.afterPrimaryId ?? null,
-        params.limit + 1
-      ) as DatabaseRow[]
+      .all(params.projectId, params.objectTypeId, ...cursor, params.limit + 1) as DatabaseRow[]
     const hasMore = rows.length > params.limit
     const objects = rows.slice(0, params.limit).map((row) => rowToObject(row))
     const last = objects.at(-1)

package/src/ontology-storage/index.ts

@@ -4,9 +4,9 @@ import { SqliteOntologyCommitStorage } from "./commits"
 import type { SqliteOntologyTransactionContext } from "./materialization-session"
 import { SqliteOntologyMaterializationStorage } from "./materializations"
 import { SqliteOntologyOutboxStorage } from "./outbox"
+import { SqliteOntologyReplacementPlanStorage } from "./replacement-plans"
 import type { SqliteRootOperation } from "./shared"
 import { SqliteOntologySourceStorage } from "./sources"
-
 import { SqliteVectorIndexingStorage } from "./vector-indexing"
 import { SqliteOntologyVectorStorage } from "./vectors"
 
@@ -14,6 +14,7 @@ export class SqliteOntologyStorage implements OntologyStorage {
   readonly commits: SqliteOntologyCommitStorage
   readonly sources: SqliteOntologySourceStorage
   readonly materializations: SqliteOntologyMaterializationStorage
+  readonly replacementPlans: SqliteOntologyReplacementPlanStorage
   readonly vectorIndexing: SqliteVectorIndexingStorage
   readonly vectors: SqliteOntologyVectorStorage
   readonly outbox: SqliteOntologyOutboxStorage
@@ -29,6 +30,10 @@ export class SqliteOntologyStorage implements OntologyStorage {
       input.db,
       input.transactionContext
     )
+    this.replacementPlans = new SqliteOntologyReplacementPlanStorage(
+      input.db,
+      input.runRootOperation
+    )
     this.vectorIndexing = new SqliteVectorIndexingStorage(
       input.db,
       input.runRootOperation,