@sixb/sqlite 0.1.16 → 0.1.18

package/src/migrations.ts

@@ -121,6 +121,11 @@ import projectionRunSupersessionSql from "./migrations/051-projection-run-supers
 import compactSourceStorageSql from "./migrations/052-compact-source-storage.sql" with {
   type: "text",
 }
+import replacementPlansSql from "./migrations/053-replacement-plans.sql" with { type: "text" }
+import slimOntologyOutboxSql from "./migrations/054-slim-ontology-outbox.sql" with { type: "text" }
+import commitTouchesSql from "./migrations/055-commit-touches.sql" with { type: "text" }
+import audioTranscriptionSql from "./migrations/056-audio-transcription.sql" with { type: "text" }
+import rerankingModelKindSql from "./migrations/057-reranking-model-kind.sql" with { type: "text" }
 
 const MIGRATIONS_TABLE_SQL = `
   CREATE TABLE IF NOT EXISTS sixb_migrations (
@@ -246,6 +251,26 @@ export const sqliteStorageMigrations = defineMigrations({
       },
       { checksum: checksum(compactSourceStorageSql) }
     ),
+    sqliteSql("053-replacement-plans", replacementPlansSql),
+    sqliteStep(
+      "054-slim-ontology-outbox",
+      (db) => {
+        // SQLite has no procedural block to refuse the conversion in SQL: check before it.
+        const mismatched = unrebuildableOutboxEvents(db)
+        if (mismatched > 0) {
+          throw new Error(
+            `[SixbSqliteStorage] ${mismatched} outbox events would not rebuild from their commit.`
+          )
+        }
+        db.run(slimOntologyOutboxSql)
+      },
+      { checksum: checksum(slimOntologyOutboxSql) }
+    ),
+    sqliteSql("055-commit-touches", commitTouchesSql),
+    sqliteSql("056-audio-transcription", audioTranscriptionSql),
+    sqliteSql("057-reranking-model-kind", rerankingModelKindSql),
+    sqliteSql("058-file-download-grants", fileDownloadGrantsSql),
+    sqliteSql("059-drained-source-versions", drainedSourceVersionsSql),
   ],
 })
 
@@ -416,6 +441,92 @@ function sourceStorageCounts(db: Database): {
     .get() as { readonly versions: number; readonly roots: number; readonly rows: number }
 }
 
+/**
+ * Stored events that would not rebuild exactly from their commit and execution. Who asked and what
+ * wrote are derived from the execution exactly as 047 derived the commit's copies; both sides were
+ * written as canonical JSON, so equal JSON compares as equal text.
+ */
+function unrebuildableOutboxEvents(db: Database): number {
+  const row = db
+    .query(
+      `SELECT COUNT(*) AS mismatched
+       FROM ontology_outbox AS outbox
+       LEFT JOIN ontology_commits AS commits
+         ON commits.project_id = outbox.project_id AND commits.id = outbox.commit_id
+       LEFT JOIN executions
+         ON executions.project_id = commits.project_id AND executions.id = commits.execution_id
+       WHERE executions.id IS NULL
+         OR outbox.envelope ->> '$.id' IS NOT outbox.id
+         OR outbox.envelope ->> '$.commitOrdinal' IS NOT outbox.commit_ordinal
+         OR outbox.envelope ->> '$.commitId' IS NOT commits.id
+         OR outbox.envelope ->> '$.projectId' IS NOT commits.project_id
+         OR outbox.envelope ->> '$.schemaVersion' IS NOT 1
+         OR outbox.envelope ->> '$.occurredAt' IS NOT commits.committed_at
+         OR outbox.envelope -> '$.origin' IS NOT json(commits.origin)
+         OR outbox.envelope ->> '$.correlationId' IS NOT executions.correlation_id
+         OR outbox.envelope -> '$.requestedBy' IS NOT CASE
+           WHEN executions.requested_by_user_id IS NOT NULL
+             THEN json_object('id', executions.requested_by_user_id, 'type', 'user')
+           WHEN executions.requested_by_service_account_id IS NOT NULL
+             THEN json_object('id', executions.requested_by_service_account_id, 'type', 'serviceAccount')
+         END
+         OR outbox.envelope -> '$.executor' IS NOT CASE executions.executor_kind
+           WHEN 'request' THEN json_object('requestId', executions.executor_id, 'type', 'request')
+           WHEN 'agent' THEN json_object('runId', executions.executor_id, 'type', 'agent')
+           WHEN 'kernel' THEN json_object(
+             'operation',
+             CASE executions.authority_kernel_operation
+               WHEN 'ontology.recover'
+                 THEN json_object('recoveryId', executions.executor_id, 'type', 'ontology.recover')
+               ELSE json_object('indexingId', executions.executor_id, 'type', 'ontology.indexVectors')
+             END,
+             'type', 'kernel'
+           )
+           ELSE json_object(
+             'id', executions.authority_primitive_id,
+             'kind', executions.executor_kind,
+             'runId', executions.executor_id,
+             'type', 'primitive'
+           )
+         END
+         OR outbox.envelope ->> '$.topic' IS NOT CASE substr(outbox.envelope ->> '$.type', 1, 4)
+           WHEN 'obje' THEN 'objects'
+           WHEN 'link' THEN 'links'
+           WHEN 'tele' THEN 'telemetry'
+         END
+         OR outbox.envelope ->> '$.partitionKey' IS NOT CASE outbox.envelope ->> '$.topic'
+           WHEN 'objects' THEN (outbox.envelope ->> '$.payload.objectTypeId') || ':' ||
+             (outbox.envelope ->> '$.payload.primaryId')
+           WHEN 'links' THEN (outbox.envelope ->> '$.payload.sourceTypeId') || ':' ||
+             (outbox.envelope ->> '$.payload.sourceId') || ':' ||
+             (outbox.envelope ->> '$.payload.linkId')
+           WHEN 'telemetry' THEN (outbox.envelope ->> '$.payload.objectTypeId') || ':' ||
+             (outbox.envelope ->> '$.payload.objectId') || ':' ||
+             (outbox.envelope ->> '$.payload.propertyId')
+         END
+         OR (
+           outbox.envelope ->> '$.type' IN ('object.created', 'link.created')
+           AND (
+             (SELECT COUNT(*) FROM json_each(outbox.envelope, '$.payload.propertyChanges'))
+               IS NOT (SELECT COUNT(*) FROM json_each(outbox.envelope, '$.payload.properties'))
+             OR EXISTS (
+               SELECT 1
+               FROM json_each(outbox.envelope, '$.payload.propertyChanges') AS change
+               LEFT JOIN json_each(outbox.envelope, '$.payload.properties') AS property
+                 ON property.key = change.key
+               WHERE property.key IS NULL
+                 OR change.value ->> '$.operation' IS NOT 'created'
+                 OR json_type(change.value, '$.after') IS NOT property.type
+                 OR change.value ->> '$.after' IS NOT property.value
+                 OR (SELECT COUNT(*) FROM json_each(change.value)) IS NOT 2
+             )
+           )
+         )`
+    )
+    .get() as { readonly mismatched: number }
+  return row.mismatched
+}
+
 function checksum(value: string): string {
   return createHash("sha256").update(value).digest("hex")
 }
@@ -482,3 +593,7 @@ import connectorOptionalPkceSql from "./migrations/039-connector-optional-pkce.s
   type: "text",
 }
 import fileUploadSessionsSql from "./migrations/048-file-upload-sessions.sql" with { type: "text" }
+import fileDownloadGrantsSql from "./migrations/058-file-download-grants.sql" with { type: "text" }
+import drainedSourceVersionsSql from "./migrations/059-drained-source-versions.sql" with {
+  type: "text",
+}

package/src/objects/query-compiler.ts

@@ -146,6 +146,11 @@ function compileObjectQueryInternal(
   ctx: CompileContext
 ): CompiledObjectQuery {
   switch (query.kind) {
+    case "rerank":
+      throw new ObjectQueryExecutionError(
+        "reranking_requires_runtime",
+        "Reranking must execute in the model runtime, not storage."
+      )
     case "start":
       return compileStart(projectId, query, ctx)
     case "refs":

package/src/objects/reader.ts

@@ -136,6 +136,7 @@ export class SqliteObjectReader {
       source,
     })
     return {
+      total: readTotal(this.db, compiled),
       facets: params.facets.map((facet) => ({
         propertyId: facet.propertyId,
         buckets: readFacetBuckets(this.db, compiled, facet),

package/src/objects/storage.ts

@@ -161,9 +161,7 @@ export class SqliteObjectStorage implements ObjectStorage {
   }
 
   async facetObjects(params: FacetObjectsInput): Promise<FacetObjectsResult> {
-    if (hasVectorProfile(params.query))
-      return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params))
-    return this.reader.facetObjects(params)
+    return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params))
   }
 
   async getByPrimaryId(params: {
@@ -252,23 +250,19 @@ export class SqliteObjectStorage implements ObjectStorage {
     limit: number
   }): Promise<{ objects: readonly ObjectRow[]; nextPrimaryId?: string }> {
     assertReconciliationPageLimit(params.limit)
+    // A cursor written as `? IS NULL OR …` cannot seek, so every page would rescan from the start.
+    const cursor = params.afterPrimaryId === undefined ? [] : [params.afterPrimaryId]
     const rows = this.db
       .query(
         `
           SELECT * FROM objects
           WHERE project_id = ? AND object_type_id = ?
-            AND (? IS NULL OR primary_id > ?)
+            ${cursor.length > 0 ? "AND primary_id > ?" : ""}
           ORDER BY primary_id ASC
           LIMIT ?
         `
       )
-      .all(
-        params.projectId,
-        params.objectTypeId,
-        params.afterPrimaryId ?? null,
-        params.afterPrimaryId ?? null,
-        params.limit + 1
-      ) as DatabaseRow[]
+      .all(params.projectId, params.objectTypeId, ...cursor, params.limit + 1) as DatabaseRow[]
     const hasMore = rows.length > params.limit
     const objects = rows.slice(0, params.limit).map((row) => rowToObject(row))
     const last = objects.at(-1)

package/src/ontology-storage/index.ts

@@ -4,9 +4,9 @@ import { SqliteOntologyCommitStorage } from "./commits"
 import type { SqliteOntologyTransactionContext } from "./materialization-session"
 import { SqliteOntologyMaterializationStorage } from "./materializations"
 import { SqliteOntologyOutboxStorage } from "./outbox"
+import { SqliteOntologyReplacementPlanStorage } from "./replacement-plans"
 import type { SqliteRootOperation } from "./shared"
 import { SqliteOntologySourceStorage } from "./sources"
-
 import { SqliteVectorIndexingStorage } from "./vector-indexing"
 import { SqliteOntologyVectorStorage } from "./vectors"
 
@@ -14,6 +14,7 @@ export class SqliteOntologyStorage implements OntologyStorage {
   readonly commits: SqliteOntologyCommitStorage
   readonly sources: SqliteOntologySourceStorage
   readonly materializations: SqliteOntologyMaterializationStorage
+  readonly replacementPlans: SqliteOntologyReplacementPlanStorage
   readonly vectorIndexing: SqliteVectorIndexingStorage
   readonly vectors: SqliteOntologyVectorStorage
   readonly outbox: SqliteOntologyOutboxStorage
@@ -29,6 +30,10 @@ export class SqliteOntologyStorage implements OntologyStorage {
       input.db,
       input.transactionContext
     )
+    this.replacementPlans = new SqliteOntologyReplacementPlanStorage(
+      input.db,
+      input.runRootOperation
+    )
     this.vectorIndexing = new SqliteVectorIndexingStorage(
       input.db,
       input.runRootOperation,