@sixb/sqlite 0.1.8 → 0.1.10

package/src/ontology-storage/source-roots.ts

@@ -0,0 +1,230 @@
+import type { Database } from "bun:sqlite"
+import type { ProjectionEntityRef } from "@sixb/core/internal/materialization"
+import {
+  MaterializationConflictError,
+  MaterializationValidationError,
+  projectionEntityKey,
+} from "@sixb/core/internal/materialization"
+import { sourceStageRoots, utf8SortKey } from "@sixb/core/internal/ontology-storage-provider"
+import type { SourceActivationWrite, StageSourceRowsInput } from "@sixb/core/storage"
+import type { SqliteOntologySourceAssertionRow } from "./shared"
+import { canonicalJson, type SqliteOntologySourceRow } from "./shared"
+
+export function replacementSourceRows(input: {
+  readonly projectId: string
+  readonly sourceId: string
+  readonly materializationId: string
+  readonly incremental: boolean
+}): { readonly sql: string; readonly values: readonly string[] } {
+  const roots = input.incremental
+    ? `ontology_source_roots AS changed
+    CROSS JOIN ontology_source_roots AS roots ON roots.project_id = changed.project_id
+      AND roots.source_id = changed.source_id AND roots.root_sort_key = changed.root_sort_key AND roots.active = 1`
+    : "ontology_source_roots AS roots"
+  const selected = input.incremental ? "changed" : "roots"
+  return {
+    sql: `SELECT * FROM ontology_source_rows WHERE project_id = ? AND source_id = ? AND materialization_id = ?
+      UNION ALL
+      SELECT rows.* FROM ${roots}
+      CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id
+        AND rows.source_id = roots.source_id AND rows.materialization_id = roots.materialization_id
+        AND rows.root_sort_key = roots.root_sort_key
+      WHERE ${selected}.project_id = ? AND ${selected}.source_id = ?
+        AND ${input.incremental ? "changed.materialization_id = ?" : "roots.active = 1"}`,
+    values: [
+      input.projectId,
+      input.sourceId,
+      input.materializationId,
+      input.projectId,
+      input.sourceId,
+      ...(input.incremental ? [input.materializationId] : []),
+    ],
+  }
+}
+
+export function stageSourceRoots(
+  db: Database,
+  manifest: SqliteOntologySourceRow,
+  input: StageSourceRowsInput
+): void {
+  const roots = sourceStageRoots(manifest.projection_kind, input)
+  if (!manifest.base_materialization_id && roots.some((root) => root.deleted)) {
+    throw new MaterializationValidationError("Root deletions require a source delta base.")
+  }
+  const existing = db.query(`SELECT staging_ordinal, deleted FROM ontology_source_roots
+    WHERE project_id = ? AND source_id = ? AND materialization_id = ? AND root_sort_key = ?`)
+  const insert = db.query(`INSERT INTO ontology_source_roots (
+    project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
+  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING RETURNING root_sort_key`)
+  for (const root of roots) {
+    const inserted = insert.get(
+      input.projectId,
+      input.source.projectionId,
+      input.materializationId,
+      root.sortKey,
+      root.root.kind,
+      root.rootKey,
+      canonicalJson(root.root),
+      root.stagingOrdinal,
+      Number(root.deleted)
+    )
+    if (inserted) continue
+    const previous = existing.get(
+      input.projectId,
+      input.source.projectionId,
+      input.materializationId,
+      root.sortKey
+    ) as { staging_ordinal: number; deleted: number } | null
+    if (
+      !previous ||
+      previous.staging_ordinal !== root.stagingOrdinal ||
+      Boolean(previous.deleted) !== root.deleted
+    ) {
+      throw new MaterializationValidationError(
+        "Source materialization repeats root or stream ordinal with different content."
+      )
+    }
+  }
+}
+
+export function activateSourceRoots(
+  db: Database,
+  projectId: string,
+  candidate: SqliteOntologySourceRow,
+  activation: SourceActivationWrite
+): void {
+  if (
+    candidate.base_materialization_id !== null &&
+    (candidate.base_materialization_id !== activation.expected.activeMaterializationId ||
+      candidate.base_commit_id !== activation.expected.lastCommitId)
+  )
+    throw new MaterializationConflictError(
+      "projection-fence",
+      "Source delta base changed before activation."
+    )
+
+  if (candidate.base_materialization_id !== null && candidate.root_count === 0) return
+  const scope = [projectId, activation.source.projectionId, activation.materializationId] as const
+  db.query(`UPDATE ontology_source_roots AS roots SET active = 0, retired_at = ?
+    WHERE project_id = ? AND source_id = ? AND active = 1
+      ${
+        candidate.base_materialization_id === null
+          ? ""
+          : `AND root_sort_key IN (
+        SELECT root_sort_key FROM ontology_source_roots
+        WHERE project_id = ? AND source_id = ? AND materialization_id = ?)`
+      }`).run(
+    activation.updatedAt,
+    projectId,
+    activation.source.projectionId,
+    ...(candidate.base_materialization_id === null ? [] : [...scope])
+  )
+  db.query(`UPDATE ontology_source_roots SET active = CASE WHEN deleted = 1 THEN 0 ELSE 1 END,
+      retired_at = CASE WHEN deleted = 1 THEN ? ELSE NULL END
+    WHERE project_id = ? AND source_id = ? AND materialization_id = ?`).run(
+    activation.updatedAt,
+    ...scope
+  )
+}
+
+export function assertSourceRootCoverage(db: Database, manifest: SqliteOntologySourceRow): void {
+  const invalid = db
+    .query(`SELECT 1 FROM ontology_source_roots AS roots
+    WHERE project_id = ? AND source_id = ? AND materialization_id = ?
+      AND roots.deleted = EXISTS (
+        SELECT 1 FROM ontology_source_rows AS rows
+        WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
+          AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
+      ) LIMIT 1`)
+    .get(manifest.project_id, manifest.source_id, manifest.materialization_id)
+  if (invalid)
+    throw new MaterializationValidationError(
+      "Source roots must have complete assertions or an explicit deletion, never both."
+    )
+}
+
+export function activeSourceRows(
+  db: Database,
+  projectId: string,
+  refs: readonly ProjectionEntityRef[]
+): SqliteOntologySourceAssertionRow[] {
+  const keys = refs.flatMap((entity) => {
+    const entitySortKey = utf8SortKey(projectionEntityKey(entity))
+    const roots =
+      entity.kind === "object"
+        ? [entitySortKey]
+        : [
+            entitySortKey,
+            utf8SortKey(projectionEntityKey({ kind: "object", ref: entity.ref.source })),
+          ]
+    return roots.map((rootSortKey) => ({ rootSortKey, entitySortKey }))
+  })
+  return db
+    .query(`WITH requested AS (
+    SELECT DISTINCT json_extract(value, '$.rootSortKey') AS root_sort_key,
+      json_extract(value, '$.entitySortKey') AS entity_sort_key FROM json_each(?)
+  ) SELECT rows.* FROM requested
+    CROSS JOIN ontology_source_roots AS roots ON roots.project_id = ?
+      AND roots.active = 1 AND roots.root_sort_key = requested.root_sort_key
+    CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
+      AND rows.materialization_id = roots.materialization_id AND rows.entity_sort_key = requested.entity_sort_key
+      AND rows.root_sort_key = roots.root_sort_key`)
+    .all(canonicalJson(keys), projectId) as SqliteOntologySourceAssertionRow[]
+}
+
+export function replacementAssertionRows(
+  db: Database,
+  input: {
+    projectId: string
+    sourceId: string
+    materializationId: string
+    incremental: boolean
+    includePrevious: boolean
+    refs: readonly ProjectionEntityRef[]
+  }
+): SqliteOntologySourceAssertionRow[] {
+  const keys = input.refs.map((ref) => ({
+    entityKind: ref.kind,
+    entityKey: projectionEntityKey(ref),
+    rootKeys:
+      ref.kind === "object"
+        ? [utf8SortKey(projectionEntityKey(ref))]
+        : [
+            utf8SortKey(projectionEntityKey(ref)),
+            utf8SortKey(projectionEntityKey({ kind: "object", ref: ref.ref.source })),
+          ],
+  }))
+  const previous = !input.includePrevious
+    ? ""
+    : `UNION ALL
+    SELECT rows.* FROM requested_entities AS requested
+    CROSS JOIN ontology_source_roots AS roots ON roots.project_id = ? AND roots.source_id = ?
+      AND roots.active = 1 AND roots.root_sort_key IN (SELECT value FROM json_each(requested.root_keys))
+    CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
+      AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
+      AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
+    ${
+      !input.incremental
+        ? ""
+        : `WHERE EXISTS (SELECT 1 FROM ontology_source_roots AS changed
+      WHERE changed.project_id = roots.project_id AND changed.source_id = roots.source_id
+        AND changed.materialization_id = ? AND changed.root_sort_key = roots.root_sort_key)`
+    }`
+  return db
+    .query(`WITH requested_entities AS (
+    SELECT DISTINCT json_extract(value, '$.entityKind') AS entity_kind,
+      json_extract(value, '$.entityKey') AS entity_key, json_extract(value, '$.rootKeys') AS root_keys FROM json_each(?)
+  ) SELECT rows.* FROM requested_entities AS requested
+    CROSS JOIN ontology_source_rows AS rows ON rows.project_id = ? AND rows.source_id = ?
+      AND rows.materialization_id = ? AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
+    ${previous}`)
+    .all(
+      canonicalJson(keys),
+      input.projectId,
+      input.sourceId,
+      input.materializationId,
+      ...(input.includePrevious
+        ? [input.projectId, input.sourceId, ...(input.incremental ? [input.materializationId] : [])]
+        : [])
+    ) as SqliteOntologySourceAssertionRow[]
+}

package/src/ontology-storage/sources.ts

@@ -47,6 +47,8 @@ import {
   sourceAssertion,
   sourceRecord,
 } from "./shared"
+import { cleanupSourceVersions } from "./source-cleanup"
+import { assertSourceRootCoverage, stageSourceRoots } from "./source-roots"
 
 export class SqliteOntologySourceStorage implements OntologySourceStorage {
   constructor(
@@ -98,9 +100,9 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
               dataset_id, dataset_version_id, dataset_version_created_at,
               projection_revision, ownership_hash, ontology_revision,
               root_count, assertion_count, created_at, ready_at, activated_at,
-              terminal_at, last_commit_id, updated_at
+              terminal_at, last_commit_id, updated_at, base_materialization_id, base_commit_id
             ) VALUES (?, ?, ?, ?, ?, 'replacement', 'staging', ?, ?, ?, ?, ?, ?, ?,
-              NULL, NULL, ?, NULL, NULL, NULL, NULL, ?)
+              NULL, NULL, ?, NULL, NULL, NULL, NULL, ?, ?, ?)
           `
         )
         .run(
@@ -117,7 +119,9 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
           input.ownershipHash,
           input.ontologyRevision,
           input.createdAt,
-          input.createdAt
+          input.createdAt,
+          input.base?.materializationId ?? null,
+          input.base?.lastCommitId ?? null
         )
       return sourceRecord(
         this.requireManifest(input.projectId, input.source.projectionId, input.materializationId)
@@ -144,6 +148,7 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
       const rows = sourceStageRows(manifest.projection_kind, input.rows)
       const existing = this.findStageRows(input, rows)
       const { pending, unchanged } = reconcileSourceStageRows(rows, existing)
+      stageSourceRoots(this.db, manifest, input)
       this.insertStageRows(input, pending)
       return { inserted: pending.length, unchanged }
     })
@@ -249,78 +254,7 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
       assertNonblank(input.projectId, "Terminal source cleanup project id")
       assertTimestamp(input.terminalBefore, "Terminal source cleanup cutoff", true)
       assertPositiveInteger(input.limit, "Terminal source cleanup limit")
-      const manifests = this.db
-        .query(
-          `
-            SELECT * FROM ontology_sources
-            WHERE project_id = ? AND status IN ('superseded', 'abandoned')
-              AND terminal_at < ?
-            ORDER BY terminal_at, source_id, materialization_id
-            LIMIT ?
-          `
-        )
-        .all(input.projectId, input.terminalBefore, input.limit) as SqliteOntologySourceRow[]
-      let remaining = input.limit
-      let rowsDeleted = 0
-      let materializationsDeleted = 0
-      for (const manifest of manifests) {
-        if (remaining === 0) break
-        const deleted = this.db
-          .query(
-            `
-              DELETE FROM ontology_source_rows
-              WHERE rowid IN (
-                SELECT rows.rowid
-                FROM ontology_source_rows AS rows
-                JOIN ontology_sources AS sources
-                  ON sources.project_id = rows.project_id
-                 AND sources.source_id = rows.source_id
-                 AND sources.materialization_id = rows.materialization_id
-                WHERE rows.project_id = ? AND rows.source_id = ? AND rows.materialization_id = ?
-                  AND sources.status IN ('superseded', 'abandoned')
-                  AND sources.terminal_at = ? AND sources.terminal_at < ?
-                ORDER BY rows.entity_sort_key
-                LIMIT ?
-              )
-            `
-          )
-          .run(
-            manifest.project_id,
-            manifest.source_id,
-            manifest.materialization_id,
-            manifest.terminal_at,
-            input.terminalBefore,
-            remaining
-          ).changes
-        rowsDeleted += deleted
-        remaining -= deleted
-        if (remaining === 0) break
-        const removed = this.db
-          .query(
-            `
-              DELETE FROM ontology_sources
-              WHERE project_id = ? AND source_id = ? AND materialization_id = ?
-                AND status IN ('superseded', 'abandoned')
-                AND terminal_at = ? AND terminal_at < ?
-                AND NOT EXISTS (
-                  SELECT 1 FROM ontology_source_rows AS rows
-                  WHERE rows.project_id = ontology_sources.project_id
-                    AND rows.source_id = ontology_sources.source_id
-                    AND rows.materialization_id = ontology_sources.materialization_id
-                )
-            `
-          )
-          .run(
-            manifest.project_id,
-            manifest.source_id,
-            manifest.materialization_id,
-            manifest.terminal_at,
-            input.terminalBefore
-          ).changes
-        materializationsDeleted += removed
-        remaining -= removed
-      }
-      return { rowsDeleted, materializationsDeleted }
+      return cleanupSourceVersions(this.db, input)
     })
   }
 
@@ -433,6 +367,10 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
         `Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`
       )
     }
+    this.db
+      .query(`UPDATE ontology_source_roots SET retired_at = ?
+      WHERE project_id = ? AND source_id = ? AND materialization_id = ? AND active = 0`)
+      .run(abandonedAt, manifest.project_id, manifest.source_id, manifest.materialization_id)
     return sourceRecord(
       this.requireManifest(manifest.project_id, manifest.source_id, manifest.materialization_id)
     )
@@ -446,14 +384,22 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
     const counts = this.db
       .query(
         `
-          SELECT COUNT(*) AS assertions, COUNT(DISTINCT root_key) AS roots,
+          SELECT (SELECT COUNT(*) FROM ontology_source_rows
+            WHERE project_id = ? AND source_id = ? AND materialization_id = ?) AS assertions, COUNT(*) AS roots,
             COUNT(DISTINCT staging_ordinal) AS ordinals,
             MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
-          FROM ontology_source_rows
+          FROM ontology_source_roots
           WHERE project_id = ? AND source_id = ? AND materialization_id = ?
         `
       )
-      .get(manifest.project_id, manifest.source_id, manifest.materialization_id) as {
+      .get(
+        manifest.project_id,
+        manifest.source_id,
+        manifest.materialization_id,
+        manifest.project_id,
+        manifest.source_id,
+        manifest.materialization_id
+      ) as {
       assertions: number
       roots: number
       ordinals: number
@@ -470,6 +416,7 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
         "Source ready counts do not match the staged roots and assertions."
       )
     }
+    assertSourceRootCoverage(this.db, manifest)
     const invalid = this.db
       .query(
         manifest.projection_kind === "link"

package/src/pipeline-run-storage.ts

@@ -1,6 +1,7 @@
 import type { Database } from "bun:sqlite"
 import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
 import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider"
+import { canRequeuePipelineRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
 import type { DatasetVersionRef } from "@sixb/core/lake-storage"
 import type {
   ExecutionStorage,
@@ -19,11 +20,7 @@ import type {
   StartPipelineRunInput,
   StartPipelineStepRunInput,
 } from "@sixb/core/storage"
-import {
-  canRequeuePipelineRunAfterEnqueueFailure,
-  PIPELINE_RUN_FAILURE_CODES,
-  PipelineRunError,
-} from "@sixb/core/storage"
+import { PIPELINE_RUN_FAILURE_CODES, PipelineRunError } from "@sixb/core/storage"
 import { queryLatestRunsByOwnerId } from "./latest-run-query"
 import { installFreshSqliteSchema } from "./migrations"
 import {

package/src/projection-run-storage.ts

@@ -114,7 +114,7 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
                 input_exhausted = ?, missing_target_object_type_id = NULL,
                 missing_target_object_id = NULL, missing_target_batch_ordinal = NULL,
                 missing_target_first_seen_at = NULL, source_rows_read = 0,
-                source_rows_skipped = 0, error = NULL
+                source_rows_skipped = 0, source_changes_read = NULL, error = NULL
               WHERE project_id = ? AND id = ? AND status = 'failed'
                 AND json_extract(error, '$.code') = 'queue.enqueue_failed'
             `
@@ -257,6 +257,7 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
             execution_token = NULL,
             source_rows_read = ?,
             source_rows_skipped = ?,
+            source_changes_read = ?,
             input_exhausted = ?,
             error = ?
           WHERE project_id = ? AND id = ? AND status = 'running' AND execution_token = ?
@@ -267,6 +268,7 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
           plan.finishedAt.toISOString(),
           plan.progress.sourceRowsRead,
           plan.progress.sourceRowsSkipped,
+          plan.progress.sourceChangesRead ?? null,
           plan.inputExhausted ? 1 : existingRow.input_exhausted,
           plan.error === undefined
             ? null
@@ -531,9 +533,10 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
     progress: ProjectionRunProgress,
     executionToken: string
   ): number {
-    const args: (string | number)[] = [
+    const args: (string | number | null)[] = [
       progress.sourceRowsRead,
       progress.sourceRowsSkipped,
+      progress.sourceChangesRead ?? null,
       existing.project_id,
       existing.id,
       executionToken,
@@ -544,7 +547,8 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
         UPDATE projection_runs
         SET
           source_rows_read = ?,
-          source_rows_skipped = ?
+          source_rows_skipped = ?,
+          source_changes_read = ?
         WHERE project_id = ? AND id = ? AND status = 'running'
           AND execution_token = ?
       `
@@ -631,6 +635,9 @@ function restoreProjectionRunRow(row: DatabaseRow): StoredProjectionRunRecord {
     progress: {
       sourceRowsRead: databaseSafeInteger(row.source_rows_read, "sourceRowsRead"),
       sourceRowsSkipped: databaseSafeInteger(row.source_rows_skipped, "sourceRowsSkipped"),
+      ...(row.source_changes_read == null
+        ? {}
+        : { sourceChangesRead: databaseSafeInteger(row.source_changes_read, "sourceChangesRead") }),
     },
     error: row.error ?? undefined,
   }
@@ -694,6 +701,7 @@ interface DatabaseRow {
   missing_target_first_seen_at: string | null
   source_rows_read: number
   source_rows_skipped: number
+  source_changes_read: number | null
   error: string | null
 }
 

package/src/sync-run-storage.ts

@@ -1,6 +1,7 @@
 import type { Database } from "bun:sqlite"
 import type { JsonValue } from "@sixb/core"
 import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
+import { canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
 import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider"
 import type {
   ExecutionStorage,
@@ -14,11 +15,7 @@ import type {
   SyncRunRecord,
   SyncRunStorage,
 } from "@sixb/core/storage"
-import {
-  canRequeueSyncRunAfterEnqueueFailure,
-  SYNC_RUN_FAILURE_CODES,
-  SyncRunError,
-} from "@sixb/core/storage"
+import { SYNC_RUN_FAILURE_CODES, SyncRunError } from "@sixb/core/storage"
 import { queryLatestRunsByOwnerId } from "./latest-run-query"
 import { installFreshSqliteSchema } from "./migrations"
 import {

package/src/workflow-run-storage.ts

@@ -1,5 +1,4 @@
 import type { Database } from "bun:sqlite"
-import { normalizeRequesterGroupIds } from "@sixb/core/internal/auth"
 import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
 import {
   assertWorkflowAgentNodeRunExecution,
@@ -112,9 +111,8 @@ export class SqliteWorkflowRunStorage implements WorkflowRunStorage {
             input,
             queued_at,
             started_at,
-            requester_group_ids,
             attempt
-          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
         `
         )
         .run(
@@ -126,7 +124,6 @@ export class SqliteWorkflowRunStorage implements WorkflowRunStorage {
           serializeRecord(input.input),
           queuedAt.toISOString(),
           queuedAt.toISOString(),
-          JSON.stringify(normalizeRequesterGroupIds(input.requesterGroupIds)),
           0
         )
     } catch (error) {
@@ -983,7 +980,6 @@ function rowToWorkflowRunRecord(row: WorkflowRunDatabaseRow): WorkflowRunRecord
     startedAt: new Date(row.started_at),
     finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
     error: row.error === null ? undefined : parseSixbFailure(row.error, WORKFLOW_RUN_FAILURE_CODES),
-    requesterGroupIds: JSON.parse(row.requester_group_ids) as string[],
     attempt: row.attempt,
     ...(row.execution_token && row.execution_queue_lease_expires_at
       ? {
@@ -1046,7 +1042,6 @@ interface WorkflowRunDatabaseRow {
   started_at: string
   finished_at: string | null
   error: string | null
-  requester_group_ids: string
   attempt: number
   execution_token: string | null
   execution_queue_lease_expires_at: string | null