@sixb/sqlite 0.1.16 → 0.1.18

package/src/ontology-storage/materializations.ts

@@ -17,32 +17,33 @@ import {
   assertExpectedObjectRevision,
   assertMaterializationFinalizationCorrelation,
   assertMaterializationHeader,
-  assertMaterializationLaneCompletion,
   assertPageRows,
-  assertPlanChunkCorrelations,
   assertSourceActivationCorrelation,
+  beginMaterializationApply,
+  beginMaterializationVectorChanges,
+  createCardinalityValidator,
   invalidCorrelation,
+  materializationOutboxWrite,
+  materializationPlanChunk,
   sameNonnegativeCounts as sameCounts,
   uniqueSorted,
 } from "@sixb/core/internal/ontology-storage-provider"
 import type {
-  ApplyMaterializationChunkInput,
+  AppliedMaterialization,
+  ApplyMaterializationInput,
   ApplyMaterializationResult,
   FinalizeMaterializationInput,
-  MaterializationObjectExistence,
   MaterializationPlanHeader,
+  MaterializationPlanWorkRecord,
   MaterializationSession,
   MaterializationStatePage,
-  MaterializationWorkPage,
+  MaterializationVectorChangePage,
   OntologyCommitRecord,
   OntologyMaterializationStorage,
-  ReadMaterializationObjectExistenceInput,
   SourceActivationWrite,
-  SourceReplacementStatePage,
   StageMaterializationWorkInput,
   StreamMaterializationStateInput,
-  StreamMaterializationWorkInput,
-  StreamSourceReplacementStateInput,
+  StreamMaterializationVectorChangesInput,
 } from "@sixb/core/storage"
 import { yieldSqliteEventLoop } from "../transactions"
 import {
@@ -50,11 +51,9 @@ import {
   SqliteMaterializationSessions,
   type SqliteOntologyTransactionContext,
 } from "./materialization-session"
-import {
-  SQLITE_MATERIALIZATION_WORK_TABLE,
-  SqliteMaterializationStateReader,
-} from "./materialization-state"
+import { SqliteMaterializationStateReader } from "./materialization-state"
 import { SqliteMaterializationWriter } from "./materialization-writer"
+import { boundReplacementPlan } from "./replacement-plans"
 import {
   assertProjectionExecution,
   canonicalJson,
@@ -67,6 +66,54 @@ import {
 } from "./shared"
 import { activateSourceRoots } from "./source-roots"
 
+/** Rows the provider reads and writes at once while it applies a staged plan. */
+const APPLY_PAGE_ROWS = 1_000
+
+/**
+ * Records the entities whose plan inputs a commit's applied work changed, once each, as the
+ * in-memory `touchPlanItem` does: a replacement plan's refresh rechecks only those.
+ */
+function touchAppliedWorkStatement(workTable: string): string {
+  return `
+  INSERT INTO ontology_commit_touches (project_id, commit_id, entity_kind, identity_key)
+  SELECT DISTINCT $projectId, $commitId, kind, CASE kind
+    WHEN 'object' THEN
+      json_array(json_extract(ref, '$.objectTypeId'), json_extract(ref, '$.primaryId'))
+    WHEN 'scope' THEN json_array(
+      json_extract(ref, '$.source.objectTypeId'), json_extract(ref, '$.source.primaryId'),
+      json_extract(ref, '$.linkId')
+    )
+    ELSE json_array(
+      json_extract(ref, '$.source.objectTypeId'), json_extract(ref, '$.source.primaryId'),
+      json_extract(ref, '$.linkId'), json_extract(ref, '$.target.objectTypeId'),
+      json_extract(ref, '$.target.primaryId')
+    )
+  END
+  FROM (
+    SELECT CASE json_extract(payload, '$.item.kind')
+        WHEN 'object-upsert' THEN 'object'
+        WHEN 'object-delete' THEN 'object'
+        WHEN 'object-override-upsert' THEN 'object'
+        WHEN 'object-override-delete' THEN 'object'
+        WHEN 'point-upsert' THEN 'object'
+        WHEN 'link-upsert' THEN 'link'
+        WHEN 'link-delete' THEN 'link'
+        WHEN 'link-override-upsert' THEN 'link'
+        WHEN 'link-override-delete' THEN 'link'
+        WHEN 'link-slot-override-upsert' THEN 'scope'
+        WHEN 'link-slot-override-delete' THEN 'scope'
+      END AS kind,
+      CASE json_extract(payload, '$.item.kind')
+        WHEN 'object-upsert' THEN json_extract(payload, '$.item.value.row.ref')
+        WHEN 'link-upsert' THEN json_extract(payload, '$.item.value.row.ref')
+        WHEN 'point-upsert' THEN json_extract(payload, '$.item.value.point.series.object')
+        ELSE json_extract(payload, '$.item.value.ref')
+      END AS ref
+    FROM ${workTable}
+    WHERE work_id = $workId AND lane = 'apply'
+  )`
+}
+
 export class SqliteOntologyMaterializationStorage implements OntologyMaterializationStorage {
   private readonly sessions: SqliteMaterializationSessions
   private readonly writer: SqliteMaterializationWriter
@@ -93,7 +140,10 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
 
   async begin(input: MaterializationPlanHeader): Promise<MaterializationSession> {
     assertMaterializationHeader(input)
-    const session = this.sessions.create(input)
+    const session = this.sessions.create(
+      input,
+      input.plan ? boundReplacementPlan(this.db, input) : null
+    )
     try {
       this.assertCommitAbsent(input)
       const reader = new SqliteMaterializationStateReader(this.db, input.commit.projectId)
@@ -178,134 +228,67 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
     }
   }
 
-  async *streamSourceReplacementState(
-    input: StreamSourceReplacementStateInput
-  ): AsyncIterable<SourceReplacementStatePage> {
-    const session = this.sessions.require(input.session)
-    assertPageRows(input.pageRows)
-    const replacement = this.requireReplacement(session, input)
-    const reader = new SqliteMaterializationStateReader(this.db, session.header.commit.projectId)
-    if (input.entityKind === "object") {
-      if (replacement.projectionKind !== "object") {
-        throw new MaterializationConflictError(
-          "source-materialization",
-          "Link projection replacement cannot stream object state."
-        )
-      }
-      if (replacement.objectStreamStarted) {
-        throw new MaterializationConflictError(
-          "effective-state",
-          "Replacement object state may only be streamed once per session."
-        )
-      }
-      replacement.objectStreamStarted = true
-      for (const identities of reader.replacementIdentities({
-        sessionId: session.id,
-        sourceId: replacement.sourceId,
-        candidateMaterializationId: replacement.candidateMaterializationId,
-        previousMaterializationId: replacement.previousMaterializationId,
-        incremental: replacement.incremental,
-        kind: "object",
-        pageRows: input.pageRows,
-      })) {
-        this.sessions.require(input.session)
-        const refs = identities.map((identity) => {
-          if (identity.kind !== "object") {
-            invalidCorrelation("Object replacement returned a link identity.")
-          }
-          return identity.ref
-        })
-        const objects = reader.replacementObjectStates(
-          replacement.sourceId,
-          replacement.candidateMaterializationId,
-          refs
-        )
-        yield { objects, links: [] }
-      }
-      this.sessions.require(input.session)
-      replacement.objectStreamCompleted = true
-      return
-    }
-
-    if (replacement.projectionKind === "object" && !replacement.objectStreamCompleted) {
-      throw new MaterializationConflictError(
-        "effective-state",
-        "Object projection replacement must fully stream object state before link state."
-      )
-    }
-    if (replacement.linkStreamStarted) {
-      throw new MaterializationConflictError(
-        "effective-state",
-        "Replacement link state may only be streamed once per session."
-      )
-    }
-    replacement.linkStreamStarted = true
-    const materializationIds = [
-      replacement.candidateMaterializationId,
-      ...(replacement.previousMaterializationId ? [replacement.previousMaterializationId] : []),
-    ]
-    for (const identities of reader.replacementIdentities({
-      sessionId: session.id,
-      sourceId: replacement.sourceId,
-      candidateMaterializationId: replacement.candidateMaterializationId,
-      previousMaterializationId: replacement.previousMaterializationId,
-      incremental: replacement.incremental,
-      kind: "link",
-      pageRows: input.pageRows,
-    })) {
-      this.sessions.require(input.session)
-      const linkIdentities = identities.map((identity) => {
-        if (identity.kind !== "link") {
-          invalidCorrelation("Link replacement returned an object identity.")
-        }
-        return identity
-      })
-      const links = reader.replacementLinkStates(
-        replacement.sourceId,
-        replacement.candidateMaterializationId,
-        materializationIds,
-        linkIdentities,
-        replacement.incremental
-      )
-      yield { objects: [], links }
-    }
-    this.sessions.require(input.session)
-    replacement.linkStreamCompleted = true
-  }
-
   async stageWork(input: StageMaterializationWorkInput): Promise<void> {
     this.sessions.stage(input)
     await yieldSqliteEventLoop()
   }
 
-  streamWork(input: StreamMaterializationWorkInput): AsyncIterable<MaterializationWorkPage> {
-    assertPageRows(input.pageRows)
-    return this.sessions.stream(input)
-  }
-
-  async readObjectExistence(
-    input: ReadMaterializationObjectExistenceInput
-  ): Promise<readonly MaterializationObjectExistence[]> {
+  async *streamVectorChanges(
+    input: StreamMaterializationVectorChangesInput
+  ): AsyncIterable<MaterializationVectorChangePage> {
     const session = this.sessions.require(input.session)
-    return this.sessions.readObjectExistence(session, input.refs)
+    assertPageRows(input.pageRows)
+    beginMaterializationVectorChanges(session)
+    for (const items of this.sessions.vectorChangePages(
+      session,
+      input.objectTypeIds,
+      input.pageRows
+    )) {
+      yield { items }
+    }
   }
 
-  async applyChunk(input: ApplyMaterializationChunkInput): Promise<void> {
+  async apply(input: ApplyMaterializationInput): Promise<AppliedMaterialization> {
     const session = this.sessions.require(input.session)
+    beginMaterializationApply(session)
     const { commit } = session.header
-    assertPlanChunkCorrelations(input.chunk, commit)
-    const sequence = this.sessions.prepareChunkSequence(session, input.chunk)
-    this.db.run("SAVEPOINT sixb_ontology_apply_chunk")
+    const cardinality = createCardinalityValidator()
+    for (const records of this.sessions.lanePages(session, "cardinality", APPLY_PAGE_ROWS)) {
+      for (const record of records) cardinality.accept(record)
+    }
+    let eventCount = 0
+    this.db.run("SAVEPOINT sixb_ontology_apply")
     try {
-      this.writer.apply(commit.projectId, commit.id, input.chunk)
-      this.db.run("RELEASE SAVEPOINT sixb_ontology_apply_chunk")
-      this.sessions.commitChunkSequence(session, sequence)
+      for (const records of this.sessions.lanePages(session, "apply", APPLY_PAGE_ROWS)) {
+        // The writer orders tables its own way within a chunk; one phase per chunk keeps the
+        // canonical phase order.
+        for (const phase of splitByPhase(records)) {
+          this.writer.apply(
+            commit.projectId,
+            commit.id,
+            materializationPlanChunk(phase.map((record) => record.item))
+          )
+        }
+        await yieldSqliteEventLoop()
+      }
+      for (const records of this.sessions.lanePages(session, "event", APPLY_PAGE_ROWS)) {
+        const outbox = records.map((record) =>
+          materializationOutboxWrite(commit, record.draft, eventCount++)
+        )
+        this.writer.apply(commit.projectId, commit.id, materializationPlanChunk([], outbox))
+        await yieldSqliteEventLoop()
+      }
+      this.db
+        .query(touchAppliedWorkStatement(session.workTable))
+        .run({ $projectId: commit.projectId, $commitId: commit.id, $workId: session.workId })
+      this.db.run("RELEASE SAVEPOINT sixb_ontology_apply")
     } catch (error) {
-      this.db.run("ROLLBACK TO SAVEPOINT sixb_ontology_apply_chunk")
-      this.db.run("RELEASE SAVEPOINT sixb_ontology_apply_chunk")
+      this.db.run("ROLLBACK TO SAVEPOINT sixb_ontology_apply")
+      this.db.run("RELEASE SAVEPOINT sixb_ontology_apply")
       throw error
     }
-    await yieldSqliteEventLoop()
+    session.appliedEventCount = eventCount
+    return { eventCount }
   }
 
   async finalize(input: FinalizeMaterializationInput): Promise<ApplyMaterializationResult> {
@@ -324,66 +307,6 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
     this.sessions.deactivateAll()
   }
 
-  private requireReplacement(
-    session: SqliteMaterializationSessionState,
-    input: StreamSourceReplacementStateInput
-  ): NonNullable<SqliteMaterializationSessionState["replacement"]> {
-    if (session.replacement) {
-      if (
-        session.replacement.sourceId !== input.source.projectionId ||
-        session.replacement.candidateMaterializationId !== input.candidateMaterializationId
-      ) {
-        throw new MaterializationConflictError(
-          "source-materialization",
-          "Materialization session already owns another replacement union."
-        )
-      }
-      return session.replacement
-    }
-    const projectId = session.header.commit.projectId
-    const candidate = this.getSource(
-      projectId,
-      input.source.projectionId,
-      input.candidateMaterializationId
-    )
-    if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
-      throw new MaterializationConflictError(
-        "source-materialization",
-        `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`
-      )
-    }
-    assertProjectionExecution(this.db, {
-      projectId,
-      sourceId: input.source.projectionId,
-      projectionRunId: candidate.projection_run_id,
-      executionToken: candidate.execution_token,
-    })
-    const previous = this.getActiveSource(projectId, input.source.projectionId)
-    if (
-      candidate.protocol !== "replacement" ||
-      (previous &&
-        (previous.protocol !== candidate.protocol ||
-          previous.projection_kind !== candidate.projection_kind))
-    ) {
-      throw new MaterializationConflictError(
-        "source-materialization",
-        "Source replacement kind or protocol does not match its active materialization."
-      )
-    }
-    session.replacement = {
-      sourceId: input.source.projectionId,
-      candidateMaterializationId: input.candidateMaterializationId,
-      previousMaterializationId: previous?.materialization_id ?? null,
-      incremental: candidate.base_materialization_id !== null,
-      projectionKind: candidate.projection_kind,
-      objectStreamStarted: false,
-      objectStreamCompleted: false,
-      linkStreamStarted: false,
-      linkStreamCompleted: false,
-    }
-    return session.replacement
-  }
-
   private assertCommitAbsent(header: MaterializationPlanHeader): void {
     const origin = originColumns(header.commit.origin)
     const duplicate = this.db
@@ -454,11 +377,9 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
     const { commit } = session.header
     const { result } = input.finalization
     assertMaterializationFinalizationCorrelation(session, input)
-    const laneCounts = this.sessions.laneCounts(session)
-    assertMaterializationLaneCompletion(session, laneCounts)
     this.assertFinalCardinality(session)
     await yieldSqliteEventLoop()
-    const eventCount = laneCounts.event
+    const eventCount = session.appliedEventCount
     const outbox = this.db
       .query(
         `
@@ -517,8 +438,8 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
                 THEN cardinality_target_type_id END) AS expected_target_type_id,
               MAX(CASE WHEN cardinality_occupied = 1
                 THEN cardinality_target_primary_id END) AS expected_target_id
-            FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
-            WHERE session_id = ? AND kind = 'cardinality'
+            FROM ${session.workTable}
+            WHERE work_id = ? AND kind = 'cardinality'
               AND cardinality_view = 'effective'
             GROUP BY sort_one, cardinality_source_type_id,
               cardinality_source_primary_id, cardinality_link_id
@@ -538,20 +459,14 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
             GROUP BY scopes.scope_sort_key, scopes.expected_count,
               scopes.expected_target_type_id, scopes.expected_target_id
           )
-          SELECT CASE WHEN expected_count > 1 THEN 'duplicate' ELSE 'mismatch' END AS reason
+          SELECT 1 AS mismatch
           FROM validated
-          WHERE expected_count > 1
-            OR actual_count <> expected_count
-            OR matching_count <> expected_count
+          WHERE actual_count <> expected_count OR matching_count <> expected_count
           LIMIT 1
         `
       )
-      .get(session.id, session.header.commit.projectId) as {
-      readonly reason: "duplicate" | "mismatch"
-    } | null
-    if (violation?.reason === "duplicate") {
-      invalidCorrelation("Materialization cardinality work violates cardinality-one.")
-    }
+      .get(session.workId, session.header.commit.projectId)
+    // Apply already rejected a scope with two staged occupants; this checks what it wrote.
     if (violation) {
       invalidCorrelation(
         "Materialization cardinality work does not match the final effective link scope."
@@ -599,7 +514,7 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
       invalidCorrelation("Source activation does not match its ready candidate identity.")
     }
     this.assertSource(activation.expected, commit.projectId)
-    activateSourceRoots(this.db, commit.projectId, candidate, activation)
+    activateSourceRoots(this.db, commit.projectId, commit.id, candidate, activation)
     const previous = this.getActiveSource(commit.projectId, activation.source.projectionId)
     if (previous) {
       assertPinnedDatasetWatermark(
@@ -673,12 +588,10 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
           `
             INSERT INTO ontology_commits (
               project_id, id, idempotency_key, request_hash, execution_id,
-              origin_kind, origin_run_id, origin_batch_ordinal, origin, requested_by, executor,
+              origin_kind, origin_run_id, origin_batch_ordinal, origin,
               ontology_revision, projection_revision, ownership_hash,
               intent, result, committed_at
-            ) VALUES (
-              ?, ?, ?, ?, ?, ?, ?, ?, json(?), json(?), json(?), ?, ?, ?, json(?), json(?), ?
-            )
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, json(?), ?, ?, ?, json(?), json(?), ?)
           `
         )
         .run(
@@ -691,8 +604,6 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
           origin.runId,
           origin.batchOrdinal,
           canonicalJson(commit.origin),
-          commit.requestedBy === undefined ? null : canonicalJson(commit.requestedBy),
-          canonicalJson(commit.executor),
           commit.ontologyRevision,
           commit.projectionRevision ?? null,
           commit.ownershipHash ?? null,
@@ -737,3 +648,15 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
       .get(projectId, sourceId, materializationId) as SqliteOntologySourceRow | null
   }
 }
+
+function splitByPhase(
+  records: readonly MaterializationPlanWorkRecord[]
+): MaterializationPlanWorkRecord[][] {
+  const phases: MaterializationPlanWorkRecord[][] = []
+  for (const record of records) {
+    const current = phases[phases.length - 1]
+    if (current && current[0]!.applyPhase === record.applyPhase) current.push(record)
+    else phases.push([record])
+  }
+  return phases
+}

package/src/ontology-storage/outbox.ts

@@ -4,6 +4,10 @@ import {
   MaterializationConflictError,
   MaterializationValidationError,
 } from "@sixb/core/internal/materialization"
+import {
+  claimedOutboxRows,
+  type OutboxEventContext,
+} from "@sixb/core/internal/ontology-storage-provider"
 import type {
   ClaimedOntologyOutboxRow,
   ClaimOntologyOutboxInput,
@@ -15,15 +19,24 @@ import type {
   SummarizeOntologyOutboxInput,
 } from "@sixb/core/storage"
 import { ONTOLOGY_OUTBOX_FAILURE_CODES } from "@sixb/core/storage"
+import { type SqliteExecutionRow, sqliteExecutionRecord } from "../execution-storage"
 import {
   assertNonblank,
   assertPositiveInteger,
   assertTimestamp,
-  outboxRecord,
+  claimedOutboxDraft,
+  parseJson,
   type SqliteOntologyOutboxRow,
   type SqliteRootOperation,
 } from "./shared"
 
+/** A commit's own fields, prefixed, beside every column of the execution it ran under. */
+interface SqliteOutboxContextRow extends SqliteExecutionRow {
+  readonly commit_id: string
+  readonly commit_committed_at: string
+  readonly commit_origin: string
+}
+
 export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
   constructor(
     private readonly db: Database,
@@ -74,8 +87,8 @@ export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
             UPDATE ontology_outbox
             SET attempts = attempts + 1, lease_id = ?, lease_expires_at = ?
             WHERE rowid IN (SELECT CAST(value AS INTEGER) FROM json_each(?))
-            RETURNING envelope, available_at, attempts, lease_id, lease_expires_at,
-              published_at, last_failure, created_at
+            RETURNING id, commit_id, commit_ordinal, event, available_at, attempts, lease_id,
+              lease_expires_at, published_at, last_failure, created_at
           `
         )
         .all(
@@ -83,22 +96,31 @@ export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
           input.leaseExpiresAt,
           JSON.stringify(selected.map((row) => row.rowId))
         ) as SqliteOntologyOutboxRow[]
-      return rows
-        .map((row) => outboxRecord(row))
+      // Each commit and its execution are read once, however many of its events the batch holds.
+      const contexts = this.db
+        .query(
+          `SELECT commits.id AS commit_id, commits.committed_at AS commit_committed_at,
+             commits.origin AS commit_origin, executions.*
+           FROM json_each(?) AS requested
+           CROSS JOIN ontology_commits AS commits
+             ON commits.project_id = ? AND commits.id = requested.value
+           CROSS JOIN executions
+             ON executions.project_id = commits.project_id AND executions.id = commits.execution_id`
+        )
+        .all(
+          JSON.stringify([...new Set(rows.map((row) => row.commit_id))]),
+          input.projectId
+        ) as SqliteOutboxContextRow[]
+      const contextOf = new Map(
+        contexts.map((row) => [row.commit_id, outboxEventContext(input.projectId, row)])
+      )
+      const drafts = rows
+        .map(claimedOutboxDraft)
         .sort(
           (left, right) =>
-            left.createdAt.localeCompare(right.createdAt) ||
-            left.envelope.id.localeCompare(right.envelope.id)
+            left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id)
         )
-        .map((record) => {
-          if (record.leaseId === null || record.leaseExpiresAt === null) {
-            throw new MaterializationConflictError(
-              "outbox-lease",
-              "Ontology outbox claim returned an unpaired lease."
-            )
-          }
-          return { ...record, leaseId: record.leaseId, leaseExpiresAt: record.leaseExpiresAt }
-        })
+      return claimedOutboxRows(drafts, (commitId) => contextOf.get(commitId))
     })
   }
 
@@ -235,6 +257,18 @@ export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
   }
 }
 
+function outboxEventContext(projectId: string, row: SqliteOutboxContextRow): OutboxEventContext {
+  return {
+    commit: {
+      projectId,
+      id: row.commit_id,
+      committedAt: row.commit_committed_at,
+      origin: parseJson<OutboxEventContext["commit"]["origin"]>(row.commit_origin),
+    },
+    execution: sqliteExecutionRecord(row),
+  }
+}
+
 function validateLeaseIds(ids: readonly string[]): string[] {
   const seen = new Set<string>()
   for (const id of ids) {