@sixb/sqlite 0.1.0 → 0.1.2

package/src/ontology-storage/shared.ts

@@ -1,4 +1,5 @@
 import type { Database } from "bun:sqlite"
+import { parseSixbFailure } from "@sixb/core/internal/errors"
 import type { ProjectionEntityRef } from "@sixb/core/internal/materialization"
 import { MaterializationConflictError } from "@sixb/core/internal/materialization"
 import type {
@@ -9,6 +10,7 @@ import type {
   OntologySourceRecord,
   StageSourceAssertion,
 } from "@sixb/core/storage"
+import { ONTOLOGY_OUTBOX_FAILURE_CODES } from "@sixb/core/storage"
 
 export {
   assertNonblank,
@@ -30,6 +32,7 @@ export interface SqliteOntologyCommitRow {
   readonly id: string
   readonly idempotency_key: string
   readonly request_hash: string
+  readonly execution_id: string
   readonly origin_kind: string
   readonly origin_run_id: string | null
   readonly origin_batch_ordinal: number | null
@@ -83,8 +86,7 @@ export interface SqliteOntologySourceAssertionRow {
   readonly assertion: string
 }
 
-export interface SqliteOntologyOverrideRow {
-  readonly entity_kind: "object" | "link"
+export interface SqliteStoredOverrideRow {
   readonly value: string
   readonly last_commit_id: string
   readonly updated_at: string
@@ -97,7 +99,7 @@ export interface SqliteOntologyOutboxRow {
   readonly lease_id: string | null
   readonly lease_expires_at: string | null
   readonly published_at: string | null
-  readonly last_error: string | null
+  readonly last_failure: string | null
   readonly created_at: string
 }
 
@@ -148,6 +150,7 @@ export function commitRecord(row: SqliteOntologyCommitRow): OntologyCommitRecord
     id: row.id,
     idempotencyKey: row.idempotency_key,
     requestHash: row.request_hash,
+    executionId: row.execution_id,
     origin: parseJson<OntologyCommitWrite["origin"]>(row.origin),
     ...(row.actor === null
       ? {}
@@ -170,7 +173,10 @@ export function outboxRecord(row: SqliteOntologyOutboxRow): OntologyOutboxRecord
     leaseId: row.lease_id,
     leaseExpiresAt: row.lease_expires_at,
     publishedAt: row.published_at,
-    lastError: row.last_error,
+    lastFailure:
+      row.last_failure === null
+        ? null
+        : parseSixbFailure(row.last_failure, ONTOLOGY_OUTBOX_FAILURE_CODES),
     createdAt: row.created_at,
   }
 }

package/src/ontology-storage/sources.ts

@@ -33,6 +33,7 @@ import type {
   SummarizeTerminalSourceMaterializationsInput,
   TerminalSourceMaterializationSummary,
 } from "@sixb/core/storage"
+import { yieldSqliteEventLoop } from "../transactions"
 import {
   assertNonblank,
   assertNonnegativeInteger,
@@ -125,7 +126,7 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
   }
 
   async stageRows(input: StageSourceRowsInput): Promise<StageSourceRowsResult> {
-    return this.runRootOperation(() => {
+    const result = await this.runRootOperation(() => {
       assertWriteIdentity(input)
       this.assertExecution(input)
       const manifest = this.requireManifest(
@@ -146,6 +147,8 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
       this.insertStageRows(input, pending)
       return { inserted: pending.length, unchanged }
     })
+    await yieldSqliteEventLoop()
+    return result
   }
 
   async markReady(input: MarkSourceMaterializationReadyInput): Promise<OntologySourceRecord> {

package/src/pipeline-run-storage.ts

@@ -1,6 +1,9 @@
 import type { Database } from "bun:sqlite"
+import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
+import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider"
 import type { DatasetVersionRef } from "@sixb/core/lake-storage"
 import type {
+  ExecutionStorage,
   FinishPipelineRunInput,
   FinishPipelineStepRunInput,
   ListLatestPipelineRunsInput,
@@ -9,14 +12,18 @@ import type {
   ListPipelineRunsResult,
   ListPipelineStepRunsInput,
   ListPipelineStepRunsResult,
-  PipelineRunFailure,
   PipelineRunRecord,
   PipelineRunStorage,
   PipelineStepRunRecord,
+  QueuePipelineRunInput,
   StartPipelineRunInput,
   StartPipelineStepRunInput,
 } from "@sixb/core/storage"
-import { PipelineRunError } from "@sixb/core/storage"
+import {
+  canRequeuePipelineRunAfterEnqueueFailure,
+  PIPELINE_RUN_FAILURE_CODES,
+  PipelineRunError,
+} from "@sixb/core/storage"
 import { queryLatestRunsByOwnerId } from "./latest-run-query"
 import { installFreshSqliteSchema } from "./migrations"
 import {
@@ -33,6 +40,8 @@ import {
 } from "./transactions"
 
 export interface SqlitePipelineRunStorageOptions {
+  /** Execution lookup sharing the same provider transaction. */
+  executions: ExecutionStorage
   /** Path to SQLite database file. Defaults to ':memory:' for in-memory database. */
   path?: string
   /** Internal shared connection used by bundled SqliteStorage. */
@@ -42,18 +51,27 @@ export interface SqlitePipelineRunStorageOptions {
 export class SqlitePipelineRunStorage implements PipelineRunStorage {
   private readonly connection: SqliteStoreConnection
   private readonly db: Database
+  private readonly executions: ExecutionStorage
 
-  constructor(options: SqlitePipelineRunStorageOptions = {}) {
+  constructor(options: SqlitePipelineRunStorageOptions) {
     this.connection = openSqliteStoreConnection(options)
     this.db = this.connection.db
+    this.executions = options.executions
 
     if (this.connection.installFreshSchema) {
       installFreshSqliteSchema(this.db)
     }
   }
 
-  async start(input: StartPipelineRunInput): Promise<PipelineRunRecord> {
-    const startedAt = input.startedAt ?? new Date()
+  async queue(input: QueuePipelineRunInput): Promise<PipelineRunRecord> {
+    const queuedAt = input.queuedAt ?? new Date()
+    await assertPipelineRunExecution({
+      executions: this.executions,
+      projectId: input.projectId,
+      executionId: input.executionId,
+      runId: input.id,
+      pipelineId: input.pipelineId,
+    })
 
     try {
       this.db
@@ -62,18 +80,25 @@ export class SqlitePipelineRunStorage implements PipelineRunStorage {
           INSERT INTO pipeline_runs (
             project_id,
             id,
+            execution_id,
             pipeline_id,
             status,
+            queued_at,
             started_at
-          ) VALUES (?, ?, ?, ?, ?)
+          ) VALUES (?, ?, ?, ?, ?, ?, NULL)
         `
         )
-        .run(input.projectId, input.id, input.pipelineId, "running", startedAt.toISOString())
+        .run(
+          input.projectId,
+          input.id,
+          input.executionId,
+          input.pipelineId,
+          "queued",
+          queuedAt.toISOString()
+        )
     } catch (error) {
       if (isUniqueConstraintError(error)) {
-        throw new PipelineRunError(
-          `[SixbSqlite] Pipeline run '${input.id}' already exists for project '${input.projectId}'.`
-        )
+        return this.requeueAfterEnqueueFailure(input, queuedAt)
       }
 
       throw error
@@ -89,6 +114,73 @@ export class SqlitePipelineRunStorage implements PipelineRunStorage {
     return record
   }
 
+  private async requeueAfterEnqueueFailure(
+    input: QueuePipelineRunInput,
+    queuedAt: Date
+  ): Promise<PipelineRunRecord> {
+    return this.db.transaction(() => {
+      const existing = this.db
+        .query("SELECT * FROM pipeline_runs WHERE project_id = ? AND id = ?")
+        .get(input.projectId, input.id) as PipelineRunDatabaseRow | null
+      if (
+        !existing ||
+        !canRequeuePipelineRunAfterEnqueueFailure(rowToPipelineRunRecord(existing), input)
+      ) {
+        throw new PipelineRunError(
+          `[SixbSqlite] Pipeline run '${input.id}' already exists for project '${input.projectId}'.`
+        )
+      }
+
+      this.db
+        .query(
+          `
+          UPDATE pipeline_runs
+          SET status = ?, queued_at = ?, started_at = NULL, finished_at = NULL,
+              output_dataset_id = NULL, output_version_id = NULL, error = NULL
+          WHERE project_id = ? AND id = ?
+        `
+        )
+        .run("queued", queuedAt.toISOString(), input.projectId, input.id)
+
+      const updated = this.db
+        .query("SELECT * FROM pipeline_runs WHERE project_id = ? AND id = ?")
+        .get(input.projectId, input.id) as PipelineRunDatabaseRow
+      return rowToPipelineRunRecord(updated)
+    })()
+  }
+
+  async start(input: StartPipelineRunInput): Promise<PipelineRunRecord> {
+    const result = this.db
+      .query(
+        `
+        UPDATE pipeline_runs
+        SET status = ?, started_at = ?, error = NULL
+        WHERE project_id = ? AND id = ? AND status = ?
+      `
+      )
+      .run(
+        "running",
+        (input.startedAt ?? new Date()).toISOString(),
+        input.projectId,
+        input.id,
+        "queued"
+      )
+    if (result.changes > 0) {
+      const record = await this.getById({ projectId: input.projectId, id: input.id })
+      if (record) return record
+    }
+
+    const existing = await this.getById({ projectId: input.projectId, id: input.id })
+    if (!existing) {
+      throw new PipelineRunError(
+        `[SixbSqlite] Pipeline run '${input.id}' not found for project '${input.projectId}'.`
+      )
+    }
+    throw new PipelineRunError(
+      `[SixbSqlite] Pipeline run '${input.id}' cannot start from status '${existing.status}'.`
+    )
+  }
+
   async finish(input: FinishPipelineRunInput): Promise<PipelineRunRecord> {
     return this.db.transaction(() => {
       const existing = this.db
@@ -101,9 +193,14 @@ export class SqlitePipelineRunStorage implements PipelineRunStorage {
         )
       }
 
-      if (existing.status !== "running") {
+      const enqueueFailure =
+        input.status === "failed" && input.error.code === "queue.enqueue_failed"
+      if (
+        (enqueueFailure && existing.status !== "queued") ||
+        (!enqueueFailure && existing.status !== "running")
+      ) {
         throw new PipelineRunError(
-          `[SixbSqlite] Pipeline run '${input.id}' for project '${input.projectId}' is already terminal.`
+          `[SixbSqlite] Pipeline run '${input.id}' cannot finish from status '${existing.status}'.`
         )
       }
 
@@ -116,8 +213,7 @@ export class SqlitePipelineRunStorage implements PipelineRunStorage {
             finished_at = ?,
             output_dataset_id = ?,
             output_version_id = ?,
-            error_name = ?,
-            error_message = ?
+            error = ?
           WHERE project_id = ? AND id = ?
         `
         )
@@ -126,8 +222,9 @@ export class SqlitePipelineRunStorage implements PipelineRunStorage {
           (input.finishedAt ?? new Date()).toISOString(),
           input.status === "succeeded" ? (input.output?.datasetId ?? null) : null,
           input.status === "succeeded" ? (input.output?.versionId ?? null) : null,
-          input.status === "succeeded" ? null : (input.error?.name ?? null),
-          input.status === "succeeded" ? null : (input.error?.message ?? null),
+          input.status === "succeeded" || input.error === undefined
+            ? null
+            : serializeSixbFailure(input.error, PIPELINE_RUN_FAILURE_CODES),
           input.projectId,
           input.id
         )
@@ -255,8 +352,7 @@ export class SqlitePipelineRunStorage implements PipelineRunStorage {
             finished_at = ?,
             output_version_id = ?,
             rows_written = ?,
-            error_name = ?,
-            error_message = ?
+            error = ?
           WHERE project_id = ? AND id = ?
         `
         )
@@ -265,8 +361,9 @@ export class SqlitePipelineRunStorage implements PipelineRunStorage {
           (input.finishedAt ?? new Date()).toISOString(),
           input.status === "succeeded" ? input.output.versionId : null,
           input.rowsWritten ?? existing.rows_written ?? null,
-          input.status === "succeeded" ? null : (input.error?.name ?? null),
-          input.status === "succeeded" ? null : (input.error?.message ?? null),
+          input.status === "succeeded" || input.error === undefined
+            ? null
+            : serializeSixbFailure(input.error, PIPELINE_RUN_FAILURE_CODES),
           input.projectId,
           input.id
         )
@@ -309,7 +406,7 @@ export class SqlitePipelineRunStorage implements PipelineRunStorage {
       args.push(...input.pipelineIds)
     }
 
-    appendRunListFilters(whereClauses, args, input)
+    appendRunListFilters(whereClauses, args, input, "COALESCE(started_at, queued_at)")
 
     const { rows, total, hasMore } = queryRunList<PipelineRunDatabaseRow>({
       db: this.db,
@@ -409,27 +506,15 @@ function parseDatasetVersionRefs(value: string): readonly DatasetVersionRef[] {
   return JSON.parse(value) as readonly DatasetVersionRef[]
 }
 
-function toFailure(row: {
-  readonly error_name: string | null
-  readonly error_message: string | null
-}): PipelineRunFailure | undefined {
-  if (!row.error_message) {
-    return undefined
-  }
-
-  return {
-    name: row.error_name ?? undefined,
-    message: row.error_message,
-  }
-}
-
 function rowToPipelineRunRecord(row: PipelineRunDatabaseRow): PipelineRunRecord {
   return {
     id: row.id,
     projectId: row.project_id,
+    executionId: row.execution_id,
     pipelineId: row.pipeline_id,
     status: row.status,
-    startedAt: new Date(row.started_at),
+    queuedAt: new Date(row.queued_at),
+    startedAt: row.started_at ? new Date(row.started_at) : undefined,
     finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
     output:
       row.output_dataset_id && row.output_version_id
@@ -438,7 +523,7 @@ function rowToPipelineRunRecord(row: PipelineRunDatabaseRow): PipelineRunRecord
             versionId: row.output_version_id,
           }
         : undefined,
-    error: toFailure(row),
+    error: row.error === null ? undefined : parseSixbFailure(row.error, PIPELINE_RUN_FAILURE_CODES),
   }
 }
 
@@ -462,7 +547,7 @@ function rowToPipelineStepRunRecord(row: PipelineStepRunDatabaseRow): PipelineSt
         }
       : undefined,
     rowsWritten: row.rows_written ?? undefined,
-    error: toFailure(row),
+    error: row.error === null ? undefined : parseSixbFailure(row.error, PIPELINE_RUN_FAILURE_CODES),
   }
 }
 
@@ -477,14 +562,15 @@ function assertOptionalNonNegativeInteger(value: number | undefined, fieldName:
 interface PipelineRunDatabaseRow {
   project_id: string
   id: string
+  execution_id: string
   pipeline_id: string
   status: PipelineRunRecord["status"]
-  started_at: string
+  queued_at: string
+  started_at: string | null
   finished_at: string | null
   output_dataset_id: string | null
   output_version_id: string | null
-  error_name: string | null
-  error_message: string | null
+  error: string | null
 }
 
 interface PipelineStepRunDatabaseRow {
@@ -501,6 +587,5 @@ interface PipelineStepRunDatabaseRow {
   inputs: string
   output_version_id: string | null
   rows_written: number | null
-  error_name: string | null
-  error_message: string | null
+  error: string | null
 }