@sixb/core 0.1.0 → 0.1.2

package/src/storage/sync-runs/in-memory.ts

@@ -1,4 +1,7 @@
+import { parseSixbFailure } from "../../errors/internal"
+import type { SixbFailure } from "../../errors/types"
 import { cloneJsonValue, type JsonValue } from "../../json"
+import type { ExecutionStorage } from "../executions"
 import {
   compareStartedAt,
   hasEmptyStatuses,
@@ -8,17 +11,21 @@ import {
   toStatusSet,
 } from "../run-listing"
 import { SyncRunError } from "./errors"
+import { canRequeueSyncRunAfterEnqueueFailure } from "./idempotency"
+import { assertSyncRunExecution } from "./provider"
 import type {
   FinishSyncRunInput,
   ListLatestSyncRunsInput,
   ListLatestSyncRunsResult,
   ListSyncRunsInput,
   ListSyncRunsResult,
+  QueueSyncRunInput,
   StartSyncRunInput,
-  SyncRunFailure,
+  SyncRunFailureCode,
   SyncRunRecord,
   SyncRunStorage,
 } from "./types"
+import { SYNC_RUN_FAILURE_CODES } from "./types"
 
 function syncRunKey(projectId: string, id: string): string {
   return `${projectId}:${id}`
@@ -32,13 +39,17 @@ function normalizeCheckpoint(checkpoint: JsonValue | undefined): JsonValue | und
   return checkpoint !== undefined ? cloneJsonValue(checkpoint) : undefined
 }
 
-function normalizeError(error: SyncRunFailure | undefined): SyncRunFailure | undefined {
-  return error ? structuredClone(error) : undefined
+function normalizeError(
+  error: SixbFailure<SyncRunFailureCode> | undefined
+): SixbFailure<SyncRunFailureCode> | undefined {
+  return error ? parseSixbFailure(error, SYNC_RUN_FAILURE_CODES) : undefined
 }
 
 export class InMemorySyncRunStorage implements SyncRunStorage {
   private readonly rows = new Map<string, SyncRunRecord>()
 
+  constructor(private readonly executions: ExecutionStorage) {}
+
   snapshot(): InMemorySyncRunStorageSnapshot {
     return structuredClone(this.rows)
   }
@@ -50,22 +61,60 @@ export class InMemorySyncRunStorage implements SyncRunStorage {
     }
   }
 
-  async start(input: StartSyncRunInput): Promise<SyncRunRecord> {
+  async queue(input: QueueSyncRunInput): Promise<SyncRunRecord> {
     const key = syncRunKey(input.projectId, input.id)
-    if (this.rows.has(key)) {
+    const existing = this.rows.get(key)
+    if (existing && !canRequeueSyncRunAfterEnqueueFailure(existing, input)) {
       throw new SyncRunError(
         `[Sixb] Sync run '${input.id}' already exists for project '${input.projectId}'.`
       )
     }
+    if (
+      [...this.rows.values()].some(
+        (run) =>
+          run.projectId === input.projectId &&
+          run.id !== input.id &&
+          run.executionId === input.executionId
+      )
+    ) {
+      throw new SyncRunError(
+        `[Sixb] Execution '${input.executionId}' already belongs to another Sync run.`
+      )
+    }
+    await assertSyncRunExecution({
+      executions: this.executions,
+      projectId: input.projectId,
+      executionId: input.executionId,
+      runId: input.id,
+      syncId: input.syncId,
+    })
+
+    const queuedAt = new Date(input.queuedAt ?? new Date())
+    if (existing) {
+      const next: SyncRunRecord = {
+        ...existing,
+        status: "queued",
+        queuedAt,
+        startedAt: undefined,
+        finishedAt: undefined,
+        rowsRead: undefined,
+        output: undefined,
+        error: undefined,
+        checkpoint: undefined,
+      }
+      this.rows.set(key, structuredClone(next))
+      return cloneSyncRunRecord(next)
+    }
 
     const record: SyncRunRecord = {
       id: input.id,
       projectId: input.projectId,
+      executionId: input.executionId,
       syncId: input.syncId,
       datasetId: input.datasetId,
       mode: input.mode,
-      status: "running",
-      startedAt: new Date(input.startedAt ?? new Date()),
+      status: "queued",
+      queuedAt,
       expectedLatestVersionId: input.expectedLatestVersionId,
       commitMessage: input.commitMessage,
     }
@@ -74,6 +123,30 @@ export class InMemorySyncRunStorage implements SyncRunStorage {
     return cloneSyncRunRecord(record)
   }
 
+  async start(input: StartSyncRunInput): Promise<SyncRunRecord> {
+    const key = syncRunKey(input.projectId, input.id)
+    const existing = this.rows.get(key)
+    if (!existing) {
+      throw new SyncRunError(
+        `[Sixb] Sync run '${input.id}' not found for project '${input.projectId}'.`
+      )
+    }
+    if (existing.status !== "queued") {
+      throw new SyncRunError(
+        `[Sixb] Sync run '${input.id}' cannot start from status '${existing.status}'.`
+      )
+    }
+
+    const next: SyncRunRecord = {
+      ...existing,
+      status: "running",
+      startedAt: new Date(input.startedAt ?? new Date()),
+      error: undefined,
+    }
+    this.rows.set(key, structuredClone(next))
+    return cloneSyncRunRecord(next)
+  }
+
   async finish(input: FinishSyncRunInput): Promise<SyncRunRecord> {
     const key = syncRunKey(input.projectId, input.id)
     const existing = this.rows.get(key)
@@ -84,15 +157,25 @@ export class InMemorySyncRunStorage implements SyncRunStorage {
       )
     }
 
+    const enqueueFailure = input.status === "failed" && input.error.code === "queue.enqueue_failed"
+    if (
+      (enqueueFailure && existing.status !== "queued") ||
+      (!enqueueFailure && existing.status !== "running")
+    ) {
+      throw new SyncRunError(
+        `[Sixb] Sync run '${input.id}' cannot finish from status '${existing.status}'.`
+      )
+    }
+
     if (input.status === "succeeded") {
       if (input.output && input.output.datasetId !== existing.datasetId) {
         throw new SyncRunError(
           `[Sixb] Sync run '${input.id}' output dataset '${input.output.datasetId}' does not match '${existing.datasetId}'.`
         )
       }
-      if (!input.output && (existing.mode !== "append" || input.rowsRead !== 0)) {
+      if (!input.output && input.rowsRead !== 0 && existing.mode !== "merge") {
         throw new SyncRunError(
-          `[Sixb] Sync run '${input.id}' may omit its output only for an empty append.`
+          `[Sixb] Sync run '${input.id}' may omit its output with rows read only for an initial merge no-op.`
         )
       }
     }

package/src/storage/sync-runs/index.ts

@@ -1,4 +1,5 @@
 export { SyncRunError } from "./errors"
+export { canRequeueSyncRunAfterEnqueueFailure } from "./idempotency"
 export { InMemorySyncRunStorage } from "./in-memory"
 export type {
   FinishSyncRunInput,
@@ -6,9 +7,12 @@ export type {
   ListLatestSyncRunsResult,
   ListSyncRunsInput,
   ListSyncRunsResult,
+  QueueSyncRunInput,
   StartSyncRunInput,
-  SyncRunFailure,
+  SyncRunFailureCode,
+  SyncRunMode,
   SyncRunRecord,
   SyncRunStatus,
   SyncRunStorage,
 } from "./types"
+export { SYNC_RUN_FAILURE_CODES } from "./types"

package/src/storage/sync-runs/provider.ts

@@ -0,0 +1,25 @@
+import type { ExecutionStorage } from "../executions"
+import { findPrimitiveRunExecution } from "../executions/run-link"
+import { SyncRunError } from "./errors"
+
+/** Validate the semantic link between a durable Sync run and its immutable execution. */
+export async function assertSyncRunExecution(input: {
+  readonly executions: ExecutionStorage
+  readonly projectId: string
+  readonly executionId: string
+  readonly runId: string
+  readonly syncId: string
+}): Promise<void> {
+  const execution = await findPrimitiveRunExecution({
+    executions: input.executions,
+    projectId: input.projectId,
+    executionId: input.executionId,
+    primitive: { kind: "sync", id: input.syncId, runId: input.runId },
+    sourceTypes: ["execution", "schedule", "event"],
+  })
+  if (!execution) {
+    throw new SyncRunError(
+      `[Sixb] Execution '${input.executionId}' does not authorize Sync run '${input.runId}'.`
+    )
+  }
+}

package/src/storage/sync-runs/types.ts

@@ -1,59 +1,87 @@
+import type { SixbErrorCode, SixbFailure } from "../../errors/types"
 import type { JsonValue } from "../../json"
 import type { DatasetVersionRef, DatasetWriteMode } from "../../lake-storage"
 
-export type SyncRunStatus = "running" | "succeeded" | "failed" | "cancelled"
+export type SyncRunMode = DatasetWriteMode | "merge"
 
-export interface SyncRunFailure {
-  readonly name?: string // Example: "AbortError"
-  readonly message: string // Example: "Sync cancelled by request"
-}
+export type SyncRunStatus = "queued" | "running" | "succeeded" | "failed" | "cancelled"
+/** Error codes a sync run can persist and expose through its public contract. */
+export const SYNC_RUN_FAILURE_CODES = [
+  "internal.unexpected",
+  "runtime.cancelled",
+  "queue.enqueue_failed",
+  "sync.execution_failed",
+] as const satisfies readonly [SixbErrorCode, ...SixbErrorCode[]]
+
+export type SyncRunFailureCode = (typeof SYNC_RUN_FAILURE_CODES)[number]
 
 export interface SyncRunRecord {
   readonly id: string
   readonly projectId: string
+  readonly executionId: string
   readonly syncId: string
   readonly datasetId: string
-  readonly mode: DatasetWriteMode
+  readonly mode: SyncRunMode
   readonly status: SyncRunStatus
-  readonly startedAt: Date
+  readonly queuedAt: Date
+  readonly startedAt?: Date
   readonly finishedAt?: Date
-  readonly rowsRead?: number // Rows produced by this run, not the dataset's full visible row count.
+  /** Source items successfully consumed. Merge runs count both upserts and deletes. */
+  readonly rowsRead?: number
   readonly output?: DatasetVersionRef
   readonly expectedLatestVersionId?: string
   readonly commitMessage?: string
-  readonly error?: SyncRunFailure
+  readonly error?: SixbFailure<SyncRunFailureCode>
   readonly checkpoint?: JsonValue
 }
 
-export interface StartSyncRunInput {
+export interface QueueSyncRunInput {
   readonly id: string
   readonly projectId: string
+  readonly executionId: string
   readonly syncId: string
   readonly datasetId: string
-  readonly mode: DatasetWriteMode
-  readonly startedAt?: Date
+  readonly mode: SyncRunMode
+  readonly queuedAt?: Date
   readonly expectedLatestVersionId?: string
   readonly commitMessage?: string
 }
 
+export interface StartSyncRunInput {
+  readonly id: string
+  readonly projectId: string
+  readonly startedAt?: Date
+}
+
 export type FinishSyncRunInput =
   | {
       readonly id: string
       readonly projectId: string
       readonly status: "succeeded"
       readonly finishedAt?: Date
+      /** Source items successfully consumed. Merge runs count both upserts and deletes. */
       readonly rowsRead: number
-      /** Absent only when the first append run succeeds without producing any rows. */
+      /** Absent when a successful run has no dataset version to reference. */
       readonly output?: DatasetVersionRef
       readonly checkpoint?: JsonValue
     }
   | {
       readonly id: string
       readonly projectId: string
-      readonly status: "failed" | "cancelled"
+      readonly status: "failed"
+      readonly finishedAt?: Date
+      /** Source items successfully consumed before the failure or cancellation. */
+      readonly rowsRead?: number
+      readonly error: SixbFailure<SyncRunFailureCode>
+    }
+  | {
+      readonly id: string
+      readonly projectId: string
+      readonly status: "cancelled"
       readonly finishedAt?: Date
+      /** Source items successfully consumed before cancellation. */
       readonly rowsRead?: number
-      readonly error?: SyncRunFailure
+      readonly error?: SixbFailure<SyncRunFailureCode>
     }
 
 export interface ListSyncRunsInput {
@@ -85,6 +113,7 @@ export interface ListLatestSyncRunsResult {
 }
 
 export interface SyncRunStorage {
+  queue(input: QueueSyncRunInput): Promise<SyncRunRecord>
   start(input: StartSyncRunInput): Promise<SyncRunRecord>
   finish(input: FinishSyncRunInput): Promise<SyncRunRecord>
   getById(params: { projectId: string; id: string }): Promise<SyncRunRecord | null>

package/src/storage/types.ts

@@ -1,6 +1,9 @@
 import type { ActionRunStorage } from "./action-runs"
 import type { AgentStorage } from "./agents"
+import type { AiUsageStorage } from "./ai-usage"
 import type { AuthStorage } from "./auth"
+import type { ConnectorConnectionStorage } from "./connector-connections"
+import type { ExecutionStorage } from "./executions"
 import type { FileUploadSessionStore } from "./file-upload-sessions"
 import type { ObjectStorage } from "./objects/types"
 import type { OntologyStorage } from "./ontology"
@@ -9,7 +12,6 @@ import type { ProjectionRunStorage } from "./projection-runs"
 import type { RulesStorage } from "./rules"
 import type { SyncRunStorage } from "./sync-runs"
 import type { TimeseriesStorage } from "./timeseries/types"
-import type { WebhookDeliveryStorage } from "./webhook-deliveries"
 import type { WebhookRunStorage } from "./webhook-runs"
 import type { WorkflowInterventionStorage } from "./workflow-interventions"
 import type { WorkflowRunStorage } from "./workflow-runs"
@@ -21,6 +23,8 @@ export type {
 } from "../materialization/model"
 export type {
   ActionRunFailure,
+  ActionRunFailureCode,
+  ActionRunFailureDetails,
   ActionRunParams,
   ActionRunPhase,
   ActionRunRecord,
@@ -33,10 +37,11 @@ export type {
   QueueActionRunInput,
   StartActionRunInput,
 } from "./action-runs"
-export { ActionRunError } from "./action-runs"
+export { ACTION_RUN_FAILURE_CODES, ACTION_RUN_PHASES, ActionRunError } from "./action-runs"
 export type {
   AgentMessageRecord,
   AgentRunExecution,
+  AgentRunFailureCode,
   AgentRunRecord,
   AgentStorage,
   AgentThreadRecord,
@@ -55,7 +60,27 @@ export type {
   ReclaimAgentRunInput,
   StartAgentRunInput,
 } from "./agents"
-export { AgentStorageError } from "./agents"
+export { AGENT_RUN_FAILURE_CODES, AgentStorageError } from "./agents"
+export type {
+  AiModelCallUsage,
+  AiModelCallUsageInput,
+  AiModelCallUsageRecord,
+  AiUsageExecutionSummary,
+  AiUsageReportingStatus,
+  AiUsageStorage,
+  AiUsageStorageErrorCode,
+  RecordAiModelCallInput,
+  RecordAiModelCallResult,
+  SummarizeAiUsageExecutionInput,
+  SummarizeAiUsageExecutionsInput,
+} from "./ai-usage"
+export {
+  AiUsageStorageError,
+  aggregateAiModelCallUsage,
+  assertAiUsageExecutionId,
+  normalizeAiModelCallRecord,
+  normalizeAiModelCallUsage,
+} from "./ai-usage"
 export type {
   AuthGroupMembershipStore,
   AuthInvitationStore,
@@ -128,11 +153,13 @@ export type {
   ListPipelineRunsResult,
   ListPipelineStepRunsInput,
   ListPipelineStepRunsResult,
-  PipelineRunFailure,
+  PipelineRunFailureCode,
   PipelineRunRecord,
   PipelineRunStatus,
   PipelineRunStorage,
   PipelineStepRunRecord,
+  PipelineStepRunStatus,
+  QueuePipelineRunInput,
   StartPipelineRunInput,
   StartPipelineStepRunInput,
 } from "./pipeline-runs"
@@ -149,6 +176,7 @@ export type {
   ObjectProjectionTarget,
   ProjectionKind,
   ProjectionRunClaim,
+  ProjectionRunFailureCode,
   ProjectionRunProgress,
   ProjectionRunRecord,
   ProjectionRunStatus,
@@ -159,7 +187,7 @@ export type {
   TelemetryProjectionRunRecord,
   UpdateProjectionRunInput,
 } from "./projection-runs"
-export { ProjectionRunError } from "./projection-runs"
+export { PROJECTION_RUN_FAILURE_CODES, ProjectionRunError } from "./projection-runs"
 export type {
   ListActiveRuleStatesInput,
   ListActiveRuleStatesResult,
@@ -170,8 +198,10 @@ export type {
   FinishSyncRunInput,
   ListSyncRunsInput,
   ListSyncRunsResult,
+  QueueSyncRunInput,
   StartSyncRunInput,
-  SyncRunFailure,
+  SyncRunFailureCode,
+  SyncRunMode,
   SyncRunRecord,
   SyncRunStatus,
   SyncRunStorage,
@@ -184,25 +214,20 @@ export type {
   TimeseriesPoint,
   TimeseriesStorage,
 } from "./timeseries/types"
-export type {
-  WebhookDeliveryClaimRecord,
-  WebhookDeliveryClaimResult,
-  WebhookDeliveryKey,
-  WebhookDeliveryRecord,
-  WebhookDeliveryStatus,
-  WebhookDeliveryStorage,
-} from "./webhook-deliveries"
 export type {
   FinishWebhookRunInput,
   FinishWebhookRunStatus,
+  GetWebhookRunByDeliveryInput,
   ListWebhookRunsInput,
   ListWebhookRunsResult,
+  RestartWebhookRunInput,
   StartWebhookRunInput,
+  WebhookRunFailureCode,
   WebhookRunRecord,
   WebhookRunStatus,
   WebhookRunStorage,
 } from "./webhook-runs"
-export { WebhookRunError } from "./webhook-runs"
+export { WEBHOOK_RUN_FAILURE_CODES, WebhookRunError } from "./webhook-runs"
 export type {
   CancelWorkflowInterventionInput,
   CreateWorkflowInterventionInput,
@@ -249,11 +274,12 @@ export type {
   WorkflowNodeRunStorage,
   WorkflowNodeRunType,
   WorkflowRunExecution,
+  WorkflowRunFailureCode,
   WorkflowRunRecord,
   WorkflowRunStatus,
   WorkflowRunStorage,
 } from "./workflow-runs"
-export { WorkflowRunError } from "./workflow-runs"
+export { WORKFLOW_RUN_FAILURE_CODES, WorkflowRunError } from "./workflow-runs"
 
 export interface StorageTransactionOptions {
   readonly isolation?: "default" | "serializable"
@@ -263,18 +289,20 @@ export interface Storage {
   objects: ObjectStorage
   timeseries: TimeseriesStorage
   ontology: OntologyStorage
+  executions: ExecutionStorage
   auth?: AuthStorage
   agents?: AgentStorage
+  aiUsage?: AiUsageStorage
   actionRuns?: ActionRunStorage
   syncRuns?: SyncRunStorage
   pipelineRuns?: PipelineRunStorage
   projectionRuns?: ProjectionRunStorage
   workflowRuns?: WorkflowRunStorage
   workflowInterventions?: WorkflowInterventionStorage
-  webhookDeliveries?: WebhookDeliveryStorage
   webhookRuns?: WebhookRunStorage
   rules?: RulesStorage
   fileUploadSessions?: FileUploadSessionStore
+  connectorConnections?: ConnectorConnectionStorage
 
   /** Lightweight reachability probe. It must not open a write transaction or run migrations. */
   ping(): Promise<void>

package/src/storage/webhook-runs/errors.ts

@@ -1,6 +1,20 @@
 /**
  * Base error for webhook-run storage operations.
  */
+export type WebhookRunErrorCode =
+  | "delivery_conflict"
+  | "duplicate_run"
+  | "invalid_execution"
+  | "invalid_transition"
+  | "not_found"
+
 export class WebhookRunError extends Error {
   readonly name = "WebhookRunError"
+
+  constructor(
+    message: string,
+    readonly code?: WebhookRunErrorCode
+  ) {
+    super(message)
+  }
 }