@sixb/core 0.1.0 → 0.1.2

package/src/storage/workflow-runs/types.ts

@@ -1,10 +1,10 @@
 import type { AgentMessagePart } from "../../agents/message"
-import type { Principal } from "../../auth"
+import type { SixbErrorCode, SixbFailure } from "../../errors/types"
 import type { JsonValue } from "../../json"
-import type { WorkflowIOSnapshot, WorkflowRunSource } from "../../workflows/types"
-import type { AgentExecutionStatus, AgentRunFinishReason, AgentRunUsage } from "../agents"
+import type { WorkflowIOSnapshot } from "../../workflows/types"
+import type { AgentExecutionStatus, AgentRunFailureCode, AgentRunFinishReason } from "../agents"
 
-export type { WorkflowIOSnapshot, WorkflowRunSource } from "../../workflows/types"
+export type { WorkflowIOSnapshot } from "../../workflows/types"
 
 export type WorkflowRunStatus =
   | "queued"
@@ -16,6 +16,15 @@ export type WorkflowRunStatus =
 export type WorkflowNodeRunStatus = Exclude<WorkflowRunStatus, "queued">
 export type WorkflowNodeRunType = "step" | "action" | "intervention" | "agent"
 
+/** Error codes a workflow or workflow-node run can persist and expose. */
+export const WORKFLOW_RUN_FAILURE_CODES = [
+  "internal.unexpected",
+  "runtime.cancelled",
+  "workflow.node_failed",
+] as const satisfies readonly [SixbErrorCode, ...SixbErrorCode[]]
+
+export type WorkflowRunFailureCode = (typeof WORKFLOW_RUN_FAILURE_CODES)[number]
+
 export interface WorkflowRunExecution {
   readonly token: string
   readonly queueLeaseExpiresAt: Date
@@ -32,16 +41,15 @@ export interface WorkflowAgentNodeRunExecution {
 export interface WorkflowAgentNodeRunRecord {
   readonly projectId: string
   readonly nodeRunId: string
+  readonly executionId: string
   readonly agentId: string
   readonly status: WorkflowAgentNodeRunStatus
   readonly prompt: string
-  readonly executionPrincipal?: Extract<Principal, { readonly type: "serviceAccount" }>
   readonly modelId?: string
   readonly finishReason?: AgentRunFinishReason
-  readonly usage?: AgentRunUsage
   readonly trace?: readonly AgentMessagePart[]
   readonly diagnostics?: readonly JsonValue[]
-  readonly error?: string
+  readonly error?: SixbFailure<AgentRunFailureCode>
   readonly attempt: number
   readonly execution?: WorkflowAgentNodeRunExecution
   readonly createdAt: Date
@@ -52,6 +60,7 @@ export interface WorkflowAgentNodeRunRecord {
 export interface CreateWorkflowAgentNodeRunInput {
   readonly projectId: string
   readonly nodeRunId: string
+  readonly executionId: string
   readonly agentId: string
   readonly prompt: string
   readonly createdAt?: Date
@@ -60,7 +69,6 @@ export interface CreateWorkflowAgentNodeRunInput {
 export interface StartWorkflowAgentNodeRunInput {
   readonly projectId: string
   readonly nodeRunId: string
-  readonly executionPrincipal?: Extract<Principal, { readonly type: "serviceAccount" }>
   readonly modelId?: string
   readonly execution: WorkflowAgentNodeRunExecution
   readonly startedAt?: Date
@@ -79,24 +87,30 @@ export interface ConfirmWorkflowAgentNodeRunExecutionOwnershipInput {
   readonly queueLeaseExpiresAt: Date
 }
 
-export type FinishWorkflowAgentNodeRunInput = {
+interface FinishWorkflowAgentNodeRunBaseInput {
   readonly projectId: string
   readonly nodeRunId: string
   readonly executionToken: string
-  readonly status: "succeeded" | "failed" | "cancelled"
   readonly modelId?: string
   readonly finishReason?: AgentRunFinishReason
-  readonly usage?: AgentRunUsage
   readonly trace?: readonly AgentMessagePart[]
   readonly diagnostics?: readonly JsonValue[]
-  readonly error?: string
   readonly completedAt?: Date
 }
 
+export type FinishWorkflowAgentNodeRunInput =
+  | (FinishWorkflowAgentNodeRunBaseInput & {
+      readonly status: "succeeded"
+    })
+  | (FinishWorkflowAgentNodeRunBaseInput & {
+      readonly status: "failed" | "cancelled"
+      readonly error?: SixbFailure<AgentRunFailureCode>
+    })
+
 export interface CancelWorkflowAgentNodeRunInput {
   readonly projectId: string
   readonly nodeRunId: string
-  readonly error?: string
+  readonly error?: SixbFailure<AgentRunFailureCode>
   readonly completedAt?: Date
 }
 
@@ -118,15 +132,17 @@ export interface ListWorkflowAgentNodeRunsResult {
 export interface WorkflowRunRecord {
   readonly id: string
   readonly projectId: string
+  readonly executionId: string
   readonly workflowId: string
   readonly status: WorkflowRunStatus
   readonly input: WorkflowIOSnapshot
+  readonly output?: WorkflowIOSnapshot
   readonly queuedAt?: Date
   readonly startedAt: Date
   readonly finishedAt?: Date
-  readonly error?: string
-  readonly source?: WorkflowRunSource
-  readonly requestedByPrincipal: Principal
+  readonly error?: SixbFailure<WorkflowRunFailureCode>
+  /** Durable group memberships snapshotted when the workflow run was admitted. */
+  readonly requesterGroupIds: readonly string[]
   readonly attempt: number
   readonly execution?: WorkflowRunExecution
 }
@@ -145,28 +161,24 @@ export interface WorkflowNodeRunRecord {
   readonly startedAt: Date
   readonly finishedAt?: Date
   readonly output?: WorkflowIOSnapshot
-  readonly error?: string
+  readonly error?: SixbFailure<WorkflowRunFailureCode>
 }
 
 export interface StartWorkflowRunInput {
   readonly id: string
   readonly projectId: string
-  readonly workflowId: string
-  readonly input: WorkflowIOSnapshot
   readonly startedAt?: Date
-  readonly source?: WorkflowRunSource
-  readonly requestedByPrincipal?: Principal
   readonly execution?: WorkflowRunExecution
 }
 
 export interface QueueWorkflowRunInput {
   readonly id: string
   readonly projectId: string
+  readonly executionId: string
   readonly workflowId: string
   readonly input: WorkflowIOSnapshot
   readonly queuedAt?: Date
-  readonly source?: WorkflowRunSource
-  readonly requestedByPrincipal?: Principal
+  readonly requesterGroupIds: readonly string[]
 }
 
 export interface ReclaimWorkflowRunInput {
@@ -201,6 +213,7 @@ export type FinishWorkflowRunInput =
       readonly id: string
       readonly projectId: string
       readonly status: "succeeded"
+      readonly output: WorkflowIOSnapshot
       readonly finishedAt?: Date
       readonly executionToken?: string
     }
@@ -209,7 +222,7 @@ export type FinishWorkflowRunInput =
       readonly projectId: string
       readonly status: "failed" | "cancelled"
       readonly finishedAt?: Date
-      readonly error?: string
+      readonly error?: SixbFailure<WorkflowRunFailureCode>
       readonly executionToken?: string
     }
 
@@ -248,7 +261,7 @@ export type FinishWorkflowNodeRunInput =
       readonly projectId: string
       readonly status: "failed" | "cancelled"
       readonly finishedAt?: Date
-      readonly error?: string
+      readonly error?: SixbFailure<WorkflowRunFailureCode>
       readonly executionToken?: string
     }
 

package/src/syncs/builders.ts

@@ -1,4 +1,4 @@
-import type { ConnectorDefinition } from "../connectors"
+import type { AnyConnectorAdapter, ConnectorDefinition } from "../connectors"
 import type { DatasetDefinition } from "../datasets"
 import type { ScheduleDefinition, ScheduleReference } from "../schedules"
 import { isScheduleDefinition } from "../schedules"
@@ -8,6 +8,7 @@ import type {
   BatchSyncDefinitionConfig,
   SyncBuilder,
   SyncDefinition,
+  SyncMode,
   SyncReadBuilder,
   SyncTargetBuilder,
 } from "./types"
@@ -18,16 +19,19 @@ function assertNonEmpty(value: string, field: string): void {
   }
 }
 
-function assertDataset(dataset: DatasetDefinition): void {
+function assertDataset(dataset: DatasetDefinition, mode: SyncMode): void {
   assertNonEmpty(dataset.id, "dataset id")
+  if (mode === "merge" && dataset.primaryKey === undefined) {
+    throw new SyncValidationError(`Merge sync dataset '${dataset.id}' must define a primaryKey.`)
+  }
 }
 
 function normalizeBatchSyncConfig(options: BatchSyncConfig | undefined): BatchSyncDefinitionConfig {
   const mode = options?.mode ?? "snapshot"
 
-  if (mode !== "snapshot" && mode !== "append") {
+  if (mode !== "snapshot" && mode !== "append" && mode !== "merge") {
     throw new SyncValidationError(
-      `Invalid sync mode '${String(mode)}'. Expected 'snapshot' or 'append'.`
+      `Invalid sync mode '${String(mode)}'. Expected 'snapshot', 'append', or 'merge'.`
     )
   }
 
@@ -37,43 +41,57 @@ function normalizeBatchSyncConfig(options: BatchSyncConfig | undefined): BatchSy
   }
 }
 
+type DefaultBatchSyncConfig = Omit<BatchSyncConfig, "mode"> & { readonly mode?: undefined }
+
 /**
  * Define a batch sync that reads from one connector and writes into one dataset.
  *
  * The returned definition is inert and safe to export from `syncs/` modules.
- * V1 supports `snapshot` and `append` modes with optional triggers and checkpoints.
+ * Supports `snapshot`, `append`, and keyed `merge` modes with optional triggers and checkpoints.
  */
+export function defineSync<TId extends string>(
+  id: TId,
+  options?: DefaultBatchSyncConfig
+): SyncBuilder<TId, never, "snapshot">
+export function defineSync<TId extends string, const TMode extends SyncMode>(
+  id: TId,
+  options: BatchSyncConfig<TMode> & { readonly mode: TMode }
+): SyncBuilder<TId, never, TMode>
+export function defineSync<TId extends string>(
+  id: TId,
+  options: BatchSyncConfig
+): SyncBuilder<TId, never, SyncMode>
 export function defineSync<TId extends string>(
   id: TId,
   options?: BatchSyncConfig
-): SyncBuilder<TId> {
+): SyncBuilder<TId, never, SyncMode> {
   assertNonEmpty(id, "id")
 
   const config = normalizeBatchSyncConfig(options)
   const triggers: ScheduleReference[] = []
 
-  function createBuilder<TCheckpoint>(): SyncBuilder<TId, TCheckpoint> {
-    const builder: SyncBuilder<TId, TCheckpoint> = {
-      when(schedule: ScheduleDefinition): SyncBuilder<TId, TCheckpoint> {
+  function createBuilder<TCheckpoint>(): SyncBuilder<TId, TCheckpoint, SyncMode> {
+    const builder: SyncBuilder<TId, TCheckpoint, SyncMode> = {
+      when(schedule: ScheduleDefinition): SyncBuilder<TId, TCheckpoint, SyncMode> {
         if (!isScheduleDefinition(schedule)) {
           throw new SyncValidationError("Sync .when(...) only accepts schedules.")
         }
         triggers.push({ type: "schedule", scheduleId: schedule.id })
         return builder
       },
-      checkpoint<TNextCheckpoint>(): SyncBuilder<TId, TNextCheckpoint> {
+      checkpoint<TNextCheckpoint>(): SyncBuilder<TId, TNextCheckpoint, SyncMode> {
         return createBuilder<TNextCheckpoint>()
       },
-      from<TConnector extends ConnectorDefinition>(
+      from<TConnector extends ConnectorDefinition<string, AnyConnectorAdapter>>(
         connector: TConnector
-      ): SyncReadBuilder<TId, TConnector, TCheckpoint> {
+      ): SyncReadBuilder<TId, TConnector, TCheckpoint, SyncMode> {
         return {
-          read(handler): SyncTargetBuilder<TId, TConnector, TCheckpoint> {
+          read(handler): SyncTargetBuilder<TId, TConnector, TCheckpoint, SyncMode> {
             return {
               intoDataset(
                 dataset: DatasetDefinition
-              ): SyncDefinition<TId, TConnector, TCheckpoint> {
-                assertDataset(dataset)
+              ): SyncDefinition<TId, TConnector, TCheckpoint, SyncMode> {
+                assertDataset(dataset, config.mode)
 
                 return {
                   kind: "sync",

package/src/syncs/execution.ts

@@ -0,0 +1,78 @@
+import { isAllowed } from "../authorization"
+import type { ExecutionContext } from "../execution"
+import type { SixbRuntimeContext } from "../runtime/types"
+import type {
+  ListLatestSyncRunsResult,
+  ListSyncRunsInput,
+  ListSyncRunsResult,
+  SyncRunRecord,
+} from "../storage/sync-runs"
+import { SyncValidationError } from "./errors"
+import { type RequestSyncRunInput, requestSyncRun, type SyncRunRequestResult } from "./request"
+import type { SyncDefinition } from "./types"
+
+export interface SyncRunsRuntime {
+  getById(runId: string): Promise<SyncRunRecord | null>
+  list(input?: Omit<ListSyncRunsInput, "projectId" | "syncIds">): Promise<ListSyncRunsResult>
+  listLatest(syncIds: readonly string[]): Promise<ListLatestSyncRunsResult>
+}
+
+export interface SyncsRuntime {
+  list(): readonly SyncDefinition[]
+  getById(syncId: string): SyncDefinition | null
+  request(input: RequestSyncRunInput): Promise<SyncRunRequestResult>
+  readonly runs: SyncRunsRuntime
+}
+
+export function createSyncsRuntime(
+  runtime: SixbRuntimeContext,
+  execution: ExecutionContext,
+  source: Pick<SyncsRuntime, "list" | "getById">
+): SyncsRuntime {
+  const allowed = (syncId: string) => isAllowed(runtime.authorization, { kind: "sync.run", syncId })
+  const visibleIds = () =>
+    source
+      .list()
+      .filter((sync) => allowed(sync.id))
+      .map((sync) => sync.id)
+
+  return {
+    list: () => source.list().filter((sync) => allowed(sync.id)),
+    getById: (syncId) => {
+      const sync = source.getById(syncId)
+      return sync && allowed(syncId) ? sync : null
+    },
+    request: async (input) => {
+      const sync = source.getById(input.syncId)
+      if (!sync) throw new SyncValidationError(`[Sixb] Unknown sync '${input.syncId}'`)
+      return requestSyncRun(runtime, execution, sync, input)
+    },
+    runs: {
+      getById: async (runId) => {
+        const run =
+          (await runtime.storage.syncRuns?.getById({
+            projectId: runtime.projectId,
+            id: runId,
+          })) ?? null
+        return run && allowed(run.syncId) ? run : null
+      },
+      list: (input = {}) => {
+        const storage = runtime.storage.syncRuns
+        if (!storage) return Promise.resolve({ runs: [], hasMore: false, total: 0 })
+        return storage.list({
+          projectId: runtime.projectId,
+          ...input,
+          syncIds: runtime.authorization ? visibleIds() : undefined,
+        })
+      },
+      listLatest: (syncIds) => {
+        const storage = runtime.storage.syncRuns
+        if (!storage || syncIds.length === 0) return Promise.resolve({ runs: [] })
+        const allowedIds = syncIds.filter(allowed)
+        return allowedIds.length === 0
+          ? Promise.resolve({ runs: [] })
+          : storage.listLatestBySyncIds({ projectId: runtime.projectId, syncIds: allowedIds })
+      },
+    },
+  }
+}

package/src/syncs/index.ts

@@ -6,13 +6,27 @@ export type {
   SyncRunRequestResult,
 } from "./request"
 export { requestSyncRun } from "./request"
+export {
+  type AutomaticSyncExecutionSource,
+  type AutomaticSyncRunDispatchInput,
+  SyncRunDispatcher,
+  type SyncRunDispatcherDependencies,
+  type SyncRunDispatchPort,
+} from "./run-dispatch"
+export {
+  resolveSyncConnectorSources,
+  type SyncConnectorSource,
+  type SyncConnectorSourceResolver,
+} from "./sources"
 export type {
   BatchSyncConfig,
   BatchSyncDefinitionConfig,
   DatasetSyncTarget,
   SyncBlobContext,
   SyncBuilder,
+  SyncConnectorConnection,
   SyncDefinition,
+  SyncMode,
   SyncReadBuilder,
   SyncReadContext,
   SyncReadHandler,

package/src/syncs/request.ts

@@ -1,15 +1,17 @@
-import { randomUUID } from "node:crypto"
 import { assertAuthorized } from "../authorization"
+import {
+  createPrimitiveExecutionRecord,
+  ensureExecutionRecord,
+  executionRecordInputFromRuntime,
+} from "../execution/durable"
+import type { ExecutionContext } from "../execution/types"
 import type { SixbRuntimeContext } from "../runtime/types"
 import { SyncValidationError } from "./errors"
+import { dispatchSyncRun } from "./run-dispatch"
 import type { SyncDefinition } from "./types"
 
 export interface SyncRunRequestOptions {
-  /**
-   * Identity of the run to create. Deduplicated when a worker claims the job, not here: two requests
-   * carrying the same id enqueue two jobs, and the second `start()` loses to the unique run id and
-   * no-ops. So one run happens, but this call cannot tell you whether it was yours.
-   */
+  /** Stable identity used to make dispatch idempotent. */
   readonly runId?: string
   /** Optimistic concurrency guard: fail the run if the dataset moved past this version. */
   readonly expectedLatestVersionId?: string
@@ -23,9 +25,10 @@ export interface RequestSyncRunInput extends SyncRunRequestOptions {
 export interface SyncRunRequestResult {
   readonly syncId: string
   readonly runId: string
-  /** When the job was enqueued. Not persisted — the run row records `startedAt` instead. */
+  /** Durable time at which the run entered the queue. */
   readonly queuedAt: string
   readonly jobId?: string
+  readonly created: boolean
 }
 
 /**
@@ -34,18 +37,11 @@ export interface SyncRunRequestResult {
  * Operates on an already-resolved definition and the shared runtime context, so the HTTP route and
  * the runtime verb cannot drift apart.
  *
- * A sync run row is created by the worker that claims the job, not here — `SyncRunStatus` has no
- * `queued` member, unlike `WorkflowRunStatus`. So this reports "the request is accepted", where
- * `requestWorkflowRun` reports "the run exists" and can therefore return `created`. Aligning the two
- * means giving sync and pipeline runs a queued state across every storage provider, which is a
- * lifecycle change rather than a field; `created` can be added here afterwards without breaking
- * anyone, since nothing implements this result type.
- *
- * The storage precondition below is checked even though nothing here writes: accepting a request the
- * worker will refuse is worse than failing now.
+ * The immutable child execution and queued run are persisted before queue publication.
  */
 export async function requestSyncRun(
   runtime: SixbRuntimeContext,
+  execution: ExecutionContext,
   sync: SyncDefinition,
   options: SyncRunRequestOptions = {}
 ): Promise<SyncRunRequestResult> {
@@ -59,33 +55,26 @@ export async function requestSyncRun(
     throw new SyncValidationError("[Sixb] Sync run queue is not configured.")
   }
 
-  const runId = createSyncRunId(options.runId)
-  const queuedAt = new Date().toISOString()
-  const [job] = await queue.enqueue({
+  return dispatchSyncRun({
+    errorReporterHost: runtime,
     projectId: runtime.projectId,
-    jobs: [
-      {
-        type: "sync.run.requested",
-        payload: {
-          syncId: sync.id,
-          runId,
-          expectedLatestVersionId: options.expectedLatestVersionId,
-          commitMessage: options.commitMessage,
-        },
-      },
-    ],
+    sync,
+    storage: runtime.storage,
+    queue,
+    ...options,
+    createExecution: async (executionId, runId) => {
+      const caller = await ensureExecutionRecord(
+        runtime.storage.executions,
+        executionRecordInputFromRuntime({
+          execution,
+          runtimeAuthorization: runtime.runtimeAuthorization,
+        })
+      )
+      return createPrimitiveExecutionRecord({
+        id: executionId,
+        primitive: { kind: "sync", id: sync.id, runId },
+        origin: { type: "execution", parent: caller },
+      })
+    },
   })
-
-  return { syncId: sync.id, runId, queuedAt, jobId: job?.id }
-}
-
-function createSyncRunId(runId: string | undefined): string {
-  if (runId !== undefined) {
-    if (!runId.trim()) {
-      throw new SyncValidationError("[Sixb] Sync run id must not be empty")
-    }
-    return runId
-  }
-
-  return `run_${randomUUID()}`
 }