@sixb/core 0.0.1

package/src/storage/workflow-runs/index.ts

@@ -0,0 +1,45 @@
+export { WorkflowRunError } from "./errors"
+export {
+  InMemoryWorkflowAgentNodeRunStorage,
+  InMemoryWorkflowNodeRunStorage,
+  InMemoryWorkflowRunStorage,
+} from "./in-memory"
+export type {
+  CancelWorkflowAgentNodeRunInput,
+  ConfirmWorkflowAgentNodeRunExecutionOwnershipInput,
+  ConfirmWorkflowRunExecutionOwnershipInput,
+  CreateWorkflowAgentNodeRunInput,
+  FinishWorkflowAgentNodeRunInput,
+  FinishWorkflowNodeRunInput,
+  FinishWorkflowRunInput,
+  ListLatestWorkflowRunsInput,
+  ListLatestWorkflowRunsResult,
+  ListWorkflowAgentNodeRunsInput,
+  ListWorkflowAgentNodeRunsResult,
+  ListWorkflowNodeRunsInput,
+  ListWorkflowNodeRunsResult,
+  ListWorkflowRunsInput,
+  ListWorkflowRunsResult,
+  QueueWorkflowRunInput,
+  ReclaimWorkflowAgentNodeRunInput,
+  ReclaimWorkflowRunInput,
+  ResumeWorkflowRunInput,
+  StartWorkflowAgentNodeRunInput,
+  StartWorkflowNodeRunInput,
+  StartWorkflowRunInput,
+  WaitWorkflowNodeRunInput,
+  WaitWorkflowRunInput,
+  WorkflowAgentNodeRunExecution,
+  WorkflowAgentNodeRunRecord,
+  WorkflowAgentNodeRunStatus,
+  WorkflowAgentNodeRunStorage,
+  WorkflowIOSnapshot,
+  WorkflowNodeRunRecord,
+  WorkflowNodeRunStatus,
+  WorkflowNodeRunStorage,
+  WorkflowNodeRunType,
+  WorkflowRunExecution,
+  WorkflowRunRecord,
+  WorkflowRunStatus,
+  WorkflowRunStorage,
+} from "./types"

package/src/storage/workflow-runs/types.ts

@@ -0,0 +1,344 @@
+import type { AgentMessagePart } from "../../agents/message"
+import type { Principal } from "../../auth"
+import type { JsonValue } from "../../json"
+import type { WorkflowIOSnapshot, WorkflowRunSource } from "../../workflows/types"
+import type { AgentExecutionStatus, AgentRunFinishReason, AgentRunUsage } from "../agents"
+
+export type { WorkflowIOSnapshot, WorkflowRunSource } from "../../workflows/types"
+
+export type WorkflowRunStatus =
+  | "queued"
+  | "running"
+  | "waiting"
+  | "succeeded"
+  | "failed"
+  | "cancelled"
+export type WorkflowNodeRunStatus = Exclude<WorkflowRunStatus, "queued">
+export type WorkflowNodeRunType = "step" | "action" | "intervention" | "agent"
+
+export interface WorkflowRunExecution {
+  readonly token: string
+  readonly queueLeaseExpiresAt: Date
+}
+
+export type WorkflowAgentNodeRunStatus = AgentExecutionStatus
+
+export interface WorkflowAgentNodeRunExecution {
+  readonly token: string
+  readonly queueLeaseExpiresAt: Date
+}
+
+/** Execution details for an agent workflow node. The generic node run owns business IO. */
+export interface WorkflowAgentNodeRunRecord {
+  readonly projectId: string
+  readonly nodeRunId: string
+  readonly agentId: string
+  readonly status: WorkflowAgentNodeRunStatus
+  readonly prompt: string
+  readonly executionPrincipal?: Extract<Principal, { readonly type: "serviceAccount" }>
+  readonly modelId?: string
+  readonly finishReason?: AgentRunFinishReason
+  readonly usage?: AgentRunUsage
+  readonly trace?: readonly AgentMessagePart[]
+  readonly diagnostics?: readonly JsonValue[]
+  readonly error?: string
+  readonly attempt: number
+  readonly execution?: WorkflowAgentNodeRunExecution
+  readonly createdAt: Date
+  readonly startedAt?: Date
+  readonly completedAt?: Date
+}
+
+export interface CreateWorkflowAgentNodeRunInput {
+  readonly projectId: string
+  readonly nodeRunId: string
+  readonly agentId: string
+  readonly prompt: string
+  readonly createdAt?: Date
+}
+
+export interface StartWorkflowAgentNodeRunInput {
+  readonly projectId: string
+  readonly nodeRunId: string
+  readonly executionPrincipal?: Extract<Principal, { readonly type: "serviceAccount" }>
+  readonly modelId?: string
+  readonly execution: WorkflowAgentNodeRunExecution
+  readonly startedAt?: Date
+}
+
+export interface ReclaimWorkflowAgentNodeRunInput {
+  readonly projectId: string
+  readonly nodeRunId: string
+  readonly execution: WorkflowAgentNodeRunExecution
+}
+
+export interface ConfirmWorkflowAgentNodeRunExecutionOwnershipInput {
+  readonly projectId: string
+  readonly nodeRunId: string
+  readonly executionToken: string
+  readonly queueLeaseExpiresAt: Date
+}
+
+export type FinishWorkflowAgentNodeRunInput = {
+  readonly projectId: string
+  readonly nodeRunId: string
+  readonly executionToken: string
+  readonly status: "succeeded" | "failed" | "cancelled"
+  readonly modelId?: string
+  readonly finishReason?: AgentRunFinishReason
+  readonly usage?: AgentRunUsage
+  readonly trace?: readonly AgentMessagePart[]
+  readonly diagnostics?: readonly JsonValue[]
+  readonly error?: string
+  readonly completedAt?: Date
+}
+
+export interface CancelWorkflowAgentNodeRunInput {
+  readonly projectId: string
+  readonly nodeRunId: string
+  readonly error?: string
+  readonly completedAt?: Date
+}
+
+export interface ListWorkflowAgentNodeRunsInput {
+  readonly projectId: string
+  readonly agentId?: string
+  readonly statuses?: readonly WorkflowAgentNodeRunStatus[]
+  readonly limit?: number
+  readonly offset?: number
+  readonly order?: "asc" | "desc"
+}
+
+export interface ListWorkflowAgentNodeRunsResult {
+  readonly runs: readonly WorkflowAgentNodeRunRecord[]
+  readonly hasMore: boolean
+  readonly total: number
+}
+
+export interface WorkflowRunRecord {
+  readonly id: string
+  readonly projectId: string
+  readonly workflowId: string
+  readonly status: WorkflowRunStatus
+  readonly input: WorkflowIOSnapshot
+  readonly queuedAt?: Date
+  readonly startedAt: Date
+  readonly finishedAt?: Date
+  readonly error?: string
+  readonly source?: WorkflowRunSource
+  readonly requestedByPrincipal: Principal
+  readonly attempt: number
+  readonly execution?: WorkflowRunExecution
+}
+
+export interface WorkflowNodeRunRecord {
+  readonly id: string
+  readonly projectId: string
+  readonly workflowRunId: string
+  readonly workflowId: string
+  readonly nodeIndex: number
+  readonly nodeType: WorkflowNodeRunType
+  readonly nodeId: string
+  readonly nodeKey: string
+  readonly status: WorkflowNodeRunStatus
+  readonly input: WorkflowIOSnapshot
+  readonly startedAt: Date
+  readonly finishedAt?: Date
+  readonly output?: WorkflowIOSnapshot
+  readonly error?: string
+}
+
+export interface StartWorkflowRunInput {
+  readonly id: string
+  readonly projectId: string
+  readonly workflowId: string
+  readonly input: WorkflowIOSnapshot
+  readonly startedAt?: Date
+  readonly source?: WorkflowRunSource
+  readonly requestedByPrincipal?: Principal
+  readonly execution?: WorkflowRunExecution
+}
+
+export interface QueueWorkflowRunInput {
+  readonly id: string
+  readonly projectId: string
+  readonly workflowId: string
+  readonly input: WorkflowIOSnapshot
+  readonly queuedAt?: Date
+  readonly source?: WorkflowRunSource
+  readonly requestedByPrincipal?: Principal
+}
+
+export interface ReclaimWorkflowRunInput {
+  readonly id: string
+  readonly projectId: string
+  readonly execution: WorkflowRunExecution
+}
+
+export interface ConfirmWorkflowRunExecutionOwnershipInput {
+  readonly id: string
+  readonly projectId: string
+  readonly executionToken: string
+  readonly queueLeaseExpiresAt: Date
+}
+
+export interface WaitWorkflowRunInput {
+  readonly id: string
+  readonly projectId: string
+  readonly waitingAt?: Date
+  readonly executionToken?: string
+}
+
+export interface ResumeWorkflowRunInput {
+  readonly id: string
+  readonly projectId: string
+  readonly resumedAt?: Date
+  readonly execution?: WorkflowRunExecution
+}
+
+export type FinishWorkflowRunInput =
+  | {
+      readonly id: string
+      readonly projectId: string
+      readonly status: "succeeded"
+      readonly finishedAt?: Date
+      readonly executionToken?: string
+    }
+  | {
+      readonly id: string
+      readonly projectId: string
+      readonly status: "failed" | "cancelled"
+      readonly finishedAt?: Date
+      readonly error?: string
+      readonly executionToken?: string
+    }
+
+export interface StartWorkflowNodeRunInput {
+  readonly id: string
+  readonly projectId: string
+  readonly workflowRunId: string
+  readonly workflowId: string
+  readonly nodeIndex: number
+  readonly nodeType: WorkflowNodeRunType
+  readonly nodeId: string
+  readonly nodeKey: string
+  readonly input: WorkflowIOSnapshot
+  readonly startedAt?: Date
+  readonly executionToken?: string
+}
+
+export interface WaitWorkflowNodeRunInput {
+  readonly id: string
+  readonly projectId: string
+  readonly waitingAt?: Date
+  readonly executionToken?: string
+}
+
+export type FinishWorkflowNodeRunInput =
+  | {
+      readonly id: string
+      readonly projectId: string
+      readonly status: "succeeded"
+      readonly finishedAt?: Date
+      readonly output?: WorkflowIOSnapshot
+      readonly executionToken?: string
+    }
+  | {
+      readonly id: string
+      readonly projectId: string
+      readonly status: "failed" | "cancelled"
+      readonly finishedAt?: Date
+      readonly error?: string
+      readonly executionToken?: string
+    }
+
+export interface ListWorkflowRunsInput {
+  readonly projectId: string
+  readonly workflowId?: string
+  readonly workflowIds?: readonly string[]
+  readonly statuses?: readonly WorkflowRunStatus[]
+  readonly startedAfter?: Date
+  readonly startedBefore?: Date
+  readonly limit?: number
+  readonly offset?: number
+  readonly order?: "asc" | "desc"
+}
+
+export interface ListWorkflowRunsResult {
+  readonly runs: readonly WorkflowRunRecord[]
+  readonly hasMore: boolean
+  readonly total: number
+}
+
+export interface ListLatestWorkflowRunsInput {
+  readonly projectId: string
+  readonly workflowIds: readonly string[]
+}
+
+export interface ListLatestWorkflowRunsResult {
+  readonly runs: readonly WorkflowRunRecord[]
+}
+
+export interface ListWorkflowNodeRunsInput {
+  readonly projectId: string
+  readonly workflowRunId?: string
+  readonly workflowId?: string
+  readonly nodeType?: WorkflowNodeRunType
+  readonly nodeId?: string
+  readonly nodeKey?: string
+  readonly statuses?: readonly WorkflowNodeRunStatus[]
+  readonly startedAfter?: Date
+  readonly startedBefore?: Date
+  readonly limit?: number
+  readonly offset?: number
+  readonly order?: "asc" | "desc"
+}
+
+export interface ListWorkflowNodeRunsResult {
+  readonly nodes: readonly WorkflowNodeRunRecord[]
+  readonly hasMore: boolean
+  readonly total: number
+}
+
+export interface WorkflowRunStorage {
+  readonly nodes: WorkflowNodeRunStorage
+  readonly agentNodes: WorkflowAgentNodeRunStorage
+
+  queue(input: QueueWorkflowRunInput): Promise<WorkflowRunRecord>
+  /** Atomically claims a queued run by transitioning it to running; competing claims must fail. */
+  start(input: StartWorkflowRunInput): Promise<WorkflowRunRecord>
+  reclaim(input: ReclaimWorkflowRunInput): Promise<WorkflowRunRecord>
+  confirmExecutionOwnership(
+    input: ConfirmWorkflowRunExecutionOwnershipInput
+  ): Promise<WorkflowRunRecord>
+  wait(input: WaitWorkflowRunInput): Promise<WorkflowRunRecord>
+  resume(input: ResumeWorkflowRunInput): Promise<WorkflowRunRecord>
+  finish(input: FinishWorkflowRunInput): Promise<WorkflowRunRecord>
+  getById(params: { projectId: string; id: string }): Promise<WorkflowRunRecord | null>
+  list(input: ListWorkflowRunsInput): Promise<ListWorkflowRunsResult>
+  listLatestByWorkflowIds(input: ListLatestWorkflowRunsInput): Promise<ListLatestWorkflowRunsResult>
+}
+
+export interface WorkflowAgentNodeRunStorage {
+  create(input: CreateWorkflowAgentNodeRunInput): Promise<WorkflowAgentNodeRunRecord>
+  start(input: StartWorkflowAgentNodeRunInput): Promise<WorkflowAgentNodeRunRecord>
+  reclaim(input: ReclaimWorkflowAgentNodeRunInput): Promise<WorkflowAgentNodeRunRecord>
+  confirmExecutionOwnership(
+    input: ConfirmWorkflowAgentNodeRunExecutionOwnershipInput
+  ): Promise<WorkflowAgentNodeRunRecord>
+  cancel(input: CancelWorkflowAgentNodeRunInput): Promise<WorkflowAgentNodeRunRecord>
+  finish(input: FinishWorkflowAgentNodeRunInput): Promise<WorkflowAgentNodeRunRecord>
+  getByNodeRunId(params: {
+    projectId: string
+    nodeRunId: string
+  }): Promise<WorkflowAgentNodeRunRecord | null>
+  list(input: ListWorkflowAgentNodeRunsInput): Promise<ListWorkflowAgentNodeRunsResult>
+}
+
+export interface WorkflowNodeRunStorage {
+  start(input: StartWorkflowNodeRunInput): Promise<WorkflowNodeRunRecord>
+  wait(input: WaitWorkflowNodeRunInput): Promise<WorkflowNodeRunRecord>
+  finish(input: FinishWorkflowNodeRunInput): Promise<WorkflowNodeRunRecord>
+  getById(params: { projectId: string; id: string }): Promise<WorkflowNodeRunRecord | null>
+  list(input: ListWorkflowNodeRunsInput): Promise<ListWorkflowNodeRunsResult>
+}

package/src/syncs/builders.ts

@@ -0,0 +1,101 @@
+import type { ConnectorDefinition } from "../connectors"
+import type { DatasetDefinition } from "../datasets"
+import type { ScheduleDefinition, ScheduleReference } from "../schedules"
+import { isScheduleDefinition } from "../schedules"
+import { SyncValidationError } from "./errors"
+import type {
+  BatchSyncConfig,
+  BatchSyncDefinitionConfig,
+  SyncBuilder,
+  SyncDefinition,
+  SyncReadBuilder,
+  SyncTargetBuilder,
+} from "./types"
+
+function assertNonEmpty(value: string, field: string): void {
+  if (!value.trim()) {
+    throw new SyncValidationError(`Sync ${field} must not be empty.`)
+  }
+}
+
+function assertDataset(dataset: DatasetDefinition): void {
+  assertNonEmpty(dataset.id, "dataset id")
+}
+
+function normalizeBatchSyncConfig(options: BatchSyncConfig | undefined): BatchSyncDefinitionConfig {
+  const mode = options?.mode ?? "snapshot"
+
+  if (mode !== "snapshot" && mode !== "append") {
+    throw new SyncValidationError(
+      `Invalid sync mode '${String(mode)}'. Expected 'snapshot' or 'append'.`
+    )
+  }
+
+  return {
+    kind: "batch",
+    mode,
+  }
+}
+
+/**
+ * Define a batch sync that reads from one connector and writes into one dataset.
+ *
+ * The returned definition is inert and safe to export from `syncs/` modules.
+ * V1 supports `snapshot` and `append` modes with optional triggers and checkpoints.
+ */
+export function defineSync<TId extends string>(
+  id: TId,
+  options?: BatchSyncConfig
+): SyncBuilder<TId> {
+  assertNonEmpty(id, "id")
+
+  const config = normalizeBatchSyncConfig(options)
+  const triggers: ScheduleReference[] = []
+
+  function createBuilder<TCheckpoint>(): SyncBuilder<TId, TCheckpoint> {
+    const builder: SyncBuilder<TId, TCheckpoint> = {
+      when(schedule: ScheduleDefinition): SyncBuilder<TId, TCheckpoint> {
+        if (!isScheduleDefinition(schedule)) {
+          throw new SyncValidationError("Sync .when(...) only accepts schedules.")
+        }
+        triggers.push({ type: "schedule", scheduleId: schedule.id })
+        return builder
+      },
+      checkpoint<TNextCheckpoint>(): SyncBuilder<TId, TNextCheckpoint> {
+        return createBuilder<TNextCheckpoint>()
+      },
+      from<TConnector extends ConnectorDefinition>(
+        connector: TConnector
+      ): SyncReadBuilder<TId, TConnector, TCheckpoint> {
+        return {
+          read(handler): SyncTargetBuilder<TId, TConnector, TCheckpoint> {
+            return {
+              intoDataset(
+                dataset: DatasetDefinition
+              ): SyncDefinition<TId, TConnector, TCheckpoint> {
+                assertDataset(dataset)
+
+                return {
+                  kind: "sync",
+                  id,
+                  config,
+                  triggers,
+                  connector,
+                  read: handler,
+                  target: {
+                    kind: "dataset",
+                    dataset,
+                  },
+                }
+              },
+            }
+          },
+        }
+      },
+    }
+
+    return builder
+  }
+
+  return createBuilder<never>()
+}

package/src/syncs/errors.ts

@@ -0,0 +1,3 @@
+export class SyncValidationError extends Error {
+  readonly name = "SyncValidationError"
+}

package/src/syncs/index.ts

@@ -0,0 +1,22 @@
+export { defineSync } from "./builders"
+export { SyncValidationError } from "./errors"
+export type {
+  RequestSyncRunInput,
+  SyncRunRequestOptions,
+  SyncRunRequestResult,
+} from "./request"
+export { requestSyncRun } from "./request"
+export type {
+  BatchSyncConfig,
+  BatchSyncDefinitionConfig,
+  DatasetSyncTarget,
+  SyncBlobContext,
+  SyncBuilder,
+  SyncDefinition,
+  SyncReadBuilder,
+  SyncReadContext,
+  SyncReadHandler,
+  SyncReadResult,
+  SyncTargetBuilder,
+} from "./types"
+export { isSyncDefinition } from "./types"

package/src/syncs/request.ts

@@ -0,0 +1,91 @@
+import { randomUUID } from "node:crypto"
+import { assertAuthorized } from "../authorization"
+import type { SixbRuntimeContext } from "../runtime/types"
+import { SyncValidationError } from "./errors"
+import type { SyncDefinition } from "./types"
+
+export interface SyncRunRequestOptions {
+  /**
+   * Identity of the run to create. Deduplicated when a worker claims the job, not here: two requests
+   * carrying the same id enqueue two jobs, and the second `start()` loses to the unique run id and
+   * no-ops. So one run happens, but this call cannot tell you whether it was yours.
+   */
+  readonly runId?: string
+  /** Optimistic concurrency guard: fail the run if the dataset moved past this version. */
+  readonly expectedLatestVersionId?: string
+  readonly commitMessage?: string
+}
+
+export interface RequestSyncRunInput extends SyncRunRequestOptions {
+  readonly syncId: string
+}
+
+export interface SyncRunRequestResult {
+  readonly syncId: string
+  readonly runId: string
+  /** When the job was enqueued. Not persisted — the run row records `startedAt` instead. */
+  readonly queuedAt: string
+  readonly jobId?: string
+}
+
+/**
+ * Authorize and queue a sync run request.
+ *
+ * Operates on an already-resolved definition and the shared runtime context, so the HTTP route and
+ * the runtime verb cannot drift apart.
+ *
+ * A sync run row is created by the worker that claims the job, not here — `SyncRunStatus` has no
+ * `queued` member, unlike `WorkflowRunStatus`. So this reports "the request is accepted", where
+ * `requestWorkflowRun` reports "the run exists" and can therefore return `created`. Aligning the two
+ * means giving sync and pipeline runs a queued state across every storage provider, which is a
+ * lifecycle change rather than a field; `created` can be added here afterwards without breaking
+ * anyone, since nothing implements this result type.
+ *
+ * The storage precondition below is checked even though nothing here writes: accepting a request the
+ * worker will refuse is worse than failing now.
+ */
+export async function requestSyncRun(
+  runtime: SixbRuntimeContext,
+  sync: SyncDefinition,
+  options: SyncRunRequestOptions = {}
+): Promise<SyncRunRequestResult> {
+  assertAuthorized(runtime, { kind: "sync.run", syncId: sync.id })
+  if (!runtime.storage.syncRuns) {
+    throw new SyncValidationError("[Sixb] Sync run storage is not configured.")
+  }
+
+  const queue = runtime.queues.syncRuns
+  if (!queue) {
+    throw new SyncValidationError("[Sixb] Sync run queue is not configured.")
+  }
+
+  const runId = createSyncRunId(options.runId)
+  const queuedAt = new Date().toISOString()
+  const [job] = await queue.enqueue({
+    projectId: runtime.projectId,
+    jobs: [
+      {
+        type: "sync.run.requested",
+        payload: {
+          syncId: sync.id,
+          runId,
+          expectedLatestVersionId: options.expectedLatestVersionId,
+          commitMessage: options.commitMessage,
+        },
+      },
+    ],
+  })
+
+  return { syncId: sync.id, runId, queuedAt, jobId: job?.id }
+}
+
+function createSyncRunId(runId: string | undefined): string {
+  if (runId !== undefined) {
+    if (!runId.trim()) {
+      throw new SyncValidationError("[Sixb] Sync run id must not be empty")
+    }
+    return runId
+  }
+
+  return `run_${randomUUID()}`
+}