@sixb/core 0.0.1

package/src/queues/in-memory.ts

@@ -0,0 +1,364 @@
+import { randomUUID } from "node:crypto"
+import { QueueError } from "./errors"
+import type {
+  ActionRunRequestedQueueJob,
+  AgentQueueJob,
+  ClaimedQueueJob,
+  NewQueueJob,
+  PipelineRunRequestedQueueJob,
+  ProjectionRunRequestedQueueJob,
+  Queue,
+  QueueJob,
+  QueueJobError,
+  Queues,
+  SyncRunRequestedQueueJob,
+  WorkflowQueueJob,
+} from "./types"
+
+type QueueRecordState = "queued" | "completed" | "failed"
+
+interface QueueRecord<TQueueJob extends QueueJob = QueueJob> {
+  readonly sequence: number
+  readonly queueId: string
+  job: TQueueJob
+  // A leased job stays "queued" until a worker completes or fails it.
+  state: QueueRecordState
+  leaseId: string | null
+  claimedAt: string | null
+  leaseExpiresAt: string | null
+}
+
+function assertNonEmpty(value: string, fieldName: string): void {
+  if (value.trim().length === 0) {
+    throw new QueueError(`[Sixb] Queue ${fieldName} must not be empty`)
+  }
+}
+
+function parseTimestamp(value: string, fieldName: string): number {
+  const timestamp = Date.parse(value)
+
+  if (Number.isNaN(timestamp)) {
+    throw new QueueError(`[Sixb] Queue ${fieldName} must be a valid timestamp`)
+  }
+
+  return timestamp
+}
+
+function assertPositiveNumber(value: number, fieldName: string): void {
+  if (!Number.isFinite(value) || value <= 0) {
+    throw new QueueError(`[Sixb] Queue ${fieldName} must be greater than 0`)
+  }
+}
+
+function clearLease(record: QueueRecord): void {
+  record.leaseId = null
+  record.claimedAt = null
+  record.leaseExpiresAt = null
+}
+
+// Best-effort ordering is part of the contract; this keeps the in-memory provider deterministic.
+function compareQueueRecords(left: QueueRecord, right: QueueRecord): number {
+  const availableDifference =
+    parseTimestamp(left.job.availableAt, "job.availableAt") -
+    parseTimestamp(right.job.availableAt, "job.availableAt")
+
+  if (availableDifference !== 0) {
+    return availableDifference
+  }
+
+  const createdDifference =
+    parseTimestamp(left.job.createdAt, "job.createdAt") -
+    parseTimestamp(right.job.createdAt, "job.createdAt")
+
+  if (createdDifference !== 0) {
+    return createdDifference
+  }
+
+  return left.sequence - right.sequence
+}
+
+function createQueueJob<TQueueJob extends QueueJob>(
+  projectId: string,
+  newJob: NewQueueJob<TQueueJob>
+): TQueueJob {
+  const createdAt = new Date().toISOString()
+  const availableAt = newJob.availableAt ?? createdAt
+  parseTimestamp(availableAt, "availableAt")
+
+  return {
+    id: newJob.id ?? randomUUID(),
+    projectId,
+    createdAt,
+    availableAt,
+    attempt: 0,
+    metadata: newJob.metadata ? structuredClone(newJob.metadata) : undefined,
+    type: newJob.type,
+    payload: structuredClone(newJob.payload),
+  } as TQueueJob
+}
+
+function leaseExpiresAtMs(record: QueueRecord): number | null {
+  if (!record.leaseId || !record.leaseExpiresAt) {
+    return null
+  }
+
+  return parseTimestamp(record.leaseExpiresAt, "leaseExpiresAt")
+}
+
+function toClaimedQueueJob<TQueueJob extends QueueJob>(
+  record: QueueRecord<TQueueJob>
+): ClaimedQueueJob<TQueueJob> {
+  if (!record.leaseId || !record.claimedAt || !record.leaseExpiresAt) {
+    throw new QueueError(`[Sixb] Queue job '${record.job.id}' is not currently leased`)
+  }
+
+  return {
+    leaseId: record.leaseId,
+    claimedAt: record.claimedAt,
+    leaseExpiresAt: record.leaseExpiresAt,
+    job: structuredClone(record.job),
+  }
+}
+
+class InMemoryQueueStore {
+  private readonly recordsByProject = new Map<string, QueueRecord[]>()
+  private nextSequence = 1
+
+  add<TQueueJob extends QueueJob>(projectId: string, queueId: string, job: TQueueJob): void {
+    const records = this.getOrCreateRecords(projectId)
+    records.push({
+      sequence: this.nextSequence,
+      queueId,
+      job,
+      state: "queued",
+      leaseId: null,
+      claimedAt: null,
+      leaseExpiresAt: null,
+    })
+    this.nextSequence += 1
+  }
+
+  list<TQueueJob extends QueueJob>(projectId: string, queueId: string): QueueRecord<TQueueJob>[] {
+    const records = this.recordsByProject.get(projectId) ?? []
+    return records.filter((record) => record.queueId === queueId) as QueueRecord<TQueueJob>[]
+  }
+
+  find<TQueueJob extends QueueJob>(
+    projectId: string,
+    queueId: string,
+    jobId: string
+  ): QueueRecord<TQueueJob> | null {
+    return (
+      this.list<TQueueJob>(projectId, queueId).find((record) => record.job.id === jobId) ?? null
+    )
+  }
+
+  private getOrCreateRecords(projectId: string): QueueRecord[] {
+    const existing = this.recordsByProject.get(projectId)
+    if (existing) {
+      return existing
+    }
+
+    const records: QueueRecord[] = []
+    this.recordsByProject.set(projectId, records)
+    return records
+  }
+}
+
+class InMemoryQueue<TQueueJob extends QueueJob> implements Queue<TQueueJob> {
+  constructor(
+    private readonly store: InMemoryQueueStore,
+    private readonly queueId: string
+  ) {
+    assertNonEmpty(queueId, "queueId")
+  }
+
+  async enqueue(params: {
+    projectId: string
+    jobs: readonly NewQueueJob<TQueueJob>[]
+  }): Promise<readonly TQueueJob[]> {
+    assertNonEmpty(params.projectId, "projectId")
+
+    if (params.jobs.length === 0) {
+      return []
+    }
+
+    const createdJobs = params.jobs.map((job) => {
+      if (job.id !== undefined) assertNonEmpty(job.id, "job.id")
+      const existing = job.id
+        ? this.store.find<TQueueJob>(params.projectId, this.queueId, job.id)
+        : null
+      if (existing) return existing.job
+
+      const created = createQueueJob(params.projectId, job)
+      this.store.add(params.projectId, this.queueId, created)
+      return created
+    })
+
+    return createdJobs.map((job) => structuredClone(job))
+  }
+
+  async claim(params: {
+    projectId: string
+    workerId: string
+    limit?: number
+    leaseMs?: number
+  }): Promise<readonly ClaimedQueueJob<TQueueJob>[]> {
+    assertNonEmpty(params.projectId, "projectId")
+    assertNonEmpty(params.workerId, "workerId")
+
+    const limit = params.limit ?? 1
+    if (limit <= 0) {
+      return []
+    }
+
+    const leaseMs = params.leaseMs ?? 30_000
+    assertPositiveNumber(leaseMs, "leaseMs")
+
+    const now = Date.now()
+    const claimable = this.store
+      .list<TQueueJob>(params.projectId, this.queueId)
+      .filter((record) => {
+        if (record.state !== "queued") {
+          return false
+        }
+
+        if (parseTimestamp(record.job.availableAt, "job.availableAt") > now) {
+          return false
+        }
+
+        const expiresAt = leaseExpiresAtMs(record)
+        return expiresAt === null || expiresAt <= now
+      })
+      .sort(compareQueueRecords)
+      .slice(0, limit)
+
+    const claimedAt = new Date(now).toISOString()
+
+    return claimable.map((record) => {
+      // Attempts count claims so redelivery after lease expiry or retry is visible to workers.
+      record.job = {
+        ...record.job,
+        attempt: record.job.attempt + 1,
+      }
+      record.leaseId = randomUUID()
+      record.claimedAt = claimedAt
+      record.leaseExpiresAt = new Date(now + leaseMs).toISOString()
+
+      return toClaimedQueueJob(record)
+    })
+  }
+
+  async complete(params: { projectId: string; jobId: string; leaseId: string }): Promise<void> {
+    const record = this.requireActiveLease(params)
+    record.state = "completed"
+    clearLease(record)
+  }
+
+  async retry(params: {
+    projectId: string
+    jobId: string
+    leaseId: string
+    availableAt?: string
+    error?: QueueJobError
+  }): Promise<void> {
+    const record = this.requireActiveLease(params)
+    const availableAt = params.availableAt ?? new Date().toISOString()
+    parseTimestamp(availableAt, "availableAt")
+
+    record.job = {
+      ...record.job,
+      availableAt,
+    }
+    clearLease(record)
+  }
+
+  async fail(params: {
+    projectId: string
+    jobId: string
+    leaseId: string
+    error: QueueJobError
+  }): Promise<void> {
+    const record = this.requireActiveLease(params)
+    record.state = "failed"
+    clearLease(record)
+  }
+
+  async renewLease(params: {
+    projectId: string
+    jobId: string
+    leaseId: string
+    leaseMs: number
+  }): Promise<ClaimedQueueJob<TQueueJob> | null> {
+    assertPositiveNumber(params.leaseMs, "leaseMs")
+
+    const record = this.store.find<TQueueJob>(params.projectId, this.queueId, params.jobId)
+    if (!record || record.state !== "queued") {
+      return null
+    }
+
+    const now = Date.now()
+    const expiresAt = leaseExpiresAtMs(record)
+    if (record.leaseId !== params.leaseId || expiresAt === null) {
+      return null
+    }
+
+    if (expiresAt <= now) {
+      return null
+    }
+
+    record.leaseExpiresAt = new Date(now + params.leaseMs).toISOString()
+    return toClaimedQueueJob(record)
+  }
+
+  private requireActiveLease(params: {
+    projectId: string
+    jobId: string
+    leaseId: string
+  }): QueueRecord<TQueueJob> {
+    assertNonEmpty(params.projectId, "projectId")
+    assertNonEmpty(params.jobId, "jobId")
+    assertNonEmpty(params.leaseId, "leaseId")
+
+    const record = this.store.find<TQueueJob>(params.projectId, this.queueId, params.jobId)
+
+    if (!record) {
+      throw new QueueError(`[Sixb] Unknown queue job '${params.jobId}'`)
+    }
+
+    if (record.state !== "queued") {
+      throw new QueueError(`[Sixb] Queue job '${params.jobId}' is no longer active`)
+    }
+
+    if (record.leaseId !== params.leaseId || !record.leaseExpiresAt) {
+      throw new QueueError(`[Sixb] Lease mismatch for queue job '${params.jobId}'`)
+    }
+
+    const expiresAt = leaseExpiresAtMs(record)
+    if (expiresAt === null || expiresAt <= Date.now()) {
+      throw new QueueError(`[Sixb] Lease for queue job '${params.jobId}' has expired`)
+    }
+
+    return record
+  }
+}
+
+export class InMemoryQueues implements Queues {
+  readonly scope = "process" as const
+  private readonly store = new InMemoryQueueStore()
+
+  readonly syncRuns = new InMemoryQueue<SyncRunRequestedQueueJob>(this.store, "sync.runs")
+  readonly pipelines = new InMemoryQueue<PipelineRunRequestedQueueJob>(this.store, "pipeline.runs")
+  readonly projections = new InMemoryQueue<ProjectionRunRequestedQueueJob>(
+    this.store,
+    "projection.runs"
+  )
+  readonly workflows = new InMemoryQueue<WorkflowQueueJob>(this.store, "workflow.runs")
+  readonly actions = new InMemoryQueue<ActionRunRequestedQueueJob>(this.store, "action.runs")
+  readonly agents = new InMemoryQueue<AgentQueueJob>(this.store, "agent.runs")
+
+  /** Nothing to reach: the store is a field of this object. */
+  health(): Promise<void> {
+    return Promise.resolve()
+  }
+}

package/src/queues/index.ts

@@ -0,0 +1,23 @@
+export type { ProviderScope } from "../provider-scope"
+export { QueueError } from "./errors"
+export { InMemoryQueues } from "./in-memory"
+export type {
+  ActionRunRequestedQueueJob,
+  AgentQueueJob,
+  AgentRunRequestedQueueJob,
+  AgentWorkflowNodeRequestedQueueJob,
+  ClaimedQueueJob,
+  NewQueueJob,
+  PipelineRunRequestedQueueJob,
+  ProjectionRunRequestedQueueJob,
+  Queue,
+  QueueJob,
+  QueueJobEnvelope,
+  QueueJobError,
+  Queues,
+  SyncRunRequestedQueueJob,
+  WorkflowQueueJob,
+  WorkflowRunRequestedQueueJob,
+  WorkflowRunResumeCause,
+  WorkflowRunResumeRequestedQueueJob,
+} from "./types"

package/src/queues/types.ts

@@ -0,0 +1,230 @@
+import type { JsonValue } from "../json"
+import type { ProjectionMaterializationIdentity } from "../materialization/model"
+import type { ProviderScope } from "../provider-scope"
+import type { WorkflowRunSource } from "../workflows/types"
+
+export interface QueueJobEnvelope {
+  readonly id: string
+  readonly projectId: string
+  readonly createdAt: string
+  readonly availableAt: string
+  /** Increments each time the job is claimed, including redelivery after retries or lease expiry. */
+  readonly attempt: number
+  readonly metadata?: Readonly<Record<string, JsonValue>>
+}
+
+export interface QueueJob<TType extends string = string, TPayload = unknown>
+  extends QueueJobEnvelope {
+  readonly type: TType
+  readonly payload: TPayload
+}
+
+export type NewQueueJob<TQueueJob extends QueueJob> =
+  TQueueJob extends QueueJob<infer TType, infer TPayload>
+    ? {
+        /** Stable provider-level id used to make enqueue idempotent when supplied. */
+        readonly id?: string
+        readonly type: TType
+        readonly payload: TPayload
+        readonly availableAt?: string
+        readonly metadata?: Readonly<Record<string, JsonValue>>
+      }
+    : never
+
+export interface QueueJobError {
+  readonly name?: string
+  readonly message: string
+}
+
+export interface ClaimedQueueJob<TQueueJob extends QueueJob = QueueJob> {
+  /** Opaque acknowledgement token for this specific claim attempt. */
+  readonly leaseId: string
+  readonly claimedAt: string
+  /** After this time, another worker may claim the job again if it was not acknowledged. */
+  readonly leaseExpiresAt: string
+  readonly job: TQueueJob
+}
+
+export interface Queue<TQueueJob extends QueueJob> {
+  /**
+   * Adds durable work to the lane. Jobs become claimable at `availableAt`, or immediately.
+   * Repeating a caller-supplied job id is idempotent while that job remains in the provider.
+   */
+  enqueue(params: {
+    projectId: string
+    jobs: readonly NewQueueJob<TQueueJob>[]
+  }): Promise<readonly TQueueJob[]>
+
+  /**
+   * Worker read path.
+   *
+   * Returns currently visible jobs and leases them to the caller so they can
+   * be processed safely. Claimed jobs must later be completed, retried, or
+   * failed with the returned lease id.
+   *
+   * `leaseMs` is the visibility timeout for this claim. If the lease expires
+   * before the job is acknowledged, the job may be delivered again.
+   */
+  claim(params: {
+    projectId: string
+    workerId: string
+    limit?: number
+    leaseMs?: number
+  }): Promise<readonly ClaimedQueueJob<TQueueJob>[]>
+
+  /** Acknowledges successful processing and removes the claimed job from delivery. */
+  complete(params: { projectId: string; jobId: string; leaseId: string }): Promise<void>
+
+  /**
+   * Releases the current lease and makes the job visible again now or at `availableAt`.
+   *
+   * The next successful claim will increment `attempt`.
+   */
+  retry(params: {
+    projectId: string
+    jobId: string
+    leaseId: string
+    availableAt?: string
+    error?: QueueJobError
+  }): Promise<void>
+
+  /** Marks the claimed job as terminally failed so it will not be delivered again. */
+  fail(params: {
+    projectId: string
+    jobId: string
+    leaseId: string
+    error: QueueJobError
+  }): Promise<void>
+
+  /** Extends an active lease. Returns `null` if the lease was already lost, expired, or completed. */
+  renewLease?(params: {
+    projectId: string
+    jobId: string
+    leaseId: string
+    leaseMs: number
+  }): Promise<ClaimedQueueJob<TQueueJob> | null>
+}
+
+export interface SyncRunRequestedQueueJob
+  extends QueueJob<
+    "sync.run.requested",
+    {
+      readonly syncId: string
+      readonly runId?: string
+      readonly expectedLatestVersionId?: string
+      readonly commitMessage?: string
+    }
+  > {}
+
+export interface PipelineRunRequestedQueueJob
+  extends QueueJob<
+    "pipeline.run.requested",
+    {
+      readonly pipelineId: string
+      readonly runId?: string
+    }
+  > {}
+
+export interface ProjectionRunRequestedQueueJob
+  extends QueueJob<"projection.run.requested", ProjectionMaterializationIdentity> {}
+
+export interface WorkflowRunRequestedQueueJob
+  extends QueueJob<
+    "workflow.run.requested",
+    {
+      readonly workflowId: string
+      readonly runId?: string
+      readonly input?: Readonly<Record<string, unknown>>
+      readonly source?: WorkflowRunSource
+    }
+  > {}
+
+export type WorkflowRunResumeCause =
+  | {
+      readonly kind: "intervention"
+      readonly interventionId: string
+    }
+  | {
+      readonly kind: "agentNode"
+      readonly nodeRunId: string
+    }
+
+export interface WorkflowRunResumeRequestedQueueJob
+  extends QueueJob<
+    "workflow.run.resume.requested",
+    {
+      readonly workflowId: string
+      readonly runId: string
+      readonly resume: WorkflowRunResumeCause
+    }
+  > {}
+
+export type WorkflowQueueJob = WorkflowRunRequestedQueueJob | WorkflowRunResumeRequestedQueueJob
+
+export interface ActionRunRequestedQueueJob
+  extends QueueJob<
+    "action.run.requested",
+    {
+      readonly actionId: string
+      readonly runId: string
+    }
+  > {}
+
+/**
+ * An agent turn is requested for a thread. The payload points to the durable queued run created
+ * with the user message; the worker transitions it to running and installs the delivery's execution
+ * token when this job is claimed.
+ */
+export interface AgentRunRequestedQueueJob
+  extends QueueJob<
+    "agent.run.requested",
+    {
+      readonly agentId: string
+      readonly threadId: string
+      readonly runId: string
+      readonly triggerMessageId: string
+    }
+  > {}
+
+export interface AgentWorkflowNodeRequestedQueueJob
+  extends QueueJob<
+    "agent.workflow-node.requested",
+    {
+      readonly agentId: string
+      readonly nodeRunId: string
+    }
+  > {}
+
+export type AgentQueueJob = AgentRunRequestedQueueJob | AgentWorkflowNodeRequestedQueueJob
+
+export interface Queues {
+  readonly syncRuns: Queue<SyncRunRequestedQueueJob>
+  readonly pipelines: Queue<PipelineRunRequestedQueueJob>
+  readonly projections: Queue<ProjectionRunRequestedQueueJob>
+  readonly workflows: Queue<WorkflowQueueJob>
+  readonly actions: Queue<ActionRunRequestedQueueJob>
+  readonly agents: Queue<AgentQueueJob>
+
+  /**
+   * Whether these queues can be shared across processes.
+   *
+   * A `"process"` provider gives every role its own private lane, so jobs are
+   * enqueued where nobody claims them and the system looks alive while doing
+   * nothing. Production roles refuse to start against one. See
+   * {@link ProviderScope} for why it is required rather than inferred.
+   */
+  readonly scope: ProviderScope
+
+  /**
+   * Reports whether the backing service is reachable, without enqueueing or claiming.
+   *
+   * The only read-only member of this contract, and required for that reason: `sixb check`
+   * used to paint its queues row green without a round trip, and an optional probe would
+   * have left the same hole open for anyone who skipped it. Resolve when reachable, throw
+   * otherwise — the message reaches an operator, so it has to name the condition.
+   */
+  health(): Promise<void>
+
+  /** Release external resources. Optional: a provider that owns none omits it. */
+  close?(): void | Promise<void>
+}