@pikku/core 0.12.107 → 0.12.112

package/src/services/in-memory-workflow-service.ts

@@ -1,595 +0,0 @@
-import { randomUUID } from 'crypto'
-import { PikkuWorkflowService } from '../wirings/workflow/pikku-workflow-service.js'
-import { isExpectedError } from '../errors/error-handler.js'
-import type { SerializedError } from '../errors/serialized-error.js'
-import type {
-  WorkflowPlannedStep,
-  WorkflowQueueOptions,
-  WorkflowRun,
-  WorkflowRunService,
-  WorkflowRunWire,
-  StepState,
-  StepStatus,
-  WorkflowStatus,
-  WorkflowVersionStatus,
-  WorkflowStepOptions,
-} from '../wirings/workflow/workflow.types.js'
-
-interface InternalStepData {
-  rpcName: string | null
-  data: any
-  stepName: string
-}
-
-// knowledge: decisions/internals/the-in-memory-workflow-service-is-inline-only-and-single-process.md
-export class InMemoryWorkflowService
-  extends PikkuWorkflowService
-  implements WorkflowRunService
-{
-  constructor(options: WorkflowQueueOptions = {}) {
-    super({ ...options, wireQueues: false })
-  }
-
-  private sleepTimers = new Set<ReturnType<typeof setTimeout>>()
-  private runs = new Map<string, WorkflowRun>()
-  private steps = new Map<string, StepState>() // keyed by `${runId}:${stepName}`
-  private stepData = new Map<string, InternalStepData>() // keyed by stepId
-  private stepHistory = new Map<
-    string,
-    Array<StepState & { stepName: string }>
-  >() // keyed by runId
-  private runState = new Map<string, Record<string, unknown>>() // keyed by runId
-  private branchKeys = new Map<string, string>() // keyed by stepId
-  private workflowVersions = new Map<
-    string,
-    { graph: any; source: string; status: WorkflowVersionStatus }
-  >() // keyed by `${name}:${graphHash}`
-
-  protected async createRunImpl(
-    workflowName: string,
-    input: any,
-    inline: boolean,
-    graphHash: string,
-    wire: WorkflowRunWire,
-    options?: {
-      deterministic?: boolean
-      plannedSteps?: WorkflowPlannedStep[]
-    }
-  ): Promise<string> {
-    const runId = randomUUID()
-    const now = new Date()
-
-    const run: WorkflowRun = {
-      id: runId,
-      workflow: workflowName,
-      status: 'running',
-      input,
-      inline,
-      graphHash,
-      deterministic: options?.deterministic,
-      plannedSteps: options?.plannedSteps,
-      wire,
-      createdAt: now,
-      updatedAt: now,
-    }
-
-    this.runs.set(runId, run)
-    this.stepHistory.set(runId, [])
-    this.runState.set(runId, {})
-
-    if (inline) {
-      this.registerInlineRun(runId)
-    }
-
-    return runId
-  }
-
-  protected override async scheduleSleep(
-    runId: string,
-    stepId: string,
-    duration: number
-  ): Promise<boolean> {
-    const timer = setTimeout(async () => {
-      this.sleepTimers.delete(timer)
-      try {
-        await this.executeWorkflowSleepCompleted(runId, stepId)
-      } catch (error: any) {
-        this.logger?.error(
-          `Failed to resume workflow sleep for runId ${runId}: ${error?.message ?? error}`
-        )
-      }
-    }, duration)
-    this.sleepTimers.add(timer)
-    return true
-  }
-
-  async getRun(id: string): Promise<WorkflowRun | null> {
-    return this.runs.get(id) || null
-  }
-
-  async getRunHistory(
-    runId: string
-  ): Promise<Array<StepState & { stepName: string }>> {
-    return this.stepHistory.get(runId) || []
-  }
-
-  protected async updateRunStatusImpl(
-    id: string,
-    status: WorkflowStatus,
-    output?: any,
-    error?: SerializedError
-  ): Promise<void> {
-    const run = this.runs.get(id)
-    if (run) {
-      run.status = status
-      run.updatedAt = new Date()
-      if (output !== undefined) run.output = output
-      if (error !== undefined) run.error = error
-    }
-  }
-
-  protected async insertStepStateImpl(
-    runId: string,
-    stepName: string,
-    rpcName: string | null,
-    data: any,
-    stepOptions?: WorkflowStepOptions,
-    fromStepName?: string
-  ): Promise<StepState> {
-    const stepId = randomUUID()
-    const now = new Date()
-
-    const step: StepState & { stepName: string } = {
-      stepId,
-      status: 'pending',
-      rpcName,
-      attemptCount: 1,
-      retries: stepOptions?.retries,
-      retryDelay: stepOptions?.retryDelay,
-      fromStepName,
-      createdAt: now,
-      updatedAt: now,
-      stepName,
-    }
-
-    const key = `${runId}:${stepName}`
-    this.steps.set(key, step)
-    this.stepData.set(stepId, { rpcName, data, stepName })
-
-    // knowledge: decisions/internals/in-memory-workflow-history-aliases-the-live-step-object.md
-    const history = this.stepHistory.get(runId) || []
-    history.push(step)
-    this.stepHistory.set(runId, history)
-
-    return step
-  }
-
-  async getStepState(runId: string, stepName: string): Promise<StepState> {
-    const key = `${runId}:${stepName}`
-    const step = this.steps.get(key)
-    if (!step) {
-      throw new Error(
-        `Step not found: runId=${runId}, stepName=${stepName}. Use insertStepState to create it.`
-      )
-    }
-    return step
-  }
-
-  protected async setStepRunningImpl(stepId: string): Promise<void> {
-    for (const step of this.steps.values()) {
-      if (step.stepId === stepId) {
-        step.status = 'running'
-        step.runningAt = new Date()
-        step.updatedAt = new Date()
-        break
-      }
-    }
-  }
-
-  protected async setStepScheduledImpl(stepId: string): Promise<void> {
-    for (const step of this.steps.values()) {
-      if (step.stepId === stepId) {
-        step.status = 'scheduled'
-        step.scheduledAt = new Date()
-        step.updatedAt = new Date()
-        break
-      }
-    }
-  }
-
-  protected async setStepResultImpl(
-    stepId: string,
-    result: any
-  ): Promise<void> {
-    for (const step of this.steps.values()) {
-      if (step.stepId === stepId) {
-        step.status = 'succeeded'
-        step.result = result
-        step.succeededAt = new Date()
-        step.updatedAt = new Date()
-        break
-      }
-    }
-  }
-
-  protected async setStepErrorImpl(
-    stepId: string,
-    error: Error
-  ): Promise<void> {
-    for (const step of this.steps.values()) {
-      if (step.stepId === stepId) {
-        step.status = 'failed'
-        step.error = {
-          name: error.name,
-          message: error.message,
-          stack: error.stack,
-          expected: isExpectedError(error),
-        }
-        step.failedAt = new Date()
-        step.updatedAt = new Date()
-        break
-      }
-    }
-  }
-
-  protected async setStepChildRunIdImpl(
-    stepId: string,
-    childRunId: string
-  ): Promise<void> {
-    for (const step of this.steps.values()) {
-      if (step.stepId === stepId) {
-        step.childRunId = childRunId
-        step.updatedAt = new Date()
-        break
-      }
-    }
-  }
-
-  protected async createRetryAttemptImpl(
-    failedStepId: string,
-    status: 'pending' | 'running'
-  ): Promise<StepState> {
-    let failedStep: StepState | undefined
-    let runId: string | undefined
-    let stepName: string | undefined
-
-    for (const [key, step] of this.steps.entries()) {
-      if (step.stepId === failedStepId) {
-        failedStep = step
-        const parts = key.split(':')
-        runId = parts[0]
-        stepName = parts.slice(1).join(':')
-        break
-      }
-    }
-
-    if (!failedStep || !runId || !stepName) {
-      throw new Error(`Step not found: ${failedStepId}`)
-    }
-
-    const failedStepData = this.stepData.get(failedStepId)
-    const newStepId = randomUUID()
-    const now = new Date()
-
-    const newStep: StepState & { stepName: string } = {
-      stepId: newStepId,
-      status,
-      rpcName: failedStep.rpcName,
-      attemptCount: failedStep.attemptCount + 1,
-      retries: failedStep.retries,
-      retryDelay: failedStep.retryDelay,
-      fromStepName: failedStep.fromStepName,
-      createdAt: now,
-      updatedAt: now,
-      stepName: stepName,
-    }
-
-    if (status === 'running') {
-      newStep.runningAt = now
-    }
-
-    const key = `${runId}:${stepName}`
-    this.steps.set(key, newStep)
-
-    if (failedStepData) {
-      this.stepData.set(newStepId, { ...failedStepData })
-    }
-
-    // knowledge: decisions/internals/in-memory-workflow-history-aliases-the-live-step-object.md
-    const history = this.stepHistory.get(runId) || []
-    history.push(newStep)
-    this.stepHistory.set(runId, history)
-
-    return newStep
-  }
-
-  /**
-   * Safe to opt in despite the pass-through `withStepLock`: this service wires
-   * no queues and runs every step inline in one process, so the relay's
-   * redundant dispatch has no second holder to race with. See
-   * knowledge: decisions/internals/the-in-memory-workflow-service-is-inline-only-and-single-process.md
-   */
-  protected async findUndispatchedSteps(
-    before: Date,
-    limit: number
-  ): Promise<Array<{ runId: string; stepId: string }>> {
-    const undispatched: Array<{ runId: string; stepId: string }> = []
-    for (const [runId, run] of this.runs) {
-      if (run.status !== 'running') continue
-      for (const step of this.stepHistory.get(runId) ?? []) {
-        if (step.status !== 'pending') continue
-        if (step.updatedAt >= before) continue
-        undispatched.push({ runId, stepId: step.stepId })
-        if (undispatched.length >= limit) return undispatched
-      }
-    }
-    return undispatched
-  }
-
-  protected async findStalledRunIds(
-    before: Date,
-    limit: number
-  ): Promise<string[]> {
-    const stalled: string[] = []
-    for (const [runId, run] of this.runs) {
-      if (run.status !== 'running') continue
-      const steps = this.stepHistory.get(runId) ?? []
-      if (
-        steps.some(
-          (step) =>
-            step.status === 'running' ||
-            step.status === 'scheduled' ||
-            step.status === 'suspended'
-        )
-      ) {
-        continue
-      }
-      const lastActivity = steps.reduce(
-        (latest, step) => (step.updatedAt > latest ? step.updatedAt : latest),
-        run.updatedAt
-      )
-      if (lastActivity >= before) continue
-      stalled.push(runId)
-      if (stalled.length >= limit) break
-    }
-    return stalled
-  }
-
-  async listRuns(options?: {
-    workflowName?: string
-    status?: string
-    limit?: number
-    offset?: number
-  }): Promise<WorkflowRun[]> {
-    let runs = Array.from(this.runs.values())
-    if (options?.workflowName) {
-      runs = runs.filter((r) => r.workflow === options.workflowName)
-    }
-    if (options?.status) {
-      runs = runs.filter((r) => r.status === options.status)
-    }
-    runs.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime())
-    const offset = options?.offset ?? 0
-    const limit = options?.limit ?? runs.length
-    return runs.slice(offset, offset + limit)
-  }
-
-  async getRunSteps(
-    runId: string
-  ): Promise<
-    Array<StepState & { stepName: string; rpcName?: string; data?: any }>
-  > {
-    const history = this.stepHistory.get(runId) || []
-    return history.map((step) => {
-      const stepDataEntry = this.stepData.get(step.stepId)
-      return {
-        ...step,
-        rpcName: stepDataEntry?.rpcName ?? undefined,
-        data: stepDataEntry?.data,
-      }
-    })
-  }
-
-  async getDistinctWorkflowNames(): Promise<string[]> {
-    const names = new Set<string>()
-    for (const run of this.runs.values()) {
-      names.add(run.workflow)
-    }
-    return Array.from(names)
-  }
-
-  async deleteRun(id: string): Promise<boolean> {
-    const existed = this.runs.has(id)
-    this.runs.delete(id)
-    this.stepHistory.delete(id)
-    this.runState.delete(id)
-    const prefix = `${id}:`
-    for (const key of this.steps.keys()) {
-      if (key.startsWith(prefix)) {
-        const step = this.steps.get(key)
-        if (step) {
-          this.stepData.delete(step.stepId)
-          this.branchKeys.delete(step.stepId)
-        }
-        this.steps.delete(key)
-      }
-    }
-    return existed
-  }
-
-  async withRunLock<T>(_id: string, fn: () => Promise<T>): Promise<T> {
-    // knowledge: decisions/internals/the-in-memory-workflow-service-is-inline-only-and-single-process.md
-    return fn()
-  }
-
-  async withStepLock<T>(
-    _runId: string,
-    _stepName: string,
-    fn: () => Promise<T>
-  ): Promise<T> {
-    return fn()
-  }
-
-  async close(): Promise<void> {
-    for (const timer of this.sleepTimers) {
-      clearTimeout(timer)
-    }
-    this.sleepTimers.clear()
-    this.runs.clear()
-    this.steps.clear()
-    this.stepData.clear()
-    this.stepHistory.clear()
-    this.runState.clear()
-    this.branchKeys.clear()
-    this.workflowVersions.clear()
-  }
-
-  async getCompletedGraphState(runId: string): Promise<{
-    completedNodeIds: string[]
-    failedNodeIds: string[]
-    branchKeys: Record<string, string>
-  }> {
-    const completedNodeIds: string[] = []
-    const failedNodeIds: string[] = []
-    const prefix = `${runId}:`
-    for (const [key, step] of this.steps.entries()) {
-      if (!key.startsWith(prefix)) continue
-      const nodeId = key.substring(prefix.length)
-      if (step.status === 'succeeded') {
-        completedNodeIds.push(nodeId)
-      } else if (step.status === 'failed') {
-        const maxAttempts = (step.retries ?? 0) + 1
-        if (step.attemptCount >= maxAttempts) {
-          failedNodeIds.push(nodeId)
-        }
-      }
-    }
-
-    const branchKeys: Record<string, string> = {}
-    for (const [stepId, branchKey] of this.branchKeys.entries()) {
-      const stepData = this.stepData.get(stepId)
-      if (stepData) {
-        branchKeys[stepData.stepName] = branchKey
-      }
-    }
-
-    return { completedNodeIds, failedNodeIds, branchKeys }
-  }
-
-  protected override async listStepStates(
-    runId: string
-  ): Promise<Array<StepState & { stepName: string }>> {
-    const prefix = `${runId}:`
-    const steps: Array<StepState & { stepName: string }> = []
-    for (const [key, step] of this.steps.entries()) {
-      if (!key.startsWith(prefix)) continue
-      steps.push({ ...step, stepName: key.substring(prefix.length) })
-    }
-    return steps
-  }
-
-  async getStepInstances(
-    runId: string
-  ): Promise<
-    Array<{ stepName: string; status: StepStatus; fromStepName?: string }>
-  > {
-    const prefix = `${runId}:`
-    const instances: Array<{
-      stepName: string
-      status: StepStatus
-      fromStepName?: string
-    }> = []
-    for (const [key, step] of this.steps.entries()) {
-      if (!key.startsWith(prefix)) continue
-      instances.push({
-        stepName: key.substring(prefix.length),
-        status: step.status,
-        fromStepName: step.fromStepName,
-      })
-    }
-    return instances
-  }
-
-  async getNodeResults(
-    runId: string,
-    nodeIds: string[]
-  ): Promise<Record<string, any>> {
-    const results: Record<string, any> = {}
-    for (const nodeId of nodeIds) {
-      const key = `${runId}:${nodeId}`
-      const step = this.steps.get(key)
-      if (step?.result !== undefined) {
-        results[nodeId] = step.result
-      }
-    }
-    return results
-  }
-
-  async setBranchKey(
-    runId: string,
-    nodeId: string,
-    branchKey: string
-  ): Promise<void> {
-    const key = `${runId}:${nodeId}`
-    const step = this.steps.get(key)
-    if (step) {
-      this.branchKeys.set(step.stepId, branchKey)
-    }
-  }
-
-  protected async setBranchTakenImpl(
-    stepId: string,
-    branchKey: string
-  ): Promise<void> {
-    this.branchKeys.set(stepId, branchKey)
-  }
-
-  protected async updateRunStateImpl(
-    runId: string,
-    name: string,
-    value: unknown
-  ): Promise<void> {
-    const state = this.runState.get(runId) || {}
-    state[name] = value
-    this.runState.set(runId, state)
-  }
-
-  async getRunState(runId: string): Promise<Record<string, unknown>> {
-    return this.runState.get(runId) || {}
-  }
-
-  protected async upsertWorkflowVersionImpl(
-    name: string,
-    graphHash: string,
-    graph: any,
-    source: string,
-    status?: WorkflowVersionStatus
-  ): Promise<void> {
-    this.workflowVersions.set(`${name}:${graphHash}`, {
-      graph,
-      source,
-      status: status ?? 'active',
-    })
-  }
-
-  protected async updateWorkflowVersionStatusImpl(
-    name: string,
-    graphHash: string,
-    status: WorkflowVersionStatus
-  ): Promise<void> {
-    const key = `${name}:${graphHash}`
-    const version = this.workflowVersions.get(key)
-    if (version) {
-      version.status = status
-    }
-  }
-
-  async getWorkflowVersion(
-    name: string,
-    graphHash: string
-  ): Promise<{ graph: any; source: string } | null> {
-    const version = this.workflowVersions.get(`${name}:${graphHash}`)
-    if (!version) return null
-    return { graph: version.graph, source: version.source }
-  }
-}