@pikku/core 0.12.79 → 0.12.80

package/src/wirings/workflow/workflow-queue-routing.ts

@@ -0,0 +1,85 @@
+import { getSingletonServices, pikkuState } from '../../pikku-state.js'
+import { getDurationInMilliseconds } from '../../time-utils.js'
+import type { JobGroup, JobOptions } from '../queue/queue.types.js'
+import type {
+  WorkflowServiceConfig,
+  WorkflowStepOptions,
+} from './workflow.types.js'
+import { DEFAULT_STEP_RETRIES } from './workflow-constants.js'
+
+export type WorkflowQueueStrategy = 'per-workflow' | 'shared-groups'
+
+const toKebab = (s: string) =>
+  s.replace(/([a-z0-9])([A-Z])/g, '$1-$2').toLowerCase()
+
+export const resolveWorkflowConfig = (): WorkflowServiceConfig => {
+  const workflow = getSingletonServices().config?.workflow
+  return {
+    retries: workflow?.retries ?? DEFAULT_STEP_RETRIES,
+    retryDelay: workflow?.retryDelay ?? 0,
+    orchestratorQueueName:
+      workflow?.orchestratorQueueName ?? 'pikku-workflow-orchestrator',
+    stepWorkerQueueName:
+      workflow?.stepWorkerQueueName ?? 'pikku-workflow-step-worker',
+    sleeperRPCName: workflow?.sleeperRPCName ?? 'pikkuWorkflowSleeper',
+  }
+}
+
+/**
+ * Falls back to the shared queue whenever the dedicated one was not generated
+ * into queue meta, so a workflow added without a rebuild still runs.
+ */
+const dedicatedQueueName = (
+  prefix: string,
+  name: string | undefined,
+  strategy: WorkflowQueueStrategy,
+  fallback: string
+): string => {
+  if (!name || strategy === 'shared-groups') {
+    return fallback
+  }
+  const dedicated = `${prefix}${toKebab(name)}`
+  return pikkuState(null, 'queue', 'meta')[dedicated] ? dedicated : fallback
+}
+
+export const orchestratorQueueName = (
+  strategy: WorkflowQueueStrategy,
+  workflowName?: string
+): string =>
+  dedicatedQueueName(
+    'wf-orchestrator-',
+    workflowName,
+    strategy,
+    resolveWorkflowConfig().orchestratorQueueName
+  )
+
+export const stepWorkerQueueName = (
+  strategy: WorkflowQueueStrategy,
+  rpcName?: string
+): string =>
+  dedicatedQueueName(
+    'wf-step-',
+    rpcName,
+    strategy,
+    resolveWorkflowConfig().stepWorkerQueueName
+  )
+
+export const jobGroupFor = (
+  strategy: WorkflowQueueStrategy,
+  id?: string
+): JobGroup | undefined =>
+  id && strategy === 'shared-groups' ? { id, tier: id } : undefined
+
+export const stepJobOptions = (
+  stepOptions?: WorkflowStepOptions
+): JobOptions => {
+  const retries = stepOptions?.retries ?? DEFAULT_STEP_RETRIES
+  const retryDelay = stepOptions?.retryDelay
+  const backoff =
+    retryDelay !== undefined && retryDelay !== 'exponential'
+      ? { type: 'fixed', delay: getDurationInMilliseconds(retryDelay) }
+      : retries > 0 || retryDelay === 'exponential'
+        ? 'exponential'
+        : undefined
+  return { attempts: retries + 1, ...(backoff ? { backoff } : {}) }
+}

package/src/wirings/workflow/workflow-queue-wiring.ts

@@ -0,0 +1,149 @@
+import { addFunction } from '../../function/function-runner.js'
+import { pikkuState } from '../../pikku-state.js'
+import { wireQueueWorker } from '../queue/queue-runner.js'
+import type {
+  GroupConcurrencyConfig,
+  PikkuWorkerConfig,
+} from '../queue/queue.types.js'
+import type { Logger } from '../../services/logger.js'
+import {
+  pikkuWorkflowOrchestratorFunc,
+  pikkuWorkflowSleeperFunc,
+  pikkuWorkflowWorkerFunc,
+} from './workflow-queue-workers.js'
+import type { WorkflowQueueStrategy } from './workflow-queue-routing.js'
+
+const ORCHESTRATOR_QUEUE_PREFIX = 'wf-orchestrator-'
+const STEP_QUEUE_PREFIX = 'wf-step-'
+
+const helperFunctionMeta = (funcId: string) => ({
+  pikkuFuncId: funcId,
+  sessionless: true,
+  functionType: 'helper' as const,
+  inputSchemaName: null,
+  outputSchemaName: null,
+})
+
+export type WorkflowQueueWiringOptions = {
+  strategy: WorkflowQueueStrategy
+  concurrency: number
+  groupConcurrency: number | GroupConcurrencyConfig
+  /**
+   * Resolved only when there is something to warn about: reading the logger
+   * throws while singleton services are uninitialized, and wiring queues before
+   * that point is legitimate.
+   */
+  getLogger: () => Logger | undefined
+}
+
+/**
+ * Registers the orchestrator, step-worker and sleeper functions and binds them
+ * to their queues. Idempotent: a queue already carrying a registered function
+ * is left alone.
+ */
+export const wireWorkflowQueueWorkers = ({
+  strategy,
+  concurrency,
+  groupConcurrency,
+  getLogger,
+}: WorkflowQueueWiringOptions): void => {
+  const functions = pikkuState(null, 'function', 'functions')
+  const functionsMeta = pikkuState(null, 'function', 'meta')
+  const queueMeta = pikkuState(null, 'queue', 'meta')
+
+  const registerWorkflowFunc = (
+    funcId: string,
+    func: { func: unknown },
+    queueName: string,
+    config?: PikkuWorkerConfig
+  ) => {
+    if (functions.has(funcId)) return
+    addFunction(funcId, func as never)
+    if (!queueMeta[queueName]) {
+      queueMeta[queueName] = { pikkuFuncId: funcId, name: queueName }
+    }
+    wireQueueWorker({ name: queueName, func, config } as never)
+    if (!functionsMeta[funcId]) {
+      functionsMeta[funcId] = helperFunctionMeta(funcId)
+    }
+  }
+
+  const sharedGroups = strategy === 'shared-groups'
+  const sharedQueueConfig: PikkuWorkerConfig | undefined = sharedGroups
+    ? { batchSize: concurrency, groupConcurrency }
+    : undefined
+
+  registerWorkflowFunc(
+    'pikkuWorkflowOrchestrator',
+    { func: pikkuWorkflowOrchestratorFunc },
+    'pikku-workflow-orchestrator',
+    sharedQueueConfig
+  )
+  registerWorkflowFunc(
+    'pikkuWorkflowStepWorker',
+    { func: pikkuWorkflowWorkerFunc },
+    'pikku-workflow-step-worker',
+    sharedQueueConfig
+  )
+
+  const registerDedicatedQueues = (meta: Record<string, any>) => {
+    for (const [queueName, entry] of Object.entries(meta)) {
+      if (functions.has(entry.pikkuFuncId)) continue
+      if (queueName.startsWith(ORCHESTRATOR_QUEUE_PREFIX)) {
+        registerWorkflowFunc(
+          entry.pikkuFuncId,
+          { func: pikkuWorkflowOrchestratorFunc },
+          queueName
+        )
+      } else if (queueName.startsWith(STEP_QUEUE_PREFIX)) {
+        registerWorkflowFunc(
+          entry.pikkuFuncId,
+          { func: pikkuWorkflowWorkerFunc },
+          queueName
+        )
+      }
+    }
+  }
+
+  if (!sharedGroups) {
+    registerDedicatedQueues(pikkuState(null, 'queue', 'meta'))
+
+    const addons = pikkuState(null, 'addons', 'packages')
+    for (const [, addon] of addons ?? []) {
+      const addonQueueMeta = pikkuState(addon.package, 'queue', 'meta')
+      if (addonQueueMeta) {
+        registerDedicatedQueues(addonQueueMeta)
+      }
+    }
+  }
+
+  warnOnMissingDedicatedQueues(queueMeta, getLogger)
+
+  if (!functions.has('pikkuWorkflowSleeper')) {
+    addFunction('pikkuWorkflowSleeper', { func: pikkuWorkflowSleeperFunc })
+  }
+  if (!functionsMeta.pikkuWorkflowSleeper) {
+    functionsMeta.pikkuWorkflowSleeper = helperFunctionMeta(
+      'pikkuWorkflowSleeper'
+    )
+  }
+}
+
+const warnOnMissingDedicatedQueues = (
+  queueMeta: Record<string, unknown>,
+  getLogger: () => Logger | undefined
+) => {
+  const workflowCount = Object.keys(
+    pikkuState(null, 'workflows', 'meta') ?? {}
+  ).length
+  const dedicatedQueues = Object.keys(queueMeta).filter((name) =>
+    name.startsWith(ORCHESTRATOR_QUEUE_PREFIX)
+  ).length
+  if (workflowCount > 0 && dedicatedQueues === 0) {
+    getLogger()?.warn?.(
+      `[pikku] ${workflowCount} workflow(s) registered but no per-workflow orchestrator queues were found in queue meta. ` +
+        `All workflows will share a single orchestrator queue, where one slow step blocks every other workflow behind it. ` +
+        `Check that the generated bootstrap imports the queue-workers meta (pikku-queue-workers-wirings-meta.gen.js).`
+    )
+  }
+}

package/src/wirings/workflow/workflow-recovery.ts

@@ -0,0 +1,162 @@
+import type { Logger } from '../../services/logger.js'
+import {
+  DEFAULT_STALLED_RUN_LIMIT,
+  DEFAULT_STALLED_RUN_MS,
+  DEFAULT_UNDISPATCHED_STEP_LIMIT,
+  DEFAULT_UNDISPATCHED_STEP_MS,
+  REDISPATCH_BACKOFF_MAX_ENTRIES,
+  REDISPATCH_BACKOFF_MAX_MS,
+  REDISPATCH_BACKOFF_MS,
+} from './workflow-constants.js'
+
+/**
+ * Per-process, advisory record of when a run may next be re-dispatched.
+ *
+ * Keyed by run rather than by step because the run is the unit of re-drive:
+ * `resumeWorkflow` replays the whole run and re-dispatches every step still
+ * owed a job, so holding off a single step while resuming its run would
+ * suppress nothing.
+ *
+ * Losing this on restart costs extra dispatches, never correctness.
+ */
+export class RedispatchBackoff {
+  private readonly eligibleAt = new Map<string, number>()
+  private readonly delays = new Map<string, number>()
+
+  public isEligible(runId: string, now: number): boolean {
+    const at = this.eligibleAt.get(runId)
+    return at === undefined || at <= now
+  }
+
+  public note(runId: string, now: number): void {
+    const previous = this.delays.get(runId)
+    const delay = Math.min(
+      previous === undefined ? REDISPATCH_BACKOFF_MS : previous * 2,
+      REDISPATCH_BACKOFF_MAX_MS
+    )
+    // A run that settles is never returned again, so entries are only evicted
+    // by this bound — oldest first, which is also least recently re-dispatched.
+    if (this.eligibleAt.size >= REDISPATCH_BACKOFF_MAX_ENTRIES) {
+      const oldest = this.eligibleAt.keys().next()
+      if (!oldest.done) {
+        this.eligibleAt.delete(oldest.value)
+        this.delays.delete(oldest.value)
+      }
+    }
+    this.delays.set(runId, delay)
+    this.eligibleAt.set(runId, now + delay)
+  }
+}
+
+type SweepDeps = {
+  resume: (runId: string) => Promise<void>
+  logger?: Logger
+}
+
+const resumeEach = async (
+  runIds: Iterable<string>,
+  { resume, logger }: SweepDeps,
+  failure: (runId: string, detail: string) => string
+): Promise<string[]> => {
+  const succeeded: string[] = []
+  for (const runId of runIds) {
+    try {
+      await resume(runId)
+      succeeded.push(runId)
+    } catch (err) {
+      // One unresumable run must not stop the sweep from recovering the rest.
+      logger?.error(
+        failure(runId, err instanceof Error ? err.message : String(err))
+      )
+    }
+  }
+  return succeeded
+}
+
+/**
+ * Re-drive runs whose next move was lost, and report which were resumed.
+ *
+ * Arming a step is two writes to two systems — the step row, then the queue or
+ * scheduler job — so a process that dies between them leaves a run that is
+ * `running` with nothing in flight. Nothing notices: the run parks on a step
+ * that will never complete and never error, so it neither finishes nor fails.
+ * (Seen on a `workflow.sleep()`: a deploy restart landed between the sleep
+ * step's insert and its timer, parking the run permanently.)
+ *
+ * Replay is the recovery — `resumeWorkflow` re-orchestrates from persisted step
+ * state, and every settled step is memoized, so resuming a run that was not
+ * actually stuck costs an orchestration pass and changes nothing. That
+ * idempotence is what makes an idle-time heuristic safe here; a run that is
+ * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
+ */
+export const sweepStalledRuns = async (
+  findStalledRunIds: (before: Date, limit: number) => Promise<string[]>,
+  options: { stalledAfterMs?: number; limit?: number } | undefined,
+  deps: SweepDeps
+): Promise<{ resumed: string[] }> => {
+  const before = new Date(
+    Date.now() - (options?.stalledAfterMs ?? DEFAULT_STALLED_RUN_MS)
+  )
+  const runIds = await findStalledRunIds(
+    before,
+    options?.limit ?? DEFAULT_STALLED_RUN_LIMIT
+  )
+  return {
+    resumed: await resumeEach(
+      runIds,
+      deps,
+      (runId, detail) =>
+        `Failed to resume stalled workflow run ${runId}: ${detail}`
+    ),
+  }
+}
+
+/**
+ * Re-drive steps whose dispatch was lost, and report which runs were nudged.
+ *
+ * The step row is the outbox record and this is the relay. Age is the only
+ * signal available — a step `pending` because its dispatch was lost is
+ * indistinguishable from one whose job is merely still queued — so a step past
+ * `undispatchedAfterMs` is re-dispatched regardless, and correctness rests on
+ * the claim in `executeWorkflowStepInner` rather than on the guess being right.
+ * A redundant dispatch costs one queue message: the loser reads `running` and
+ * returns without invoking anything.
+ *
+ * Re-dispatches back off per run (doubling from 30s, capped at 10m) so a
+ * genuine queue backlog is not amplified by a tick that keeps firing at the
+ * steps the backlog is already delaying.
+ */
+export const sweepUndispatchedSteps = async (
+  findUndispatchedSteps: (
+    before: Date,
+    limit: number
+  ) => Promise<Array<{ runId: string; stepId: string }>>,
+  backoff: RedispatchBackoff,
+  options: { undispatchedAfterMs?: number; limit?: number } | undefined,
+  deps: SweepDeps
+): Promise<{ redispatched: string[] }> => {
+  const before = new Date(
+    Date.now() - (options?.undispatchedAfterMs ?? DEFAULT_UNDISPATCHED_STEP_MS)
+  )
+  const steps = await findUndispatchedSteps(
+    before,
+    options?.limit ?? DEFAULT_UNDISPATCHED_STEP_LIMIT
+  )
+
+  const now = Date.now()
+  const runIds = new Set<string>()
+  for (const { runId } of steps) {
+    if (!backoff.isEligible(runId, now)) continue
+    backoff.note(runId, now)
+    runIds.add(runId)
+  }
+
+  return {
+    redispatched: await resumeEach(
+      runIds,
+      deps,
+      (runId, detail) =>
+        `Failed to re-dispatch workflow run ${runId}: ${detail}`
+    ),
+  }
+}

package/src/wirings/workflow/workflow-retry-policy.test.ts

@@ -3,7 +3,7 @@ import assert from 'node:assert/strict'
 
 import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
 import { pikkuState } from '../../pikku-state.js'
-import { DEFAULT_STEP_RETRIES } from './pikku-workflow-service.js'
+import { DEFAULT_STEP_RETRIES } from './workflow-constants.js'
 import type { PikkuWorkflowService } from './pikku-workflow-service.js'
 import { deriveInvocationId } from './workflow-invocation-id.js'
 import type { WorkflowStepOptions } from './workflow.types.js'

package/src/wirings/workflow/workflow-run-authority.test.ts

@@ -0,0 +1,215 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState } from '../../pikku-state.js'
+import { runWorkflowGraph } from './graph/graph-runner.js'
+import { assertWorkflowRunOwner } from './workflow-run-ownership.js'
+import { WorkflowStepFunctionMismatchError } from './workflow-errors.js'
+import { ForbiddenError } from '../../errors/errors.js'
+
+const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
+
+const seedSingletons = () =>
+  pikkuState(null, 'package', 'singletonServices', {
+    logger: silentLogger,
+    // `resumeWorkflow` hands the run back to the orchestrator queue; the tests
+    // here assert on what happened before that hand-off.
+    queueService: { add: async () => 'job-1' },
+  } as any)
+
+describe('a graph run starts at an entry node the graph declared', () => {
+  const meta = {
+    name: 'payout',
+    pikkuFuncId: 'payout',
+    source: 'graph' as const,
+    entryNodeIds: ['validate'],
+    graphHash: 'payout-hash',
+    nodes: {
+      validate: { nodeId: 'validate', rpcName: 'validatePayout' },
+      // Reachable only after validation, but its input is a literal — so
+      // `areDependenciesSatisfied` alone would happily start the run here.
+      transfer: {
+        nodeId: 'transfer',
+        rpcName: 'transferFunds',
+        input: { amount: 1000 },
+      },
+    },
+  }
+
+  test('a startNode outside entryNodeIds is rejected', async () => {
+    seedSingletons()
+    const ws = new InMemoryWorkflowService()
+
+    await assert.rejects(
+      runWorkflowGraph(
+        ws,
+        'payout',
+        {},
+        { rpcWithWire: async () => ({}) },
+        true,
+        'transfer',
+        undefined,
+        meta
+      ),
+      /'transfer' is not an entry node/
+    )
+  })
+
+  test('a declared entry node still starts the run', async () => {
+    seedSingletons()
+    const ws = new InMemoryWorkflowService()
+
+    const { runId } = await runWorkflowGraph(
+      ws,
+      'payout',
+      {},
+      { rpcWithWire: async () => ({}) },
+      true,
+      'validate',
+      undefined,
+      meta
+    )
+
+    assert.ok(runId, 'the graph ran from the entry node it declared')
+  })
+})
+
+describe('a step runs the function the workflow dispatched it with', () => {
+  test('a queue message naming another function is rejected', async () => {
+    seedSingletons()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'queue',
+    })
+    await ws.insertStepState(runId, 'notify', 'sendReceipt', {})
+
+    const invoked: string[] = []
+    await assert.rejects(
+      ws.executeWorkflowStep(runId, 'notify', 'deleteAllUsers', {}, {
+        rpcWithWire: async (rpcName: string) => {
+          invoked.push(rpcName)
+          return {}
+        },
+      } as any),
+      WorkflowStepFunctionMismatchError
+    )
+    assert.deepEqual(invoked, [], 'nothing was invoked')
+
+    const stepState = await ws.getStepState(runId, 'notify')
+    assert.equal(
+      stepState.status,
+      'pending',
+      'the forged message left the step untouched'
+    )
+  })
+
+  test('the dispatched function still runs', async () => {
+    seedSingletons()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'queue',
+    })
+    await ws.insertStepState(runId, 'notify', 'sendReceipt', {})
+
+    const invoked: string[] = []
+    await ws.executeWorkflowStep(runId, 'notify', 'sendReceipt', {}, {
+      rpcWithWire: async (rpcName: string) => {
+        invoked.push(rpcName)
+        return {}
+      },
+    } as any)
+
+    assert.deepEqual(invoked, ['sendReceipt'])
+  })
+})
+
+describe('workflow run ownership', () => {
+  test('a run started by a session is readable only by that session', () => {
+    assert.throws(
+      () =>
+        assertWorkflowRunOwner(
+          { type: 'http', pikkuUserId: 'user-a' },
+          {
+            userId: 'user-b',
+          }
+        ),
+      ForbiddenError
+    )
+    assert.throws(
+      () =>
+        assertWorkflowRunOwner(
+          { type: 'http', pikkuUserId: 'user-a' },
+          undefined
+        ),
+      ForbiddenError
+    )
+    assert.doesNotThrow(() =>
+      assertWorkflowRunOwner(
+        { type: 'http', pikkuUserId: 'user-a' },
+        {
+          userId: 'user-a',
+        }
+      )
+    )
+  })
+
+  test('a run with no recorded owner has no ownership to enforce', () => {
+    assert.doesNotThrow(() =>
+      assertWorkflowRunOwner({ type: 'trigger' }, { userId: 'user-b' })
+    )
+  })
+
+  test('approveStep refuses a caller who does not own the run', async () => {
+    seedSingletons()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'http',
+      pikkuUserId: 'owner',
+    })
+
+    await assert.rejects(
+      ws.approveStep(
+        runId,
+        'release-funds',
+        { ok: true },
+        {
+          userId: 'attacker',
+        }
+      ),
+      ForbiddenError
+    )
+
+    const state = await ws.getRunState(runId)
+    assert.deepEqual(
+      Object.keys(state),
+      [],
+      'no decision was recorded for the run'
+    )
+  })
+
+  test('approveStep accepts the run owner', async () => {
+    seedSingletons()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'http',
+      pikkuUserId: 'owner',
+    })
+
+    await ws.approveStep(
+      runId,
+      'release-funds',
+      { ok: true },
+      {
+        userId: 'owner',
+      }
+    )
+
+    const state = await ws.getRunState(runId)
+    assert.equal(
+      Object.keys(state).length,
+      1,
+      'the owner’s decision was recorded'
+    )
+  })
+})