@pikku/core 0.12.78 → 0.12.80

package/src/wirings/workflow/workflow-meta-resolver.ts

@@ -0,0 +1,45 @@
+import { pikkuState } from '../../pikku-state.js'
+import type { WorkflowRuntimeMeta } from './workflow.types.js'
+
+export type ResolvedWorkflowMeta = {
+  meta: WorkflowRuntimeMeta
+  packageName: string | null
+  resolvedName: string
+}
+
+/**
+ * Resolves a workflow name against the root registry, then — for a
+ * `namespace:name` form — against the addon package that claims the namespace.
+ */
+export const resolveWorkflowMeta = (
+  name: string
+): ResolvedWorkflowMeta | null => {
+  const rootMeta = pikkuState(null, 'workflows', 'meta')
+  if (rootMeta[name]) {
+    return { meta: rootMeta[name], packageName: null, resolvedName: name }
+  }
+
+  const colonIndex = name.indexOf(':')
+  if (colonIndex === -1) {
+    return null
+  }
+
+  const namespace = name.substring(0, colonIndex)
+  const localName = name.substring(colonIndex + 1)
+  const addons = pikkuState(null, 'addons', 'packages')
+  const pkgConfig = addons?.get(namespace)
+  if (!pkgConfig) {
+    return null
+  }
+
+  const addonMeta = pikkuState(pkgConfig.package, 'workflows', 'meta')
+  if (!addonMeta?.[localName]) {
+    return null
+  }
+
+  return {
+    meta: addonMeta[localName],
+    packageName: pkgConfig.package,
+    resolvedName: localName,
+  }
+}

package/src/wirings/workflow/workflow-missing-meta.test.ts

@@ -0,0 +1,92 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState, resetPikkuState } from '../../pikku-state.js'
+import { PikkuMissingMetaError } from '../../errors/errors.js'
+import { WorkflowNotFoundError } from './workflow-errors.js'
+
+const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
+
+const startRun = async (workflowName: string) => {
+  pikkuState(null, 'package', 'singletonServices', {
+    logger: silentLogger,
+  } as any)
+  const service = new InMemoryWorkflowService()
+  const runId = await service.createRun(workflowName, {}, false, '', {
+    type: 'test',
+  })
+  return { service, runId }
+}
+
+describe('a run whose workflow has no generated meta', () => {
+  test('reports missing metadata rather than crashing on an undefined meta', async () => {
+    resetPikkuState()
+    const { service, runId } = await startRun('flowWithoutMeta')
+
+    // The registration exists, so this is not an unknown workflow — codegen or
+    // the generated bootstrap import is what is missing.
+    pikkuState(null, 'workflows', 'registrations').set('flowWithoutMeta', {
+      name: 'flowWithoutMeta',
+      func: async () => undefined,
+    } as any)
+
+    await assert.rejects(
+      service.runWorkflowJob(runId, {}),
+      (error: unknown) => {
+        assert.ok(
+          error instanceof PikkuMissingMetaError,
+          `expected PikkuMissingMetaError, got ${(error as Error)?.constructor?.name}: ${(error as Error)?.message}`
+        )
+        assert.match(String((error as Error).message), /flowWithoutMeta/)
+        return true
+      }
+    )
+  })
+
+  test('a workflow with neither meta nor a registration is still missing meta', async () => {
+    resetPikkuState()
+    const { service, runId } = await startRun('unknownFlow')
+
+    await assert.rejects(
+      service.runWorkflowJob(runId, {}),
+      (error: unknown) => {
+        assert.ok(
+          error instanceof PikkuMissingMetaError,
+          `expected PikkuMissingMetaError, got ${(error as Error)?.constructor?.name}`
+        )
+        return true
+      }
+    )
+  })
+
+  test('a registered workflow that does have meta gets past both guards', async () => {
+    resetPikkuState()
+    const { service, runId } = await startRun('flowWithMeta')
+
+    pikkuState(null, 'workflows', 'meta', {
+      flowWithMeta: {
+        name: 'flowWithMeta',
+        pikkuFuncId: 'flowWithMeta',
+        source: 'dsl',
+      },
+    } as any)
+
+    // Guards the tests above: without this the assertions would pass for a
+    // service that threw PikkuMissingMetaError unconditionally.
+    await assert.rejects(
+      service.runWorkflowJob(runId, {}),
+      (error: unknown) => {
+        assert.ok(
+          !(error instanceof PikkuMissingMetaError),
+          'meta was present, so the missing-meta guard must not fire'
+        )
+        assert.ok(
+          error instanceof WorkflowNotFoundError,
+          `expected WorkflowNotFoundError, got ${(error as Error)?.constructor?.name}`
+        )
+        return true
+      }
+    )
+  })
+})

package/src/wirings/workflow/workflow-queue-routing.ts

@@ -0,0 +1,85 @@
+import { getSingletonServices, pikkuState } from '../../pikku-state.js'
+import { getDurationInMilliseconds } from '../../time-utils.js'
+import type { JobGroup, JobOptions } from '../queue/queue.types.js'
+import type {
+  WorkflowServiceConfig,
+  WorkflowStepOptions,
+} from './workflow.types.js'
+import { DEFAULT_STEP_RETRIES } from './workflow-constants.js'
+
+export type WorkflowQueueStrategy = 'per-workflow' | 'shared-groups'
+
+const toKebab = (s: string) =>
+  s.replace(/([a-z0-9])([A-Z])/g, '$1-$2').toLowerCase()
+
+export const resolveWorkflowConfig = (): WorkflowServiceConfig => {
+  const workflow = getSingletonServices().config?.workflow
+  return {
+    retries: workflow?.retries ?? DEFAULT_STEP_RETRIES,
+    retryDelay: workflow?.retryDelay ?? 0,
+    orchestratorQueueName:
+      workflow?.orchestratorQueueName ?? 'pikku-workflow-orchestrator',
+    stepWorkerQueueName:
+      workflow?.stepWorkerQueueName ?? 'pikku-workflow-step-worker',
+    sleeperRPCName: workflow?.sleeperRPCName ?? 'pikkuWorkflowSleeper',
+  }
+}
+
+/**
+ * Falls back to the shared queue whenever the dedicated one was not generated
+ * into queue meta, so a workflow added without a rebuild still runs.
+ */
+const dedicatedQueueName = (
+  prefix: string,
+  name: string | undefined,
+  strategy: WorkflowQueueStrategy,
+  fallback: string
+): string => {
+  if (!name || strategy === 'shared-groups') {
+    return fallback
+  }
+  const dedicated = `${prefix}${toKebab(name)}`
+  return pikkuState(null, 'queue', 'meta')[dedicated] ? dedicated : fallback
+}
+
+export const orchestratorQueueName = (
+  strategy: WorkflowQueueStrategy,
+  workflowName?: string
+): string =>
+  dedicatedQueueName(
+    'wf-orchestrator-',
+    workflowName,
+    strategy,
+    resolveWorkflowConfig().orchestratorQueueName
+  )
+
+export const stepWorkerQueueName = (
+  strategy: WorkflowQueueStrategy,
+  rpcName?: string
+): string =>
+  dedicatedQueueName(
+    'wf-step-',
+    rpcName,
+    strategy,
+    resolveWorkflowConfig().stepWorkerQueueName
+  )
+
+export const jobGroupFor = (
+  strategy: WorkflowQueueStrategy,
+  id?: string
+): JobGroup | undefined =>
+  id && strategy === 'shared-groups' ? { id, tier: id } : undefined
+
+export const stepJobOptions = (
+  stepOptions?: WorkflowStepOptions
+): JobOptions => {
+  const retries = stepOptions?.retries ?? DEFAULT_STEP_RETRIES
+  const retryDelay = stepOptions?.retryDelay
+  const backoff =
+    retryDelay !== undefined && retryDelay !== 'exponential'
+      ? { type: 'fixed', delay: getDurationInMilliseconds(retryDelay) }
+      : retries > 0 || retryDelay === 'exponential'
+        ? 'exponential'
+        : undefined
+  return { attempts: retries + 1, ...(backoff ? { backoff } : {}) }
+}

package/src/wirings/workflow/workflow-queue-wiring.ts

@@ -0,0 +1,149 @@
+import { addFunction } from '../../function/function-runner.js'
+import { pikkuState } from '../../pikku-state.js'
+import { wireQueueWorker } from '../queue/queue-runner.js'
+import type {
+  GroupConcurrencyConfig,
+  PikkuWorkerConfig,
+} from '../queue/queue.types.js'
+import type { Logger } from '../../services/logger.js'
+import {
+  pikkuWorkflowOrchestratorFunc,
+  pikkuWorkflowSleeperFunc,
+  pikkuWorkflowWorkerFunc,
+} from './workflow-queue-workers.js'
+import type { WorkflowQueueStrategy } from './workflow-queue-routing.js'
+
+const ORCHESTRATOR_QUEUE_PREFIX = 'wf-orchestrator-'
+const STEP_QUEUE_PREFIX = 'wf-step-'
+
+const helperFunctionMeta = (funcId: string) => ({
+  pikkuFuncId: funcId,
+  sessionless: true,
+  functionType: 'helper' as const,
+  inputSchemaName: null,
+  outputSchemaName: null,
+})
+
+export type WorkflowQueueWiringOptions = {
+  strategy: WorkflowQueueStrategy
+  concurrency: number
+  groupConcurrency: number | GroupConcurrencyConfig
+  /**
+   * Resolved only when there is something to warn about: reading the logger
+   * throws while singleton services are uninitialized, and wiring queues before
+   * that point is legitimate.
+   */
+  getLogger: () => Logger | undefined
+}
+
+/**
+ * Registers the orchestrator, step-worker and sleeper functions and binds them
+ * to their queues. Idempotent: a queue already carrying a registered function
+ * is left alone.
+ */
+export const wireWorkflowQueueWorkers = ({
+  strategy,
+  concurrency,
+  groupConcurrency,
+  getLogger,
+}: WorkflowQueueWiringOptions): void => {
+  const functions = pikkuState(null, 'function', 'functions')
+  const functionsMeta = pikkuState(null, 'function', 'meta')
+  const queueMeta = pikkuState(null, 'queue', 'meta')
+
+  const registerWorkflowFunc = (
+    funcId: string,
+    func: { func: unknown },
+    queueName: string,
+    config?: PikkuWorkerConfig
+  ) => {
+    if (functions.has(funcId)) return
+    addFunction(funcId, func as never)
+    if (!queueMeta[queueName]) {
+      queueMeta[queueName] = { pikkuFuncId: funcId, name: queueName }
+    }
+    wireQueueWorker({ name: queueName, func, config } as never)
+    if (!functionsMeta[funcId]) {
+      functionsMeta[funcId] = helperFunctionMeta(funcId)
+    }
+  }
+
+  const sharedGroups = strategy === 'shared-groups'
+  const sharedQueueConfig: PikkuWorkerConfig | undefined = sharedGroups
+    ? { batchSize: concurrency, groupConcurrency }
+    : undefined
+
+  registerWorkflowFunc(
+    'pikkuWorkflowOrchestrator',
+    { func: pikkuWorkflowOrchestratorFunc },
+    'pikku-workflow-orchestrator',
+    sharedQueueConfig
+  )
+  registerWorkflowFunc(
+    'pikkuWorkflowStepWorker',
+    { func: pikkuWorkflowWorkerFunc },
+    'pikku-workflow-step-worker',
+    sharedQueueConfig
+  )
+
+  const registerDedicatedQueues = (meta: Record<string, any>) => {
+    for (const [queueName, entry] of Object.entries(meta)) {
+      if (functions.has(entry.pikkuFuncId)) continue
+      if (queueName.startsWith(ORCHESTRATOR_QUEUE_PREFIX)) {
+        registerWorkflowFunc(
+          entry.pikkuFuncId,
+          { func: pikkuWorkflowOrchestratorFunc },
+          queueName
+        )
+      } else if (queueName.startsWith(STEP_QUEUE_PREFIX)) {
+        registerWorkflowFunc(
+          entry.pikkuFuncId,
+          { func: pikkuWorkflowWorkerFunc },
+          queueName
+        )
+      }
+    }
+  }
+
+  if (!sharedGroups) {
+    registerDedicatedQueues(pikkuState(null, 'queue', 'meta'))
+
+    const addons = pikkuState(null, 'addons', 'packages')
+    for (const [, addon] of addons ?? []) {
+      const addonQueueMeta = pikkuState(addon.package, 'queue', 'meta')
+      if (addonQueueMeta) {
+        registerDedicatedQueues(addonQueueMeta)
+      }
+    }
+  }
+
+  warnOnMissingDedicatedQueues(queueMeta, getLogger)
+
+  if (!functions.has('pikkuWorkflowSleeper')) {
+    addFunction('pikkuWorkflowSleeper', { func: pikkuWorkflowSleeperFunc })
+  }
+  if (!functionsMeta.pikkuWorkflowSleeper) {
+    functionsMeta.pikkuWorkflowSleeper = helperFunctionMeta(
+      'pikkuWorkflowSleeper'
+    )
+  }
+}
+
+const warnOnMissingDedicatedQueues = (
+  queueMeta: Record<string, unknown>,
+  getLogger: () => Logger | undefined
+) => {
+  const workflowCount = Object.keys(
+    pikkuState(null, 'workflows', 'meta') ?? {}
+  ).length
+  const dedicatedQueues = Object.keys(queueMeta).filter((name) =>
+    name.startsWith(ORCHESTRATOR_QUEUE_PREFIX)
+  ).length
+  if (workflowCount > 0 && dedicatedQueues === 0) {
+    getLogger()?.warn?.(
+      `[pikku] ${workflowCount} workflow(s) registered but no per-workflow orchestrator queues were found in queue meta. ` +
+        `All workflows will share a single orchestrator queue, where one slow step blocks every other workflow behind it. ` +
+        `Check that the generated bootstrap imports the queue-workers meta (pikku-queue-workers-wirings-meta.gen.js).`
+    )
+  }
+}

package/src/wirings/workflow/workflow-recovery.ts

@@ -0,0 +1,162 @@
+import type { Logger } from '../../services/logger.js'
+import {
+  DEFAULT_STALLED_RUN_LIMIT,
+  DEFAULT_STALLED_RUN_MS,
+  DEFAULT_UNDISPATCHED_STEP_LIMIT,
+  DEFAULT_UNDISPATCHED_STEP_MS,
+  REDISPATCH_BACKOFF_MAX_ENTRIES,
+  REDISPATCH_BACKOFF_MAX_MS,
+  REDISPATCH_BACKOFF_MS,
+} from './workflow-constants.js'
+
+/**
+ * Per-process, advisory record of when a run may next be re-dispatched.
+ *
+ * Keyed by run rather than by step because the run is the unit of re-drive:
+ * `resumeWorkflow` replays the whole run and re-dispatches every step still
+ * owed a job, so holding off a single step while resuming its run would
+ * suppress nothing.
+ *
+ * Losing this on restart costs extra dispatches, never correctness.
+ */
+export class RedispatchBackoff {
+  private readonly eligibleAt = new Map<string, number>()
+  private readonly delays = new Map<string, number>()
+
+  public isEligible(runId: string, now: number): boolean {
+    const at = this.eligibleAt.get(runId)
+    return at === undefined || at <= now
+  }
+
+  public note(runId: string, now: number): void {
+    const previous = this.delays.get(runId)
+    const delay = Math.min(
+      previous === undefined ? REDISPATCH_BACKOFF_MS : previous * 2,
+      REDISPATCH_BACKOFF_MAX_MS
+    )
+    // A run that settles is never returned again, so entries are only evicted
+    // by this bound — oldest first, which is also least recently re-dispatched.
+    if (this.eligibleAt.size >= REDISPATCH_BACKOFF_MAX_ENTRIES) {
+      const oldest = this.eligibleAt.keys().next()
+      if (!oldest.done) {
+        this.eligibleAt.delete(oldest.value)
+        this.delays.delete(oldest.value)
+      }
+    }
+    this.delays.set(runId, delay)
+    this.eligibleAt.set(runId, now + delay)
+  }
+}
+
+type SweepDeps = {
+  resume: (runId: string) => Promise<void>
+  logger?: Logger
+}
+
+const resumeEach = async (
+  runIds: Iterable<string>,
+  { resume, logger }: SweepDeps,
+  failure: (runId: string, detail: string) => string
+): Promise<string[]> => {
+  const succeeded: string[] = []
+  for (const runId of runIds) {
+    try {
+      await resume(runId)
+      succeeded.push(runId)
+    } catch (err) {
+      // One unresumable run must not stop the sweep from recovering the rest.
+      logger?.error(
+        failure(runId, err instanceof Error ? err.message : String(err))
+      )
+    }
+  }
+  return succeeded
+}
+
+/**
+ * Re-drive runs whose next move was lost, and report which were resumed.
+ *
+ * Arming a step is two writes to two systems — the step row, then the queue or
+ * scheduler job — so a process that dies between them leaves a run that is
+ * `running` with nothing in flight. Nothing notices: the run parks on a step
+ * that will never complete and never error, so it neither finishes nor fails.
+ * (Seen on a `workflow.sleep()`: a deploy restart landed between the sleep
+ * step's insert and its timer, parking the run permanently.)
+ *
+ * Replay is the recovery — `resumeWorkflow` re-orchestrates from persisted step
+ * state, and every settled step is memoized, so resuming a run that was not
+ * actually stuck costs an orchestration pass and changes nothing. That
+ * idempotence is what makes an idle-time heuristic safe here; a run that is
+ * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
+ */
+export const sweepStalledRuns = async (
+  findStalledRunIds: (before: Date, limit: number) => Promise<string[]>,
+  options: { stalledAfterMs?: number; limit?: number } | undefined,
+  deps: SweepDeps
+): Promise<{ resumed: string[] }> => {
+  const before = new Date(
+    Date.now() - (options?.stalledAfterMs ?? DEFAULT_STALLED_RUN_MS)
+  )
+  const runIds = await findStalledRunIds(
+    before,
+    options?.limit ?? DEFAULT_STALLED_RUN_LIMIT
+  )
+  return {
+    resumed: await resumeEach(
+      runIds,
+      deps,
+      (runId, detail) =>
+        `Failed to resume stalled workflow run ${runId}: ${detail}`
+    ),
+  }
+}
+
+/**
+ * Re-drive steps whose dispatch was lost, and report which runs were nudged.
+ *
+ * The step row is the outbox record and this is the relay. Age is the only
+ * signal available — a step `pending` because its dispatch was lost is
+ * indistinguishable from one whose job is merely still queued — so a step past
+ * `undispatchedAfterMs` is re-dispatched regardless, and correctness rests on
+ * the claim in `executeWorkflowStepInner` rather than on the guess being right.
+ * A redundant dispatch costs one queue message: the loser reads `running` and
+ * returns without invoking anything.
+ *
+ * Re-dispatches back off per run (doubling from 30s, capped at 10m) so a
+ * genuine queue backlog is not amplified by a tick that keeps firing at the
+ * steps the backlog is already delaying.
+ */
+export const sweepUndispatchedSteps = async (
+  findUndispatchedSteps: (
+    before: Date,
+    limit: number
+  ) => Promise<Array<{ runId: string; stepId: string }>>,
+  backoff: RedispatchBackoff,
+  options: { undispatchedAfterMs?: number; limit?: number } | undefined,
+  deps: SweepDeps
+): Promise<{ redispatched: string[] }> => {
+  const before = new Date(
+    Date.now() - (options?.undispatchedAfterMs ?? DEFAULT_UNDISPATCHED_STEP_MS)
+  )
+  const steps = await findUndispatchedSteps(
+    before,
+    options?.limit ?? DEFAULT_UNDISPATCHED_STEP_LIMIT
+  )
+
+  const now = Date.now()
+  const runIds = new Set<string>()
+  for (const { runId } of steps) {
+    if (!backoff.isEligible(runId, now)) continue
+    backoff.note(runId, now)
+    runIds.add(runId)
+  }
+
+  return {
+    redispatched: await resumeEach(
+      runIds,
+      deps,
+      (runId, detail) =>
+        `Failed to re-dispatch workflow run ${runId}: ${detail}`
+    ),
+  }
+}

package/src/wirings/workflow/workflow-retry-policy.test.ts

@@ -3,7 +3,7 @@ import assert from 'node:assert/strict'
 
 import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
 import { pikkuState } from '../../pikku-state.js'
-import { DEFAULT_STEP_RETRIES } from './pikku-workflow-service.js'
+import { DEFAULT_STEP_RETRIES } from './workflow-constants.js'
 import type { PikkuWorkflowService } from './pikku-workflow-service.js'
 import { deriveInvocationId } from './workflow-invocation-id.js'
 import type { WorkflowStepOptions } from './workflow.types.js'