@pikku/core 0.12.88 → 0.12.90

package/src/wirings/virtual-user/virtual-user-schedule.ts

@@ -0,0 +1,156 @@
+import type { VirtualUserRunStore } from './virtual-user-run-store.js'
+import type {
+  VirtualUserScheduleRecord,
+  VirtualUserScheduleStore,
+} from './virtual-user-schedule-store.js'
+
+/**
+ * How long a run may sit at `running` before it is read as dead rather than
+ * busy.
+ *
+ * A run holds no process across a restart — see {@link VirtualUserRunRecord} —
+ * so a deploy mid-run strands the record, and a stranded record would block its
+ * persona's schedule for good. Twice the longest duration budget anyone sets in
+ * practice, because the failure this guards against is cheap to recover from
+ * and expensive to trigger early: reaping a run that was still working loses
+ * its findings.
+ */
+export const STALE_RUN_AFTER_MS = 2 * 60 * 60 * 1000
+
+/**
+ * The cadence a schedule gets when it is written without one: roughly a run a
+ * day, at an hour nobody can predict.
+ *
+ * Sparse on purpose. Every tick spends model budget with no caller present to
+ * notice, so the default is the one an app can leave switched on and forget,
+ * not the one that finds the most.
+ */
+export const DEFAULT_MIN_INTERVAL_MS = 6 * 60 * 60 * 1000
+export const DEFAULT_MAX_INTERVAL_MS = 24 * 60 * 60 * 1000
+
+/** Whether a row is the tick's business, for stores that cannot ask in a query. */
+export const isDue = (schedule: VirtualUserScheduleRecord, now: Date) =>
+  schedule.enabled && schedule.nextRunAt.getTime() <= now.getTime()
+
+/**
+ * When this persona should next appear, drawn from its own interval.
+ *
+ * Uniform between the two bounds. Reversed bounds are read as a range rather
+ * than rejected, because a schedule is configuration and a swapped pair is a
+ * typo, not an attack.
+ */
+export const nextRunAt = (
+  schedule: Pick<VirtualUserScheduleRecord, 'minIntervalMs' | 'maxIntervalMs'>,
+  now: Date,
+  random: () => number
+) => {
+  const low = Math.max(
+    0,
+    Math.min(schedule.minIntervalMs, schedule.maxIntervalMs)
+  )
+  const high = Math.max(
+    0,
+    Math.max(schedule.minIntervalMs, schedule.maxIntervalMs)
+  )
+  return new Date(now.getTime() + low + random() * (high - low))
+}
+
+export type VirtualUserSkipReason =
+  'in-flight' | 'dispatch-failed' | 'claimed-elsewhere'
+
+export interface VirtualUserTickResult {
+  dispatched: { persona: string; runId: string }[]
+  skipped: { persona: string; reason: VirtualUserSkipReason }[]
+  /** Runs found stranded at `running` and marked failed. */
+  reaped: string[]
+}
+
+export interface VirtualUserTickParams {
+  schedules: VirtualUserScheduleStore
+  runs: VirtualUserRunStore
+  /** Starts one run and answers with its id. Nothing here knows how. */
+  dispatch: (schedule: VirtualUserScheduleRecord) => Promise<string>
+  now?: Date
+  random?: () => number
+  staleAfterMs?: number
+}
+
+/**
+ * Acts on whichever personas are due, once.
+ *
+ * The whole cadence lives in this one call, so what schedules it is the host's
+ * choice — a cron wiring, a platform scheduler, or a person clicking a button.
+ * Pikku does not start a timer on an app's behalf; a scaffold that did would
+ * begin spending model budget the moment a project ran `pikku all`.
+ *
+ * A persona is skipped, not queued, while its previous run is still going. Two
+ * copies of the same user acting at once is not a heavier test, it is a
+ * different one, and every finding it produces is unreproducible. Two ticks
+ * running at once are held to the same rule by the claim, which only lands for
+ * whichever of them still sees the `nextRunAt` it read.
+ */
+export const tickVirtualUserSchedules = async ({
+  schedules,
+  runs,
+  dispatch,
+  now = new Date(),
+  random = Math.random,
+  staleAfterMs = STALE_RUN_AFTER_MS,
+}: VirtualUserTickParams): Promise<VirtualUserTickResult> => {
+  const result: VirtualUserTickResult = {
+    dispatched: [],
+    skipped: [],
+    reaped: [],
+  }
+
+  for (const schedule of await schedules.due(now)) {
+    const [latest] = await runs.list({ persona: schedule.persona, limit: 1 })
+    if (latest?.status === 'running') {
+      if (now.getTime() - latest.createdAt.getTime() < staleAfterMs) {
+        result.skipped.push({ persona: schedule.persona, reason: 'in-flight' })
+        continue
+      }
+      await runs.fail(
+        latest.runId,
+        `Abandoned: still running ${Math.round((now.getTime() - latest.createdAt.getTime()) / 60000)}m after it started, which is longer than any budget allows.`
+      )
+      result.reaped.push(latest.runId)
+    }
+
+    const due = nextRunAt(schedule, now, random)
+    const acquired = await schedules.claim(schedule.persona, {
+      from: schedule.nextRunAt,
+      nextRunAt: due,
+      runId: null,
+      at: now,
+    })
+    if (!acquired) {
+      result.skipped.push({
+        persona: schedule.persona,
+        reason: 'claimed-elsewhere',
+      })
+      continue
+    }
+
+    let runId: string
+    try {
+      runId = await dispatch(schedule)
+    } catch {
+      result.skipped.push({
+        persona: schedule.persona,
+        reason: 'dispatch-failed',
+      })
+      continue
+    }
+
+    await schedules.claim(schedule.persona, {
+      from: due,
+      nextRunAt: due,
+      runId,
+      at: now,
+    })
+    result.dispatched.push({ persona: schedule.persona, runId })
+  }
+
+  return result
+}

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -59,6 +59,7 @@ import {
   WORKFLOW_POLL_FACTOR,
   WORKFLOW_POLL_MIN_MS,
   WORKFLOW_TERMINAL_STATES,
+  isRunSettled,
 } from './workflow-constants.js'
 import {
   WorkflowAsyncException,
@@ -643,8 +644,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    * overriding this, or no concurrency for one to exclude: the relay makes
    * duplicate dispatch routine, and the claim is what keeps a duplicate from
    * becoming a second execution. Every `@pikku/kysely` dialect qualifies on its
-   * status-guarded claim, `in-memory` on being inline and single-process;
-   * `mongodb` still qualifies on neither.
+   * status-guarded claim, `mongodb` on the same claim expressed as a
+   * single-document update, `in-memory` on being inline and single-process —
+   * none of them overrides this yet.
    */
   protected async findUndispatchedSteps(
     _before: Date,
@@ -672,6 +674,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
   }): Promise<{ resumed: string[] }> {
     return sweepStalledRuns(
       (before, limit) => this.findStalledRunIds(before, limit),
+      this.redispatchBackoff,
       options,
       this.sweepDeps
     )
@@ -1084,6 +1087,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     if (!run) {
       throw new WorkflowRunNotFoundError(runId)
     }
+    if (isRunSettled(run.status)) return
 
     const resolved = resolveWorkflowMeta(run.workflow)
     const workflowMeta = resolved?.meta

package/src/wirings/workflow/workflow-constants.ts

@@ -15,6 +15,25 @@ export const WORKFLOW_TERMINAL_STATES: ReadonlySet<string> = new Set([
   'cancelled',
 ])
 
+/**
+ * True for a run that will never move again, whatever arrives for it.
+ *
+ * Worth checking before doing anything with an orchestrator message, because
+ * such a message is routine rather than exceptional: the queue is
+ * at-least-once, the relay re-dispatches on purpose, and a run can settle
+ * while a message for it is still in flight. Replaying one is not free —
+ * `runWorkflowJob` takes the run lock and re-enters the workflow body, and a
+ * body re-entered after its run failed can park on a wait that nothing will
+ * ever satisfy, holding the lock and the connection under it until something
+ * external gives up. Every leaked advisory lock seen in production traced back
+ * to that: a granted lock, an idle session, and a run already `failed`.
+ *
+ * Note `suspended` is deliberately absent. It ends a run's *current* pass but
+ * not the run, which resumes when its approval or signal arrives.
+ */
+export const isRunSettled = (status: string): boolean =>
+  WORKFLOW_TERMINAL_STATES.has(status)
+
 export const WORKFLOW_POLL_MIN_MS = 10
 
 export const WORKFLOW_POLL_FACTOR = 1.6

package/src/wirings/workflow/workflow-recovery.ts

@@ -17,6 +17,15 @@ import {
  * owed a job, so holding off a single step while resuming its run would
  * suppress nothing.
  *
+ * For the same reason one instance is shared by every sweep rather than kept
+ * per sweep. The record is of the action, not of the signal that prompted it:
+ * a stalled run and an undispatched step are different observations, but both
+ * are answered by the one orchestrator message, so a run the relay re-drove a
+ * moment ago gains nothing from the stalled sweep re-driving it again. Sharing
+ * is what makes the guarantee a message-per-run-per-window instead of one per
+ * sweep, and no recovery is lost by it: whichever sweep gets there first
+ * performs the identical re-drive, and the cap keeps the delay at 10m.
+ *
  * Losing this on restart costs extra dispatches, never correctness.
  */
 export class RedispatchBackoff {
@@ -88,19 +97,40 @@ const resumeEach = async (
  * actually stuck costs an orchestration pass and changes nothing. That
  * idempotence is what makes an idle-time heuristic safe here; a run that is
  * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
+ *
+ * Idempotent is not free, though: a run stays stalled until something clears
+ * the reason it stalled, so an unconditional sweep re-queues the same runs on
+ * every tick forever. That is how a handful of wedged runs became a queue of
+ * thousands that could not drain — each pass added work the previous pass had
+ * not finished. The same per-run backoff the relay uses bounds it: a run is
+ * re-driven, then held off for a doubling delay, so a sweep costs at most one
+ * message per run per window rather than one per run per tick.
  */
 export const sweepStalledRuns = async (
   findStalledRunIds: (before: Date, limit: number) => Promise<string[]>,
+  backoff: RedispatchBackoff,
   options: { stalledAfterMs?: number; limit?: number } | undefined,
   deps: SweepDeps
 ): Promise<{ resumed: string[] }> => {
   const before = new Date(
     Date.now() - (options?.stalledAfterMs ?? DEFAULT_STALLED_RUN_MS)
   )
-  const runIds = await findStalledRunIds(
+  const found = await findStalledRunIds(
     before,
     options?.limit ?? DEFAULT_STALLED_RUN_LIMIT
   )
+
+  const now = Date.now()
+  const runIds: string[] = []
+  for (const runId of found) {
+    if (!backoff.isEligible(runId, now)) continue
+    // Noted before the resume, not after: a run whose resume throws is exactly
+    // the run most likely to still be here next tick, and re-driving it every
+    // tick is the amplification this backoff exists to stop.
+    backoff.note(runId, now)
+    runIds.push(runId)
+  }
+
   return {
     resumed: await resumeEach(
       runIds,

package/src/wirings/workflow/workflow-stalled-recovery.test.ts

@@ -89,6 +89,52 @@ describe('stalled run recovery', () => {
     assert.deepEqual(resumed, [], 'a run that just moved is not swept')
   })
 
+  test('backs off rather than re-resuming the same run every tick', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+    await backdate(ws, runId, 10 * 60_000)
+
+    const first = await ws.recoverStalledRuns()
+    // Still stalled: a resume does not clear whatever wedged the run, so an
+    // unconditional sweep would re-queue it on this tick and on every tick
+    // after it.
+    const second = await ws.recoverStalledRuns()
+
+    assert.deepEqual(first.resumed, [runId], 'the first sweep resumes it')
+    assert.deepEqual(
+      second.resumed,
+      [],
+      'the next sweep is held off by the backoff'
+    )
+    assert.deepEqual(resumes.runIds, [runId], 'only one queue message')
+  })
+
+  test('does not re-drive a run the relay has just re-dispatched', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    // Old enough to read as both an undispatched step and a stalled run.
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+    await backdate(ws, runId, 10 * 60_000)
+
+    const relayed = await ws.relayUndispatchedSteps()
+    const swept = await ws.recoverStalledRuns()
+
+    assert.deepEqual(relayed.redispatched, [runId], 'the relay re-drives it')
+    assert.deepEqual(
+      swept.resumed,
+      [],
+      'the sweep adds nothing the relay has not already queued'
+    )
+    assert.deepEqual(resumes.runIds, [runId], 'only one queue message')
+  })
+
   test('leaves a finished run alone', async () => {
     trackResumes()
     const ws = new InMemoryWorkflowService()

package/src/wirings/workflow/workflow-terminal-run-guard.test.ts

@@ -0,0 +1,105 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState, resetPikkuState } from '../../pikku-state.js'
+
+const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
+
+/** Records every run lock taken, which is the whole point of the guard. */
+class LockSpyWorkflowService extends InMemoryWorkflowService {
+  public readonly locked: string[] = []
+
+  public override async withRunLock<T>(
+    id: string,
+    fn: () => Promise<T>
+  ): Promise<T> {
+    this.locked.push(id)
+    return super.withRunLock(id, fn)
+  }
+}
+
+/** A registered, fully described workflow, so nothing else can short-circuit. */
+const startRun = async (): Promise<{
+  service: LockSpyWorkflowService
+  runId: string
+  entered: () => boolean
+}> => {
+  resetPikkuState()
+  pikkuState(null, 'package', 'singletonServices', {
+    logger: silentLogger,
+    queueService: { add: async () => 'job-1' },
+  } as any)
+
+  let bodyEntered = false
+  const func = async () => {
+    bodyEntered = true
+  }
+  pikkuState(null, 'workflows', 'meta', {
+    flow: { name: 'flow', pikkuFuncId: 'flow', source: 'dsl' },
+  } as any)
+  pikkuState(null, 'workflows', 'registrations').set('flow', {
+    name: 'flow',
+    func,
+  } as any)
+  pikkuState(null, 'function', 'meta', {
+    flow: {
+      pikkuFuncId: 'flow',
+      inputSchemaName: null,
+      outputSchemaName: null,
+      sessionless: true,
+    },
+  } as any)
+  pikkuState(null, 'function', 'functions').set('flow', { func } as any)
+
+  const service = new LockSpyWorkflowService()
+  const runId = await service.createRun('flow', {}, false, '', { type: 'test' })
+  return { service, runId, entered: () => bodyEntered }
+}
+
+/**
+ * The leak this guards, read straight off production: every granted advisory
+ * lock held by an idle session mapped to a run that was already `failed`. The
+ * orchestrator queue is at-least-once, so a message for a settled run is
+ * routine — and answering it by taking the run lock and replaying the body is
+ * how a run that can never move again ends up holding a lock and a pooled
+ * connection while it waits on something that will never arrive.
+ */
+describe('an orchestrator message for a run that already settled', () => {
+  for (const status of ['failed', 'completed', 'cancelled'] as const) {
+    test(`a ${status} run is answered without taking its lock`, async () => {
+      const { service, runId, entered } = await startRun()
+      await service.updateRunStatus(runId, status)
+
+      await service.runWorkflowJob(runId, {} as any)
+
+      assert.deepEqual(
+        service.locked,
+        [],
+        'the run lock was taken for a run that can never move again'
+      )
+      assert.equal(
+        entered(),
+        false,
+        'the workflow body was replayed after the run had settled'
+      )
+    })
+  }
+
+  /**
+   * Guards the tests above: they would pass just as well for a service that
+   * refused to orchestrate anything at all.
+   */
+  test('a suspended run is still orchestrated', async () => {
+    const { service, runId } = await startRun()
+    await service.updateRunStatus(runId, 'suspended')
+
+    await service.runWorkflowJob(runId, {} as any)
+
+    assert.deepEqual(
+      service.locked,
+      [runId],
+      'suspended ends a pass, not the run — it resumes when its signal arrives'
+    )
+  })
+})