@pikku/core 0.12.148 → 0.12.151

package/dist/wirings/workflow/workflow-step-claim.js

@@ -1,10 +1,52 @@
 import { DEFAULT_STEP_RETRIES, isStepLeaseLive } from './workflow-constants.js';
 import { WorkflowStepFunctionMismatchError, WorkflowStepLeaseExpiredError, } from './workflow-errors.js';
+import { deriveInvocationId } from './workflow-invocation-id.js';
+import { parseTraceparent, tracedLogger } from '../../trace.js';
+const warnLostStep = async (log, runId, stepName, attempt, message) => {
+    // The step has already moved on, so no failure here may reach the claim.
+    try {
+        const logger = log.logger();
+        if (!logger)
+            return;
+        const traceparent = await log.traceparent(runId).catch(() => undefined);
+        tracedLogger(logger, parseTraceparent(traceparent)?.traceId).warn({
+            message,
+            runId,
+            stepName,
+            attempt,
+            invocationId: deriveInvocationId(runId, stepName),
+        });
+    }
+    catch { }
+};
+export const warnStepRerun = (log, runId, stepName, attempt) => warnLostStep(log, runId, stepName, attempt, `Workflow step '${stepName}' (run ${runId}) lost its worker and is running again as attempt ${attempt}`);
+export const warnStepAttemptsExhausted = (log, runId, stepName, attempts) => warnLostStep(log, runId, stepName, attempts, `Workflow step '${stepName}' (run ${runId}) lost its worker on attempt ${attempts} with no attempts left, so it failed`);
 /**
  * Whether a step whose worker vanished may be handed to another one, or has
  * run out of attempts to spend on that.
  */
 export const leaseAttemptsExhausted = (stepState) => stepState.attemptCount >= (stepState.retries ?? DEFAULT_STEP_RETRIES) + 1;
+/**
+ * A pass holds its run's lease for as long as it works a step inline, so one
+ * that meets its own step already `running` is looking at a pass that died
+ * mid-step. Running it again is a new attempt, counted as one so a step that
+ * takes its process down every time runs out instead of looping forever. The
+ * invocation id comes from the step's name, so the attempt keeps it.
+ */
+export const takeBackLostInlineStep = async (store, runId, stepName, stepState) => {
+    if (stepState.status !== 'running') {
+        return stepState;
+    }
+    if (leaseAttemptsExhausted(stepState)) {
+        const error = new WorkflowStepLeaseExpiredError(runId, stepName, stepState.attemptCount);
+        await store.setStepError(stepState.stepId, error, stepState.attemptCount);
+        await store.logStepAttemptsExhausted?.(runId, stepName, stepState.attemptCount);
+        throw error;
+    }
+    const attempt = await store.createRetryAttempt(stepState.stepId, 'pending');
+    await store.logStepRerun?.(runId, stepName, attempt.attemptCount);
+    return attempt;
+};
 /**
  * Decide whether this dispatch owns the step, by reading its state and then
  * writing it — which only excludes a concurrent dispatch when the caller holds
@@ -29,6 +71,7 @@ export const claimStepByReadThenWrite = async (store, runId, stepName, rpcName,
         }
         if (leaseAttemptsExhausted(stepState)) {
             await store.setStepError(stepState.stepId, new WorkflowStepLeaseExpiredError(runId, stepName, stepState.attemptCount), stepState.attemptCount);
+            await store.logStepAttemptsExhausted?.(runId, stepName, stepState.attemptCount);
             await store.resumeWorkflow?.(runId);
             return null;
         }
@@ -41,6 +84,10 @@ export const claimStepByReadThenWrite = async (store, runId, stepName, rpcName,
     if (stepState.status === 'failed' || stepState.status === 'running') {
         const attempt = await store.createRetryAttempt(stepState.stepId, 'running');
         await store.refreshStepLease(attempt.stepId, leaseMs);
+        // A `failed` step threw, and this is its ordinary retry.
+        if (stepState.status === 'running') {
+            await store.logStepRerun?.(runId, stepName, attempt.attemptCount);
+        }
         return attempt;
     }
     if (stepState.status === 'pending' || stepState.status === 'scheduled') {

package/dist/wirings/workflow/workflow-step-retry.d.ts

@@ -9,6 +9,10 @@ export type StepRetryStore = {
 /**
  * Spend a step's retry budget here and now, rather than by handing the run back
  * to the orchestrator between attempts. For an inline run there is nothing to
- * hand it back to.
+ * hand it back to. Nothing ends the run between attempts either, so a forward
+ * step passes `beforeAttempt` to stop once the run has been cancelled.
+ *
+ * `retries` counts what follows the first attempt, as it does on the queue, so
+ * a step gets `retries + 1` attempts.
  */
-export declare const runInlineRetryLoop: (store: StepRetryStore, stepState: StepState, retries: number, retryDelay: WorkflowStepOptions['retryDelay'], doWork: (currentStepState: StepState) => Promise<any>, onError?: (error: any) => Promise<void>) => Promise<any>;
+export declare const runInlineRetryLoop: (store: StepRetryStore, stepState: StepState, retries: number, retryDelay: WorkflowStepOptions['retryDelay'], doWork: (currentStepState: StepState) => Promise<any>, onError?: (error: any) => Promise<void>, beforeAttempt?: () => Promise<void>) => Promise<any>;

package/dist/wirings/workflow/workflow-step-retry.js

@@ -2,11 +2,16 @@ import { getDurationInMilliseconds } from '../../time-utils.js';
 /**
  * Spend a step's retry budget here and now, rather than by handing the run back
  * to the orchestrator between attempts. For an inline run there is nothing to
- * hand it back to.
+ * hand it back to. Nothing ends the run between attempts either, so a forward
+ * step passes `beforeAttempt` to stop once the run has been cancelled.
+ *
+ * `retries` counts what follows the first attempt, as it does on the queue, so
+ * a step gets `retries + 1` attempts.
  */
-export const runInlineRetryLoop = async (store, stepState, retries, retryDelay, doWork, onError) => {
+export const runInlineRetryLoop = async (store, stepState, retries, retryDelay, doWork, onError, beforeAttempt) => {
     let currentStepState = stepState;
     while (true) {
+        await beforeAttempt?.();
         let result;
         try {
             await store.setStepRunning(currentStepState.stepId);
@@ -16,7 +21,7 @@ export const runInlineRetryLoop = async (store, stepState, retries, retryDelay,
             if (onError)
                 await onError(error);
             await store.setStepError(currentStepState.stepId, error);
-            if (currentStepState.attemptCount >= retries) {
+            if (currentStepState.attemptCount > retries) {
                 throw error;
             }
             currentStepState = await store.createRetryAttempt(currentStepState.stepId, 'pending');

package/dist/wirings/workflow/workflow-wire.d.ts

@@ -0,0 +1,12 @@
+import type { PikkuWorkflowWire, WorkflowApprovalOptions, WorkflowRun } from './workflow.types.js';
+/** The steps a wire drives, already bound to its run. */
+export type WorkflowWireSteps = {
+    getRun: () => Promise<WorkflowRun | null | undefined>;
+    verifyStepName: (stepName: string) => void;
+    rpcStep: (stepName: string, rpcName: string, data: any, options?: any) => Promise<any>;
+    inlineStep: (stepName: string, fn: any, options?: any) => Promise<any>;
+    sleepStep: (stepName: string, ms: number) => Promise<void>;
+    suspendStep: (reason: string) => Promise<void>;
+    approvalStep: (reason: string, options: WorkflowApprovalOptions) => Promise<any>;
+};
+export declare const buildWorkflowWire: (name: string, runId: string, addonNamespace: string | null | undefined, steps: WorkflowWireSteps) => PikkuWorkflowWire;

package/dist/wirings/workflow/workflow-wire.js

@@ -0,0 +1,33 @@
+import { getDurationInMilliseconds } from '../../time-utils.js';
+import { milestoneStepName } from './workflow-constants.js';
+export const buildWorkflowWire = (name, runId, addonNamespace, steps) => ({
+    name,
+    runId,
+    getRun: async () => (await steps.getRun()),
+    do: async (stepName, rpcNameOrFn, dataOrOptions, options) => {
+        steps.verifyStepName(stepName);
+        if (typeof rpcNameOrFn === 'string') {
+            const resolvedRpcName = addonNamespace && !rpcNameOrFn.includes(':')
+                ? `${addonNamespace}:${rpcNameOrFn}`
+                : rpcNameOrFn;
+            return await steps.rpcStep(stepName, resolvedRpcName, dataOrOptions, options);
+        }
+        return await steps.inlineStep(stepName, rpcNameOrFn, dataOrOptions);
+    },
+    sleep: async (stepName, duration) => {
+        steps.verifyStepName(stepName);
+        await steps.sleepStep(stepName, getDurationInMilliseconds(duration));
+    },
+    suspend: async (reason) => {
+        steps.verifyStepName(reason);
+        await steps.suspendStep(reason);
+    },
+    milestone: async (name) => {
+        steps.verifyStepName(name);
+        await steps.inlineStep(milestoneStepName(name), async () => ({ name }));
+    },
+    approval: (async (reason, options) => {
+        steps.verifyStepName(reason);
+        return await steps.approvalStep(reason, options);
+    }),
+});

package/dist/wirings/workflow/workflow.types.d.ts

@@ -16,6 +16,8 @@ export interface WorkflowRunWire {
     parentRunId?: string;
     parentStepId?: string;
     pikkuUserId?: string;
+    /** The W3C trace every hop of the run stays on. Set when the run is created. */
+    traceparent?: string;
 }
 /**
  * The resolved `workflow` section of the app config: default retries and the queue names the engine uses.
@@ -26,6 +28,20 @@ export interface WorkflowServiceConfig {
     orchestratorQueueName: string;
     stepWorkerQueueName: string;
     sleeperRPCName: string;
+    /**
+     * How `pikku dev` and `pikku serve` sweep for work a dead process left
+     * behind. They do by default; `false` turns it off.
+     */
+    recovery?: WorkflowRecoveryOptions | false;
+}
+/** Tuning for `startRecovery`. */
+export interface WorkflowRecoveryOptions {
+    /** How often to sweep after the startup pass. Default 30s. */
+    intervalMs?: number;
+    /** Idle time before a `running` run with nothing in flight is re-driven. Default 30s. */
+    stalledAfterMs?: number;
+    /** Age at which a dispatched step's job is presumed lost. Default 30s. */
+    undispatchedAfterMs?: number;
 }
 /**
  * How a workflow service routes its jobs. `per-workflow` (the default) uses a

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/core",
-  "version": "0.12.148",
+  "version": "0.12.151",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/pikkujs/pikku.git",
@@ -76,6 +76,7 @@
     "./oauth2": "./dist/wirings/oauth2/index.js",
     "./errors": "./dist/errors/index.js",
     "./services": "./dist/services/index.js",
+    "./trace": "./dist/trace.js",
     "./flag": "./dist/wirings/flag/index.js",
     "./services/local-meta": "./dist/services/meta-service.js",
     "./services/in-memory-queue": "./dist/wirings/queue/in-memory-queue-service.js",