@pikku/core 0.12.84 → 0.12.88

package/src/wirings/workflow/pikku-scenario-service.ts

@@ -36,10 +36,57 @@ import type {
   WorkflowExpectErrorOptions,
   WorkflowExpectScoreOptions,
   WorkflowExpectServiceOptions,
+  WorkflowStepOptions,
 } from './dsl/workflow-dsl.types.js'
 
+/**
+ * The retry default for the `expect*` family: none, the same as `given`,
+ * `when` and `then` take, and for the same reason — a failed assertion is a
+ * result, not a transient fault.
+ *
+ * Left to the workflow-wide `DEFAULT_STEP_RETRIES` they get six attempts each,
+ * which re-runs the RPC behind a failing `expectError`, restarts a whole
+ * `expectEventually` poll after it has already spent its own deadline, and
+ * lets a live judge re-roll a grade until one lands inside the band. The last
+ * of those is how it was found: a judge scored a deliberately useless answer
+ * 1, and the scenario went green anyway on the next attempt.
+ *
+ * A caller that genuinely wants attempts still asks for them.
+ */
+const assertionStep = <T extends WorkflowStepOptions>(
+  options?: T
+): T & { retries: number } =>
+  ({ ...options, retries: options?.retries ?? 0 }) as T & { retries: number }
+
 export { addFeature, resolveFeatureScenarios } from './feature.js'
-export type * from './scenario.types.js'
+export type {
+  CoreFeature,
+  CoreFeatureScenario,
+  FeatureMeta,
+  FeaturesMeta,
+  PikkuBrowserWire,
+  PikkuScenarioWire,
+  ScenarioBrowserFailure,
+  ScenarioBrowserProvider,
+  ScenarioEnvironment,
+  ScenarioStepKind,
+  ScenarioStepMeta,
+  ScenarioStepOptions,
+  ScenarioStepPhase,
+  ScenarioSurface,
+  TestIdSelector,
+} from './scenario.types.js'
+export type {
+  ScenarioArtifact,
+  ScenarioFailureDetail,
+  ScenarioResult,
+  ScenarioRunRecord,
+  ScenarioRunReport,
+  ScenarioRunStatus,
+  ScenarioRunStore,
+  ScenarioRunSummary,
+  ScenarioStepRow,
+} from './scenario-run.types.js'
 export { SCENARIO_SURFACES } from './scenario-step.types.js'
 
 // Which of a step's bindings run: one for an action, every witness for a `then`
@@ -50,7 +97,7 @@ export { createCookieJar } from './scenario-cookie-jar.js'
 export type { ScenarioCookieJar } from './scenario-cookie-jar.js'
 export { composeStepProse, renderStepTemplate } from './scenario-prose.js'
 
-export { requireActor, requireScenarioEnv } from './scenario-step-guards.js'
+export { requireScenarioEnv } from './scenario-step-guards.js'
 
 export type {
   ScenarioHttpResponse,
@@ -171,20 +218,24 @@ addError(ScenarioBrowserUnavailable, {
 })
 
 /**
- * A browser binding was reached without an actor, so there is no persona whose
- * window it would run in.
+ * A step that is driven by a persona was dispatched without one.
+ *
+ * Raised before the body runs rather than partway through it: the step declared
+ * that it needs an actor — by having a `browser` binding, which cannot be
+ * provisioned without a window to open, or by `actor: true` — so a call site
+ * that omits one is a mistake at the call site.
  */
-export class ScenarioBrowserActorRequired extends PikkuError {
+export class ScenarioActorRequired extends PikkuError {
   constructor(public readonly stepFunc: string) {
     super(
-      `[scenario] step '${stepFunc}' resolved to its browser binding but was called without an actor. ` +
-        `Pass { actor: actors.<name> } so the browser signs in as that persona.`
+      `[scenario] step '${stepFunc}' is driven by a persona but was called without an actor. ` +
+        `Pass { actor: actors.<name> } so it runs as that persona.`
     )
   }
 }
-addError(ScenarioBrowserActorRequired, {
+addError(ScenarioActorRequired, {
   status: 500,
-  message: 'A browser binding needs an actor.',
+  message: 'This step needs an actor.',
 })
 
 /**
@@ -480,8 +531,7 @@ export class PikkuScenarioService implements WorkflowRunExtension {
     let wireServices: Record<string, unknown> | undefined
     try {
       wireServices = (await createWireServices?.(singletonServices, wire)) as
-        | Record<string, unknown>
-        | undefined
+        Record<string, unknown> | undefined
       const services =
         wireServices && Object.keys(wireServices).length > 0
           ? { ...singletonServices, ...wireServices }
@@ -602,7 +652,7 @@ export class PikkuScenarioService implements WorkflowRunExtension {
               await new Promise((resolve) => setTimeout(resolve, interval))
             }
           },
-          options
+          assertionStep(options)
         )
       },
 
@@ -645,7 +695,7 @@ export class PikkuScenarioService implements WorkflowRunExtension {
               `[workflow] expectError '${stepName}' ('${resolvedRpcName}') expected an error but the call succeeded: ${JSON.stringify(result)?.slice(0, 300)}`
             )
           },
-          options
+          assertionStep(options)
         )
       },
 
@@ -700,7 +750,7 @@ export class PikkuScenarioService implements WorkflowRunExtension {
               )
             }
           },
-          options
+          assertionStep(options)
         )
       },
 
@@ -745,7 +795,7 @@ export class PikkuScenarioService implements WorkflowRunExtension {
             }
             return grade
           },
-          options
+          assertionStep(options)
         )
       },
 
@@ -814,6 +864,10 @@ export class PikkuScenarioService implements WorkflowRunExtension {
       this.scenarioStepDescription(packageName, resolvedStepFunc) ??
       stepName
 
+    if (!actor && this.requiresActor(packageName, resolvedStepFunc)) {
+      throw new ScenarioActorRequired(resolvedStepFunc)
+    }
+
     const declared = this.declaredSurfaces(packageName, resolvedStepFunc)
     const resolution = resolveScenarioSurfaces(phase, declared, this.runSurface)
 
@@ -828,13 +882,13 @@ export class PikkuScenarioService implements WorkflowRunExtension {
             scenario: workflowWire,
             pikkuUserId: workflowWire.pikkuUserId,
             actors: this.runActors.get(runId),
+            ...(actor ? { actor } : {}),
             scenarioStep: {
               name: resolvedStepFunc,
               stepName,
               runId,
               phase,
               surface,
-              actor,
               env: this.scenarioEnvironment,
             },
           }
@@ -842,11 +896,10 @@ export class PikkuScenarioService implements WorkflowRunExtension {
             if (!this.scenarioBrowserProvider) {
               throw new ScenarioBrowserUnavailable(resolvedStepFunc)
             }
-            if (!actor) {
-              throw new ScenarioBrowserActorRequired(resolvedStepFunc)
-            }
+            // The dispatch guard above already refused an actor-less call, and a
+            // browser binding always requires one.
             wire.browser = await this.scenarioBrowserProvider.sessionFor(
-              actor.name
+              actor!.name
             )
           }
           return await runPikkuFunc(
@@ -947,4 +1000,14 @@ export class PikkuScenarioService implements WorkflowRunExtension {
     const surfaces = this.scenarioStepConfig(packageName, stepFunc)?.surfaces
     return surfaces?.length ? surfaces : ['default']
   }
+
+  /**
+   * Whether a step is driven by a persona, and so must be given one. Stamped by
+   * the definer from a `browser` binding or an explicit `actor: true`.
+   */
+  private requiresActor(packageName: string | null, stepFunc: string): boolean {
+    return (
+      this.scenarioStepConfig(packageName, stepFunc)?.requiresActor === true
+    )
+  }
 }

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -5,11 +5,8 @@ import {
   pikkuState,
 } from '../../pikku-state.js'
 import { getDurationInMilliseconds } from '../../time-utils.js'
-import type {
-  CoreUserSession,
-  PikkuRawWire,
-  SerializedError,
-} from '../../types/core.types.js'
+import type { CoreUserSession, PikkuRawWire } from '../../types/core.types.js'
+import type { SerializedError } from '../../errors/serialized-error.js'
 import type {
   GroupConcurrencyConfig,
   JobGroup,
@@ -39,6 +36,7 @@ import {
   executeGraphStep,
   runWorkflowGraph,
   runFromMeta,
+  stripInstanceOrdinal,
 } from './graph/graph-runner.js'
 import type { WorkflowService } from '../../services/workflow-service.js'
 import type { ScenarioPersonas } from '../../services/personas-service.js'
@@ -70,7 +68,6 @@ import {
   WorkflowRunCancelledError,
   WorkflowRunFailedError,
   WorkflowRunNotFoundError,
-  WorkflowStepFunctionMismatchError,
   WorkflowStepNameNotString,
   WorkflowSuspendedException,
 } from './workflow-errors.js'
@@ -97,6 +94,7 @@ import {
 } from './workflow-approval.js'
 import { auditApprovalDecision } from './workflow-approval-audit.js'
 import { recordSuspension, suspendStepNameFor } from './workflow-suspend.js'
+import { claimStepByReadThenWrite } from './workflow-step-claim.js'
 import {
   RedispatchBackoff,
   sweepStalledRuns,
@@ -641,12 +639,12 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    *
    * Returns nothing by default so a store that cannot express the query keeps
    * working unchanged — and, because it does not opt in, gains no re-dispatches
-   * either. A store must have an atomic `withStepLock` before overriding this,
-   * or no concurrency for one to exclude: the relay makes duplicate dispatch
-   * routine, and the claim in `executeWorkflowStepInner` is what keeps a
-   * duplicate from becoming a second execution. `kysely-postgres` and
-   * `kysely-mysql` qualify on the lock, `in-memory` on being inline and
-   * single-process; `mongodb` and `kysely-sqlite` qualify on neither.
+   * either. A store must have an atomic `claimStepForExecution` before
+   * overriding this, or no concurrency for one to exclude: the relay makes
+   * duplicate dispatch routine, and the claim is what keeps a duplicate from
+   * becoming a second execution. Every `@pikku/kysely` dialect qualifies on its
+   * status-guarded claim, `in-memory` on being inline and single-process;
+   * `mongodb` still qualifies on neither.
    */
   protected async findUndispatchedSteps(
     _before: Date,
@@ -1302,6 +1300,29 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     }
   }
 
+  /**
+   * Take sole ownership of a step before it runs, returning the state to run
+   * under — or `null` when another dispatch already owns it.
+   *
+   * Dispatch is at-least-once by design: the relay re-dispatches steps it
+   * believes were dropped, and a queue can redeliver a job it already handed
+   * out. This is the one place that keeps a duplicate dispatch from becoming a
+   * second execution of a side-effecting step, so it is only as strong as the
+   * exclusion it is built on — and `withStepLock` excludes nothing unless the
+   * store backs it with a real primitive. A store able to express the decision
+   * as one conditional write should override this rather than reach for a lock,
+   * which is what `@pikku/kysely` does with a status-guarded `UPDATE`.
+   */
+  protected async claimStepForExecution(
+    runId: string,
+    stepName: string,
+    rpcName: string
+  ): Promise<StepState | null> {
+    return this.withStepLock(runId, stepName, () =>
+      claimStepByReadThenWrite(this, runId, stepName, rpcName)
+    )
+  }
+
   private async executeWorkflowStepInner(
     runId: string,
     stepName: string,
@@ -1309,26 +1330,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     data: any,
     rpcService: PikkuRPC
   ): Promise<void> {
-    const claimed = await this.withStepLock(runId, stepName, async () => {
-      const stepState = await this.getStepState(runId, stepName)
-      // knowledge: decisions/security/a-step-runs-the-function-the-workflow-dispatched-it-with.md
-      if (
-        stepState.rpcName !== undefined &&
-        stepState.rpcName !== (rpcName ?? null)
-      ) {
-        throw new WorkflowStepFunctionMismatchError(runId, stepName)
-      }
-      if (stepState.status === 'succeeded' || stepState.status === 'running') {
-        return null
-      }
-      if (stepState.status === 'failed') {
-        return this.createRetryAttempt(stepState.stepId, 'running')
-      }
-      if (stepState.status === 'pending' || stepState.status === 'scheduled') {
-        await this.setStepRunning(stepState.stepId)
-      }
-      return stepState
-    })
+    const claimed = await this.claimStepForExecution(runId, stepName, rpcName)
 
     if (!claimed) {
       return
@@ -1352,9 +1354,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         if (stepName in workflowMeta.nodes) {
           graphNodeId = stepName
         } else {
-          const hash = stepName.lastIndexOf('#')
-          const base = hash > 0 ? stepName.slice(0, hash) : undefined
-          if (base && base in workflowMeta.nodes) graphNodeId = base
+          const base = stripInstanceOrdinal(stepName)
+          if (base !== stepName && base in workflowMeta.nodes)
+            graphNodeId = base
         }
       }
       if (graphNodeId) {

package/src/wirings/workflow/run-timeline.ts

@@ -1,4 +1,4 @@
-import type { SerializedError } from '../../types/core.types.js'
+import type { SerializedError } from '../../errors/serialized-error.js'
 import type { StepState, StepStatus } from './workflow.types.js'
 
 export interface RunTimelineEvent {

package/src/wirings/workflow/scenario-expectations.test.ts

@@ -65,7 +65,7 @@ describe('workflow.expectError', () => {
     })
 
     await assert.rejects(
-      wire.expectError('should have failed', 'deleteTodo', {}, { retries: 0 }),
+      wire.expectError('should have failed', 'deleteTodo', {}),
       /expected an error but the call succeeded/
     )
   })
@@ -82,7 +82,7 @@ describe('workflow.expectError', () => {
         'wrong reason',
         'deleteTodo',
         {},
-        { matches: /Forbidden/, retries: 0 }
+        { matches: /Forbidden/ }
       ),
       /did not match .*database offline/
     )
@@ -114,10 +114,7 @@ describe('workflow.expectService', () => {
     const { wire } = await scenarioWire(oneEmail())
 
     await assert.rejects(
-      wire.expectService('two emails', 'email.send', {
-        times: 2,
-        retries: 0,
-      }),
+      wire.expectService('two emails', 'email.send', { times: 2 }),
       /expected 2 call\(s\).*found 1.*email\.send/s
     )
   })
@@ -131,7 +128,6 @@ describe('workflow.expectService', () => {
     await assert.rejects(
       wire.expectService('emailed someone else', 'email.send', {
         calledWith: { to: 'x@y.z' },
-        retries: 0,
       }),
       /found 0/
     )
@@ -172,7 +168,7 @@ describe('workflow.expectScore', () => {
     const { wire } = await scenarioWire(grades({ score: 0 }))
 
     await assert.rejects(
-      wire.expectScore('was brief', 'run-1', 'brevity', { retries: 0 }),
+      wire.expectScore('was brief', 'run-1', 'brevity'),
       /expected 'brevity' to grade run run-1 at least 0\.5, got 0/
     )
   })
@@ -183,10 +179,7 @@ describe('workflow.expectScore', () => {
     )
 
     await assert.rejects(
-      wire.expectScore('was brief', 'run-1', 'brevity', {
-        atLeast: 0.8,
-        retries: 0,
-      }),
+      wire.expectScore('was brief', 'run-1', 'brevity', { atLeast: 0.8 }),
       /It rambled for four paragraphs/
     )
   })
@@ -198,12 +191,34 @@ describe('workflow.expectScore', () => {
       wire.expectScore('was not sycophantic', 'run-1', 'sycophancy', {
         atLeast: 0,
         atMost: 0.3,
-        retries: 0,
       }),
       /between 0 and 0\.3, got 0\.95/
     )
   })
 
+  test('a grade that misses the bound is not re-rolled until it lands', async () => {
+    let graded = 0
+    const { wire } = await scenarioWire({
+      rpcWithWire: async () => {
+        graded += 1
+        return { score: 1 }
+      },
+    })
+
+    await assert.rejects(
+      wire.expectScore('was not sycophantic', 'run-1', 'sycophancy', {
+        atLeast: 0,
+        atMost: 0.3,
+      }),
+      /got 1/
+    )
+
+    // The whole point of a judge is that it is not deterministic. Six attempts
+    // at it is six chances for one to land inside the band, which grades the
+    // sampling rather than the run.
+    assert.equal(graded, 1)
+  })
+
   test('a reference answer reaches the grader', async () => {
     let seen: any
     const { wire } = await scenarioWire({

package/src/wirings/workflow/scenario-run.types.ts

@@ -0,0 +1,186 @@
+/**
+ * What one `pikku scenario run` leaves behind.
+ *
+ * A run is history: the scenarios it selected, the sentences they were made of,
+ * and the images and footage they produced. The sentences are snapshotted into
+ * the record rather than looked up later, because a scenario is code and the
+ * code moves — a run from last week describes a suite that no longer exists,
+ * and re-deriving its ladder from today's source would quietly rewrite it.
+ *
+ * Everything here is plain serialisable data. That is what lets one record be
+ * a JSON file on disk in OSS and a row in Postgres on a hosted project, read
+ * back by the same console screen either way.
+ */
+import type { ScenarioBrowserFailure } from './scenario-step.types.js'
+
+/** What a run filed, in the order it filed it. */
+export type ScenarioArtifactKind = 'screenshot' | 'failure' | 'video'
+
+/**
+ * One image or recording a scenario produced.
+ *
+ * `path` is relative to the run's artifact root and always uses forward
+ * slashes: it is a content key, not a filesystem path. Locally it resolves
+ * under the run's directory; on a hosted project it is the object key in a
+ * bucket, and the same record works unchanged.
+ */
+export interface ScenarioArtifact {
+  /** The scenario label this belongs to, as the report names it. */
+  scenario: string
+  kind: ScenarioArtifactKind
+  path: string
+  /** Whose window it was — a two-actor scenario produces two of everything. */
+  actor?: string
+  /** The caption the scenario author took a screenshot under. */
+  name?: string
+}
+
+/** One step of a run, already joined to the prose that declared it. */
+export interface ScenarioStepRow {
+  sentence: string
+  status: string
+  durationMs?: number
+  error?: string
+}
+
+/** Everything known about why one scenario failed. */
+export interface ScenarioFailureDetail {
+  /** The rendered sentence of the failing step; absent if no step failed. */
+  sentence?: string
+  message: string
+  stack?: string
+  /**
+   * True when the failure was a deliberate one (a PikkuError). Its message is
+   * the whole story, so the stack is noise.
+   */
+  expected?: boolean
+  browser?: ScenarioBrowserFailure[]
+}
+
+export interface ScenarioResult {
+  name: string
+  status: 'passed' | 'failed'
+  durationMs: number
+  output?: unknown
+  error?: string
+  steps?: ScenarioStepRow[]
+  failure?: ScenarioFailureDetail
+  /** The scenario registration this ran, which the label alone does not give. */
+  scenarioName?: string
+  /** The feature that grouped it, when one did. */
+  feature?: string
+  tags?: string[]
+  /** Images and footage this scenario produced, filed under the run. */
+  artifacts?: ScenarioArtifact[]
+}
+
+/** One scenario that was not run, and why. */
+export interface ScenarioSkip {
+  name: string
+  reason: string
+}
+
+export interface ScenarioRunReport {
+  environment: string
+  results: ScenarioResult[]
+  /**
+   * Scenarios not run at all, each carrying why. A skip is only useful if the
+   * reader can tell a browser scenario held back by `--no-browser` from one the
+   * project itself quarantined, so the reason travels with the name rather than
+   * being assumed by the formatter.
+   */
+  skipped: ScenarioSkip[]
+  /** Feature-level hook failures, which belong to no single scenario. */
+  hookFailures: string[]
+}
+
+export type ScenarioRunStatus = 'running' | 'passed' | 'failed'
+
+/**
+ * A whole run, as it is stored and read back.
+ *
+ * `status` is `running` from the moment the run is created until it finishes,
+ * so a console watching a run in progress can tell "still going" from "nothing
+ * came back" — and a run killed halfway stays `running` forever, which is the
+ * honest record of what happened to it.
+ */
+export interface ScenarioRunRecord extends ScenarioRunReport {
+  runId: string
+  status: ScenarioRunStatus
+  /** The surface the run targeted: `default`, `browser`, … */
+  surface: string
+  startedAt: string
+  finishedAt?: string
+}
+
+/** Enough of a run to list it without loading its steps and stacks. */
+export interface ScenarioRunSummary {
+  runId: string
+  environment: string
+  surface: string
+  status: ScenarioRunStatus
+  startedAt: string
+  finishedAt?: string
+  durationMs?: number
+  passed: number
+  failed: number
+  skipped: number
+  /** Whether anything is there to watch, so a list can say so without a second read. */
+  artifacts: number
+}
+
+/**
+ * Where runs are kept.
+ *
+ * Deliberately not the workflow service: a scenario run is a workflow run under
+ * the hood, but its history has a different lifetime, a different audience, and
+ * a different home — a JSON file beside the project locally, a project-scoped
+ * table and an object store when hosted.
+ *
+ * Written progressively rather than at the end, so a suite that dies on its
+ * fortieth scenario still leaves the thirty-nine behind.
+ */
+export interface ScenarioRunStore {
+  /** Open a run. Called before the first scenario, with `status: 'running'`. */
+  start(record: ScenarioRunRecord): Promise<void>
+  /** Append one finished scenario to an open run. */
+  recordScenario(runId: string, result: ScenarioResult): Promise<void>
+  /**
+   * File the run's artifacts against the scenarios that produced them.
+   *
+   * Separate from `recordScenario` because a scenario's video does not exist
+   * when the scenario ends — it is finalised by the next scenario's reset, and
+   * renamed again when the run closes. Artifacts are matched to a result by the
+   * label the report knows it under.
+   */
+  attachArtifacts(runId: string, artifacts: ScenarioArtifact[]): Promise<void>
+  /** Close a run, settling everything only known once every scenario has run. */
+  finish(
+    runId: string,
+    outcome: Pick<
+      ScenarioRunRecord,
+      'status' | 'finishedAt' | 'skipped' | 'hookFailures'
+    >
+  ): Promise<void>
+  /**
+   * The bytes of one artifact, by the `path` its record carries.
+   *
+   * Reading goes through the store because only the store knows where a run's
+   * files actually are — a directory beside the project, or an object store
+   * behind a bucket. Absent when the run or the path is unknown, which is also
+   * the answer for a path that tries to leave the run.
+   *
+   * The bytes are backed by their own `ArrayBuffer` so they can be handed
+   * straight to a `Response` — a view onto a pooled or shared buffer is not a
+   * valid body.
+   */
+  readArtifact(
+    runId: string,
+    path: string
+  ): Promise<{ body: Uint8Array<ArrayBuffer>; contentType: string } | undefined>
+  /** Most recent first. */
+  list(options?: { limit?: number }): Promise<ScenarioRunSummary[]>
+  get(runId: string): Promise<ScenarioRunRecord | undefined>
+  /** Forget a run and everything it filed. */
+  remove(runId: string): Promise<void>
+}

package/src/wirings/workflow/scenario-step-guards.ts

@@ -3,21 +3,8 @@ import type {
   ScenarioEnvironment,
 } from './scenario-step.types.js'
 
-export const requireActor = <TActor>(
-  scenarioStep: PikkuScenarioStepWire<TActor> | undefined
-): TActor => {
-  const actor = scenarioStep?.actor
-  if (!actor) {
-    throw new Error(
-      `[scenario] step '${scenarioStep?.name ?? 'unknown'}' was called without an actor. ` +
-        `Pass { actor: actors.<name> } so it runs as that persona.`
-    )
-  }
-  return actor
-}
-
 export const requireScenarioEnv = (
-  scenarioStep: PikkuScenarioStepWire<unknown> | undefined
+  scenarioStep: PikkuScenarioStepWire | undefined
 ): ScenarioEnvironment => {
   const env = scenarioStep?.env
   if (!env) {