@pikku/core 0.12.83 → 0.12.86

package/src/wirings/workflow/pikku-scenario-service.ts

@@ -36,10 +36,31 @@ import type {
   WorkflowExpectErrorOptions,
   WorkflowExpectScoreOptions,
   WorkflowExpectServiceOptions,
+  WorkflowStepOptions,
 } from './dsl/workflow-dsl.types.js'
 
+/**
+ * The retry default for the `expect*` family: none, the same as `given`,
+ * `when` and `then` take, and for the same reason — a failed assertion is a
+ * result, not a transient fault.
+ *
+ * Left to the workflow-wide `DEFAULT_STEP_RETRIES` they get six attempts each,
+ * which re-runs the RPC behind a failing `expectError`, restarts a whole
+ * `expectEventually` poll after it has already spent its own deadline, and
+ * lets a live judge re-roll a grade until one lands inside the band. The last
+ * of those is how it was found: a judge scored a deliberately useless answer
+ * 1, and the scenario went green anyway on the next attempt.
+ *
+ * A caller that genuinely wants attempts still asks for them.
+ */
+const assertionStep = <T extends WorkflowStepOptions>(
+  options?: T
+): T & { retries: number } =>
+  ({ ...options, retries: options?.retries ?? 0 }) as T & { retries: number }
+
 export { addFeature, resolveFeatureScenarios } from './feature.js'
 export type * from './scenario.types.js'
+export type * from './scenario-run.types.js'
 export { SCENARIO_SURFACES } from './scenario-step.types.js'
 
 // Which of a step's bindings run: one for an action, every witness for a `then`
@@ -50,7 +71,7 @@ export { createCookieJar } from './scenario-cookie-jar.js'
 export type { ScenarioCookieJar } from './scenario-cookie-jar.js'
 export { composeStepProse, renderStepTemplate } from './scenario-prose.js'
 
-export { requireActor, requireScenarioEnv } from './scenario-step-guards.js'
+export { requireScenarioEnv } from './scenario-step-guards.js'
 
 export type {
   ScenarioHttpResponse,
@@ -171,20 +192,24 @@ addError(ScenarioBrowserUnavailable, {
 })
 
 /**
- * A browser binding was reached without an actor, so there is no persona whose
- * window it would run in.
+ * A step that is driven by a persona was dispatched without one.
+ *
+ * Raised before the body runs rather than partway through it: the step declared
+ * that it needs an actor — by having a `browser` binding, which cannot be
+ * provisioned without a window to open, or by `actor: true` — so a call site
+ * that omits one is a mistake at the call site.
  */
-export class ScenarioBrowserActorRequired extends PikkuError {
+export class ScenarioActorRequired extends PikkuError {
   constructor(public readonly stepFunc: string) {
     super(
-      `[scenario] step '${stepFunc}' resolved to its browser binding but was called without an actor. ` +
-        `Pass { actor: actors.<name> } so the browser signs in as that persona.`
+      `[scenario] step '${stepFunc}' is driven by a persona but was called without an actor. ` +
+        `Pass { actor: actors.<name> } so it runs as that persona.`
     )
   }
 }
-addError(ScenarioBrowserActorRequired, {
+addError(ScenarioActorRequired, {
   status: 500,
-  message: 'A browser binding needs an actor.',
+  message: 'This step needs an actor.',
 })
 
 /**
@@ -480,8 +505,7 @@ export class PikkuScenarioService implements WorkflowRunExtension {
     let wireServices: Record<string, unknown> | undefined
     try {
       wireServices = (await createWireServices?.(singletonServices, wire)) as
-        | Record<string, unknown>
-        | undefined
+        Record<string, unknown> | undefined
       const services =
         wireServices && Object.keys(wireServices).length > 0
           ? { ...singletonServices, ...wireServices }
@@ -602,7 +626,7 @@ export class PikkuScenarioService implements WorkflowRunExtension {
               await new Promise((resolve) => setTimeout(resolve, interval))
             }
           },
-          options
+          assertionStep(options)
         )
       },
 
@@ -645,7 +669,7 @@ export class PikkuScenarioService implements WorkflowRunExtension {
               `[workflow] expectError '${stepName}' ('${resolvedRpcName}') expected an error but the call succeeded: ${JSON.stringify(result)?.slice(0, 300)}`
             )
           },
-          options
+          assertionStep(options)
         )
       },
 
@@ -700,7 +724,7 @@ export class PikkuScenarioService implements WorkflowRunExtension {
               )
             }
           },
-          options
+          assertionStep(options)
         )
       },
 
@@ -745,7 +769,7 @@ export class PikkuScenarioService implements WorkflowRunExtension {
             }
             return grade
           },
-          options
+          assertionStep(options)
         )
       },
 
@@ -814,6 +838,10 @@ export class PikkuScenarioService implements WorkflowRunExtension {
       this.scenarioStepDescription(packageName, resolvedStepFunc) ??
       stepName
 
+    if (!actor && this.requiresActor(packageName, resolvedStepFunc)) {
+      throw new ScenarioActorRequired(resolvedStepFunc)
+    }
+
     const declared = this.declaredSurfaces(packageName, resolvedStepFunc)
     const resolution = resolveScenarioSurfaces(phase, declared, this.runSurface)
 
@@ -828,13 +856,13 @@ export class PikkuScenarioService implements WorkflowRunExtension {
             scenario: workflowWire,
             pikkuUserId: workflowWire.pikkuUserId,
             actors: this.runActors.get(runId),
+            ...(actor ? { actor } : {}),
             scenarioStep: {
               name: resolvedStepFunc,
               stepName,
               runId,
               phase,
               surface,
-              actor,
               env: this.scenarioEnvironment,
             },
           }
@@ -842,11 +870,10 @@ export class PikkuScenarioService implements WorkflowRunExtension {
             if (!this.scenarioBrowserProvider) {
               throw new ScenarioBrowserUnavailable(resolvedStepFunc)
             }
-            if (!actor) {
-              throw new ScenarioBrowserActorRequired(resolvedStepFunc)
-            }
+            // The dispatch guard above already refused an actor-less call, and a
+            // browser binding always requires one.
             wire.browser = await this.scenarioBrowserProvider.sessionFor(
-              actor.name
+              actor!.name
             )
           }
           return await runPikkuFunc(
@@ -947,4 +974,17 @@ export class PikkuScenarioService implements WorkflowRunExtension {
     const surfaces = this.scenarioStepConfig(packageName, stepFunc)?.surfaces
     return surfaces?.length ? surfaces : ['default']
   }
+
+  /**
+   * Whether a step is driven by a persona, and so must be given one. Stamped by
+   * the definer from a `browser` binding or an explicit `actor: true`.
+   */
+  private requiresActor(
+    packageName: string | null,
+    stepFunc: string
+  ): boolean {
+    return (
+      this.scenarioStepConfig(packageName, stepFunc)?.requiresActor === true
+    )
+  }
 }

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -5,11 +5,8 @@ import {
   pikkuState,
 } from '../../pikku-state.js'
 import { getDurationInMilliseconds } from '../../time-utils.js'
-import type {
-  CoreUserSession,
-  PikkuRawWire,
-  SerializedError,
-} from '../../types/core.types.js'
+import type { CoreUserSession, PikkuRawWire } from '../../types/core.types.js'
+import type { SerializedError } from '../../errors/serialized-error.js'
 import type {
   GroupConcurrencyConfig,
   JobGroup,
@@ -39,6 +36,7 @@ import {
   executeGraphStep,
   runWorkflowGraph,
   runFromMeta,
+  stripInstanceOrdinal,
 } from './graph/graph-runner.js'
 import type { WorkflowService } from '../../services/workflow-service.js'
 import type { ScenarioPersonas } from '../../services/personas-service.js'
@@ -1352,9 +1350,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         if (stepName in workflowMeta.nodes) {
           graphNodeId = stepName
         } else {
-          const hash = stepName.lastIndexOf('#')
-          const base = hash > 0 ? stepName.slice(0, hash) : undefined
-          if (base && base in workflowMeta.nodes) graphNodeId = base
+          const base = stripInstanceOrdinal(stepName)
+          if (base !== stepName && base in workflowMeta.nodes)
+            graphNodeId = base
         }
       }
       if (graphNodeId) {

package/src/wirings/workflow/run-timeline.ts

@@ -1,4 +1,4 @@
-import type { SerializedError } from '../../types/core.types.js'
+import type { SerializedError } from '../../errors/serialized-error.js'
 import type { StepState, StepStatus } from './workflow.types.js'
 
 export interface RunTimelineEvent {
@@ -17,7 +17,7 @@ export interface RunTimelineEvent {
 
 export type RunTimeline = RunTimelineEvent[]
 
-type HistoryEntry = StepState & { stepName: string }
+export type HistoryEntry = StepState & { stepName: string }
 
 const LIFECYCLE_ORDER: Record<RunTimelineEvent['type'], number> = {
   pending: 0,

package/src/wirings/workflow/scenario-expectations.test.ts

@@ -65,7 +65,7 @@ describe('workflow.expectError', () => {
     })
 
     await assert.rejects(
-      wire.expectError('should have failed', 'deleteTodo', {}, { retries: 0 }),
+      wire.expectError('should have failed', 'deleteTodo', {}),
       /expected an error but the call succeeded/
     )
   })
@@ -82,7 +82,7 @@ describe('workflow.expectError', () => {
         'wrong reason',
         'deleteTodo',
         {},
-        { matches: /Forbidden/, retries: 0 }
+        { matches: /Forbidden/ }
       ),
       /did not match .*database offline/
     )
@@ -114,10 +114,7 @@ describe('workflow.expectService', () => {
     const { wire } = await scenarioWire(oneEmail())
 
     await assert.rejects(
-      wire.expectService('two emails', 'email.send', {
-        times: 2,
-        retries: 0,
-      }),
+      wire.expectService('two emails', 'email.send', { times: 2 }),
       /expected 2 call\(s\).*found 1.*email\.send/s
     )
   })
@@ -131,7 +128,6 @@ describe('workflow.expectService', () => {
     await assert.rejects(
       wire.expectService('emailed someone else', 'email.send', {
         calledWith: { to: 'x@y.z' },
-        retries: 0,
       }),
       /found 0/
     )
@@ -172,7 +168,7 @@ describe('workflow.expectScore', () => {
     const { wire } = await scenarioWire(grades({ score: 0 }))
 
     await assert.rejects(
-      wire.expectScore('was brief', 'run-1', 'brevity', { retries: 0 }),
+      wire.expectScore('was brief', 'run-1', 'brevity'),
       /expected 'brevity' to grade run run-1 at least 0\.5, got 0/
     )
   })
@@ -183,10 +179,7 @@ describe('workflow.expectScore', () => {
     )
 
     await assert.rejects(
-      wire.expectScore('was brief', 'run-1', 'brevity', {
-        atLeast: 0.8,
-        retries: 0,
-      }),
+      wire.expectScore('was brief', 'run-1', 'brevity', { atLeast: 0.8 }),
       /It rambled for four paragraphs/
     )
   })
@@ -198,12 +191,34 @@ describe('workflow.expectScore', () => {
       wire.expectScore('was not sycophantic', 'run-1', 'sycophancy', {
         atLeast: 0,
         atMost: 0.3,
-        retries: 0,
       }),
       /between 0 and 0\.3, got 0\.95/
     )
   })
 
+  test('a grade that misses the bound is not re-rolled until it lands', async () => {
+    let graded = 0
+    const { wire } = await scenarioWire({
+      rpcWithWire: async () => {
+        graded += 1
+        return { score: 1 }
+      },
+    })
+
+    await assert.rejects(
+      wire.expectScore('was not sycophantic', 'run-1', 'sycophancy', {
+        atLeast: 0,
+        atMost: 0.3,
+      }),
+      /got 1/
+    )
+
+    // The whole point of a judge is that it is not deterministic. Six attempts
+    // at it is six chances for one to land inside the band, which grades the
+    // sampling rather than the run.
+    assert.equal(graded, 1)
+  })
+
   test('a reference answer reaches the grader', async () => {
     let seen: any
     const { wire } = await scenarioWire({

package/src/wirings/workflow/scenario-run.types.ts

@@ -0,0 +1,186 @@
+/**
+ * What one `pikku scenario run` leaves behind.
+ *
+ * A run is history: the scenarios it selected, the sentences they were made of,
+ * and the images and footage they produced. The sentences are snapshotted into
+ * the record rather than looked up later, because a scenario is code and the
+ * code moves — a run from last week describes a suite that no longer exists,
+ * and re-deriving its ladder from today's source would quietly rewrite it.
+ *
+ * Everything here is plain serialisable data. That is what lets one record be
+ * a JSON file on disk in OSS and a row in Postgres on a hosted project, read
+ * back by the same console screen either way.
+ */
+import type { ScenarioBrowserFailure } from './scenario-step.types.js'
+
+/** What a run filed, in the order it filed it. */
+export type ScenarioArtifactKind = 'screenshot' | 'failure' | 'video'
+
+/**
+ * One image or recording a scenario produced.
+ *
+ * `path` is relative to the run's artifact root and always uses forward
+ * slashes: it is a content key, not a filesystem path. Locally it resolves
+ * under the run's directory; on a hosted project it is the object key in a
+ * bucket, and the same record works unchanged.
+ */
+export interface ScenarioArtifact {
+  /** The scenario label this belongs to, as the report names it. */
+  scenario: string
+  kind: ScenarioArtifactKind
+  path: string
+  /** Whose window it was — a two-actor scenario produces two of everything. */
+  actor?: string
+  /** The caption the scenario author took a screenshot under. */
+  name?: string
+}
+
+/** One step of a run, already joined to the prose that declared it. */
+export interface ScenarioStepRow {
+  sentence: string
+  status: string
+  durationMs?: number
+  error?: string
+}
+
+/** Everything known about why one scenario failed. */
+export interface ScenarioFailureDetail {
+  /** The rendered sentence of the failing step; absent if no step failed. */
+  sentence?: string
+  message: string
+  stack?: string
+  /**
+   * True when the failure was a deliberate one (a PikkuError). Its message is
+   * the whole story, so the stack is noise.
+   */
+  expected?: boolean
+  browser?: ScenarioBrowserFailure[]
+}
+
+export interface ScenarioResult {
+  name: string
+  status: 'passed' | 'failed'
+  durationMs: number
+  output?: unknown
+  error?: string
+  steps?: ScenarioStepRow[]
+  failure?: ScenarioFailureDetail
+  /** The scenario registration this ran, which the label alone does not give. */
+  scenarioName?: string
+  /** The feature that grouped it, when one did. */
+  feature?: string
+  tags?: string[]
+  /** Images and footage this scenario produced, filed under the run. */
+  artifacts?: ScenarioArtifact[]
+}
+
+/** One scenario that was not run, and why. */
+export interface ScenarioSkip {
+  name: string
+  reason: string
+}
+
+export interface ScenarioRunReport {
+  environment: string
+  results: ScenarioResult[]
+  /**
+   * Scenarios not run at all, each carrying why. A skip is only useful if the
+   * reader can tell a browser scenario held back by `--no-browser` from one the
+   * project itself quarantined, so the reason travels with the name rather than
+   * being assumed by the formatter.
+   */
+  skipped: ScenarioSkip[]
+  /** Feature-level hook failures, which belong to no single scenario. */
+  hookFailures: string[]
+}
+
+export type ScenarioRunStatus = 'running' | 'passed' | 'failed'
+
+/**
+ * A whole run, as it is stored and read back.
+ *
+ * `status` is `running` from the moment the run is created until it finishes,
+ * so a console watching a run in progress can tell "still going" from "nothing
+ * came back" — and a run killed halfway stays `running` forever, which is the
+ * honest record of what happened to it.
+ */
+export interface ScenarioRunRecord extends ScenarioRunReport {
+  runId: string
+  status: ScenarioRunStatus
+  /** The surface the run targeted: `default`, `browser`, … */
+  surface: string
+  startedAt: string
+  finishedAt?: string
+}
+
+/** Enough of a run to list it without loading its steps and stacks. */
+export interface ScenarioRunSummary {
+  runId: string
+  environment: string
+  surface: string
+  status: ScenarioRunStatus
+  startedAt: string
+  finishedAt?: string
+  durationMs?: number
+  passed: number
+  failed: number
+  skipped: number
+  /** Whether anything is there to watch, so a list can say so without a second read. */
+  artifacts: number
+}
+
+/**
+ * Where runs are kept.
+ *
+ * Deliberately not the workflow service: a scenario run is a workflow run under
+ * the hood, but its history has a different lifetime, a different audience, and
+ * a different home — a JSON file beside the project locally, a project-scoped
+ * table and an object store when hosted.
+ *
+ * Written progressively rather than at the end, so a suite that dies on its
+ * fortieth scenario still leaves the thirty-nine behind.
+ */
+export interface ScenarioRunStore {
+  /** Open a run. Called before the first scenario, with `status: 'running'`. */
+  start(record: ScenarioRunRecord): Promise<void>
+  /** Append one finished scenario to an open run. */
+  recordScenario(runId: string, result: ScenarioResult): Promise<void>
+  /**
+   * File the run's artifacts against the scenarios that produced them.
+   *
+   * Separate from `recordScenario` because a scenario's video does not exist
+   * when the scenario ends — it is finalised by the next scenario's reset, and
+   * renamed again when the run closes. Artifacts are matched to a result by the
+   * label the report knows it under.
+   */
+  attachArtifacts(runId: string, artifacts: ScenarioArtifact[]): Promise<void>
+  /** Close a run, settling everything only known once every scenario has run. */
+  finish(
+    runId: string,
+    outcome: Pick<
+      ScenarioRunRecord,
+      'status' | 'finishedAt' | 'skipped' | 'hookFailures'
+    >
+  ): Promise<void>
+  /**
+   * The bytes of one artifact, by the `path` its record carries.
+   *
+   * Reading goes through the store because only the store knows where a run's
+   * files actually are — a directory beside the project, or an object store
+   * behind a bucket. Absent when the run or the path is unknown, which is also
+   * the answer for a path that tries to leave the run.
+   *
+   * The bytes are backed by their own `ArrayBuffer` so they can be handed
+   * straight to a `Response` — a view onto a pooled or shared buffer is not a
+   * valid body.
+   */
+  readArtifact(
+    runId: string,
+    path: string
+  ): Promise<{ body: Uint8Array<ArrayBuffer>; contentType: string } | undefined>
+  /** Most recent first. */
+  list(options?: { limit?: number }): Promise<ScenarioRunSummary[]>
+  get(runId: string): Promise<ScenarioRunRecord | undefined>
+  /** Forget a run and everything it filed. */
+  remove(runId: string): Promise<void>
+}

package/src/wirings/workflow/scenario-step-guards.ts

@@ -3,21 +3,8 @@ import type {
   ScenarioEnvironment,
 } from './scenario-step.types.js'
 
-export const requireActor = <TActor>(
-  scenarioStep: PikkuScenarioStepWire<TActor> | undefined
-): TActor => {
-  const actor = scenarioStep?.actor
-  if (!actor) {
-    throw new Error(
-      `[scenario] step '${scenarioStep?.name ?? 'unknown'}' was called without an actor. ` +
-        `Pass { actor: actors.<name> } so it runs as that persona.`
-    )
-  }
-  return actor
-}
-
 export const requireScenarioEnv = (
-  scenarioStep: PikkuScenarioStepWire<unknown> | undefined
+  scenarioStep: PikkuScenarioStepWire | undefined
 ): ScenarioEnvironment => {
   const env = scenarioStep?.env
   if (!env) {

package/src/wirings/workflow/scenario-step.test.ts

@@ -8,7 +8,7 @@ import { addFunction } from '../../function/function-runner.js'
 import type { ScenarioPersona } from '../../services/personas-service.js'
 import type { PikkuWire } from '../../types/core.types.js'
 import type { ScenarioSurface } from './scenario-step.types.js'
-import { requireActor, requireScenarioEnv } from './scenario-step-guards.js'
+import { requireScenarioEnv } from './scenario-step-guards.js'
 
 const noopLogger = { error() {}, info() {}, warn() {}, debug() {} }
 
@@ -145,6 +145,7 @@ const registerStep = (
   config: {
     description?: string
     surfaces?: ScenarioSurface[]
+    requiresActor?: boolean
     func: (services: any, data: any, wire: PikkuWire) => Promise<unknown>
   }
 ) => {
@@ -215,7 +216,7 @@ describe('pikkuScenarioStep (scenario.given/when/then)', () => {
     let received: unknown
     registerStep('checksOut', {
       func: async (_services, _data, wire) => {
-        received = wire.scenarioStep!.actor
+        received = wire.actor
         return null
       },
     })
@@ -712,16 +713,17 @@ describe('the scenario environment reaches the step wire', () => {
   })
 })
 
-describe('requireActor / requireScenarioEnv', () => {
+describe('a step driven by a persona is given its actor', () => {
   beforeEach(() => resetPikkuState())
 
-  test('requireActor returns the actor a step was called with', async () => {
+  test('the actor is on the wire, not something the body unwraps', async () => {
     const { workflowService: ws, scenarioService } = createScenarioRunner()
     const shopper = fakeActor('shopper', async () => null)
     let resolved: unknown
     registerStep('needsAnActor', {
+      requiresActor: true,
       func: async (_services, _data, wire) => {
-        resolved = requireActor(wire.scenarioStep)
+        resolved = wire.actor
         return null
       },
     })
@@ -735,10 +737,15 @@ describe('requireActor / requireScenarioEnv', () => {
     assert.equal(resolved, shopper)
   })
 
-  test('requireActor names the step when it was called without one', async () => {
+  test('one dispatched without an actor is refused before its body runs', async () => {
     const { workflowService: ws, scenarioService } = createScenarioRunner()
+    let bodyRan = false
     registerStep('needsAnActor', {
-      func: async (_services, _data, wire) => requireActor(wire.scenarioStep),
+      requiresActor: true,
+      func: async () => {
+        bodyRan = true
+        return null
+      },
     })
 
     const runId = await setup(ws)
@@ -747,6 +754,24 @@ describe('requireActor / requireScenarioEnv', () => {
       wire.when('nobody acts', 'needsAnActor'),
       /needsAnActor.*actor/s
     )
+    assert.equal(bodyRan, false)
+  })
+
+  test('a step that declares no persona runs without one', async () => {
+    const { workflowService: ws, scenarioService } = createScenarioRunner()
+    let seen: unknown = 'unset'
+    registerStep('assertsSomething', {
+      func: async (_services, _data, wire) => {
+        seen = wire.actor
+        return null
+      },
+    })
+
+    const runId = await setup(ws)
+    const wire = ws.createWorkflowWire('scenarioTest', runId, {})
+    await wire.then('it holds', 'assertsSomething')
+
+    assert.equal(seen, undefined)
   })
 
   test('requireScenarioEnv returns the environment, or says how to declare one', async () => {