@pikku/core 0.12.79 → 0.12.82

package/src/wirings/workflow/scenario-expectations.test.ts

@@ -146,3 +146,78 @@ describe('workflow.expectService', () => {
     )
   })
 })
+
+describe('workflow.expectScore', () => {
+  beforeEach(() => resetPikkuState())
+
+  const grades = (grade: { score: number; reason?: string }) => ({
+    rpcWithWire: async (rpcName: string, data: any) => {
+      assert.equal(rpcName, 'pikkuScenarioGradeRun')
+      assert.equal(data.runId, 'run-1')
+      return grade
+    },
+  })
+
+  test('returns the grade when the score clears the bound', async () => {
+    const { wire } = await scenarioWire(grades({ score: 0.9 }))
+
+    const grade = await wire.expectScore('was brief', 'run-1', 'brevity', {
+      atLeast: 0.8,
+    })
+
+    assert.equal(grade.score, 0.9)
+  })
+
+  test('a scorer that graded zero fails without a bound being stated', async () => {
+    const { wire } = await scenarioWire(grades({ score: 0 }))
+
+    await assert.rejects(
+      wire.expectScore('was brief', 'run-1', 'brevity', { retries: 0 }),
+      /expected 'brevity' to grade run run-1 at least 0\.5, got 0/
+    )
+  })
+
+  test('the failure carries the reason the judge gave', async () => {
+    const { wire } = await scenarioWire(
+      grades({ score: 0.2, reason: 'It rambled for four paragraphs' })
+    )
+
+    await assert.rejects(
+      wire.expectScore('was brief', 'run-1', 'brevity', {
+        atLeast: 0.8,
+        retries: 0,
+      }),
+      /It rambled for four paragraphs/
+    )
+  })
+
+  test('atMost fails a score that is too high', async () => {
+    const { wire } = await scenarioWire(grades({ score: 0.95 }))
+
+    await assert.rejects(
+      wire.expectScore('was not sycophantic', 'run-1', 'sycophancy', {
+        atLeast: 0,
+        atMost: 0.3,
+        retries: 0,
+      }),
+      /between 0 and 0\.3, got 0\.95/
+    )
+  })
+
+  test('a reference answer reaches the grader', async () => {
+    let seen: any
+    const { wire } = await scenarioWire({
+      rpcWithWire: async (_rpcName: string, data: any) => {
+        seen = data
+        return { score: 1 }
+      },
+    })
+
+    await wire.expectScore('matched the answer key', 'run-1', 'correctness', {
+      reference: 'Paris',
+    })
+
+    assert.equal(seen.reference, 'Paris')
+    assert.equal(seen.scorer, 'correctness')
+  })
+})

package/src/wirings/workflow/scenario-hooks.test.ts

@@ -184,6 +184,58 @@ describe('scenario before/after hooks', () => {
     ])
   })
 
+  test('after reads what the body wrote to the scenario context', async () => {
+    // The reason the context exists: a hook's `data` is only ever the run's
+    // *input*, never what the body produced. The context is the channel the
+    // body writes that through, and it reaches the hook on the shared wire.
+    let seen: any
+    await runScenario('contextHandover', {
+      after: async (_services, _data, wire) => {
+        seen = { ...wire.scenario.context }
+      },
+      func: async (_services, _data, wire) => {
+        wire.scenario.context.projectId = 'p_1'
+        return { projectId: 'p_1' }
+      },
+    })
+    assert.deepEqual(seen, { projectId: 'p_1' })
+  })
+
+  test('after sees the context of a body that threw after writing it', async () => {
+    // The failing path is the one that matters — a passing run could have
+    // returned the ids instead.
+    let seen: any
+    const { error } = await runScenario('contextOnFailure', {
+      after: async (_services, _data, wire) => {
+        seen = { ...wire.scenario.context }
+      },
+      func: async (_services, _data, wire) => {
+        wire.scenario.context.projectId = 'p_2'
+        throw new Error('body blew up')
+      },
+    })
+    assert.equal(error?.message, 'body blew up')
+    assert.deepEqual(seen, { projectId: 'p_2' })
+  })
+
+  test('before, the body and after all share one context object', async () => {
+    let seen: any
+    await runScenario('contextShared', {
+      before: async (_services, _data, wire) => {
+        wire.scenario.context.seededBy = 'before'
+      },
+      after: async (_services, _data, wire) => {
+        seen = { ...wire.scenario.context }
+      },
+      func: async (_services, _data, wire) => {
+        assert.equal(wire.scenario.context.seededBy, 'before')
+        wire.scenario.context.addedBy = 'func'
+        return { ok: true }
+      },
+    })
+    assert.deepEqual(seen, { seededBy: 'before', addedBy: 'func' })
+  })
+
   test('hooks are scenario-only — a plain DSL workflow never runs them', async () => {
     const order: string[] = []
     await runScenario(

package/src/wirings/workflow/scenario-step.types.ts

@@ -229,6 +229,14 @@ export interface ScenarioBrowserProvider {
    * scenario cannot leave the next signed in as somebody else.
    */
   reset?(): Promise<void>
+  /**
+   * Name the scenario about to run, so anything it captures is filed under it.
+   *
+   * Called before each scenario. A provider that is never told has to fall
+   * back to one shared label, which puts every run's artifacts in a single
+   * folder — findable only by timestamp.
+   */
+  beginScenario?(scenario: string): void
   /**
    * Snapshot every open window for a failed scenario. `label` identifies the
    * scenario in artifact filenames. Never throws: a failure to capture must

package/src/wirings/workflow/workflow-approval-audit.ts

@@ -0,0 +1,47 @@
+import { getSingletonServices } from '../../pikku-state.js'
+import type { ApprovalAuditEvent } from './workflow-approval.js'
+
+/** The audit type an answer to an approval gate is recorded under. */
+export const APPROVAL_AUDIT_TYPE = 'workflow.approval.decided'
+
+/**
+ * Record an answer to an approval gate where it outlives the run.
+ *
+ * The step result carries the settled decision, but it is deleted with the run
+ * — `deleteRun` cascades to steps and to history — and a refused attempt never
+ * reaches a step at all. An approval is asked for precisely so it can be
+ * answered for afterwards, so the answer also goes to the audit sink, which
+ * holds no foreign key to the run.
+ *
+ * A project with no sink wired records nothing, and a sink that fails is logged
+ * rather than thrown: the trail must not be the reason a decision is lost.
+ */
+export const auditApprovalDecision = async (
+  event: ApprovalAuditEvent
+): Promise<void> => {
+  const services = getSingletonServices()
+  if (!services?.audit) {
+    return
+  }
+  try {
+    await services.audit.audit({
+      type: APPROVAL_AUDIT_TYPE,
+      source: 'explicit',
+      outcome: event.outcome,
+      occurredAt: new Date().toISOString(),
+      wireType: 'workflow',
+      userIdentity: { pikkuUserId: event.decidedBy?.userId },
+      metadata: {
+        runId: event.runId,
+        reason: event.reason,
+        scopes: event.decidedBy?.scopes,
+        refusal: event.refusal,
+      },
+    })
+  } catch (error) {
+    services.logger?.warn(
+      `Failed to audit the decision on approval '${event.reason}' for run ${event.runId}`,
+      error
+    )
+  }
+}