@pikku/core 0.12.80 → 0.12.83

package/src/wirings/workflow/scenario-expectations.test.ts

@@ -146,3 +146,78 @@ describe('workflow.expectService', () => {
     )
   })
 })
+
+describe('workflow.expectScore', () => {
+  beforeEach(() => resetPikkuState())
+
+  const grades = (grade: { score: number; reason?: string }) => ({
+    rpcWithWire: async (rpcName: string, data: any) => {
+      assert.equal(rpcName, 'pikkuScenarioGradeRun')
+      assert.equal(data.runId, 'run-1')
+      return grade
+    },
+  })
+
+  test('returns the grade when the score clears the bound', async () => {
+    const { wire } = await scenarioWire(grades({ score: 0.9 }))
+
+    const grade = await wire.expectScore('was brief', 'run-1', 'brevity', {
+      atLeast: 0.8,
+    })
+
+    assert.equal(grade.score, 0.9)
+  })
+
+  test('a scorer that graded zero fails without a bound being stated', async () => {
+    const { wire } = await scenarioWire(grades({ score: 0 }))
+
+    await assert.rejects(
+      wire.expectScore('was brief', 'run-1', 'brevity', { retries: 0 }),
+      /expected 'brevity' to grade run run-1 at least 0\.5, got 0/
+    )
+  })
+
+  test('the failure carries the reason the judge gave', async () => {
+    const { wire } = await scenarioWire(
+      grades({ score: 0.2, reason: 'It rambled for four paragraphs' })
+    )
+
+    await assert.rejects(
+      wire.expectScore('was brief', 'run-1', 'brevity', {
+        atLeast: 0.8,
+        retries: 0,
+      }),
+      /It rambled for four paragraphs/
+    )
+  })
+
+  test('atMost fails a score that is too high', async () => {
+    const { wire } = await scenarioWire(grades({ score: 0.95 }))
+
+    await assert.rejects(
+      wire.expectScore('was not sycophantic', 'run-1', 'sycophancy', {
+        atLeast: 0,
+        atMost: 0.3,
+        retries: 0,
+      }),
+      /between 0 and 0\.3, got 0\.95/
+    )
+  })
+
+  test('a reference answer reaches the grader', async () => {
+    let seen: any
+    const { wire } = await scenarioWire({
+      rpcWithWire: async (_rpcName: string, data: any) => {
+        seen = data
+        return { score: 1 }
+      },
+    })
+
+    await wire.expectScore('matched the answer key', 'run-1', 'correctness', {
+      reference: 'Paris',
+    })
+
+    assert.equal(seen.reference, 'Paris')
+    assert.equal(seen.scorer, 'correctness')
+  })
+})

package/src/wirings/workflow/scenario-hooks.test.ts

@@ -185,8 +185,9 @@ describe('scenario before/after hooks', () => {
   })
 
   test('after reads what the body wrote to the scenario context', async () => {
-    // The reason the context exists: teardown needs the ids the body minted,
-    // and a hook only ever receives the run's *input*.
+    // The reason the context exists: a hook's `data` is only ever the run's
+    // *input*, never what the body produced. The context is the channel the
+    // body writes that through, and it reaches the hook on the shared wire.
     let seen: any
     await runScenario('contextHandover', {
       after: async (_services, _data, wire) => {

package/src/wirings/workflow/scenario-step.types.ts

@@ -229,6 +229,14 @@ export interface ScenarioBrowserProvider {
    * scenario cannot leave the next signed in as somebody else.
    */
   reset?(): Promise<void>
+  /**
+   * Name the scenario about to run, so anything it captures is filed under it.
+   *
+   * Called before each scenario. A provider that is never told has to fall
+   * back to one shared label, which puts every run's artifacts in a single
+   * folder — findable only by timestamp.
+   */
+  beginScenario?(scenario: string): void
   /**
    * Snapshot every open window for a failed scenario. `label` identifies the
    * scenario in artifact filenames. Never throws: a failure to capture must

package/src/wirings/workflow/scenario.types.ts

@@ -0,0 +1,63 @@
+import type {
+  CorePikkuFunctionConfig,
+  CorePikkuFunctionHook,
+} from '../../function/functions.types.js'
+
+export type {
+  ScenarioStepInvocation,
+  ScenarioStepMeta,
+  PikkuScenarioWire,
+} from './dsl/workflow-dsl.types.js'
+
+export type {
+  ScenarioStepPhase,
+  ScenarioStepKind,
+  ScenarioStepOptions,
+  PikkuScenarioStepWire,
+  ScenarioEnvironment,
+  ScenarioSurface,
+  ScenarioSurfaceResolution,
+  PikkuBrowserWire,
+  TestIdSelector,
+  ScenarioBrowserProvider,
+  ScenarioBrowserFailure,
+} from './scenario-step.types.js'
+
+export type CoreFeatureScenario =
+  | CorePikkuFunctionConfig<any, any, any>
+  | { scenario: CorePikkuFunctionConfig<any, any, any>; data: unknown }
+
+export type CoreFeature = {
+  name: string
+  description?: string
+  tags?: string[]
+  scenarios: readonly CoreFeatureScenario[]
+  before?: CorePikkuFunctionHook
+  after?: CorePikkuFunctionHook
+}
+
+export type FeatureMetaEntry = {
+  scenario: string
+  data?: unknown
+}
+
+export type FeatureMeta = {
+  id: string
+  name: string
+  description?: string
+  tags: string[]
+  entries: FeatureMetaEntry[]
+  unresolvedEntries: number
+  hasBefore: boolean
+  hasAfter: boolean
+}
+
+export type FeaturesMeta = Record<string, FeatureMeta>
+
+export type FeaturePlanEntry = {
+  featureId: string
+  featureName: string
+  scenarioName: string
+  data?: unknown
+  tags: string[]
+}

package/src/wirings/workflow/workflow-approval-audit.ts

@@ -0,0 +1,47 @@
+import { getSingletonServices } from '../../pikku-state.js'
+import type { ApprovalAuditEvent } from './workflow-approval.js'
+
+/** The audit type an answer to an approval gate is recorded under. */
+export const APPROVAL_AUDIT_TYPE = 'workflow.approval.decided'
+
+/**
+ * Record an answer to an approval gate where it outlives the run.
+ *
+ * The step result carries the settled decision, but it is deleted with the run
+ * — `deleteRun` cascades to steps and to history — and a refused attempt never
+ * reaches a step at all. An approval is asked for precisely so it can be
+ * answered for afterwards, so the answer also goes to the audit sink, which
+ * holds no foreign key to the run.
+ *
+ * A project with no sink wired records nothing, and a sink that fails is logged
+ * rather than thrown: the trail must not be the reason a decision is lost.
+ */
+export const auditApprovalDecision = async (
+  event: ApprovalAuditEvent
+): Promise<void> => {
+  const services = getSingletonServices()
+  if (!services?.audit) {
+    return
+  }
+  try {
+    await services.audit.audit({
+      type: APPROVAL_AUDIT_TYPE,
+      source: 'explicit',
+      outcome: event.outcome,
+      occurredAt: new Date().toISOString(),
+      wireType: 'workflow',
+      userIdentity: { pikkuUserId: event.decidedBy?.userId },
+      metadata: {
+        runId: event.runId,
+        reason: event.reason,
+        scopes: event.decidedBy?.scopes,
+        refusal: event.refusal,
+      },
+    })
+  } catch (error) {
+    services.logger?.warn(
+      `Failed to audit the decision on approval '${event.reason}' for run ${event.runId}`,
+      error
+    )
+  }
+}