@pikku/core 0.12.72 → 0.12.77

package/src/wirings/virtual-user/virtual-user-rng.ts

@@ -0,0 +1,50 @@
+/**
+ * A seeded PRNG, so a run replays exactly.
+ *
+ * Reproducibility is the whole point: a virtual user that finds something has
+ * to be re-runnable into the same finding before it can be shrunk into a
+ * regression scenario. `Math.random` would make every finding a one-off story.
+ *
+ * mulberry32 — 32 bits of state, uniform enough for scheduling weights, and
+ * short enough to read.
+ */
+export const createRng = (seed: number) => {
+  let state = seed >>> 0
+  const next = () => {
+    state = (state + 0x6d2b79f5) >>> 0
+    let t = state
+    t = Math.imul(t ^ (t >>> 15), t | 1)
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296
+  }
+  return {
+    next,
+    /** True with probability `p`. */
+    chance: (p: number) => next() < p,
+    /** A uniform pick, or undefined for an empty list. */
+    pick: <T>(items: readonly T[]): T | undefined =>
+      items.length === 0 ? undefined : items[Math.floor(next() * items.length)],
+    /**
+     * A weighted pick over `[key, weight]` pairs. Zero-weight keys can never be
+     * drawn, which is how a disposition switches a move off entirely.
+     */
+    weighted: <K extends string>(
+      weights: Readonly<Record<K, number>>
+    ): K | undefined => {
+      const entries = (Object.entries(weights) as [K, number][]).filter(
+        ([, weight]) => weight > 0
+      )
+      const total = entries.reduce((sum, [, weight]) => sum + weight, 0)
+      if (total <= 0) return undefined
+      let roll = next() * total
+      for (const [key, weight] of entries) {
+        roll -= weight
+        if (roll <= 0) return key
+      }
+      return entries[entries.length - 1]?.[0]
+    },
+  }
+}
+
+/** The seeded generator the engine threads through a whole run. */
+export type VirtualUserRng = ReturnType<typeof createRng>

package/src/wirings/virtual-user/virtual-user-run-store.ts

@@ -0,0 +1,98 @@
+import type {
+  VirtualUserDisposition,
+  VirtualUserFinding,
+  VirtualUserTally,
+} from './virtual-user.types.js'
+
+/**
+ * One recorded run: who ran, what they were told, and what came back.
+ *
+ * A run is dispatched and answered for later, so the record is created before
+ * the work starts and is the thing the returned `runId` addresses.
+ *
+ * This record is a run's ONLY trace. A virtual user is not a workflow — it
+ * explores, so no two attempts take the same steps and there is nothing to
+ * replay — and it is not queued either, because the record already carries the
+ * progress a queue would only be holding on the way here.
+ *
+ * The cost of that is the one thing to know when reading `status`: a restart
+ * mid-run leaves a record at `running` with nothing left to finish it. A run
+ * older than its budget window and still `running` is dead, not working.
+ */
+export interface VirtualUserRunRecord {
+  runId: string
+  persona: string
+  disposition: VirtualUserDisposition
+  /** What makes a run replayable at all — a finding without it is an anecdote. */
+  seed: number
+  /**
+   * `running` until the engine returns. Not derived from `finishedAt` being
+   * unset: a crashed run has no finish time either, and the two are not the
+   * same result.
+   */
+  status: 'running' | 'completed' | 'failed'
+  /** The caller's situational goals, run alongside the derived intents. */
+  goals: string[]
+  /**
+   * Ids and slugs the user carried in, and whatever it learned on the way out.
+   * Kept because a finding only reproduces alongside the notes that produced it.
+   */
+  memory: Record<string, string>
+  findings: VirtualUserFinding[]
+  tally: VirtualUserTally | null
+  /** Which budget or stopping rule ended the run. */
+  stoppedBy: string | null
+  /**
+   * Why the run itself failed, as opposed to what it found. A run that could
+   * not start has no findings and is not a clean empty result.
+   */
+  error: string | null
+  /** The session that started it, where the host tracks one. */
+  startedBy: string | null
+  createdAt: Date
+  finishedAt: Date | null
+}
+
+/** What a run is created with — everything else is filled in by the outcome. */
+export interface VirtualUserRunStart {
+  persona: string
+  disposition: VirtualUserDisposition
+  seed: number
+  goals?: readonly string[]
+  memory?: Record<string, string>
+  startedBy?: string | null
+}
+
+/** The outcome of a run that reached the end of its budget without throwing. */
+export interface VirtualUserRunOutcome {
+  findings: readonly VirtualUserFinding[]
+  tally: VirtualUserTally
+  memory: Record<string, string>
+  stoppedBy: string | null
+}
+
+/**
+ * Where runs are kept. Declared here rather than in a database package so the
+ * scaffolded RPCs depend on the shape and not on kysely — `@pikku/kysely` ships
+ * one implementation, and an app with its own store satisfies this instead.
+ *
+ * SECURITY: findings from an `adversarial` run are working exploits carrying
+ * live ids. An implementation is a privileged store; the scaffold gates every
+ * read behind a scope for that reason, and a host exposing these records more
+ * widely is publishing its own exploits.
+ */
+export interface VirtualUserRunStore {
+  /** Records a run as `running` and returns its id. */
+  start(run: VirtualUserRunStart): Promise<string>
+  /** Marks a run `completed` and stores what it found. */
+  complete(runId: string, outcome: VirtualUserRunOutcome): Promise<void>
+  /** Marks a run `failed`. The run itself broke; it has no findings. */
+  fail(runId: string, error: string): Promise<void>
+  get(runId: string): Promise<VirtualUserRunRecord | null>
+  /** Newest first. `persona` narrows to one persona's history. */
+  list(options?: {
+    persona?: string
+    limit?: number
+    offset?: number
+  }): Promise<VirtualUserRunRecord[]>
+}

package/src/wirings/virtual-user/virtual-user-target.ts

@@ -0,0 +1,47 @@
+import type { ScenarioPersona } from '../../services/personas-service.js'
+import type { VirtualUserTarget } from './virtual-user.types.js'
+
+export interface PersonaTargetOptions {
+  /** Model the persona uses for its side of a conversation with an agent. */
+  model?: string
+  /** Agents this user may talk to. Omit to leave `talkTo` unavailable. */
+  agents?: readonly string[]
+}
+
+/**
+ * Drive a virtual user through a signed-in persona.
+ *
+ * This is the only place the engine meets the network. A persona already signs
+ * in as a real user over the app's own auth, keeps its cookies, re-logs-in when
+ * a session expires mid-run and reports a status as data rather than throwing —
+ * everything a user imitating a user needs, and none of it worth building twice.
+ *
+ * `invokeRaw` rather than `invoke` matters: to a virtual user a 403 is not an
+ * error, it is the system saying no, which is information it should be allowed
+ * to act on.
+ */
+export const personaVirtualUserTarget = (
+  persona: ScenarioPersona,
+  { model, agents }: PersonaTargetOptions = {}
+): VirtualUserTarget => ({
+  call: (rpcName, args) => persona.invokeRaw(rpcName, args),
+  talkTo: agents?.length
+    ? async (agent, task) => {
+        if (!agents.includes(agent)) {
+          return {
+            passed: false,
+            reasoning: `There is no assistant called '${agent}'.`,
+            transcript: [],
+          }
+        }
+        return persona.converse({
+          agent,
+          task,
+          // A user asking for something judges the answer by whether they got
+          // what they asked for. There is no separate criterion to hold it to.
+          evaluate: `The assistant did what was asked: ${task}`,
+          model,
+        })
+      }
+    : undefined,
+})

package/src/wirings/virtual-user/virtual-user.types.ts

@@ -0,0 +1,219 @@
+import type { ScenarioHttpResponse } from '../../services/personas-service.js'
+import type { ActorFlowVerdict } from '../actor-flow/actor-flow.types.js'
+
+/**
+ * How a virtual user behaves, mechanically. A disposition changes the loop, the
+ * input generation, the memory and the oracle — anything that only changes how
+ * the user *talks* is persona prose (`ResolvedPersona.personality`) and does
+ * not belong here.
+ *
+ * - `realistic` — schema-first, goal-directed, plausible values (default).
+ * - `careless` — repeats, abandons, double-submits, boundary values.
+ * - `newcomer` — starts with empty memory and must discover everything.
+ * - `stale` — seeded from an earlier run's ids, some no longer valid.
+ * - `auditor` — pursues no goal; reads one truth from several places and compares.
+ * - `adversarial` — probes authorization and tenancy; a 2xx can be the finding.
+ * - `accountable` — pursues its goals for real; the only one production allows.
+ *
+ * `accountable` sits opposite `adversarial` on an *intent* axis — bad faith
+ * against good faith — where `careless` and `thorough` are a *care* axis. It is
+ * the one disposition whose actions carry consequences: no oracle, no rollback,
+ * and every call attributed to the persona in the audit log like anyone else's.
+ */
+export type VirtualUserDisposition =
+  | 'realistic'
+  | 'careless'
+  | 'newcomer'
+  | 'stale'
+  | 'auditor'
+  | 'adversarial'
+  | 'accountable'
+
+/**
+ * The one disposition a production environment accepts.
+ *
+ * Every other disposition exists to find out what the product does wrong, which
+ * is not a thing to do to real customers' data. Named here so the declaration
+ * check and the sign-in check cannot drift apart.
+ */
+export const PRODUCTION_DISPOSITION: VirtualUserDisposition = 'accountable'
+
+/**
+ * Hard caps the engine enforces itself. Anything the engine cannot count —
+ * money above all — belongs in {@link RunVirtualUserParams.stop} instead.
+ */
+export interface VirtualUserBudget {
+  /** Model turns, including ones that produced no call. */
+  steps?: number
+  /** Non-read RPC calls. */
+  mutations?: number
+  /** Wall clock, as ms or a duration string (`'30m'`). */
+  duration?: number | string
+}
+
+/**
+ * The running count handed to {@link RunVirtualUserParams.stop}. Pikku collects
+ * it because only the engine sees every step; what it is *worth* is the app's,
+ * which is why no price or currency appears anywhere in core.
+ */
+export interface VirtualUserTally {
+  steps: number
+  calls: number
+  mutations: number
+  tokensIn: number
+  tokensOut: number
+  model: string
+  elapsedMs: number
+  findings: number
+}
+
+/**
+ * One RPC as the virtual user sees it before reading its schema: enough to
+ * choose from, not enough to call with. The whole catalogue goes in the prompt,
+ * so this stays deliberately compact — a project with 430 RPCs renders to
+ * roughly 8k tokens, which is cheaper than any retrieval step and hides nothing.
+ */
+export interface ApiCatalogueEntry {
+  name: string
+  /** Top-level input property names, for choosing between similar endpoints. */
+  inputKeys?: readonly string[]
+  /** Top-level output property names. */
+  outputKeys?: readonly string[]
+  description?: string
+  /**
+   * Whether the call only reads. `undefined` means nobody annotated it — which
+   * is not the same as safe, so the engine falls back to a name heuristic and
+   * reports how many entries it had to guess at.
+   */
+  readonly?: boolean
+  /** Declared by the app as needing a human's approval — denied by default. */
+  approvalRequired?: boolean
+  /**
+   * Scopes the session must hold to call this, all of them.
+   *
+   * Splits the catalogue in two: what a persona's roles entitle it to reach,
+   * and what they do not. Scopes rather than permission-function names because
+   * a scope is declared and checkable, whereas a permission is arbitrary code
+   * whose verdict nobody can predict without running it.
+   */
+  scopes?: readonly string[]
+  tags?: readonly string[]
+  /** JSON schema returned by `describe`, before the user may call. */
+  inputSchema?: Record<string, unknown>
+  outputSchema?: Record<string, unknown>
+}
+
+/**
+ * Where an intent comes from: a feature or scenario in the app's own words.
+ *
+ * Deliberately prose. The virtual user never receives the scenario's step graph
+ * — no rpc names, no `$ref` dataflow — because finding the API itself is the
+ * behaviour under test, and those graphs are a build-time artifact that no
+ * deployed stage carries.
+ */
+export interface IntentSource {
+  id: string
+  title: string
+  description?: string
+  /** Step prose, as a person would be told it. Never rpc names. */
+  steps?: readonly string[]
+  tags?: readonly string[]
+  /** Persona ids this intent is available to. Empty means everyone. */
+  personas?: readonly string[]
+}
+
+/** Why the engine flagged something. */
+export type VirtualUserFindingKind =
+  | 'server-error'
+  | 'transport-error'
+  | 'schema-violation'
+  | 'unexpected-success'
+  | 'custom'
+
+/** Something worth a human's attention, found while imitating a user. */
+export interface VirtualUserFinding {
+  kind: VirtualUserFindingKind
+  detail: string
+  rpcName?: string
+  status?: number
+  intentId?: string
+  /** Step index within the run, so the transcript can be replayed to here. */
+  step: number
+}
+
+/**
+ * The transport the virtual user acts through. HTTP in production (an
+ * `HttpPersona`), but the engine only sees this contract — which is what
+ * lets the whole loop be tested without a network.
+ */
+export interface VirtualUserTarget {
+  /** Invoke an RPC as this user, reporting the status as data rather than throwing. */
+  call(rpcName: string, args: unknown): Promise<ScenarioHttpResponse>
+  /** Converse with an AI agent in persona. Omitted by apps that have no agents. */
+  talkTo?(agent: string, task: string): Promise<ActorFlowVerdict>
+  /** Send a fixture file. Omitted by apps with no upload surface. */
+  upload?(file: string): Promise<ScenarioHttpResponse>
+}
+
+/** What the model chose to do this turn. */
+export type VirtualUserAction =
+  | { kind: 'describe'; rpcName: string }
+  | { kind: 'call'; rpcName: string; args?: unknown }
+  | { kind: 'talkTo'; agent: string; task: string }
+  | { kind: 'upload'; file: string }
+  | { kind: 'complete'; summary?: string }
+  | { kind: 'stuck'; reason?: string }
+
+/** How an intent ended, or that it has not. */
+export type IntentStatus =
+  | 'open'
+  | 'suspended'
+  | 'completed'
+  | 'abandoned'
+  | 'stuck'
+
+/** One intent's life within a run. */
+export interface IntentRecord {
+  id: string
+  sourceId: string
+  title: string
+  status: IntentStatus
+  /** Step indices at which this intent was the active one. */
+  steps: number[]
+  /** How many times it was put down and picked back up. */
+  suspensions: number
+  summary?: string
+}
+
+/** One turn: what the engine scheduled, what the model did, what came back. */
+export interface StepRecord {
+  index: number
+  intentId?: string
+  action: VirtualUserAction | { kind: 'invalid'; detail: string }
+  status?: number
+  ok?: boolean
+  /** Truncated response text, so a run record stays readable. */
+  response?: string
+  findingKinds?: VirtualUserFindingKind[]
+  tokensIn: number
+  tokensOut: number
+}
+
+/** Everything a run produced. Seeded, so it replays. */
+export interface VirtualUserRunResult {
+  seed: number
+  tally: VirtualUserTally
+  findings: VirtualUserFinding[]
+  intents: IntentRecord[]
+  steps: StepRecord[]
+  /** Ids and slugs the user saw, for seeding a later `stale` run. */
+  memory: Record<string, string>
+  /** Why the run ended. */
+  stoppedBy:
+    | 'no-intents'
+    | 'budget-steps'
+    | 'budget-mutations'
+    | 'budget-duration'
+    | 'stop-hook'
+    | 'exhausted'
+}

package/src/wirings/workflow/dsl/workflow-dsl.types.ts

@@ -6,10 +6,11 @@
 import type { StandardSchemaV1 } from '@standard-schema/spec'
 
 import type { WorkflowRun } from '../workflow.types.js'
-import type { ScenarioActor } from '../../../services/scenario-actors-service.js'
+import type { ScenarioPersona } from '../../../services/personas-service.js'
 import type {
   ScenarioStepOptions,
   ScenarioStepPhase,
+  ScenarioSurface,
 } from '../scenario-step.types.js'
 
 /**
@@ -35,7 +36,7 @@ export interface WorkflowStepOptions {
    * internally — so auth middleware and permissions are exercised end-to-end.
    * The step is recorded durably like any RPC step.
    */
-  actor?: ScenarioActor
+  actor?: ScenarioPersona
   // Future: timeout, failFast, priority
 }
 
@@ -84,7 +85,7 @@ export type WorkflowWireDoInline = <T>(
 ) => Promise<T>
 
 /**
- * Type signature for scenario.step/given/when/then - used by inspector.
+ * Type signature for scenario.given/when/then - used by inspector.
  *
  * Deliberately mirrors WorkflowWireDoRPC's shape: the target is a string, not
  * an imported symbol, so the extractor reads it as a literal.
@@ -198,7 +199,7 @@ export interface RpcStepMeta {
 }
 
 /**
- * Scenario step metadata — a call to `scenario.step/given/when/then`.
+ * Scenario step metadata — a call to `scenario.given/when/then`.
  *
  * Distinct from RpcStepMeta on purpose: a step runs locally through
  * runPikkuFunc and must never be treated as dispatchable on the queue/replay
@@ -211,7 +212,7 @@ export interface ScenarioStepMeta {
   stepName: string
   /** Registered name of the step function being run */
   stepFunc: string
-  /** Which keyword the reporter prefixes — given/when/then, or none for `step` */
+  /** Which keyword the reporter prefixes — Given, When or Then */
   phase: ScenarioStepPhase
   /** Output variable name (if assigned) */
   outputVar?: string
@@ -221,8 +222,8 @@ export interface ScenarioStepMeta {
   options?: WorkflowStepOptions
   /** Scenario actor name this step runs as ({ actor: actors.x }) */
   actor?: string
-  /** Mirrors the step function's `browser: true` declaration */
-  browser?: boolean
+  /** Mirrors the surfaces the step function declares a binding for */
+  surfaces?: ScenarioSurface[]
 }
 
 /**
@@ -556,20 +557,14 @@ export interface PikkuScenarioWire extends PikkuWorkflowWire {
   ) => Promise<void>
 
   /**
-   * Run a registered scenario step. Shaped exactly like `do`'s RPC form —
-   * `(stepName, target, data, options)` — so the inspector reads the target as
-   * a string literal rather than resolving an imported symbol.
+   * Run a registered scenario step, as the setup the scenario starts from.
+   *
+   * Shaped exactly like `do`'s RPC form — `(stepName, target, data, options)` —
+   * so the inspector reads the target as a string literal rather than resolving
+   * an imported symbol.
    *
    * The generated `TypedScenario` narrows these over `FlattenedScenarioStepMap`.
    */
-  step(
-    stepName: string,
-    stepFunc: string,
-    data?: any,
-    options?: ScenarioStepOptions
-  ): Promise<any>
-
-  /** `step` with a "Given" prefix in the rendered prose */
   given(
     stepName: string,
     stepFunc: string,
@@ -577,7 +572,7 @@ export interface PikkuScenarioWire extends PikkuWorkflowWire {
     options?: ScenarioStepOptions
   ): Promise<any>
 
-  /** `step` with a "When" prefix in the rendered prose */
+  /** `when`: the same call as `given`, rendered as the action under test */
   when(
     stepName: string,
     stepFunc: string,
@@ -585,7 +580,11 @@ export interface PikkuScenarioWire extends PikkuWorkflowWire {
     options?: ScenarioStepOptions
   ): Promise<any>
 
-  /** `step` with a "Then" prefix in the rendered prose */
+  /**
+   * `then`: a claim about what the action left behind. Unlike `given`/`when`
+   * this is not prose alone — the step's bindings become witnesses, so every
+   * declared surface is observed and the observations must agree.
+   */
   then(
     stepName: string,
     stepFunc: string,

package/src/wirings/workflow/dsl/workflow-runner.ts

@@ -1,10 +1,6 @@
 import { pikkuState } from '../../../pikku-state.js'
 import { addFunction } from '../../../function/function-runner.js'
 
-/**
- * Add a workflow to the system
- * This is called by the generated workflow wirings
- */
 export const addWorkflow = (
   workflowName: string,
   workflowFunc: any,

package/src/wirings/workflow/feature.ts

@@ -5,10 +5,6 @@ import type {
   FeaturePlanEntry,
 } from './workflow.types.js'
 
-/**
- * Register a feature under the name it is exported as. Called from generated
- * wiring, the same way `addWorkflow` is.
- */
 export const addFeature = (
   featureId: string,
   feature: CoreFeature,
@@ -17,21 +13,6 @@ export const addFeature = (
   pikkuState(packageName, 'workflows', 'features').set(featureId, feature)
 }
 
-/**
- * Resolve every feature's scenario references back to the names their
- * scenarios are registered under.
- *
- * Matching is by **object identity**: `pikkuScenario` returns its config
- * verbatim and `addWorkflow` registers that same object, so a feature holding
- * the imported identifier holds the very object that was registered. Nothing
- * is matched by shape, by name, or by any other guess — which is also why a
- * scenario built inline inside a feature (and therefore never registered)
- * comes back as unresolved rather than silently running as something else.
- *
- * Entries are returned in declaration order, features in registration order.
- * A scenario's effective tags are its own plus the containing feature's, so
- * `--tags credential` selects through the feature.
- */
 export const resolveFeatureScenarios = (
   features: Map<string, CoreFeature>,
   registrations: Map<string, CoreWorkflow>