@pikku/core 0.12.71 → 0.12.74

package/src/wirings/virtual-user/run-virtual-user.ts

@@ -0,0 +1,671 @@
+import type { ResolvedPersona } from '../../services/personas-service.js'
+import type { AIMessage } from '../ai-agent/ai-agent.types.js'
+import type { ActorLLM } from '../actor-flow/run-conversation.js'
+import { getDurationInMilliseconds } from '../../time-utils.js'
+import {
+  catalogueIndex,
+  catalogueLookup,
+  describeEntry,
+  isReadOnly,
+  reachableCatalogue,
+  renderCatalogue,
+} from './virtual-user-catalogue.js'
+import {
+  dispositionProfile,
+  type DispositionProfile,
+  type VirtualUserTuning,
+} from './virtual-user-dispositions.js'
+import { IntentStack, intentsForPersona } from './virtual-user-intents.js'
+import { createRng } from './virtual-user-rng.js'
+import type {
+  ApiCatalogueEntry,
+  IntentSource,
+  StepRecord,
+  VirtualUserAction,
+  VirtualUserBudget,
+  VirtualUserDisposition,
+  VirtualUserFinding,
+  VirtualUserRunResult,
+  VirtualUserTally,
+  VirtualUserTarget,
+} from './virtual-user.types.js'
+
+/** One turn: what the user decided to do next, and why. */
+const ACTION_SCHEMA = {
+  type: 'object',
+  properties: {
+    thought: { type: 'string' },
+    kind: {
+      type: 'string',
+      enum: ['describe', 'call', 'talkTo', 'upload', 'complete', 'stuck'],
+    },
+    rpcName: { type: 'string' },
+    /**
+     * A JSON object, as a string. Not `type: 'object'`: structured output is
+     * strict, and a schema-less object is the one thing strict mode cannot
+     * express — OpenAI rejects the whole request with "'additionalProperties'
+     * is required to be supplied and to be false", so every turn 400s and the
+     * run never takes a step. The arguments differ per endpoint and are only
+     * known from the `describe` the user just did, so there is no schema to
+     * give here; a string is how a free-form object crosses that boundary.
+     */
+    args: { type: 'string' },
+    agent: { type: 'string' },
+    task: { type: 'string' },
+    file: { type: 'string' },
+    summary: { type: 'string' },
+  },
+  required: ['kind'],
+} as const
+
+const DEFAULT_MAX_STEPS = 40
+const DEFAULT_MAX_STEPS_PER_INTENT = 20
+const RESPONSE_EXCERPT = 600
+
+/** What an app-specific oracle is shown about one call. */
+export interface VirtualUserCallContext {
+  rpcName: string
+  args: unknown
+  status: number
+  ok: boolean
+  body: unknown
+  intentId?: string
+  step: number
+  disposition: VirtualUserDisposition
+}
+
+export interface RunVirtualUserParams {
+  /** The person being run — name, job title, personality, goals. */
+  persona: ResolvedPersona
+  /** The id they were declared under, as the intent sources refer to them. */
+  personaId: string
+  /** How it behaves. Default `realistic`. */
+  disposition?: VirtualUserDisposition
+  /** Per-user overrides for that disposition's dials. */
+  tuning?: VirtualUserTuning
+  /** Every RPC the app exposes. Narrowed internally; nothing is hidden by default. */
+  catalogue: readonly ApiCatalogueEntry[]
+  /** Features and scenarios in the app's own words — never their step graphs. */
+  intents: readonly IntentSource[]
+  /** Extra goals in the user's own words, run alongside the derived intents. */
+  goals?: readonly string[]
+  /** The transport it acts through. */
+  target: VirtualUserTarget
+  /** The user's own LLM. */
+  llm: ActorLLM
+  model: string
+  budget?: VirtualUserBudget
+  /**
+   * The app's own stopping rule, given the running tally. Pikku counts steps,
+   * calls, mutations and tokens because only the engine sees them; what they
+   * cost is the app's to decide, which is why no price appears in core.
+   */
+  stop?: (tally: VirtualUserTally) => boolean | Promise<boolean>
+  /** Fixed seed, so a run replays into the same finding. */
+  seed?: number
+  /**
+   * Check a response against the RPC's own output schema. Supplied by the CLI
+   * from generated schemas; a mismatch means the contract and the implementation
+   * disagree, which nothing else in a normal test suite looks for.
+   */
+  validateOutput?: (rpcName: string, body: unknown) => string | null
+  /** App-specific oracle — ownership, invariants, anything core cannot know. */
+  classify?: (
+    context: VirtualUserCallContext
+  ) => VirtualUserFinding[] | null | undefined
+  /** Files this user may upload. */
+  fixtures?: readonly string[]
+  /** Agents it may talk to. */
+  agents?: readonly { name: string; description?: string }[]
+  /** Ids and slugs remembered from an earlier run, for a `stale` user. */
+  memory?: Record<string, string>
+  /**
+   * The scopes this persona actually holds, resolved from its roles at sign-in.
+   *
+   * Used two ways: to narrow a large catalogue to what it is entitled to reach,
+   * and as an oracle — a call whose scopes it does not hold that nonetheless
+   * succeeds is authorization drift.
+   */
+  scopes?: readonly string[]
+  /**
+   * Offer the endpoints the app marked as needing a human's approval. Off by
+   * default: those are the ones that spend money and move real traffic.
+   */
+  allowApprovalRequired?: boolean
+  maxStepsPerIntent?: number
+}
+
+const msg = (role: AIMessage['role'], content: string): AIMessage => ({
+  id: globalThis.crypto.randomUUID(),
+  role,
+  content,
+  createdAt: new Date(),
+})
+
+/** Read a structured result, falling back to parsing JSON out of the text. */
+const readObject = <T>(result: { object?: unknown; text?: string }): T | null => {
+  if (result.object && typeof result.object === 'object') return result.object as T
+  if (result.text) {
+    try {
+      return JSON.parse(result.text) as T
+    } catch {
+      return null
+    }
+  }
+  return null
+}
+
+/**
+ * The arguments for a call, however the model chose to spell them.
+ *
+ * `ACTION_SCHEMA` asks for a JSON string because strict structured output
+ * cannot carry a schema-less object, but a model reading the endpoint's own
+ * schema will sometimes send the object anyway, and a scripted or non-strict
+ * provider always will. Both are the same intent, so both are accepted; text
+ * that is not JSON at all is not arguments, and calling with it would only
+ * produce a 400 attributable to the wrong thing.
+ */
+const readArgs = (args: unknown): Record<string, unknown> => {
+  if (args && typeof args === 'object') return args as Record<string, unknown>
+  if (typeof args === 'string' && args.trim()) {
+    try {
+      const parsed = JSON.parse(args)
+      if (parsed && typeof parsed === 'object') {
+        return parsed as Record<string, unknown>
+      }
+    } catch {
+      // Not JSON — fall through to no arguments.
+    }
+  }
+  return {}
+}
+
+/** Keys worth remembering across intents — the things a person writes down. */
+const MEMORABLE = /(id|slug|token|email|name|key|url|code)$/i
+
+/**
+ * Pull identifiers out of a response so later intents can refer to them.
+ *
+ * This is what makes a run cohesive without ever handing the user a dataflow
+ * map: it discovers that a project has an id by looking at one, the same way a
+ * person reads it off the screen.
+ */
+export const rememberIds = (
+  body: unknown,
+  into: Record<string, string>,
+  depth = 0
+): Record<string, string> => {
+  if (depth > 4 || body === null || typeof body !== 'object') return into
+  if (Array.isArray(body)) {
+    for (const item of body.slice(0, 5)) rememberIds(item, into, depth + 1)
+    return into
+  }
+  for (const [key, value] of Object.entries(body)) {
+    if (typeof value === 'string' && MEMORABLE.test(key) && value.length < 200) {
+      into[key] = value
+    } else if (typeof value === 'object') {
+      rememberIds(value, into, depth + 1)
+    }
+  }
+  return into
+}
+
+const personaInstructions = (
+  persona: ResolvedPersona,
+  profile: DispositionProfile,
+  catalogue: string,
+  agents: readonly { name: string; description?: string }[],
+  fixtures: readonly string[]
+): string =>
+  [
+    `You are a real person using a product through its API. Stay in character; you are the user, not the system.`,
+    `Your name is ${persona.name}.`,
+    persona.jobTitle ? `Your role: ${persona.jobTitle}.` : '',
+    persona.personality ? `Your manner: ${persona.personality}.` : '',
+    '',
+    profile.instructions,
+    '',
+    `Each turn you take exactly ONE action:`,
+    `- describe: read an endpoint's schema. You MUST describe an endpoint before you may call it.`,
+    `- call: invoke an endpoint with arguments that fit its schema. Put them in 'args' as a JSON object written as a string, e.g. "{\\"id\\":\\"abc\\"}".`,
+    agents.length
+      ? `- talkTo: ask one of the product's assistants to do something for you in plain language.`
+      : '',
+    fixtures.length ? `- upload: send one of your files.` : '',
+    `- complete: you got what you came for.`,
+    `- stuck: you cannot get there, and you can say why.`,
+    '',
+    agents.length
+      ? `Assistants you can talk to:\n${agents
+          .map((agent) => `- ${agent.name}${agent.description ? `: ${agent.description}` : ''}`)
+          .join('\n')}`
+      : '',
+    fixtures.length ? `Files you have: ${fixtures.join(', ')}` : '',
+    '',
+    `The endpoints available to you, as name(inputs) -> outputs:`,
+    catalogue,
+  ]
+    .filter(Boolean)
+    .join('\n')
+
+/**
+ * Run one virtual user against a live target until its budget runs out.
+ *
+ * The engine schedules which intent is live and the model decides what to do
+ * within it — that split is what lets a run be replayed from its seed while the
+ * work inside it stays genuinely open-ended.
+ */
+export const runVirtualUser = async (
+  params: RunVirtualUserParams
+): Promise<VirtualUserRunResult> => {
+  const {
+    persona,
+    personaId,
+    disposition = 'realistic',
+    tuning,
+    target,
+    llm,
+    model,
+    budget,
+    stop,
+    validateOutput,
+    classify,
+    fixtures = [],
+    agents = [],
+    scopes,
+    allowApprovalRequired = false,
+    maxStepsPerIntent = DEFAULT_MAX_STEPS_PER_INTENT,
+  } = params
+
+  const profile = dispositionProfile(disposition, tuning)
+  const seed = params.seed ?? Math.floor(Math.random() * 2 ** 31)
+  const rng = createRng(seed)
+  const startedAt = Date.now()
+
+  // An adversarial user is offered the whole surface on purpose — probing what
+  // it should not reach is the behaviour under test — while `scopes` stays live
+  // below as the oracle that says a success there was a finding.
+  const reachable = reachableCatalogue(params.catalogue, {
+    readOnly: profile.readOnly,
+    allowApprovalRequired,
+    scopes: profile.invertedOracle ? undefined : scopes,
+  })
+  const index = catalogueIndex(reachable)
+
+  const sources: IntentSource[] = [
+    ...intentsForPersona(params.intents, personaId),
+    ...(params.goals ?? []).map((goal, i) => ({
+      id: `goal_${i + 1}`,
+      title: goal,
+    })),
+  ]
+  const stack = new IntentStack(sources, rng, profile)
+
+  const memory: Record<string, string> = profile.emptyMemory
+    ? {}
+    : { ...(params.memory ?? {}) }
+  const described = new Set<string>()
+  const findings: VirtualUserFinding[] = []
+  const steps: StepRecord[] = []
+  const history = new Map<string, AIMessage[]>()
+
+  const tally: VirtualUserTally = {
+    steps: 0,
+    calls: 0,
+    mutations: 0,
+    tokensIn: 0,
+    tokensOut: 0,
+    model,
+    elapsedMs: 0,
+    findings: 0,
+  }
+
+  const maxSteps = budget?.steps ?? DEFAULT_MAX_STEPS
+  const maxDurationMs =
+    budget?.duration === undefined
+      ? undefined
+      : typeof budget.duration === 'number'
+        ? budget.duration
+        : getDurationInMilliseconds(budget.duration)
+
+  const instructions = personaInstructions(
+    persona,
+    profile,
+    renderCatalogue(reachable),
+    agents,
+    fixtures
+  )
+
+  const addFinding = (finding: VirtualUserFinding) => {
+    findings.push(finding)
+    tally.findings++
+  }
+
+  // Every other exit sets this and breaks; running the loop out means the step
+  // budget is what ended it.
+  let stoppedBy: VirtualUserRunResult['stoppedBy'] = 'budget-steps'
+
+  for (let step = 0; step < maxSteps; step++) {
+    tally.elapsedMs = Date.now() - startedAt
+    if (maxDurationMs !== undefined && tally.elapsedMs >= maxDurationMs) {
+      stoppedBy = 'budget-duration'
+      break
+    }
+    if (budget?.mutations !== undefined && tally.mutations >= budget.mutations) {
+      stoppedBy = 'budget-mutations'
+      break
+    }
+    if (stop && (await stop({ ...tally }))) {
+      stoppedBy = 'stop-hook'
+      break
+    }
+
+    const tick = stack.next(step)
+    if (!tick) {
+      // Nothing left to schedule: either this user was given no intents at all,
+      // or it saw every one of them through.
+      stoppedBy = stack.records().length === 0 ? 'no-intents' : 'exhausted'
+      break
+    }
+
+    // A user who cannot get anywhere gives up rather than looping forever.
+    if (tick.intent.steps.length > maxStepsPerIntent) {
+      stack.stuck('gave up after going in circles')
+      continue
+    }
+
+    const intentId = tick.intent.id
+    if (!history.has(intentId)) {
+      const source = tick.intent.source
+      history.set(intentId, [
+        msg(
+          'user',
+          [
+            `What you want to do right now: ${source.title}`,
+            source.description ?? '',
+            source.steps?.length
+              ? `Roughly how it goes:\n${source.steps.map((s) => `- ${s}`).join('\n')}`
+              : '',
+            `Work it out through the API yourself — nobody has told you which endpoints do this.`,
+          ]
+            .filter(Boolean)
+            .join('\n')
+        ),
+      ])
+    }
+    const messages = history.get(intentId)!
+    if (tick.move === 'resume' || tick.move === 'suspend') {
+      messages.push(
+        msg(
+          'user',
+          `You were pulled away and have come back to this. Things may have changed since.`
+        )
+      )
+    }
+    // Whether this user trusts its notes this turn or goes and looks again.
+    // `stale` almost always trusts them, a newcomer has none to trust, and an
+    // auditor checks nearly everything — which is the whole difference between
+    // those runs, expressed as one roll rather than as prose.
+    if (Object.keys(memory).length && !rng.chance(profile.reReadRate)) {
+      messages.push(
+        msg(
+          'user',
+          `Things you have noted down: ${Object.entries(memory)
+            .slice(-25)
+            .map(([key, value]) => `${key}=${value}`)
+            .join(', ')}`
+        )
+      )
+    } else if (Object.keys(memory).length) {
+      messages.push(
+        msg('user', `You are not sure your notes are current. Look things up.`)
+      )
+    }
+
+    const result = await llm({
+      model,
+      temperature: profile.temperature,
+      instructions,
+      messages,
+      tools: [],
+      maxSteps: 1,
+      toolChoice: 'none',
+      outputSchema: ACTION_SCHEMA as unknown as Record<string, unknown>,
+    })
+
+    tally.steps++
+    tally.tokensIn += result.usage?.inputTokens ?? 0
+    tally.tokensOut += result.usage?.outputTokens ?? 0
+
+    const raw = readObject<Record<string, unknown>>(result)
+    const record: StepRecord = {
+      index: step,
+      intentId,
+      action: { kind: 'invalid', detail: 'no action produced' },
+      tokensIn: result.usage?.inputTokens ?? 0,
+      tokensOut: result.usage?.outputTokens ?? 0,
+    }
+
+    if (!raw || typeof raw.kind !== 'string') {
+      steps.push(record)
+      messages.push(msg('user', 'That was not a valid action. Choose one action.'))
+      continue
+    }
+
+    const action = { ...raw, kind: raw.kind } as unknown as VirtualUserAction
+    record.action = action
+    messages.push(msg('assistant', JSON.stringify(raw)))
+
+    /** Tell the user what happened, so the next turn is informed. */
+    const reply = (text: string) => messages.push(msg('user', text))
+
+    if (action.kind === 'complete') {
+      stack.complete(action.summary)
+      steps.push(record)
+      continue
+    }
+    if (action.kind === 'stuck') {
+      stack.stuck(action.reason)
+      steps.push(record)
+      continue
+    }
+
+    if (action.kind === 'describe') {
+      const entry = catalogueLookup(index, action.rpcName)
+      if (!entry) {
+        reply(`There is no endpoint called '${action.rpcName}'.`)
+        steps.push(record)
+        continue
+      }
+      described.add(entry.name)
+      // Saying the gate is now open matters: reading a schema is free and
+      // calling is not, so a model left to infer it will happily spend a whole
+      // budget describing one endpoint after another and never touch the app.
+      reply(
+        `${JSON.stringify(describeEntry(entry))}\nYou have read this one, so you may now call '${entry.name}'.`
+      )
+      steps.push(record)
+      continue
+    }
+
+    if (action.kind === 'talkTo') {
+      if (!target.talkTo) {
+        reply('There is nobody to talk to here.')
+        steps.push(record)
+        continue
+      }
+      const verdict = await target.talkTo(action.agent, action.task)
+      record.ok = verdict.passed
+      record.response = verdict.reasoning.slice(0, RESPONSE_EXCERPT)
+      reply(
+        `You spoke to ${action.agent}. ${verdict.passed ? 'It sorted it out.' : 'It did not.'} ${verdict.reasoning}`
+      )
+      steps.push(record)
+      continue
+    }
+
+    if (action.kind === 'upload') {
+      if (!target.upload) {
+        reply('You have nowhere to send a file.')
+        steps.push(record)
+        continue
+      }
+      const response = await target.upload(action.file)
+      record.status = response.status
+      record.ok = response.ok
+      record.response = response.serialized.slice(0, RESPONSE_EXCERPT)
+      if (response.status >= 500) {
+        record.findingKinds = ['server-error']
+        addFinding({
+          kind: 'server-error',
+          detail: `upload '${action.file}' returned ${response.status}`,
+          status: response.status,
+          intentId,
+          step,
+        })
+      }
+      reply(`Upload came back ${response.status}: ${record.response}`)
+      steps.push(record)
+      continue
+    }
+
+    // Everything below is a call.
+    const entry = catalogueLookup(index, action.rpcName)
+    if (!entry) {
+      reply(`There is no endpoint called '${action.rpcName}'.`)
+      steps.push(record)
+      continue
+    }
+    if (!described.has(entry.name)) {
+      // Enforced rather than merely asked for: guessing field names is
+      // unproductive fuzz that turns every 400 into a typo. Handing back the
+      // schema makes this self-correcting in one turn.
+      reply(
+        `You do not know this endpoint's fields yet. Its schema is: ${JSON.stringify(
+          describeEntry(entry)
+        )}`
+      )
+      described.add(entry.name)
+      steps.push(record)
+      continue
+    }
+
+    // What may be called is decided once, when the catalogue is narrowed: a
+    // read-only disposition is never offered a mutation and an approval-gated
+    // endpoint is never offered at all, so `index` is the gate and there is no
+    // second one here to drift out of step with it.
+    const mutating = !isReadOnly(entry)
+
+    const args = readArgs((action as { args?: unknown }).args)
+    const repeats =
+      profile.repeatRate > 0 && rng.chance(profile.repeatRate) ? 2 : 1
+    let lastResponseText = ''
+
+    for (let attempt = 0; attempt < repeats; attempt++) {
+      let response
+      try {
+        response = await target.call(entry.name, args)
+      } catch (error) {
+        record.findingKinds = [...(record.findingKinds ?? []), 'transport-error']
+        addFinding({
+          kind: 'transport-error',
+          detail: `${entry.name} threw: ${(error as Error).message}`,
+          rpcName: entry.name,
+          intentId,
+          step,
+        })
+        lastResponseText = `The request failed outright: ${(error as Error).message}`
+        break
+      }
+
+      tally.calls++
+      if (mutating) tally.mutations++
+      record.status = response.status
+      record.ok = response.ok
+      record.response = response.serialized.slice(0, RESPONSE_EXCERPT)
+      lastResponseText = `${response.status}: ${record.response}`
+
+      const kinds: NonNullable<StepRecord['findingKinds']> = []
+      if (response.status >= 500) {
+        kinds.push('server-error')
+        addFinding({
+          kind: 'server-error',
+          detail: `${entry.name} returned ${response.status}`,
+          rpcName: entry.name,
+          status: response.status,
+          intentId,
+          step,
+        })
+      }
+      if (response.ok && validateOutput) {
+        const problem = validateOutput(entry.name, response.body)
+        if (problem) {
+          kinds.push('schema-violation')
+          addFinding({
+            kind: 'schema-violation',
+            detail: `${entry.name} answered outside its own output schema: ${problem}`,
+            rpcName: entry.name,
+            status: response.status,
+            intentId,
+            step,
+          })
+        }
+      }
+      // This persona does not hold the scopes this endpoint declares, and the
+      // server served it anyway — the declaration and the enforcement disagree.
+      if (
+        response.ok &&
+        scopes &&
+        entry.scopes?.some((scope) => !scopes.includes(scope))
+      ) {
+        kinds.push('unexpected-success')
+        addFinding({
+          kind: 'unexpected-success',
+          detail: `${entry.name} succeeded for '${personaId}', who does not hold ${entry.scopes
+            .filter((scope) => !scopes.includes(scope))
+            .join(', ')}`,
+          rpcName: entry.name,
+          status: response.status,
+          intentId,
+          step,
+        })
+      }
+      for (const finding of classify?.({
+        rpcName: entry.name,
+        args,
+        status: response.status,
+        ok: response.ok,
+        body: response.body,
+        intentId,
+        step,
+        disposition,
+      }) ?? []) {
+        kinds.push(finding.kind)
+        addFinding({ ...finding, step })
+      }
+      if (kinds.length) {
+        record.findingKinds = [...(record.findingKinds ?? []), ...kinds]
+      }
+
+      if (response.ok) rememberIds(response.body, memory)
+      if (attempt + 1 < repeats) {
+        lastResponseText += ` (you were not sure it went through, so you sent it again)`
+      }
+    }
+
+    reply(lastResponseText)
+    steps.push(record)
+  }
+
+  tally.elapsedMs = Date.now() - startedAt
+
+  return {
+    seed,
+    tally,
+    findings,
+    intents: stack.records(),
+    steps,
+    memory,
+    stoppedBy,
+  }
+}