@pikku/core 0.12.71 → 0.12.74

package/src/wirings/virtual-user/virtual-user-derive.test.ts

@@ -0,0 +1,398 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { deriveCatalogue, deriveIntents } from './virtual-user-derive.js'
+import { personaVirtualUserTarget } from './virtual-user-target.js'
+import type { FunctionsMeta } from '../../types/core.types.js'
+import type { WorkflowsMeta } from '../workflow/workflow.types.js'
+import type {
+  ScenarioPersona,
+  ScenarioHttpResponse,
+} from '../../services/personas-service.js'
+
+/**
+ * `expose: true` by default, because these fixtures stand for functions a
+ * virtual user could actually call. The two cases that matter — explicitly
+ * unexposed, and the far commoner un-annotated — are spelled out where they
+ * are tested.
+ */
+const fn = (
+  extra: Partial<FunctionsMeta[string]> = {}
+): FunctionsMeta[string] =>
+  ({
+    pikkuFuncId: 'f',
+    inputSchemaName: null,
+    outputSchemaName: null,
+    expose: true,
+    ...extra,
+  }) as FunctionsMeta[string]
+
+describe('deriving the catalogue from generated meta', () => {
+  const functions: FunctionsMeta = {
+    createProject: fn({
+      description: 'Create a project',
+      inputSchemaName: 'CreateProjectInput',
+      outputSchemaName: 'CreateProjectOutput',
+      scopes: ['org:admin', 'org:admin'],
+      tags: ['projects'],
+    }),
+    getProject: fn({ readonly: true, summary: 'Read a project' }),
+    deployStage: fn({ approvalRequired: true }),
+    internalReindex: fn({ expose: false }),
+    // No `expose` at all — the shape almost half the e2e app's functions are
+    // in, and the one that used to reach the model and 404.
+    agentCaller: fn({ expose: undefined }),
+    userSeesTheDashboard: fn({ scenarioStep: true }),
+    onboardingScenario: fn({ scenario: true }),
+    // The two subjects nobody signs in as. Both spelled with `expose: true` on
+    // purpose: the point is that being marked as a step kind is what withholds
+    // them, so the exclusion survives someone exposing one by mistake.
+    trialHasExpired: fn({ scenarioStep: true, scenarioStepKind: 'platform' }),
+    stripeWebhookArrives: fn({
+      scenarioStep: true,
+      scenarioStepKind: 'addon',
+      scenarioStepAddon: 'stripe',
+    }),
+  }
+
+  const schemas = {
+    CreateProjectInput: {
+      type: 'object',
+      properties: { name: { type: 'string' }, orgId: { type: 'string' } },
+      required: ['name'],
+    },
+    CreateProjectOutput: {
+      type: 'object',
+      properties: { projectId: { type: 'string' } },
+    },
+  }
+
+  test('an rpc arrives with its schema, scopes and gates attached', () => {
+    const [entry] = deriveCatalogue(functions, schemas)
+    assert.deepEqual(entry, {
+      name: 'createProject',
+      description: 'Create a project',
+      readonly: undefined,
+      approvalRequired: undefined,
+      scopes: ['org:admin'],
+      tags: ['projects'],
+      inputKeys: ['name', 'orgId'],
+      outputKeys: ['projectId'],
+      inputSchema: schemas.CreateProjectInput,
+      outputSchema: schemas.CreateProjectOutput,
+    })
+  })
+
+  test('nothing that cannot be called over the wire is offered', () => {
+    const names = deriveCatalogue(functions, schemas).map((e) => e.name)
+    assert.deepEqual(names, [
+      'createProject',
+      'getProject',
+      'deployStage',
+      // internalReindex is not exposed and agentCaller was never put on the rpc
+      // transport at all; the scenario body and its step are not shipped.
+    ])
+  })
+
+  // Found by running one: 4 of 11 calls in the first live run were 404s, all of
+  // them functions with no `expose` annotation. Absent has to mean no.
+  test('a function that was never exposed is not offered either', () => {
+    const names = deriveCatalogue(functions, schemas).map((e) => e.name)
+    assert.ok(!names.includes('agentCaller'))
+  })
+
+  // Oracle integrity, not tidiness. A virtual user that can invoke "Stripe's
+  // webhook arrives" forges its own payment success, and every finding
+  // downstream of that is worthless — so this is enforced at derivation rather
+  // than left to whoever remembers not to expose a step.
+  test('the platform and an addon are not things a virtual user can be', () => {
+    const names = deriveCatalogue(functions, schemas).map((e) => e.name)
+    assert.ok(!names.includes('trialHasExpired'))
+    assert.ok(!names.includes('stripeWebhookArrives'))
+  })
+
+  test('a step kind is withheld even if nothing else marked it a step', () => {
+    const names = deriveCatalogue(
+      { stripeWebhookArrives: fn({ scenarioStepKind: 'addon' }) },
+      {}
+    ).map((e) => e.name)
+    assert.deepEqual(names, [])
+  })
+
+  test('an approval gate and a readonly flag survive the trip', () => {
+    const entries = deriveCatalogue(functions, schemas)
+    assert.equal(entries.find((e) => e.name === 'getProject')?.readonly, true)
+    assert.equal(
+      entries.find((e) => e.name === 'deployStage')?.approvalRequired,
+      true
+    )
+  })
+
+  test('a summary stands in when nobody wrote a description', () => {
+    const entries = deriveCatalogue(functions, schemas)
+    assert.equal(
+      entries.find((e) => e.name === 'getProject')?.description,
+      'Read a project'
+    )
+  })
+
+  test('a missing schema is absent rather than invented', () => {
+    const entries = deriveCatalogue(
+      { orphan: fn({ inputSchemaName: 'NotGenerated' }) },
+      {}
+    )
+    assert.equal(entries[0]!.inputSchema, undefined)
+    assert.equal(entries[0]!.inputKeys, undefined)
+  })
+
+  test('meta the project never produced derives an empty catalogue, not a crash', () => {
+    assert.deepEqual(deriveCatalogue({} as FunctionsMeta), [])
+  })
+})
+
+describe('deriving intents from scenarios', () => {
+  const functions: FunctionsMeta = {
+    aSignedInAdmin: fn({ scenarioStepTemplate: 'is signed in' }),
+    invitesATeammate: fn({ scenarioStepTemplate: 'invites {email}' }),
+    seesTheMemberList: fn({ description: 'sees the new member in the list' }),
+    unnamedStep: fn({}),
+  }
+
+  const workflows = {
+    inviteFlow: {
+      name: 'inviteFlow',
+      scenario: true,
+      title: 'Invite a teammate',
+      description: 'An admin brings someone new into the org',
+      tags: ['org'],
+      actors: ['orgAdmin'],
+      steps: [
+        {
+          type: 'scenarioStep',
+          stepName: 'a',
+          stepFunc: 'aSignedInAdmin',
+          phase: 'given',
+          actor: 'orgAdmin',
+        },
+        {
+          type: 'scenarioStep',
+          stepName: 'b',
+          stepFunc: 'invitesATeammate',
+          phase: 'when',
+          actor: 'orgAdmin',
+        },
+        {
+          type: 'scenarioStep',
+          stepName: 'c',
+          stepFunc: 'seesTheMemberList',
+          phase: 'then',
+          actor: 'orgAdmin',
+        },
+        {
+          type: 'scenarioStep',
+          stepName: 'd',
+          stepFunc: 'unnamedStep',
+          phase: 'then',
+        },
+        { type: 'rpc', stepName: 'e', rpcName: 'createProject' },
+      ],
+    },
+    quarantined: {
+      name: 'quarantined',
+      scenario: true,
+      title: 'Flaky thing',
+      skip: 'fails on CI',
+      steps: [],
+    },
+    nightlyReport: { name: 'nightlyReport', steps: [] },
+  } as unknown as WorkflowsMeta
+
+  test('a scenario becomes an intent in its own words', () => {
+    const [intent] = deriveIntents(workflows, functions)
+    assert.equal(intent!.id, 'inviteFlow')
+    assert.equal(intent!.title, 'Invite a teammate')
+    assert.equal(
+      intent!.description,
+      'An admin brings someone new into the org'
+    )
+    assert.deepEqual(intent!.personas, ['orgAdmin'])
+    assert.deepEqual(intent!.tags, ['org'])
+  })
+
+  test('the prose comes through with its placeholders left open', () => {
+    const [intent] = deriveIntents(workflows, functions)
+    assert.deepEqual(intent!.steps, [
+      'Given the orgAdmin is signed in',
+      // The scenario knows which address it invites. The user has to pick one.
+      'When the orgAdmin invites {email}',
+      'Then the orgAdmin sees the new member in the list',
+    ])
+  })
+
+  test('a step with no sentence is dropped rather than named', () => {
+    const [intent] = deriveIntents(workflows, functions)
+    assert.ok(intent!.steps!.every((step) => !step.includes('unnamedStep')))
+  })
+
+  test('the rpcs a scenario calls are never passed on — finding them is the test', () => {
+    const [intent] = deriveIntents(workflows, functions)
+    assert.ok(
+      !JSON.stringify(intent).includes('createProject'),
+      'a scenario’s own rpc leaked into the intent'
+    )
+  })
+
+  test('a quarantined scenario does not get to drive real traffic', () => {
+    const ids = deriveIntents(workflows, functions).map((i) => i.id)
+    assert.ok(!ids.includes('quarantined'))
+  })
+
+  test('ordinary workflows are not scenarios and are left alone', () => {
+    const ids = deriveIntents(workflows, functions).map((i) => i.id)
+    assert.deepEqual(ids, ['inviteFlow'])
+  })
+
+  test('a scenario with no prose still yields an intent from its title', () => {
+    const intents = deriveIntents(
+      {
+        bare: { name: 'bare', scenario: true, steps: [] },
+      } as unknown as WorkflowsMeta,
+      {}
+    )
+    assert.equal(intents.length, 1)
+    assert.equal(intents[0]!.title, 'bare')
+    assert.equal(intents[0]!.steps, undefined)
+  })
+
+  // What `getWorkflowMeta()` returns is the graph the CLI wrote, not the
+  // inspector state the runner holds: same scenario, `nodes` instead of
+  // `steps` and `source` instead of `scenario`. The console derives its screen
+  // from this shape, so reading only the runner's would show every user with
+  // nothing it wants.
+  test('a scenario read back off disk yields the same intent as the runner sees', () => {
+    const intents = deriveIntents(
+      {
+        inviteFlow: {
+          name: 'inviteFlow',
+          source: 'scenario',
+          title: 'Invite a teammate',
+          actors: ['orgAdmin'],
+          nodes: {
+            step_0: { nodeId: 'step_0', flow: 'branch', branches: [] },
+            a: {
+              nodeId: 'a',
+              rpcName: 'aSignedInAdmin',
+              scenarioStepPhase: 'given',
+              actor: 'orgAdmin',
+            },
+            b: {
+              nodeId: 'b',
+              rpcName: 'invitesATeammate',
+              scenarioStepPhase: 'when',
+              actor: 'orgAdmin',
+            },
+          },
+        },
+      } as unknown as WorkflowsMeta,
+      functions
+    )
+
+    assert.deepEqual(intents[0]!.steps, [
+      'Given the orgAdmin is signed in',
+      'When the orgAdmin invites {email}',
+    ])
+    assert.deepEqual(intents[0]!.personas, ['orgAdmin'])
+  })
+
+  test('an ordinary workflow read off disk is still not a scenario', () => {
+    const intents = deriveIntents(
+      {
+        nightly: { name: 'nightly', source: 'workflow', nodes: {} },
+      } as unknown as WorkflowsMeta,
+      functions
+    )
+    assert.deepEqual(intents, [])
+  })
+})
+
+describe('driving a virtual user through a signed-in actor', () => {
+  const response: ScenarioHttpResponse = {
+    status: 200,
+    ok: true,
+    body: {},
+    serialized: '{}',
+  }
+
+  const stubPersona = (
+    overrides: Partial<ScenarioPersona> = {}
+  ): ScenarioPersona =>
+    ({
+      name: 'orgAdmin',
+      email: 'admin@personas.invalid',
+      invoke: async () => ({}),
+      invokeRaw: async () => response,
+      converse: async () => ({ passed: true, reasoning: 'ok', transcript: [] }),
+      sessionRoles: async () => [],
+      ...overrides,
+    }) as ScenarioPersona
+
+  test('a call goes out as the persona, and a refusal comes back as data', async () => {
+    const seen: unknown[] = []
+    const target = personaVirtualUserTarget(
+      stubPersona({
+        invokeRaw: async (rpcName: string, data: unknown) => {
+          seen.push([rpcName, data])
+          return { status: 403, ok: false, body: {}, serialized: '{}' }
+        },
+      })
+    )
+
+    const result = await target.call('createProject', { name: 'x' })
+    assert.deepEqual(seen, [['createProject', { name: 'x' }]])
+    assert.equal(result.status, 403)
+  })
+
+  test('an app with no agents gives its users nobody to talk to', () => {
+    assert.equal(personaVirtualUserTarget(stubPersona()).talkTo, undefined)
+  })
+
+  test('an agent it was told about is reached in persona', async () => {
+    let asked: unknown = null
+    const target = personaVirtualUserTarget(
+      stubPersona({
+        converse: async (options: unknown) => {
+          asked = options
+          return { passed: true, reasoning: 'sorted', transcript: [] }
+        },
+      }),
+      { agents: ['support'], model: 'test-model' }
+    )
+
+    const verdict = await target.talkTo!('support', 'cancel my plan')
+    assert.equal(verdict.passed, true)
+    assert.deepEqual(asked, {
+      agent: 'support',
+      task: 'cancel my plan',
+      evaluate: 'The assistant did what was asked: cancel my plan',
+      model: 'test-model',
+    })
+  })
+
+  test('an agent it invented is answered, not dialled', async () => {
+    let called = false
+    const target = personaVirtualUserTarget(
+      stubPersona({
+        converse: async () => {
+          called = true
+          return { passed: true, reasoning: '', transcript: [] }
+        },
+      }),
+      { agents: ['support'] }
+    )
+
+    const verdict = await target.talkTo!('concierge', 'anything')
+    assert.equal(called, false)
+    assert.equal(verdict.passed, false)
+    assert.match(verdict.reasoning, /no assistant called 'concierge'/)
+  })
+})

package/src/wirings/virtual-user/virtual-user-derive.ts

@@ -0,0 +1,173 @@
+import type { FunctionsMeta } from '../../types/core.types.js'
+import type { WorkflowsMeta } from '../workflow/workflow.types.js'
+import type { ScenarioStepPhase } from '../workflow/scenario-step.types.js'
+import { composeStepProse } from '../workflow/scenario-prose.js'
+import type { ApiCatalogueEntry, IntentSource } from './virtual-user.types.js'
+
+/** A generated JSON schema, keyed by the name the function meta refers to. */
+export type SchemaMap = Record<string, Record<string, unknown> | undefined>
+
+const topLevelKeys = (
+  schema: Record<string, unknown> | undefined
+): string[] | undefined => {
+  const properties = schema?.properties
+  if (!properties || typeof properties !== 'object') return undefined
+  const keys = Object.keys(properties as Record<string, unknown>)
+  return keys.length ? keys : undefined
+}
+
+/**
+ * Every RPC a virtual user could reach, read off the meta a project already
+ * generates.
+ *
+ * Nothing here is new authoring, which is the whole point: the scopes, the
+ * `readonly` flags, the approval gates and the schemas were written for other
+ * reasons, and a virtual user is another reader of them. A project that has
+ * never heard of virtual users already ships everything one needs.
+ */
+export const deriveCatalogue = (
+  functions: FunctionsMeta,
+  schemas: SchemaMap = {}
+): ApiCatalogueEntry[] => {
+  const entries: ApiCatalogueEntry[] = []
+  for (const [name, meta] of Object.entries(functions ?? {})) {
+    // Scenario bodies and their steps are never network-callable — they are
+    // held out of every deployed unit — so offering one would only ever waste
+    // a turn on a 404.
+    //
+    // For a platform or addon step this is not an efficiency argument but the
+    // oracle: a virtual user's findings are worth something only because it
+    // cannot manufacture the outcomes it is meant to discover. One that could
+    // invoke "Stripe's webhook arrives" would forge its own payment success,
+    // and every finding downstream of that is worthless. Same class of argument
+    // as `allowApprovalRequired` defaulting to false — and enforced here, at
+    // derivation, rather than by convention.
+    if (meta.scenario || meta.scenarioStep || meta.scenarioStepKind) continue
+    // `expose: true` is what puts a function on the rpc transport, and that is
+    // what the shipped target calls over. Absent is not permissive: an unexposed
+    // function 404s, and offering one spends a step to learn nothing about the
+    // product. Measured on the e2e app, 34 of its 72 functions are in exactly
+    // that state — nearly half a catalogue that cannot be called.
+    if (meta.expose !== true) continue
+
+    const inputSchema = meta.inputSchemaName
+      ? schemas[meta.inputSchemaName]
+      : undefined
+    const outputSchema = meta.outputSchemaName
+      ? schemas[meta.outputSchemaName]
+      : undefined
+
+    entries.push({
+      name,
+      description: meta.description ?? meta.summary ?? meta.title,
+      readonly: meta.readonly,
+      approvalRequired: meta.approvalRequired,
+      scopes: meta.scopes?.length ? [...new Set(meta.scopes)] : undefined,
+      tags: meta.tags?.length ? meta.tags : undefined,
+      inputKeys: topLevelKeys(inputSchema),
+      outputKeys: topLevelKeys(outputSchema),
+      inputSchema,
+      outputSchema,
+    })
+  }
+  return entries
+}
+
+/** A scenario step reduced to what prose needs, whichever shape it arrived in. */
+type IntentStep = {
+  stepFunc: string
+  phase: ScenarioStepPhase
+  actor?: string
+}
+
+/**
+ * The same scenario, spelled either way.
+ *
+ * One scenario has two meta shapes and both reach this function: the runner
+ * hands over the inspector's DSL meta, where steps are a `steps[]` of
+ * `scenarioStep`s, while the console reads what was written to disk through
+ * `getWorkflowMeta()`, where the same steps are the graph's `nodes` and the
+ * scenario declares itself with `source` rather than `scenario`. Reading only
+ * the first shape is not a narrower reading — it is an empty one, and a screen
+ * built on it shows every user wanting nothing at all.
+ */
+const scenarioSteps = (meta: {
+  steps?: WorkflowsMeta[string]['steps']
+  nodes?: Record<string, unknown>
+}): IntentStep[] => {
+  if (meta.steps?.length) {
+    return meta.steps.flatMap((step) =>
+      step.type === 'scenarioStep'
+        ? [{ stepFunc: step.stepFunc, phase: step.phase, actor: step.actor }]
+        : []
+    )
+  }
+
+  // Insertion order is the order the CLI wrote the nodes in, which is the order
+  // the scenario declares them. Following `next` instead would buy an ordering
+  // that is already true and cost a traversal that has to decide what a branch
+  // means to a reader who will never take one.
+  return Object.values(meta.nodes ?? {}).flatMap((node) => {
+    const { rpcName, scenarioStepPhase, actor } = (node ?? {}) as {
+      rpcName?: string
+      scenarioStepPhase?: ScenarioStepPhase
+      actor?: string
+    }
+    if (!rpcName || !scenarioStepPhase) return []
+    return [{ stepFunc: rpcName, phase: scenarioStepPhase, actor }]
+  })
+}
+
+/**
+ * The things a virtual user might want, read off the scenarios a project
+ * already declares.
+ *
+ * Only the prose survives the trip. A scenario knows which RPCs it calls and
+ * how their outputs feed each other, and none of that is passed on: finding
+ * the API is the behaviour under test, and a user handed the answer tests
+ * nothing. A step with no sentence to its name is dropped rather than
+ * described by its function name, which would leak exactly that answer.
+ */
+export const deriveIntents = (
+  workflows: WorkflowsMeta,
+  functions: FunctionsMeta = {}
+): IntentSource[] => {
+  const intents: IntentSource[] = []
+  for (const [name, meta] of Object.entries(workflows ?? {})) {
+    if (!meta.scenario && (meta as { source?: string }).source !== 'scenario')
+      continue
+    // A quarantined scenario is one someone decided not to run. Driving real
+    // traffic with it would be a louder version of the same mistake.
+    if (meta.skip) continue
+
+    const steps: string[] = []
+    for (const step of scenarioSteps(meta)) {
+      const stepMeta = functions[step.stepFunc]
+      const template = stepMeta?.scenarioStepTemplate
+      const description = stepMeta?.description ?? stepMeta?.summary
+      if (!template && !description) continue
+      steps.push(
+        composeStepProse({
+          phase: step.phase,
+          // The template goes in unfilled, braces and all. A reporter fills its
+          // placeholders from a run that happened; there is no run yet, and
+          // "invites {email}" tells a user to pick someone where the filled-in
+          // form would have told it whom — which is the scenario's answer, not
+          // the user's own.
+          description: template ?? description ?? '',
+          actor: step.actor,
+        })
+      )
+    }
+
+    intents.push({
+      id: name,
+      title: meta.title ?? meta.summary ?? name,
+      description: meta.description,
+      steps: steps.length ? steps : undefined,
+      tags: meta.tags?.length ? meta.tags : undefined,
+      personas: meta.actors?.length ? meta.actors : undefined,
+    })
+  }
+  return intents
+}

package/src/wirings/virtual-user/virtual-user-dispositions.test.ts

@@ -0,0 +1,63 @@
+import { strict as assert } from 'assert'
+import { describe, test } from 'node:test'
+import { DISPOSITIONS, dispositionProfile } from './virtual-user-dispositions.js'
+
+describe('resolving a disposition profile', () => {
+  test('an unknown disposition falls back to realistic rather than throwing', () => {
+    assert.equal(
+      dispositionProfile('nonsense' as never),
+      DISPOSITIONS.realistic
+    )
+  })
+
+  test('with no tuning the profile is the shared one, untouched', () => {
+    assert.equal(dispositionProfile('careless'), DISPOSITIONS.careless)
+  })
+
+  test('a tuned dial overrides, and the rest of the disposition survives', () => {
+    const profile = dispositionProfile('careless', { repeatRate: 0.35 })
+    assert.equal(profile.repeatRate, 0.35)
+    assert.equal(profile.temperature, DISPOSITIONS.careless.temperature)
+    assert.deepEqual(profile.moves, DISPOSITIONS.careless.moves)
+    // Tuning one user must not tune every user sharing the disposition.
+    assert.equal(DISPOSITIONS.careless.repeatRate, 0.18)
+  })
+
+  test('moves merge per weight, so overriding one keeps the other three', () => {
+    const profile = dispositionProfile('realistic', { moves: { suspend: 30 } })
+    assert.deepEqual(profile.moves, {
+      ...DISPOSITIONS.realistic.moves,
+      suspend: 30,
+    })
+  })
+
+  // Spreading an optional value in is ordinary code; it must not read as a
+  // request to blank the dial.
+  test('an explicit undefined leaves the default alone', () => {
+    const profile = dispositionProfile('careless', {
+      temperature: undefined,
+      moves: { abandon: undefined },
+    })
+    assert.equal(profile.temperature, DISPOSITIONS.careless.temperature)
+    assert.equal(profile.moves.abandon, DISPOSITIONS.careless.moves.abandon)
+  })
+
+  test('instructions append, so a tuned careless user is still careless', () => {
+    const profile = dispositionProfile('careless', {
+      instructions: 'Paste ids from other tabs.',
+    })
+    assert.ok(profile.instructions.startsWith(DISPOSITIONS.careless.instructions))
+    assert.ok(profile.instructions.endsWith('Paste ids from other tabs.'))
+  })
+
+  test('the oracle and the read-only guard are tunable like any other dial', () => {
+    // An auditor allowed to write is a legitimate user to declare; the point is
+    // that it is declared rather than being a surprise at run time.
+    const writingAuditor = dispositionProfile('auditor', { readOnly: false })
+    assert.equal(writingAuditor.readOnly, false)
+    assert.equal(writingAuditor.reReadRate, DISPOSITIONS.auditor.reReadRate)
+
+    const suspicious = dispositionProfile('realistic', { invertedOracle: true })
+    assert.equal(suspicious.invertedOracle, true)
+  })
+})