@pikku/core 0.12.51 → 0.12.53

package/src/wirings/actor-flow/run-conversation.test.ts

@@ -0,0 +1,176 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { runConversation, type PersonaLLM } from './run-conversation.js'
+import type {
+  TargetAgentDriver,
+  TargetAgentReply,
+  ActorFlowApprovalPolicy,
+} from './actor-flow.types.js'
+import type { AIAgentStepResult } from '../../services/ai-agent-runner-service.js'
+
+const stepResult = (object: unknown): AIAgentStepResult => ({
+  text: '',
+  object,
+  toolCalls: [],
+  toolResults: [],
+  usage: { inputTokens: 0, outputTokens: 0 },
+  finishReason: 'stop',
+})
+
+/** A persona LLM scripted by the outputSchema it's asked for. */
+const scriptedLLM = (script: {
+  turns: Array<{ message: string; done: boolean }>
+  decisions: Array<{ toolCallId: string; approved: boolean }>
+  evaluation: { passed: boolean; reasoning: string }
+}): { llm: PersonaLLM; calls: string[] } => {
+  let turn = 0
+  const calls: string[] = []
+  const llm: PersonaLLM = async (params) => {
+    const props =
+      (params.outputSchema as { properties?: Record<string, unknown> })
+        ?.properties ?? {}
+    if ('message' in props) {
+      // Providers reject an empty prompt — every persona turn must carry messages.
+      assert.ok(
+        params.messages.length > 0,
+        'persona turn requested with empty messages'
+      )
+      const t = script.turns[turn] ?? script.turns[script.turns.length - 1]
+      turn++
+      calls.push('turn')
+      return stepResult(t)
+    }
+    if ('decisions' in props) {
+      calls.push('approval')
+      return stepResult({ decisions: script.decisions })
+    }
+    if ('passed' in props) {
+      calls.push('eval')
+      return stepResult(script.evaluation)
+    }
+    throw new Error('unexpected schema')
+  }
+  return { llm, calls }
+}
+
+/** A target that suspends once for approval, then completes. */
+const suspendingTarget = (): {
+  target: TargetAgentDriver
+  approvedWith: Array<{ toolCallId: string; approved: boolean }[]>
+} => {
+  let runs = 0
+  const approvedWith: Array<{ toolCallId: string; approved: boolean }[]> = []
+  const target: TargetAgentDriver = {
+    run: async (): Promise<TargetAgentReply> => {
+      runs++
+      if (runs === 1) {
+        return {
+          text: 'Let me do that.',
+          runId: 'run-1',
+          status: 'suspended',
+          pendingApprovals: [
+            { toolCallId: 'tc1', toolName: 'createTodo', args: { title: 'x' } },
+          ],
+        }
+      }
+      return { text: 'All set.', runId: `run-${runs}`, status: 'completed' }
+    },
+    approve: async (runId, decisions): Promise<TargetAgentReply> => {
+      approvedWith.push(decisions)
+      return { text: 'Created it.', runId, status: 'completed' }
+    },
+  }
+  return { target, approvedWith }
+}
+
+/** A target that never completes — always re-suspends for the same approval. */
+const alwaysSuspendingTarget = (): {
+  target: TargetAgentDriver
+  approveCalls: () => number
+} => {
+  let approveCalls = 0
+  const suspended: TargetAgentReply = {
+    text: 'working on it',
+    runId: 'run-stuck',
+    status: 'suspended',
+    pendingApprovals: [
+      { toolCallId: 'tc1', toolName: 'createTodo', args: { title: 'x' } },
+    ],
+  }
+  const target: TargetAgentDriver = {
+    run: async () => suspended,
+    approve: async () => {
+      approveCalls++
+      return suspended
+    },
+  }
+  return { target, approveCalls: () => approveCalls }
+}
+
+const base = {
+  persona: { email: 'pm@example.com', name: 'Priya', personality: 'concise' },
+  personaName: 'Priya',
+  agentName: 'todoBot',
+  task: 'Get a todo created',
+  evaluate: 'A todo now exists',
+  model: 'test/test-model',
+}
+
+describe('runConversation', () => {
+  test('drives turns, approves in-persona, evaluates', async () => {
+    const { llm, calls } = scriptedLLM({
+      turns: [
+        { message: 'please make a todo', done: false },
+        { message: 'thanks', done: true },
+      ],
+      decisions: [{ toolCallId: 'tc1', approved: true }],
+      evaluation: { passed: true, reasoning: 'todo created' },
+    })
+    const { target, approvedWith } = suspendingTarget()
+
+    const verdict = await runConversation({ ...base, llm, target })
+
+    assert.equal(verdict.passed, true)
+    assert.match(verdict.reasoning, /todo/i)
+    assert.ok(verdict.transcript.length >= 2)
+    assert.deepEqual(approvedWith, [[{ toolCallId: 'tc1', approved: true }]])
+    assert.ok(calls.includes('approval'))
+    assert.ok(calls.includes('eval'))
+  })
+
+  test("approvals policy 'never' denies without asking the persona", async () => {
+    const { llm, calls } = scriptedLLM({
+      turns: [{ message: 'make a todo', done: true }],
+      decisions: [{ toolCallId: 'tc1', approved: true }],
+      evaluation: { passed: false, reasoning: 'blocked' },
+    })
+    const { target, approvedWith } = suspendingTarget()
+
+    const verdict = await runConversation({
+      ...base,
+      approvals: 'never' as ActorFlowApprovalPolicy,
+      llm,
+      target,
+    })
+
+    assert.deepEqual(approvedWith, [[{ toolCallId: 'tc1', approved: false }]])
+    assert.ok(!calls.includes('approval'))
+    assert.equal(verdict.passed, false)
+  })
+
+  test('caps the approval loop when the target never stops suspending', async () => {
+    const { llm } = scriptedLLM({
+      turns: [{ message: 'please make a todo', done: false }],
+      decisions: [{ toolCallId: 'tc1', approved: true }],
+      evaluation: { passed: false, reasoning: 'never got there' },
+    })
+    const { target, approveCalls } = alwaysSuspendingTarget()
+
+    await assert.rejects(
+      runConversation({ ...base, approvals: 'always', maxApprovalRounds: 3, llm, target }),
+      /approval rounds/
+    )
+    assert.equal(approveCalls(), 3)
+  })
+})

package/src/wirings/actor-flow/run-conversation.ts

@@ -0,0 +1,285 @@
+import type {
+  ActorFlowApprovalPolicy,
+  ActorFlowVerdict,
+  TargetAgentDriver,
+  TargetPendingApproval,
+} from './actor-flow.types.js'
+import type { ScenarioActorConfig } from '../../services/scenario-actors-service.js'
+import type { AIMessage } from '../ai-agent/ai-agent.types.js'
+import type {
+  AIAgentRunnerParams,
+  AIAgentStepResult,
+} from '../../services/ai-agent-runner-service.js'
+
+/** One turn the persona takes: the message to send and whether it's finished. */
+const PERSONA_TURN_SCHEMA = {
+  type: 'object',
+  properties: {
+    message: { type: 'string' },
+    done: { type: 'boolean' },
+  },
+  required: ['message', 'done'],
+} as const
+
+/** The persona's approve/deny decision for each pending tool request. */
+const APPROVAL_DECISION_SCHEMA = {
+  type: 'object',
+  properties: {
+    decisions: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          toolCallId: { type: 'string' },
+          approved: { type: 'boolean' },
+          reason: { type: 'string' },
+        },
+        required: ['toolCallId', 'approved'],
+      },
+    },
+  },
+  required: ['decisions'],
+} as const
+
+/** The persona's final verdict on whether the task was accomplished. */
+const EVALUATION_SCHEMA = {
+  type: 'object',
+  properties: {
+    passed: { type: 'boolean' },
+    reasoning: { type: 'string' },
+  },
+  required: ['passed', 'reasoning'],
+} as const
+
+const DEFAULT_MAX_TURNS = 12
+
+const DEFAULT_MAX_APPROVAL_ROUNDS = 16
+
+/** The LLM call the persona uses for its own turns/decisions/evaluation. */
+export type PersonaLLM = (
+  params: AIAgentRunnerParams
+) => Promise<AIAgentStepResult>
+
+export interface RunConversationParams {
+  /** Persona config (personality/jobTitle/name) that shapes how the actor talks. */
+  persona: ScenarioActorConfig
+  /** Stable persona name (for transcript labelling). */
+  personaName: string
+  /** What the actor is trying to get the target agent to accomplish. */
+  task: string
+  /** Natural-language success criterion the actor evaluates at the end. */
+  evaluate: string
+  /** How the actor answers the target agent's tool-approval requests. */
+  approvals?: ActorFlowApprovalPolicy
+  /** Model the persona uses for its own turns/decisions. */
+  model: string
+  /** Hard cap on conversation turns. Default 12. */
+  maxTurns?: number
+  /** Hard cap on tool-approval rounds within a single target turn. Default 16. */
+  maxApprovalRounds?: number
+  /** Transport that drives the target agent (HTTP in production). */
+  target: TargetAgentDriver
+  /** The persona's own LLM. */
+  llm: PersonaLLM
+  /** Display name of the target agent (transcript labelling). */
+  agentName: string
+}
+
+function msg(role: AIMessage['role'], content: string): AIMessage {
+  return {
+    id: globalThis.crypto.randomUUID(),
+    role,
+    content,
+    createdAt: new Date(),
+  }
+}
+
+/** Read a structured `run` result, falling back to parsing JSON from text. */
+function readObject<T>(result: { object?: unknown; text?: string }): T | null {
+  if (result.object && typeof result.object === 'object') {
+    return result.object as T
+  }
+  if (result.text) {
+    try {
+      return JSON.parse(result.text) as T
+    } catch {
+      return null
+    }
+  }
+  return null
+}
+
+function personaInstructions(
+  persona: ScenarioActorConfig,
+  task: string
+): string {
+  return [
+    `You are role-playing a real user interacting with an AI assistant. Stay in character at all times — you are the user, not the assistant.`,
+    persona.name ? `Your name is ${persona.name}.` : '',
+    persona.jobTitle ? `Your role: ${persona.jobTitle}.` : '',
+    persona.personality
+      ? `Your personality and communication style: ${persona.personality}. Match this tone, vocabulary, and level of detail exactly.`
+      : '',
+    `Your goal in this conversation: ${task}.`,
+    `Send one message at a time. Set "done" to true only once your goal is clearly accomplished, or clearly impossible.`,
+  ]
+    .filter(Boolean)
+    .join('\n')
+}
+
+/** Route the target agent's pending tool approvals through the persona. */
+async function decideApprovals(
+  params: RunConversationParams,
+  instructions: string,
+  pending: TargetPendingApproval[]
+): Promise<{ toolCallId: string; approved: boolean }[]> {
+  const policy = params.approvals ?? 'in-persona'
+  if (policy === 'always') {
+    return pending.map((p) => ({ toolCallId: p.toolCallId, approved: true }))
+  }
+  if (policy === 'never') {
+    return pending.map((p) => ({ toolCallId: p.toolCallId, approved: false }))
+  }
+
+  const summary = pending
+    .map(
+      (p) =>
+        `- toolCallId "${p.toolCallId}": ${p.toolName}(${JSON.stringify(p.args)})${p.reason ? ` — ${p.reason}` : ''}`
+    )
+    .join('\n')
+
+  const result = await params.llm({
+    model: params.model,
+    instructions: `${instructions}\nThe assistant is asking permission to run tools on your behalf. Decide whether YOU, as this persona, would allow each one.`,
+    messages: [
+      msg(
+        'user',
+        `The assistant wants to run these tools:\n${summary}\nApprove or deny each toolCallId.`
+      ),
+    ],
+    tools: [],
+    maxSteps: 1,
+    toolChoice: 'none',
+    outputSchema: APPROVAL_DECISION_SCHEMA as unknown as Record<
+      string,
+      unknown
+    >,
+  })
+
+  const decided =
+    readObject<{
+      decisions?: { toolCallId: string; approved: boolean }[]
+    }>(result)?.decisions ?? []
+
+  // Every pending call must get a decision; a missing one defaults to denied.
+  return pending.map((p) => {
+    const match = decided.find((d) => d.toolCallId === p.toolCallId)
+    return { toolCallId: p.toolCallId, approved: match?.approved ?? false }
+  })
+}
+
+/** Drive the target to a non-suspended reply, routing approvals to the persona. */
+async function converseWithTarget(
+  params: RunConversationParams,
+  instructions: string,
+  message: string
+) {
+  const maxRounds = params.maxApprovalRounds ?? DEFAULT_MAX_APPROVAL_ROUNDS
+  let reply = await params.target.run(message)
+  let rounds = 0
+  while (
+    reply.status === 'suspended' &&
+    reply.pendingApprovals &&
+    reply.pendingApprovals.length > 0
+  ) {
+    if (rounds >= maxRounds) {
+      throw new Error(
+        `Target agent "${params.agentName}" stayed suspended after ${maxRounds} approval rounds (runId ${reply.runId}); aborting to avoid an unbounded approve loop.`
+      )
+    }
+    rounds++
+    const decisions = await decideApprovals(
+      params,
+      instructions,
+      reply.pendingApprovals
+    )
+    reply = await params.target.approve(reply.runId, decisions)
+  }
+  return reply
+}
+
+/**
+ * Run a conversation: an LLM-driven persona holds a real multi-turn exchange
+ * with a target agent (driven via the injected transport), answers the target's
+ * tool-approval requests in-persona, then evaluates whether the task was met.
+ * Deterministic checks are the caller's responsibility.
+ */
+export async function runConversation(
+  params: RunConversationParams
+): Promise<ActorFlowVerdict> {
+  const maxTurns = params.maxTurns ?? DEFAULT_MAX_TURNS
+  const instructions = personaInstructions(params.persona, params.task)
+  // Seed a kickoff so the very first persona turn has a non-empty message list
+  // (providers reject an empty prompt). It's an instruction TO the persona, so
+  // it never appears in the transcript.
+  const personaMessages: AIMessage[] = [
+    msg(
+      'user',
+      'Begin the conversation now — send your first message to the assistant to work towards your goal.'
+    ),
+  ]
+  const transcript: string[] = []
+
+  for (let turn = 0; turn < maxTurns; turn++) {
+    const personaResult = await params.llm({
+      model: params.model,
+      instructions,
+      messages: personaMessages,
+      tools: [],
+      maxSteps: 1,
+      toolChoice: 'none',
+      outputSchema: PERSONA_TURN_SCHEMA as unknown as Record<string, unknown>,
+    })
+
+    const turnData = readObject<{ message: string; done: boolean }>(
+      personaResult
+    )
+    const personaMessage = turnData?.message?.trim()
+    if (!personaMessage) {
+      break
+    }
+
+    personaMessages.push(msg('assistant', personaMessage))
+    transcript.push(`${params.personaName}: ${personaMessage}`)
+
+    const reply = await converseWithTarget(params, instructions, personaMessage)
+    personaMessages.push(msg('user', reply.text ?? ''))
+    transcript.push(`${params.agentName}: ${reply.text ?? ''}`)
+
+    if (turnData?.done) {
+      break
+    }
+  }
+
+  const evalResult = await params.llm({
+    model: params.model,
+    instructions: `${instructions}\nThe conversation has ended. Judge honestly whether your goal was accomplished.`,
+    messages: [
+      msg(
+        'user',
+        `Here is the full conversation:\n\n${transcript.join('\n')}\n\nSuccess criterion: ${params.evaluate}\n\nWas it met?`
+      ),
+    ],
+    tools: [],
+    maxSteps: 1,
+    toolChoice: 'none',
+    outputSchema: EVALUATION_SCHEMA as unknown as Record<string, unknown>,
+  })
+
+  const verdict = readObject<{ passed: boolean; reasoning: string }>(evalResult)
+  return {
+    passed: verdict?.passed ?? false,
+    reasoning: verdict?.reasoning ?? 'No evaluation produced.',
+    transcript,
+  }
+}

package/src/wirings/workflow/dsl/workflow-dsl.types.ts

@@ -4,7 +4,7 @@
  */
 
 import type { WorkflowRun } from '../workflow.types.js'
-import type { UserFlowActor } from '../../../services/user-flow-actors-service.js'
+import type { ScenarioActor } from '../../../services/scenario-actors-service.js'
 
 /**
  * Workflow step options
@@ -17,18 +17,18 @@ export interface WorkflowStepOptions {
   /** Delay between retry attempts (e.g., '1s', '2s', '2min') */
   retryDelay?: string | number
   /**
-   * Run this step as an actor (user flows). The RPC is sent through the
+   * Run this step as an actor (scenarios). The RPC is sent through the
    * actor's authenticated client over the REAL transport — never dispatched
    * internally — so auth middleware and permissions are exercised end-to-end.
    * The step is recorded durably like any RPC step.
    */
-  actor?: UserFlowActor
+  actor?: ScenarioActor
   // Future: timeout, failFast, priority
 }
 
 /**
  * Options for workflow.expectEventually() — a durable polling step used by
- * user flows to await async effects (a notification landing, a job finishing).
+ * scenarios to await async effects (a notification landing, a job finishing).
  */
 export interface WorkflowExpectEventuallyOptions extends WorkflowStepOptions {
   /** Give up after this long (e.g. '30s'). Default '30s'. */
@@ -37,6 +37,20 @@ export interface WorkflowExpectEventuallyOptions extends WorkflowStepOptions {
   interval?: string | number
 }
 
+/** Options for workflow.expectError() */
+export interface WorkflowExpectErrorOptions extends WorkflowStepOptions {
+  /** Assert the error message matches (string = substring match). */
+  matches?: string | RegExp
+}
+
+/** Options for workflow.expectService() */
+export interface WorkflowExpectServiceOptions extends WorkflowStepOptions {
+  /** Assert a recorded call's first argument deep-equals this value. */
+  calledWith?: unknown
+  /** Assert the exact number of matching calls. Default: at least one. */
+  times?: number
+}
+
 /**
  * Type signature for workflow.do() RPC form - used by inspector
  */
@@ -109,7 +123,7 @@ export interface RpcStepMeta {
   inputs?: Record<string, InputSource> | 'passthrough'
   /** Step options */
   options?: WorkflowStepOptions
-  /** User-flow actor name this step runs as ({ actor: actors.x }) */
+  /** Scenario actor name this step runs as ({ actor: actors.x }) */
   actor?: string
   /** True for workflow.expectEventually polling steps */
   expectEventually?: boolean
@@ -370,7 +384,7 @@ export interface PikkuWorkflowWire {
   do: WorkflowWireDoRPC & WorkflowWireDoInline
 
   /**
-   * Durable polling step (user flows): invoke `rpcName` (as an actor when
+   * Durable polling step (scenarios): invoke `rpcName` (as an actor when
    * `options.as` is set) until `predicate` passes or `options.within` elapses.
    */
   expectEventually: <TOutput = any, TInput = any>(
@@ -381,6 +395,21 @@ export interface PikkuWorkflowWire {
     options?: WorkflowExpectEventuallyOptions
   ) => Promise<TOutput>
 
+  /** Error-path step (scenarios): succeeds only when the RPC throws; returns the message */
+  expectError: <TInput = any>(
+    stepName: string,
+    rpcName: string,
+    data: TInput,
+    options?: WorkflowExpectErrorOptions
+  ) => Promise<string>
+
+  /** Stub-assertion step (scenarios): asserts `service.method` was called on the target server */
+  expectService: (
+    stepName: string,
+    serviceMethod: string,
+    options?: WorkflowExpectServiceOptions
+  ) => Promise<void>
+
   /** Sleep for a duration */
   sleep: WorkflowWireSleep