@pikku/core 0.12.50 → 0.12.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +93 -0
- package/dist/index.d.ts +1 -1
- package/dist/services/http-scenario-actors.d.ts +67 -0
- package/dist/services/http-scenario-actors.js +193 -0
- package/dist/services/http-user-flow-actors.d.ts +20 -0
- package/dist/services/http-user-flow-actors.js +100 -0
- package/dist/services/index.d.ts +2 -2
- package/dist/services/index.js +1 -1
- package/dist/services/meta-service.d.ts +4 -4
- package/dist/services/meta-service.js +8 -8
- package/dist/services/scenario-actors-service.d.ts +39 -0
- package/dist/services/scenario-actors-service.js +1 -0
- package/dist/services/user-flow-actors-service.d.ts +11 -1
- package/dist/types/core.types.d.ts +48 -4
- package/dist/wirings/actor-flow/actor-flow.types.d.ts +68 -0
- package/dist/wirings/actor-flow/actor-flow.types.js +1 -0
- package/dist/wirings/actor-flow/index.d.ts +11 -0
- package/dist/wirings/actor-flow/index.js +1 -0
- package/dist/wirings/actor-flow/run-conversation.d.ts +36 -0
- package/dist/wirings/actor-flow/run-conversation.js +181 -0
- package/dist/wirings/http/http.types.d.ts +1 -0
- package/dist/wirings/workflow/dsl/workflow-dsl.types.d.ts +6 -6
- package/dist/wirings/workflow/pikku-workflow-service.d.ts +10 -2
- package/dist/wirings/workflow/pikku-workflow-service.js +42 -2
- package/dist/wirings/workflow/workflow.types.d.ts +6 -6
- package/package.json +2 -1
- package/run-tests.sh +4 -4
- package/src/index.ts +6 -0
- package/src/services/http-scenario-actors-converse.test.ts +222 -0
- package/src/services/{http-user-flow-actors.test.ts → http-scenario-actors.test.ts} +17 -7
- package/src/services/http-scenario-actors.ts +269 -0
- package/src/services/index.ts +8 -8
- package/src/services/meta-service.ts +9 -9
- package/src/services/{user-flow-actors-service.ts → scenario-actors-service.ts} +18 -4
- package/src/types/core.types.ts +53 -4
- package/src/wirings/actor-flow/actor-flow.types.ts +73 -0
- package/src/wirings/actor-flow/index.ts +22 -0
- package/src/wirings/actor-flow/run-conversation.test.ts +176 -0
- package/src/wirings/actor-flow/run-conversation.ts +285 -0
- package/src/wirings/http/http.types.ts +1 -0
- package/src/wirings/workflow/dsl/workflow-dsl.types.ts +6 -6
- package/src/wirings/workflow/pikku-workflow-service.ts +50 -5
- package/src/wirings/workflow/{user-flow-step.test.ts → scenario-step.test.ts} +19 -14
- package/src/wirings/workflow/workflow.types.ts +6 -6
- package/tsconfig.tsbuildinfo +1 -1
- package/src/services/http-user-flow-actors.ts +0 -132
package/src/types/core.types.ts
CHANGED
|
@@ -29,7 +29,7 @@ import type { DeploymentService } from '../services/deployment-service.js'
|
|
|
29
29
|
import type { AIStorageService } from '../services/ai-storage-service.js'
|
|
30
30
|
|
|
31
31
|
import type { ContentService } from '../services/content-service.js'
|
|
32
|
-
import type {
|
|
32
|
+
import type { ScenarioActors } from '../services/scenario-actors-service.js'
|
|
33
33
|
import type { AIAgentRunnerService } from '../services/ai-agent-runner-service.js'
|
|
34
34
|
import type { AIRunStateService } from '../services/ai-run-state-service.js'
|
|
35
35
|
import type { AgentRunService } from '../wirings/ai-agent/ai-agent.types.js'
|
|
@@ -244,7 +244,7 @@ export type CoreConfig<Config extends Record<string, unknown> = {}> = {
|
|
|
244
244
|
export interface CoreUserSession {
|
|
245
245
|
userId?: string
|
|
246
246
|
orgId?: string
|
|
247
|
-
/** True when the session belongs to a synthetic
|
|
247
|
+
/** True when the session belongs to a synthetic scenario actor — lets audits/analytics address synthetic traffic */
|
|
248
248
|
actor?: boolean
|
|
249
249
|
}
|
|
250
250
|
|
|
@@ -339,8 +339,8 @@ export type PikkuWire<
|
|
|
339
339
|
queue: PikkuQueue
|
|
340
340
|
cli: PikkuCLI
|
|
341
341
|
workflow: TypedWorkflow
|
|
342
|
-
/**
|
|
343
|
-
actors:
|
|
342
|
+
/** Scenario actor registry (scenario runs only) — pass into workflow.do as `{ actor: actors.x }` */
|
|
343
|
+
actors: ScenarioActors
|
|
344
344
|
workflowStep: WorkflowStepWire
|
|
345
345
|
graph: PikkuGraphWire
|
|
346
346
|
trigger: PikkuTrigger<TriggerOutput>
|
|
@@ -609,6 +609,55 @@ export type CommonWireMeta = {
|
|
|
609
609
|
permissions?: PermissionMetadata[]
|
|
610
610
|
}
|
|
611
611
|
|
|
612
|
+
/**
|
|
613
|
+
* Dependency security audit artifact — the shape of `.pikku/audit.json` written
|
|
614
|
+
* by `pikku audit`. Canonical home for the type shared by the CLI (writer), the
|
|
615
|
+
* console addon (reader), and the console UI (renderer).
|
|
616
|
+
*/
|
|
617
|
+
export type SecuritySeverity = 'critical' | 'high' | 'moderate' | 'low' | 'info'
|
|
618
|
+
export type SecurityUpdateLevel = 'major' | 'minor' | 'patch' | 'unknown'
|
|
619
|
+
|
|
620
|
+
export interface SecurityAuditIssue {
|
|
621
|
+
package: string
|
|
622
|
+
severity: SecuritySeverity
|
|
623
|
+
title: string
|
|
624
|
+
advisoryId: string
|
|
625
|
+
url: string
|
|
626
|
+
vulnerableVersions: string
|
|
627
|
+
cwe: string[]
|
|
628
|
+
cvssScore: number | null
|
|
629
|
+
recommendedVersion: string | null
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
export interface SecurityAuditUpdate {
|
|
633
|
+
package: string
|
|
634
|
+
current: string
|
|
635
|
+
latest: string
|
|
636
|
+
level: SecurityUpdateLevel
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
export interface SecurityAuditSummary {
|
|
640
|
+
totalIssues: number
|
|
641
|
+
critical: number
|
|
642
|
+
high: number
|
|
643
|
+
moderate: number
|
|
644
|
+
low: number
|
|
645
|
+
totalUpdates: number
|
|
646
|
+
major: number
|
|
647
|
+
minor: number
|
|
648
|
+
patch: number
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
export interface SecurityAuditReport {
|
|
652
|
+
schemaVersion: number
|
|
653
|
+
tool: string
|
|
654
|
+
generatedAt: string
|
|
655
|
+
note?: string
|
|
656
|
+
issues: SecurityAuditIssue[]
|
|
657
|
+
updates: SecurityAuditUpdate[]
|
|
658
|
+
summary: SecurityAuditSummary
|
|
659
|
+
}
|
|
660
|
+
|
|
612
661
|
/**
|
|
613
662
|
* Serialized error for storage
|
|
614
663
|
*/
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How an actor agent answers the target agent's tool-approval requests during
|
|
3
|
+
* a conversation.
|
|
4
|
+
* - `'in-persona'` — the actor agent decides as the persona would (default).
|
|
5
|
+
* - `'always'` — approve every request (stress the happy path).
|
|
6
|
+
* - `'never'` — deny every request (exercise refusal handling).
|
|
7
|
+
*/
|
|
8
|
+
export type ActorFlowApprovalPolicy = 'in-persona' | 'always' | 'never'
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Options for `actor.converse(...)` — a dynamic conversation an actor holds
|
|
12
|
+
* with a target Pikku AI agent, in the actor's own persona. `TAgentName` is
|
|
13
|
+
* bound to the generated union of agent names in a typed project.
|
|
14
|
+
*/
|
|
15
|
+
export interface ConverseOptions<TAgentName extends string = string> {
|
|
16
|
+
/** Target Pikku AI agent name to converse with. */
|
|
17
|
+
agent: TAgentName
|
|
18
|
+
/** What the actor is trying to get the agent to accomplish. */
|
|
19
|
+
task: string
|
|
20
|
+
/** Natural-language success criterion the actor evaluates at the end. */
|
|
21
|
+
evaluate: string
|
|
22
|
+
/** How the actor answers the agent's tool-approval requests. Default `'in-persona'`. */
|
|
23
|
+
approvals?: ActorFlowApprovalPolicy
|
|
24
|
+
/** Model the persona uses for its own turns/decisions. Falls back to the actor service default. */
|
|
25
|
+
model?: string
|
|
26
|
+
/** Hard cap on conversation turns before forcing evaluation. Default 12. */
|
|
27
|
+
maxTurns?: number
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The verdict a conversation produces: the persona's LLM self-evaluation of
|
|
32
|
+
* whether the task was met. Deterministic checks are the caller's job — they
|
|
33
|
+
* already hold the actor and can `actor.invoke(...)` afterwards.
|
|
34
|
+
*/
|
|
35
|
+
export interface ActorFlowVerdict {
|
|
36
|
+
/** Whether the actor judged the task accomplished. */
|
|
37
|
+
passed: boolean
|
|
38
|
+
/** The actor's reasoning for its verdict. */
|
|
39
|
+
reasoning: string
|
|
40
|
+
/** The conversation transcript, for debugging/reporting. */
|
|
41
|
+
transcript: string[]
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** A pending tool-approval request surfaced by the target agent. */
|
|
45
|
+
export interface TargetPendingApproval {
|
|
46
|
+
toolCallId: string
|
|
47
|
+
toolName: string
|
|
48
|
+
args: unknown
|
|
49
|
+
reason?: string
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** A normalized reply from the target agent, independent of transport. */
|
|
53
|
+
export interface TargetAgentReply {
|
|
54
|
+
text: string
|
|
55
|
+
runId: string
|
|
56
|
+
status?: 'completed' | 'suspended'
|
|
57
|
+
pendingApprovals?: TargetPendingApproval[]
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Drives the target agent. In production this is HTTP-backed (the actor's
|
|
62
|
+
* `agentRun` / `agentApprove` calls as the signed-in actor); the conversation
|
|
63
|
+
* engine only sees this transport-agnostic contract.
|
|
64
|
+
*/
|
|
65
|
+
export interface TargetAgentDriver {
|
|
66
|
+
/** Send a message, starting or continuing the target agent's run. */
|
|
67
|
+
run(message: string): Promise<TargetAgentReply>
|
|
68
|
+
/** Answer the target agent's pending approvals and continue its run. */
|
|
69
|
+
approve(
|
|
70
|
+
runId: string,
|
|
71
|
+
decisions: { toolCallId: string; approved: boolean }[]
|
|
72
|
+
): Promise<TargetAgentReply>
|
|
73
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Actor flow module exports.
|
|
3
|
+
*
|
|
4
|
+
* An actor holds a dynamic conversation with a target Pikku AI agent via
|
|
5
|
+
* `actor.converse(...)` — playing its own persona, approving the agent's tool
|
|
6
|
+
* requests in-persona, and evaluating whether the task was accomplished. The
|
|
7
|
+
* conversation engine here is transport-agnostic; the actor drives the target
|
|
8
|
+
* over HTTP.
|
|
9
|
+
*/
|
|
10
|
+
export type {
|
|
11
|
+
ActorFlowApprovalPolicy,
|
|
12
|
+
ActorFlowVerdict,
|
|
13
|
+
ConverseOptions,
|
|
14
|
+
TargetAgentReply,
|
|
15
|
+
TargetPendingApproval,
|
|
16
|
+
TargetAgentDriver,
|
|
17
|
+
} from './actor-flow.types.js'
|
|
18
|
+
export {
|
|
19
|
+
runConversation,
|
|
20
|
+
type RunConversationParams,
|
|
21
|
+
type PersonaLLM,
|
|
22
|
+
} from './run-conversation.js'
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { describe, test } from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
|
|
4
|
+
import { runConversation, type PersonaLLM } from './run-conversation.js'
|
|
5
|
+
import type {
|
|
6
|
+
TargetAgentDriver,
|
|
7
|
+
TargetAgentReply,
|
|
8
|
+
ActorFlowApprovalPolicy,
|
|
9
|
+
} from './actor-flow.types.js'
|
|
10
|
+
import type { AIAgentStepResult } from '../../services/ai-agent-runner-service.js'
|
|
11
|
+
|
|
12
|
+
const stepResult = (object: unknown): AIAgentStepResult => ({
|
|
13
|
+
text: '',
|
|
14
|
+
object,
|
|
15
|
+
toolCalls: [],
|
|
16
|
+
toolResults: [],
|
|
17
|
+
usage: { inputTokens: 0, outputTokens: 0 },
|
|
18
|
+
finishReason: 'stop',
|
|
19
|
+
})
|
|
20
|
+
|
|
21
|
+
/** A persona LLM scripted by the outputSchema it's asked for. */
|
|
22
|
+
const scriptedLLM = (script: {
|
|
23
|
+
turns: Array<{ message: string; done: boolean }>
|
|
24
|
+
decisions: Array<{ toolCallId: string; approved: boolean }>
|
|
25
|
+
evaluation: { passed: boolean; reasoning: string }
|
|
26
|
+
}): { llm: PersonaLLM; calls: string[] } => {
|
|
27
|
+
let turn = 0
|
|
28
|
+
const calls: string[] = []
|
|
29
|
+
const llm: PersonaLLM = async (params) => {
|
|
30
|
+
const props =
|
|
31
|
+
(params.outputSchema as { properties?: Record<string, unknown> })
|
|
32
|
+
?.properties ?? {}
|
|
33
|
+
if ('message' in props) {
|
|
34
|
+
// Providers reject an empty prompt — every persona turn must carry messages.
|
|
35
|
+
assert.ok(
|
|
36
|
+
params.messages.length > 0,
|
|
37
|
+
'persona turn requested with empty messages'
|
|
38
|
+
)
|
|
39
|
+
const t = script.turns[turn] ?? script.turns[script.turns.length - 1]
|
|
40
|
+
turn++
|
|
41
|
+
calls.push('turn')
|
|
42
|
+
return stepResult(t)
|
|
43
|
+
}
|
|
44
|
+
if ('decisions' in props) {
|
|
45
|
+
calls.push('approval')
|
|
46
|
+
return stepResult({ decisions: script.decisions })
|
|
47
|
+
}
|
|
48
|
+
if ('passed' in props) {
|
|
49
|
+
calls.push('eval')
|
|
50
|
+
return stepResult(script.evaluation)
|
|
51
|
+
}
|
|
52
|
+
throw new Error('unexpected schema')
|
|
53
|
+
}
|
|
54
|
+
return { llm, calls }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** A target that suspends once for approval, then completes. */
|
|
58
|
+
const suspendingTarget = (): {
|
|
59
|
+
target: TargetAgentDriver
|
|
60
|
+
approvedWith: Array<{ toolCallId: string; approved: boolean }[]>
|
|
61
|
+
} => {
|
|
62
|
+
let runs = 0
|
|
63
|
+
const approvedWith: Array<{ toolCallId: string; approved: boolean }[]> = []
|
|
64
|
+
const target: TargetAgentDriver = {
|
|
65
|
+
run: async (): Promise<TargetAgentReply> => {
|
|
66
|
+
runs++
|
|
67
|
+
if (runs === 1) {
|
|
68
|
+
return {
|
|
69
|
+
text: 'Let me do that.',
|
|
70
|
+
runId: 'run-1',
|
|
71
|
+
status: 'suspended',
|
|
72
|
+
pendingApprovals: [
|
|
73
|
+
{ toolCallId: 'tc1', toolName: 'createTodo', args: { title: 'x' } },
|
|
74
|
+
],
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return { text: 'All set.', runId: `run-${runs}`, status: 'completed' }
|
|
78
|
+
},
|
|
79
|
+
approve: async (runId, decisions): Promise<TargetAgentReply> => {
|
|
80
|
+
approvedWith.push(decisions)
|
|
81
|
+
return { text: 'Created it.', runId, status: 'completed' }
|
|
82
|
+
},
|
|
83
|
+
}
|
|
84
|
+
return { target, approvedWith }
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** A target that never completes — always re-suspends for the same approval. */
|
|
88
|
+
const alwaysSuspendingTarget = (): {
|
|
89
|
+
target: TargetAgentDriver
|
|
90
|
+
approveCalls: () => number
|
|
91
|
+
} => {
|
|
92
|
+
let approveCalls = 0
|
|
93
|
+
const suspended: TargetAgentReply = {
|
|
94
|
+
text: 'working on it',
|
|
95
|
+
runId: 'run-stuck',
|
|
96
|
+
status: 'suspended',
|
|
97
|
+
pendingApprovals: [
|
|
98
|
+
{ toolCallId: 'tc1', toolName: 'createTodo', args: { title: 'x' } },
|
|
99
|
+
],
|
|
100
|
+
}
|
|
101
|
+
const target: TargetAgentDriver = {
|
|
102
|
+
run: async () => suspended,
|
|
103
|
+
approve: async () => {
|
|
104
|
+
approveCalls++
|
|
105
|
+
return suspended
|
|
106
|
+
},
|
|
107
|
+
}
|
|
108
|
+
return { target, approveCalls: () => approveCalls }
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const base = {
|
|
112
|
+
persona: { email: 'pm@example.com', name: 'Priya', personality: 'concise' },
|
|
113
|
+
personaName: 'Priya',
|
|
114
|
+
agentName: 'todoBot',
|
|
115
|
+
task: 'Get a todo created',
|
|
116
|
+
evaluate: 'A todo now exists',
|
|
117
|
+
model: 'test/test-model',
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
describe('runConversation', () => {
|
|
121
|
+
test('drives turns, approves in-persona, evaluates', async () => {
|
|
122
|
+
const { llm, calls } = scriptedLLM({
|
|
123
|
+
turns: [
|
|
124
|
+
{ message: 'please make a todo', done: false },
|
|
125
|
+
{ message: 'thanks', done: true },
|
|
126
|
+
],
|
|
127
|
+
decisions: [{ toolCallId: 'tc1', approved: true }],
|
|
128
|
+
evaluation: { passed: true, reasoning: 'todo created' },
|
|
129
|
+
})
|
|
130
|
+
const { target, approvedWith } = suspendingTarget()
|
|
131
|
+
|
|
132
|
+
const verdict = await runConversation({ ...base, llm, target })
|
|
133
|
+
|
|
134
|
+
assert.equal(verdict.passed, true)
|
|
135
|
+
assert.match(verdict.reasoning, /todo/i)
|
|
136
|
+
assert.ok(verdict.transcript.length >= 2)
|
|
137
|
+
assert.deepEqual(approvedWith, [[{ toolCallId: 'tc1', approved: true }]])
|
|
138
|
+
assert.ok(calls.includes('approval'))
|
|
139
|
+
assert.ok(calls.includes('eval'))
|
|
140
|
+
})
|
|
141
|
+
|
|
142
|
+
test("approvals policy 'never' denies without asking the persona", async () => {
|
|
143
|
+
const { llm, calls } = scriptedLLM({
|
|
144
|
+
turns: [{ message: 'make a todo', done: true }],
|
|
145
|
+
decisions: [{ toolCallId: 'tc1', approved: true }],
|
|
146
|
+
evaluation: { passed: false, reasoning: 'blocked' },
|
|
147
|
+
})
|
|
148
|
+
const { target, approvedWith } = suspendingTarget()
|
|
149
|
+
|
|
150
|
+
const verdict = await runConversation({
|
|
151
|
+
...base,
|
|
152
|
+
approvals: 'never' as ActorFlowApprovalPolicy,
|
|
153
|
+
llm,
|
|
154
|
+
target,
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
assert.deepEqual(approvedWith, [[{ toolCallId: 'tc1', approved: false }]])
|
|
158
|
+
assert.ok(!calls.includes('approval'))
|
|
159
|
+
assert.equal(verdict.passed, false)
|
|
160
|
+
})
|
|
161
|
+
|
|
162
|
+
test('caps the approval loop when the target never stops suspending', async () => {
|
|
163
|
+
const { llm } = scriptedLLM({
|
|
164
|
+
turns: [{ message: 'please make a todo', done: false }],
|
|
165
|
+
decisions: [{ toolCallId: 'tc1', approved: true }],
|
|
166
|
+
evaluation: { passed: false, reasoning: 'never got there' },
|
|
167
|
+
})
|
|
168
|
+
const { target, approveCalls } = alwaysSuspendingTarget()
|
|
169
|
+
|
|
170
|
+
await assert.rejects(
|
|
171
|
+
runConversation({ ...base, approvals: 'always', maxApprovalRounds: 3, llm, target }),
|
|
172
|
+
/approval rounds/
|
|
173
|
+
)
|
|
174
|
+
assert.equal(approveCalls(), 3)
|
|
175
|
+
})
|
|
176
|
+
})
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
ActorFlowApprovalPolicy,
|
|
3
|
+
ActorFlowVerdict,
|
|
4
|
+
TargetAgentDriver,
|
|
5
|
+
TargetPendingApproval,
|
|
6
|
+
} from './actor-flow.types.js'
|
|
7
|
+
import type { ScenarioActorConfig } from '../../services/scenario-actors-service.js'
|
|
8
|
+
import type { AIMessage } from '../ai-agent/ai-agent.types.js'
|
|
9
|
+
import type {
|
|
10
|
+
AIAgentRunnerParams,
|
|
11
|
+
AIAgentStepResult,
|
|
12
|
+
} from '../../services/ai-agent-runner-service.js'
|
|
13
|
+
|
|
14
|
+
/** One turn the persona takes: the message to send and whether it's finished. */
|
|
15
|
+
const PERSONA_TURN_SCHEMA = {
|
|
16
|
+
type: 'object',
|
|
17
|
+
properties: {
|
|
18
|
+
message: { type: 'string' },
|
|
19
|
+
done: { type: 'boolean' },
|
|
20
|
+
},
|
|
21
|
+
required: ['message', 'done'],
|
|
22
|
+
} as const
|
|
23
|
+
|
|
24
|
+
/** The persona's approve/deny decision for each pending tool request. */
|
|
25
|
+
const APPROVAL_DECISION_SCHEMA = {
|
|
26
|
+
type: 'object',
|
|
27
|
+
properties: {
|
|
28
|
+
decisions: {
|
|
29
|
+
type: 'array',
|
|
30
|
+
items: {
|
|
31
|
+
type: 'object',
|
|
32
|
+
properties: {
|
|
33
|
+
toolCallId: { type: 'string' },
|
|
34
|
+
approved: { type: 'boolean' },
|
|
35
|
+
reason: { type: 'string' },
|
|
36
|
+
},
|
|
37
|
+
required: ['toolCallId', 'approved'],
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
},
|
|
41
|
+
required: ['decisions'],
|
|
42
|
+
} as const
|
|
43
|
+
|
|
44
|
+
/** The persona's final verdict on whether the task was accomplished. */
|
|
45
|
+
const EVALUATION_SCHEMA = {
|
|
46
|
+
type: 'object',
|
|
47
|
+
properties: {
|
|
48
|
+
passed: { type: 'boolean' },
|
|
49
|
+
reasoning: { type: 'string' },
|
|
50
|
+
},
|
|
51
|
+
required: ['passed', 'reasoning'],
|
|
52
|
+
} as const
|
|
53
|
+
|
|
54
|
+
const DEFAULT_MAX_TURNS = 12
|
|
55
|
+
|
|
56
|
+
const DEFAULT_MAX_APPROVAL_ROUNDS = 16
|
|
57
|
+
|
|
58
|
+
/** The LLM call the persona uses for its own turns/decisions/evaluation. */
|
|
59
|
+
export type PersonaLLM = (
|
|
60
|
+
params: AIAgentRunnerParams
|
|
61
|
+
) => Promise<AIAgentStepResult>
|
|
62
|
+
|
|
63
|
+
export interface RunConversationParams {
|
|
64
|
+
/** Persona config (personality/jobTitle/name) that shapes how the actor talks. */
|
|
65
|
+
persona: ScenarioActorConfig
|
|
66
|
+
/** Stable persona name (for transcript labelling). */
|
|
67
|
+
personaName: string
|
|
68
|
+
/** What the actor is trying to get the target agent to accomplish. */
|
|
69
|
+
task: string
|
|
70
|
+
/** Natural-language success criterion the actor evaluates at the end. */
|
|
71
|
+
evaluate: string
|
|
72
|
+
/** How the actor answers the target agent's tool-approval requests. */
|
|
73
|
+
approvals?: ActorFlowApprovalPolicy
|
|
74
|
+
/** Model the persona uses for its own turns/decisions. */
|
|
75
|
+
model: string
|
|
76
|
+
/** Hard cap on conversation turns. Default 12. */
|
|
77
|
+
maxTurns?: number
|
|
78
|
+
/** Hard cap on tool-approval rounds within a single target turn. Default 16. */
|
|
79
|
+
maxApprovalRounds?: number
|
|
80
|
+
/** Transport that drives the target agent (HTTP in production). */
|
|
81
|
+
target: TargetAgentDriver
|
|
82
|
+
/** The persona's own LLM. */
|
|
83
|
+
llm: PersonaLLM
|
|
84
|
+
/** Display name of the target agent (transcript labelling). */
|
|
85
|
+
agentName: string
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function msg(role: AIMessage['role'], content: string): AIMessage {
|
|
89
|
+
return {
|
|
90
|
+
id: globalThis.crypto.randomUUID(),
|
|
91
|
+
role,
|
|
92
|
+
content,
|
|
93
|
+
createdAt: new Date(),
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Read a structured `run` result, falling back to parsing JSON from text. */
|
|
98
|
+
function readObject<T>(result: { object?: unknown; text?: string }): T | null {
|
|
99
|
+
if (result.object && typeof result.object === 'object') {
|
|
100
|
+
return result.object as T
|
|
101
|
+
}
|
|
102
|
+
if (result.text) {
|
|
103
|
+
try {
|
|
104
|
+
return JSON.parse(result.text) as T
|
|
105
|
+
} catch {
|
|
106
|
+
return null
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return null
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function personaInstructions(
|
|
113
|
+
persona: ScenarioActorConfig,
|
|
114
|
+
task: string
|
|
115
|
+
): string {
|
|
116
|
+
return [
|
|
117
|
+
`You are role-playing a real user interacting with an AI assistant. Stay in character at all times — you are the user, not the assistant.`,
|
|
118
|
+
persona.name ? `Your name is ${persona.name}.` : '',
|
|
119
|
+
persona.jobTitle ? `Your role: ${persona.jobTitle}.` : '',
|
|
120
|
+
persona.personality
|
|
121
|
+
? `Your personality and communication style: ${persona.personality}. Match this tone, vocabulary, and level of detail exactly.`
|
|
122
|
+
: '',
|
|
123
|
+
`Your goal in this conversation: ${task}.`,
|
|
124
|
+
`Send one message at a time. Set "done" to true only once your goal is clearly accomplished, or clearly impossible.`,
|
|
125
|
+
]
|
|
126
|
+
.filter(Boolean)
|
|
127
|
+
.join('\n')
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Route the target agent's pending tool approvals through the persona. */
|
|
131
|
+
async function decideApprovals(
|
|
132
|
+
params: RunConversationParams,
|
|
133
|
+
instructions: string,
|
|
134
|
+
pending: TargetPendingApproval[]
|
|
135
|
+
): Promise<{ toolCallId: string; approved: boolean }[]> {
|
|
136
|
+
const policy = params.approvals ?? 'in-persona'
|
|
137
|
+
if (policy === 'always') {
|
|
138
|
+
return pending.map((p) => ({ toolCallId: p.toolCallId, approved: true }))
|
|
139
|
+
}
|
|
140
|
+
if (policy === 'never') {
|
|
141
|
+
return pending.map((p) => ({ toolCallId: p.toolCallId, approved: false }))
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const summary = pending
|
|
145
|
+
.map(
|
|
146
|
+
(p) =>
|
|
147
|
+
`- toolCallId "${p.toolCallId}": ${p.toolName}(${JSON.stringify(p.args)})${p.reason ? ` — ${p.reason}` : ''}`
|
|
148
|
+
)
|
|
149
|
+
.join('\n')
|
|
150
|
+
|
|
151
|
+
const result = await params.llm({
|
|
152
|
+
model: params.model,
|
|
153
|
+
instructions: `${instructions}\nThe assistant is asking permission to run tools on your behalf. Decide whether YOU, as this persona, would allow each one.`,
|
|
154
|
+
messages: [
|
|
155
|
+
msg(
|
|
156
|
+
'user',
|
|
157
|
+
`The assistant wants to run these tools:\n${summary}\nApprove or deny each toolCallId.`
|
|
158
|
+
),
|
|
159
|
+
],
|
|
160
|
+
tools: [],
|
|
161
|
+
maxSteps: 1,
|
|
162
|
+
toolChoice: 'none',
|
|
163
|
+
outputSchema: APPROVAL_DECISION_SCHEMA as unknown as Record<
|
|
164
|
+
string,
|
|
165
|
+
unknown
|
|
166
|
+
>,
|
|
167
|
+
})
|
|
168
|
+
|
|
169
|
+
const decided =
|
|
170
|
+
readObject<{
|
|
171
|
+
decisions?: { toolCallId: string; approved: boolean }[]
|
|
172
|
+
}>(result)?.decisions ?? []
|
|
173
|
+
|
|
174
|
+
// Every pending call must get a decision; a missing one defaults to denied.
|
|
175
|
+
return pending.map((p) => {
|
|
176
|
+
const match = decided.find((d) => d.toolCallId === p.toolCallId)
|
|
177
|
+
return { toolCallId: p.toolCallId, approved: match?.approved ?? false }
|
|
178
|
+
})
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Drive the target to a non-suspended reply, routing approvals to the persona. */
|
|
182
|
+
async function converseWithTarget(
|
|
183
|
+
params: RunConversationParams,
|
|
184
|
+
instructions: string,
|
|
185
|
+
message: string
|
|
186
|
+
) {
|
|
187
|
+
const maxRounds = params.maxApprovalRounds ?? DEFAULT_MAX_APPROVAL_ROUNDS
|
|
188
|
+
let reply = await params.target.run(message)
|
|
189
|
+
let rounds = 0
|
|
190
|
+
while (
|
|
191
|
+
reply.status === 'suspended' &&
|
|
192
|
+
reply.pendingApprovals &&
|
|
193
|
+
reply.pendingApprovals.length > 0
|
|
194
|
+
) {
|
|
195
|
+
if (rounds >= maxRounds) {
|
|
196
|
+
throw new Error(
|
|
197
|
+
`Target agent "${params.agentName}" stayed suspended after ${maxRounds} approval rounds (runId ${reply.runId}); aborting to avoid an unbounded approve loop.`
|
|
198
|
+
)
|
|
199
|
+
}
|
|
200
|
+
rounds++
|
|
201
|
+
const decisions = await decideApprovals(
|
|
202
|
+
params,
|
|
203
|
+
instructions,
|
|
204
|
+
reply.pendingApprovals
|
|
205
|
+
)
|
|
206
|
+
reply = await params.target.approve(reply.runId, decisions)
|
|
207
|
+
}
|
|
208
|
+
return reply
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Run a conversation: an LLM-driven persona holds a real multi-turn exchange
|
|
213
|
+
* with a target agent (driven via the injected transport), answers the target's
|
|
214
|
+
* tool-approval requests in-persona, then evaluates whether the task was met.
|
|
215
|
+
* Deterministic checks are the caller's responsibility.
|
|
216
|
+
*/
|
|
217
|
+
export async function runConversation(
|
|
218
|
+
params: RunConversationParams
|
|
219
|
+
): Promise<ActorFlowVerdict> {
|
|
220
|
+
const maxTurns = params.maxTurns ?? DEFAULT_MAX_TURNS
|
|
221
|
+
const instructions = personaInstructions(params.persona, params.task)
|
|
222
|
+
// Seed a kickoff so the very first persona turn has a non-empty message list
|
|
223
|
+
// (providers reject an empty prompt). It's an instruction TO the persona, so
|
|
224
|
+
// it never appears in the transcript.
|
|
225
|
+
const personaMessages: AIMessage[] = [
|
|
226
|
+
msg(
|
|
227
|
+
'user',
|
|
228
|
+
'Begin the conversation now — send your first message to the assistant to work towards your goal.'
|
|
229
|
+
),
|
|
230
|
+
]
|
|
231
|
+
const transcript: string[] = []
|
|
232
|
+
|
|
233
|
+
for (let turn = 0; turn < maxTurns; turn++) {
|
|
234
|
+
const personaResult = await params.llm({
|
|
235
|
+
model: params.model,
|
|
236
|
+
instructions,
|
|
237
|
+
messages: personaMessages,
|
|
238
|
+
tools: [],
|
|
239
|
+
maxSteps: 1,
|
|
240
|
+
toolChoice: 'none',
|
|
241
|
+
outputSchema: PERSONA_TURN_SCHEMA as unknown as Record<string, unknown>,
|
|
242
|
+
})
|
|
243
|
+
|
|
244
|
+
const turnData = readObject<{ message: string; done: boolean }>(
|
|
245
|
+
personaResult
|
|
246
|
+
)
|
|
247
|
+
const personaMessage = turnData?.message?.trim()
|
|
248
|
+
if (!personaMessage) {
|
|
249
|
+
break
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
personaMessages.push(msg('assistant', personaMessage))
|
|
253
|
+
transcript.push(`${params.personaName}: ${personaMessage}`)
|
|
254
|
+
|
|
255
|
+
const reply = await converseWithTarget(params, instructions, personaMessage)
|
|
256
|
+
personaMessages.push(msg('user', reply.text ?? ''))
|
|
257
|
+
transcript.push(`${params.agentName}: ${reply.text ?? ''}`)
|
|
258
|
+
|
|
259
|
+
if (turnData?.done) {
|
|
260
|
+
break
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
const evalResult = await params.llm({
|
|
265
|
+
model: params.model,
|
|
266
|
+
instructions: `${instructions}\nThe conversation has ended. Judge honestly whether your goal was accomplished.`,
|
|
267
|
+
messages: [
|
|
268
|
+
msg(
|
|
269
|
+
'user',
|
|
270
|
+
`Here is the full conversation:\n\n${transcript.join('\n')}\n\nSuccess criterion: ${params.evaluate}\n\nWas it met?`
|
|
271
|
+
),
|
|
272
|
+
],
|
|
273
|
+
tools: [],
|
|
274
|
+
maxSteps: 1,
|
|
275
|
+
toolChoice: 'none',
|
|
276
|
+
outputSchema: EVALUATION_SCHEMA as unknown as Record<string, unknown>,
|
|
277
|
+
})
|
|
278
|
+
|
|
279
|
+
const verdict = readObject<{ passed: boolean; reasoning: string }>(evalResult)
|
|
280
|
+
return {
|
|
281
|
+
passed: verdict?.passed ?? false,
|
|
282
|
+
reasoning: verdict?.reasoning ?? 'No evaluation produced.',
|
|
283
|
+
transcript,
|
|
284
|
+
}
|
|
285
|
+
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import type { WorkflowRun } from '../workflow.types.js'
|
|
7
|
-
import type {
|
|
7
|
+
import type { ScenarioActor } from '../../../services/scenario-actors-service.js'
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* Workflow step options
|
|
@@ -17,18 +17,18 @@ export interface WorkflowStepOptions {
|
|
|
17
17
|
/** Delay between retry attempts (e.g., '1s', '2s', '2min') */
|
|
18
18
|
retryDelay?: string | number
|
|
19
19
|
/**
|
|
20
|
-
* Run this step as an actor (
|
|
20
|
+
* Run this step as an actor (scenarios). The RPC is sent through the
|
|
21
21
|
* actor's authenticated client over the REAL transport — never dispatched
|
|
22
22
|
* internally — so auth middleware and permissions are exercised end-to-end.
|
|
23
23
|
* The step is recorded durably like any RPC step.
|
|
24
24
|
*/
|
|
25
|
-
actor?:
|
|
25
|
+
actor?: ScenarioActor
|
|
26
26
|
// Future: timeout, failFast, priority
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
/**
|
|
30
30
|
* Options for workflow.expectEventually() — a durable polling step used by
|
|
31
|
-
*
|
|
31
|
+
* scenarios to await async effects (a notification landing, a job finishing).
|
|
32
32
|
*/
|
|
33
33
|
export interface WorkflowExpectEventuallyOptions extends WorkflowStepOptions {
|
|
34
34
|
/** Give up after this long (e.g. '30s'). Default '30s'. */
|
|
@@ -109,7 +109,7 @@ export interface RpcStepMeta {
|
|
|
109
109
|
inputs?: Record<string, InputSource> | 'passthrough'
|
|
110
110
|
/** Step options */
|
|
111
111
|
options?: WorkflowStepOptions
|
|
112
|
-
/**
|
|
112
|
+
/** Scenario actor name this step runs as ({ actor: actors.x }) */
|
|
113
113
|
actor?: string
|
|
114
114
|
/** True for workflow.expectEventually polling steps */
|
|
115
115
|
expectEventually?: boolean
|
|
@@ -370,7 +370,7 @@ export interface PikkuWorkflowWire {
|
|
|
370
370
|
do: WorkflowWireDoRPC & WorkflowWireDoInline
|
|
371
371
|
|
|
372
372
|
/**
|
|
373
|
-
* Durable polling step (
|
|
373
|
+
* Durable polling step (scenarios): invoke `rpcName` (as an actor when
|
|
374
374
|
* `options.as` is set) until `predicate` passes or `options.within` elapses.
|
|
375
375
|
*/
|
|
376
376
|
expectEventually: <TOutput = any, TInput = any>(
|