@particle-academy/prism-harness 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -0
- package/dist/errors.d.ts +35 -1
- package/dist/errors.js +32 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +2 -0
- package/dist/runtime.d.ts +68 -19
- package/dist/runtime.js +218 -82
- package/dist/structured.d.ts +37 -0
- package/dist/structured.js +158 -0
- package/dist/thread-rows.d.ts +71 -0
- package/dist/thread-rows.js +135 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -115,6 +115,117 @@ new ModeRegistry({
|
|
|
115
115
|
A value that is not a map is refused when the mode is resolved, with
|
|
116
116
|
`mode_malformed`.
|
|
117
117
|
|
|
118
|
+
## What your model client receives
|
|
119
|
+
|
|
120
|
+
`request.messages` is the thread, oldest first, in the shape the PHP reference
|
|
121
|
+
stores Prism's messages:
|
|
122
|
+
|
|
123
|
+
| `type` | Carries |
|
|
124
|
+
|---|---|
|
|
125
|
+
| `user` | `content`, and `additional_content` when the turn has attachments |
|
|
126
|
+
| `assistant` | `content`, `tool_calls` (`id`, `name`, `arguments`, `result_id`, `reasoning_id`, `reasoning_summary`), `additional_content`, `tool_approval_requests` |
|
|
127
|
+
| `tool_result` | `tool_results` (`tool_call_id`, `tool_name`, `args`, `result`, `tool_call_result_id`, `artifacts`), `tool_approval_responses` |
|
|
128
|
+
|
|
129
|
+
Consecutive tool result rows reach the client as one, so each call's result is
|
|
130
|
+
sent to the provider once. Return a call's provider ids as `resultId`,
|
|
131
|
+
`reasoningId` and `reasoningSummary`, and provider state for the turn as
|
|
132
|
+
`additionalContent`: they are recorded and come back in `messages`.
|
|
133
|
+
|
|
134
|
+
## Structured turns
|
|
135
|
+
|
|
136
|
+
When the answer is a document rather than prose, hand the turn a schema:
|
|
137
|
+
|
|
138
|
+
```ts
|
|
139
|
+
const schema = {
|
|
140
|
+
name: 'plan',
|
|
141
|
+
type: 'object',
|
|
142
|
+
properties: {
|
|
143
|
+
title: { type: 'string' },
|
|
144
|
+
steps: { type: 'array', items: { type: 'string' } },
|
|
145
|
+
},
|
|
146
|
+
required: ['title', 'steps'],
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
const response = await runtime.sendStructured(session, 'Plan the release', schema);
|
|
150
|
+
|
|
151
|
+
response.structured; // { title: 'Ship it', steps: ['write', 'test'] }
|
|
152
|
+
response.text; // the document as the model wrote it
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
It is the same run as `send()` — the mode's system prompt, its tools, the step
|
|
156
|
+
budget, the events — asking the provider for structured output. Your client
|
|
157
|
+
receives the schema as `request.schema` and returns what it parsed as
|
|
158
|
+
`response.structured`; with `prism-ts` that is `Prism.structured().withSchema()`
|
|
159
|
+
and the response's `structured`.
|
|
160
|
+
|
|
161
|
+
**The thread keeps the text, with the parsed object beside it.** The assistant
|
|
162
|
+
message is the raw document, and `structured` rides along in the row's
|
|
163
|
+
`additional_content`. A later turn replays the conversation as messages and reads
|
|
164
|
+
the text, so a thread that contains a structured answer reads like any other.
|
|
165
|
+
|
|
166
|
+
**A document that misses the schema is refused, not repaired.**
|
|
167
|
+
|
|
168
|
+
```ts
|
|
169
|
+
try {
|
|
170
|
+
const plan = (await runtime.sendStructured(session, brief, schema)).structured;
|
|
171
|
+
} catch (error) {
|
|
172
|
+
if (error instanceof HarnessError) {
|
|
173
|
+
error.code; // structured_schema_violation, or structured_unreadable
|
|
174
|
+
error.problems; // every way it missed, not the first
|
|
175
|
+
error.document; // what the model actually said
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Nothing is coerced, nothing is trimmed to the fields that fit, and the result is
|
|
181
|
+
never an empty document. An empty plan settles a batch as `done`, which reads
|
|
182
|
+
exactly like a considered answer of "nothing to propose" — the failure this
|
|
183
|
+
refusal exists to prevent. The exchange is still recorded, and the run is marked
|
|
184
|
+
failed: a thread that omits the answer it did not like cannot explain the retry
|
|
185
|
+
sitting next to it. The run row and the `run.failed` event name the CODE, not the
|
|
186
|
+
message — the message quotes the values that missed, and an event carrying those
|
|
187
|
+
would ship model output to every listener, which is the same reason tool
|
|
188
|
+
arguments are names-only here.
|
|
189
|
+
|
|
190
|
+
The check reads the schema's own JSON Schema, so a hand-written schema is held to
|
|
191
|
+
the same terms. `schemaProblems()` is exported if you want it directly. It checks
|
|
192
|
+
declared types, required keys, enum members, array items, and — where a schema
|
|
193
|
+
closes itself — keys nobody declared. It does not read `$ref`, `allOf`, `oneOf`
|
|
194
|
+
or the numeric and string facets; what it cannot read, it passes, rather than
|
|
195
|
+
reporting a constraint it did not actually check. The same rules as the PHP
|
|
196
|
+
reference, message for message, so a document refused in one language is refused
|
|
197
|
+
in the other for the same stated reason.
|
|
198
|
+
|
|
199
|
+
## Approvals
|
|
200
|
+
|
|
201
|
+
A mode names the tools a person must approve:
|
|
202
|
+
|
|
203
|
+
```ts
|
|
204
|
+
guarded: { system_prompt: '...', tools: ['read', 'delete'], requires_approval: ['delete'] },
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
A step that calls one stops with `finishReason: 'awaiting_approval'`. The calls
|
|
208
|
+
that need nobody have already run. Record a decision for every pending approval,
|
|
209
|
+
then resume with an empty prompt:
|
|
210
|
+
|
|
211
|
+
```ts
|
|
212
|
+
const response = await runtime.send(session, 'Clean up the failed run');
|
|
213
|
+
|
|
214
|
+
if (response.finishReason === 'awaiting_approval') {
|
|
215
|
+
for (const pending of response.pendingApprovals) {
|
|
216
|
+
await recordApproval(session, pending.id, true); // or false, 'not on production'
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
await runtime.send(session, '');
|
|
220
|
+
}
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
On resume the approved calls run once and denied calls return their reason, and
|
|
224
|
+
then the model continues. The model is not asked to make the calls again. A
|
|
225
|
+
pending call with no decision is refused with "No approval response provided".
|
|
226
|
+
A call that has a result never runs again. Who may approve is your application's
|
|
227
|
+
decision: authorize before calling `recordApproval()`.
|
|
228
|
+
|
|
118
229
|
## Agent task lists
|
|
119
230
|
|
|
120
231
|
An agent given a goal has to keep working across many requests. It needs a list
|
package/dist/errors.d.ts
CHANGED
|
@@ -68,12 +68,30 @@ export type HarnessErrorCode =
|
|
|
68
68
|
/** A turn attachment carries no bytes, no file id and no chunks. */
|
|
69
69
|
| 'attachment_empty'
|
|
70
70
|
/** Attachments were offered with an empty prompt, which sends no user message. */
|
|
71
|
-
| 'attachment_without_prompt'
|
|
71
|
+
| 'attachment_without_prompt'
|
|
72
|
+
/** A structured turn came back with text holding no readable document. */
|
|
73
|
+
| 'structured_unreadable'
|
|
74
|
+
/** A structured turn returned a document its schema refuses. */
|
|
75
|
+
| 'structured_schema_violation';
|
|
72
76
|
export interface HarnessErrorOptions {
|
|
73
77
|
cause?: unknown;
|
|
78
|
+
/** The model's own text, on a structured failure. See `HarnessError.document`. */
|
|
79
|
+
document?: string;
|
|
80
|
+
/** Every way a document missed its schema, on a structured failure. */
|
|
81
|
+
problems?: readonly string[];
|
|
74
82
|
}
|
|
75
83
|
export declare class HarnessError extends Error {
|
|
76
84
|
readonly code: HarnessErrorCode;
|
|
85
|
+
/**
|
|
86
|
+
* The model's own text, on a structured failure.
|
|
87
|
+
*
|
|
88
|
+
* It travels on the error because the first question anyone asks is "what did
|
|
89
|
+
* it actually say", and an error that answers it turns a support thread into
|
|
90
|
+
* a log line. It is model output: log it where you log model output.
|
|
91
|
+
*/
|
|
92
|
+
readonly document?: string;
|
|
93
|
+
/** Every way a document missed its schema, rather than only the first. */
|
|
94
|
+
readonly problems: readonly string[];
|
|
77
95
|
constructor(code: HarnessErrorCode, message: string, options?: HarnessErrorOptions);
|
|
78
96
|
static sessionLocked(key: string, waitSeconds: number): HarnessError;
|
|
79
97
|
/**
|
|
@@ -216,4 +234,20 @@ export declare class HarnessError extends Error {
|
|
|
216
234
|
static attachmentEmpty(index: number): HarnessError;
|
|
217
235
|
/** An empty prompt resumes a paused run and sends no user message, so attachments would vanish. */
|
|
218
236
|
static attachmentWithoutPrompt(): HarnessError;
|
|
237
|
+
/**
|
|
238
|
+
* A structured turn whose text held no document at all.
|
|
239
|
+
*
|
|
240
|
+
* Separate from the schema violation because the caller's next move differs:
|
|
241
|
+
* an apology in prose or a truncated answer is a prompting or a budget
|
|
242
|
+
* problem, and a document with the wrong shape is a schema one.
|
|
243
|
+
*/
|
|
244
|
+
static structuredUnreadable(document: string): HarnessError;
|
|
245
|
+
/**
|
|
246
|
+
* Valid JSON, wrong shape.
|
|
247
|
+
*
|
|
248
|
+
* NOT coerced, not trimmed to the fields that fit, and never an empty
|
|
249
|
+
* document: one that fails the contract arriving as `{}` reads to the code
|
|
250
|
+
* receiving it exactly like a considered answer of "nothing".
|
|
251
|
+
*/
|
|
252
|
+
static structuredSchemaViolation(document: string, problems: readonly string[]): HarnessError;
|
|
219
253
|
}
|
package/dist/errors.js
CHANGED
|
@@ -1,9 +1,21 @@
|
|
|
1
1
|
export class HarnessError extends Error {
|
|
2
2
|
code;
|
|
3
|
+
/**
|
|
4
|
+
* The model's own text, on a structured failure.
|
|
5
|
+
*
|
|
6
|
+
* It travels on the error because the first question anyone asks is "what did
|
|
7
|
+
* it actually say", and an error that answers it turns a support thread into
|
|
8
|
+
* a log line. It is model output: log it where you log model output.
|
|
9
|
+
*/
|
|
10
|
+
document;
|
|
11
|
+
/** Every way a document missed its schema, rather than only the first. */
|
|
12
|
+
problems;
|
|
3
13
|
constructor(code, message, options = {}) {
|
|
4
14
|
super(message, options);
|
|
5
15
|
this.name = 'HarnessError';
|
|
6
16
|
this.code = code;
|
|
17
|
+
this.document = options.document;
|
|
18
|
+
this.problems = options.problems ?? [];
|
|
7
19
|
}
|
|
8
20
|
static sessionLocked(key, waitSeconds) {
|
|
9
21
|
return new HarnessError('session_locked', `Could not acquire the lock on session [${key}] within ${waitSeconds}s. ` +
|
|
@@ -220,4 +232,24 @@ export class HarnessError extends Error {
|
|
|
220
232
|
return new HarnessError('attachment_without_prompt', 'Attachments need a prompt to travel with. An empty prompt resumes a paused run and sends no user ' +
|
|
221
233
|
'message, so these attachments would be dropped without a word.');
|
|
222
234
|
}
|
|
235
|
+
/**
|
|
236
|
+
* A structured turn whose text held no document at all.
|
|
237
|
+
*
|
|
238
|
+
* Separate from the schema violation because the caller's next move differs:
|
|
239
|
+
* an apology in prose or a truncated answer is a prompting or a budget
|
|
240
|
+
* problem, and a document with the wrong shape is a schema one.
|
|
241
|
+
*/
|
|
242
|
+
static structuredUnreadable(document) {
|
|
243
|
+
return new HarnessError('structured_unreadable', 'The model returned no readable document for a structured turn. Its text is on this error as document.', { document });
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Valid JSON, wrong shape.
|
|
247
|
+
*
|
|
248
|
+
* NOT coerced, not trimmed to the fields that fit, and never an empty
|
|
249
|
+
* document: one that fails the contract arriving as `{}` reads to the code
|
|
250
|
+
* receiving it exactly like a considered answer of "nothing".
|
|
251
|
+
*/
|
|
252
|
+
static structuredSchemaViolation(document, problems) {
|
|
253
|
+
return new HarnessError('structured_schema_violation', `The model returned a document that does not satisfy the schema: ${problems.join(' ')}`, { document, problems });
|
|
254
|
+
}
|
|
223
255
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -24,7 +24,10 @@ export type { HarnessEvent, HarnessListener, RunFailed, RunFinished, RunStarted
|
|
|
24
24
|
export { diagnose } from './doctor.js';
|
|
25
25
|
export type { DoctorFinding, DoctorReport, DoctorSubjects } from './doctor.js';
|
|
26
26
|
export { AgentRuntime, recordApproval } from './runtime.js';
|
|
27
|
-
export
|
|
27
|
+
export { assistantRow, threadView, toolResultEntry, toolResultRow } from './thread-rows.js';
|
|
28
|
+
export type { ApprovalDecisionEntry, ApprovalRequestEntry, AssistantRow, ToolCallInput, ToolCallRow, ToolResultEntry, ToolResultRow, } from './thread-rows.js';
|
|
29
|
+
export type { AgentResponse, AgentRuntimeOptions, LlmClient, LlmRequest, LlmResponse, LlmToolCall, PendingApproval, StructuredAgentResponse, } from './runtime.js';
|
|
30
|
+
export { schemaName, schemaProblems } from './structured.js';
|
|
28
31
|
export { DEFAULT_LEASE_SECONDS, TASK_STATES, asAgentTask, attributesToRecord, canonicalTaskJson, isTaskState, isTerminalState, recordToAttributes, taskRecordFrom, withAgentTask, } from './tasks.js';
|
|
29
32
|
export type { AgentTask, AgentTaskAttributeSource, AgentTaskAttributes, AgentTaskRecord, AgentTaskSource, LeasedAgentTask, TaskOutcome, TaskState, } from './tasks.js';
|
|
30
33
|
export { StoreTaskSource, StoredAgentTask } from './store-task-source.js';
|
package/dist/index.js
CHANGED
|
@@ -13,6 +13,8 @@ export { ToolAuthorizer, ToolRegistry, authorizedTool } from './tools.js';
|
|
|
13
13
|
export { HarnessEvents } from './events.js';
|
|
14
14
|
export { diagnose } from './doctor.js';
|
|
15
15
|
export { AgentRuntime, recordApproval } from './runtime.js';
|
|
16
|
+
export { assistantRow, threadView, toolResultEntry, toolResultRow } from './thread-rows.js';
|
|
17
|
+
export { schemaName, schemaProblems } from './structured.js';
|
|
16
18
|
export { DEFAULT_LEASE_SECONDS, TASK_STATES, asAgentTask, attributesToRecord, canonicalTaskJson, isTaskState, isTerminalState, recordToAttributes, taskRecordFrom, withAgentTask, } from './tasks.js';
|
|
17
19
|
export { StoreTaskSource, StoredAgentTask } from './store-task-source.js';
|
|
18
20
|
export { agentCompletionTool } from './task-tools.js';
|
package/dist/runtime.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { JsonObject } from './json.js';
|
|
1
|
+
import type { JsonObject, JsonValue } from './json.js';
|
|
2
2
|
import type { HarnessEvents } from './events.js';
|
|
3
3
|
import type { ModeRegistry } from './modes.js';
|
|
4
4
|
import type { Session } from './session.js';
|
|
@@ -28,11 +28,29 @@ export interface LlmRequest {
|
|
|
28
28
|
* interpret them.
|
|
29
29
|
*/
|
|
30
30
|
providerOptions: Readonly<JsonObject>;
|
|
31
|
+
/**
|
|
32
|
+
* The JSON Schema a structured turn asks the answer to satisfy.
|
|
33
|
+
*
|
|
34
|
+
* Absent on an ordinary turn. A client passes it to its provider's structured
|
|
35
|
+
* mode (for prism-ts, `Prism.structured().withSchema()`) and returns what it
|
|
36
|
+
* parsed as `structured`; the harness checks that against this same schema
|
|
37
|
+
* before the caller sees it.
|
|
38
|
+
*/
|
|
39
|
+
schema?: Readonly<JsonObject>;
|
|
31
40
|
}
|
|
32
41
|
export interface LlmToolCall {
|
|
33
42
|
id: string;
|
|
34
43
|
name: string;
|
|
35
44
|
arguments: JsonObject;
|
|
45
|
+
/**
|
|
46
|
+
* The ids a provider keys the call's result and reasoning by, when they differ
|
|
47
|
+
* from `id`. OpenAI's Responses API answers a `function_call` by its `call_id`,
|
|
48
|
+
* and replays the reasoning item it came from by id. Recorded on the call, as
|
|
49
|
+
* prism's ToolCall stores them, so the next request can send them back.
|
|
50
|
+
*/
|
|
51
|
+
resultId?: string | null;
|
|
52
|
+
reasoningId?: string | null;
|
|
53
|
+
reasoningSummary?: readonly JsonValue[] | null;
|
|
36
54
|
}
|
|
37
55
|
export interface LlmResponse {
|
|
38
56
|
text: string;
|
|
@@ -49,6 +67,14 @@ export interface LlmResponse {
|
|
|
49
67
|
* straight through and read it back from `messages`.
|
|
50
68
|
*/
|
|
51
69
|
additionalContent?: Readonly<JsonObject>;
|
|
70
|
+
/**
|
|
71
|
+
* The document a structured turn parsed out of `text`.
|
|
72
|
+
*
|
|
73
|
+
* Null when the text held none — an apology in prose, a truncated answer, a
|
|
74
|
+
* fence that never closed. The harness tells that case apart from a document
|
|
75
|
+
* with the wrong shape, because the caller's next move differs.
|
|
76
|
+
*/
|
|
77
|
+
structured?: JsonValue | null;
|
|
52
78
|
}
|
|
53
79
|
export type LlmClient = (request: LlmRequest) => Promise<LlmResponse>;
|
|
54
80
|
export interface AgentResponse {
|
|
@@ -63,8 +89,21 @@ export interface AgentResponse {
|
|
|
63
89
|
/** Set when the run stopped because the tree ran out of budget, or was cancelled. */
|
|
64
90
|
stoppedBecause: string | null;
|
|
65
91
|
}
|
|
92
|
+
/**
|
|
93
|
+
* One structured run's result: the document, and the text it was read from.
|
|
94
|
+
*
|
|
95
|
+
* BOTH, not one. `structured` is what the caller asked for, and `text` is what
|
|
96
|
+
* the model actually sent — which is also exactly what the thread stored, so a
|
|
97
|
+
* transcript and a parse can be compared rather than trusted.
|
|
98
|
+
*/
|
|
99
|
+
export interface StructuredAgentResponse extends AgentResponse {
|
|
100
|
+
/** Checked against the schema before it got here; a run that could not produce one threw. */
|
|
101
|
+
structured: JsonValue;
|
|
102
|
+
}
|
|
66
103
|
export interface PendingApproval {
|
|
104
|
+
/** The APPROVAL id: what `recordApproval()` answers. Not the tool call id. */
|
|
67
105
|
id: string;
|
|
106
|
+
toolCallId: string;
|
|
68
107
|
tool: string;
|
|
69
108
|
arguments: JsonObject;
|
|
70
109
|
}
|
|
@@ -75,24 +114,6 @@ export interface AgentRuntimeOptions {
|
|
|
75
114
|
authorizer?: ToolAuthorizer;
|
|
76
115
|
events?: HarnessEvents;
|
|
77
116
|
}
|
|
78
|
-
/**
|
|
79
|
-
* The loop: prompt in, turns out, everything recorded.
|
|
80
|
-
*
|
|
81
|
-
* Three properties matter more than the mechanics.
|
|
82
|
-
*
|
|
83
|
-
* **Every step is checked against the budget BEFORE it is taken**, not after.
|
|
84
|
-
* Checking afterwards means the step that broke the limit has already been
|
|
85
|
-
* paid for, which makes a budget a report rather than a control.
|
|
86
|
-
*
|
|
87
|
-
* **An approval stops the run and is written to the THREAD**, not held in
|
|
88
|
-
* memory. That is what makes it survive: the approval a person grants this
|
|
89
|
-
* morning is a durable row, so the worker that resumes tonight — a different
|
|
90
|
-
* process, possibly after a deploy — reads the same answer.
|
|
91
|
-
*
|
|
92
|
-
* **A tool that needs approval and has none DOES NOT RUN.** Failing closed is
|
|
93
|
-
* the only safe direction: an unanswered approval that executed anyway is
|
|
94
|
-
* exactly the outcome the whole mechanism exists to prevent.
|
|
95
|
-
*/
|
|
96
117
|
export declare class AgentRuntime {
|
|
97
118
|
#private;
|
|
98
119
|
constructor(options: AgentRuntimeOptions);
|
|
@@ -105,6 +126,28 @@ export declare class AgentRuntime {
|
|
|
105
126
|
* second instruction competing with the one the tool call came from.
|
|
106
127
|
*/
|
|
107
128
|
send(session: Session, prompt: string, toolNames?: readonly string[], context?: RunContext, additionalContent?: readonly unknown[]): Promise<AgentResponse>;
|
|
129
|
+
/**
|
|
130
|
+
* A turn whose answer is a document.
|
|
131
|
+
*
|
|
132
|
+
* The same run as `send()` — same mode, same tools, same budget, same events,
|
|
133
|
+
* same approvals — with a schema the answer has to satisfy. The schema
|
|
134
|
+
* travels on the request for the client to hand its provider, and what comes
|
|
135
|
+
* back is checked against it here before the caller sees it.
|
|
136
|
+
*
|
|
137
|
+
* WHAT THE THREAD KEEPS IS THE TEXT, with the parsed document beside it as
|
|
138
|
+
* `structured` in the assistant row's `additional_content`. Never instead of
|
|
139
|
+
* it: a later turn replays this conversation as messages, and a transcript
|
|
140
|
+
* that reads differently because of the SHAPE of the request that produced it
|
|
141
|
+
* is a difference nothing reports.
|
|
142
|
+
*
|
|
143
|
+
* A FAILED DOCUMENT IS STILL RECORDED, and then thrown. The exchange
|
|
144
|
+
* happened, and a thread that omits the answer it did not like cannot explain
|
|
145
|
+
* the retry sitting next to it. The run is marked failed and `run.failed` is
|
|
146
|
+
* emitted, as for any other failure.
|
|
147
|
+
*
|
|
148
|
+
* @throws HarnessError `structured_unreadable` or `structured_schema_violation`
|
|
149
|
+
*/
|
|
150
|
+
sendStructured(session: Session, prompt: string, schema: Readonly<JsonObject>, toolNames?: readonly string[], context?: RunContext, additionalContent?: readonly unknown[]): Promise<StructuredAgentResponse>;
|
|
108
151
|
}
|
|
109
152
|
/**
|
|
110
153
|
* Answer a pending approval, durably.
|
|
@@ -114,5 +157,11 @@ export declare class AgentRuntime {
|
|
|
114
157
|
* already scoped to a participant, so nobody can answer another participant's
|
|
115
158
|
* approval through it, but "this user may approve THIS action" is a question
|
|
116
159
|
* only the host can answer. Authorize before calling.
|
|
160
|
+
*
|
|
161
|
+
* Nothing runs until the next `send()`, which acts on every decision recorded
|
|
162
|
+
* by then and refuses any pending call still without one. With several pending,
|
|
163
|
+
* record them all first.
|
|
164
|
+
*
|
|
165
|
+
* `approvalId` is `PendingApproval.id`, not the tool call id.
|
|
117
166
|
*/
|
|
118
167
|
export declare function recordApproval(session: Session, approvalId: string, approved: boolean, reason?: string | null): Promise<void>;
|
package/dist/runtime.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { assistantRow, threadView, toolResultEntry, toolResultRow, } from './thread-rows.js';
|
|
2
3
|
import { admitAttachments } from './attachments.js';
|
|
3
4
|
import { HarnessError } from './errors.js';
|
|
5
|
+
import { schemaName, schemaProblems } from './structured.js';
|
|
4
6
|
import { RunBudget, RunContext } from './subagents.js';
|
|
5
7
|
/**
|
|
6
8
|
* The loop: prompt in, turns out, everything recorded.
|
|
@@ -20,6 +22,10 @@ import { RunBudget, RunContext } from './subagents.js';
|
|
|
20
22
|
* the only safe direction: an unanswered approval that executed anyway is
|
|
21
23
|
* exactly the outcome the whole mechanism exists to prevent.
|
|
22
24
|
*/
|
|
25
|
+
/** How long resolving approvals may hold the session lock: long enough for a tool to run. */
|
|
26
|
+
const RESOLUTION_LOCK_SECONDS = 300;
|
|
27
|
+
/** How long a second worker waits for that lock before giving up without running anything. */
|
|
28
|
+
const RESOLUTION_WAIT_SECONDS = 30;
|
|
23
29
|
export class AgentRuntime {
|
|
24
30
|
#client;
|
|
25
31
|
#modes;
|
|
@@ -42,6 +48,35 @@ export class AgentRuntime {
|
|
|
42
48
|
* second instruction competing with the one the tool call came from.
|
|
43
49
|
*/
|
|
44
50
|
async send(session, prompt, toolNames, context, additionalContent = []) {
|
|
51
|
+
return await this.#turn(session, prompt, toolNames, context, additionalContent);
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* A turn whose answer is a document.
|
|
55
|
+
*
|
|
56
|
+
* The same run as `send()` — same mode, same tools, same budget, same events,
|
|
57
|
+
* same approvals — with a schema the answer has to satisfy. The schema
|
|
58
|
+
* travels on the request for the client to hand its provider, and what comes
|
|
59
|
+
* back is checked against it here before the caller sees it.
|
|
60
|
+
*
|
|
61
|
+
* WHAT THE THREAD KEEPS IS THE TEXT, with the parsed document beside it as
|
|
62
|
+
* `structured` in the assistant row's `additional_content`. Never instead of
|
|
63
|
+
* it: a later turn replays this conversation as messages, and a transcript
|
|
64
|
+
* that reads differently because of the SHAPE of the request that produced it
|
|
65
|
+
* is a difference nothing reports.
|
|
66
|
+
*
|
|
67
|
+
* A FAILED DOCUMENT IS STILL RECORDED, and then thrown. The exchange
|
|
68
|
+
* happened, and a thread that omits the answer it did not like cannot explain
|
|
69
|
+
* the retry sitting next to it. The run is marked failed and `run.failed` is
|
|
70
|
+
* emitted, as for any other failure.
|
|
71
|
+
*
|
|
72
|
+
* @throws HarnessError `structured_unreadable` or `structured_schema_violation`
|
|
73
|
+
*/
|
|
74
|
+
async sendStructured(session, prompt, schema, toolNames, context, additionalContent = []) {
|
|
75
|
+
const response = await this.#turn(session, prompt, toolNames, context, additionalContent, schema);
|
|
76
|
+
// Narrowed by #turn: with a schema it always returns the structured shape.
|
|
77
|
+
return response;
|
|
78
|
+
}
|
|
79
|
+
async #turn(session, prompt, toolNames, context, additionalContent = [], schema) {
|
|
45
80
|
// Refused before a run exists: a bad attachment is a mistake in the call,
|
|
46
81
|
// and it should not cost a run, events or budget.
|
|
47
82
|
const attachments = admitAttachments(prompt, additionalContent);
|
|
@@ -67,20 +102,38 @@ export class AgentRuntime {
|
|
|
67
102
|
depth: run.depth,
|
|
68
103
|
at: new Date().toISOString(),
|
|
69
104
|
});
|
|
70
|
-
if (prompt !== '') {
|
|
71
|
-
// With attachments, the shape prism-ts's UserMessage.toObject() writes: the
|
|
72
|
-
// media parts, then the turn's own text as a trailing text part, which
|
|
73
|
-
// fromObject() strips back off. Without them, unchanged.
|
|
74
|
-
const turn = attachments.length === 0
|
|
75
|
-
? { type: 'user', content: prompt }
|
|
76
|
-
: { type: 'user', content: prompt, additional_content: [...attachments, { text: prompt }], additional_attributes: {} };
|
|
77
|
-
await thread.record([turn], runId);
|
|
78
|
-
}
|
|
79
105
|
try {
|
|
80
|
-
|
|
106
|
+
const resolved = await this.#tools.resolve(toolNames ?? mode.tools, session);
|
|
107
|
+
const offered = this.#authorizer ? await this.#authorizer.allowed(session, resolved) : [...resolved.values()];
|
|
108
|
+
const called = [];
|
|
109
|
+
// Decisions recorded since the run stopped are acted on FIRST, before a
|
|
110
|
+
// new prompt is recorded, so the results land after the calls they answer
|
|
111
|
+
// rather than after the new turn.
|
|
112
|
+
await this.#resolveApprovals(session, mode, offered, runId, called);
|
|
113
|
+
if (prompt !== '') {
|
|
114
|
+
// With attachments, the shape prism-ts's UserMessage.toObject() writes:
|
|
115
|
+
// the media parts, then the turn's own text as a trailing text part,
|
|
116
|
+
// which fromObject() strips back off. Without them, unchanged.
|
|
117
|
+
const turn = attachments.length === 0
|
|
118
|
+
? { type: 'user', content: prompt }
|
|
119
|
+
: { type: 'user', content: prompt, additional_content: [...attachments, { text: prompt }], additional_attributes: {} };
|
|
120
|
+
await thread.record([turn], runId);
|
|
121
|
+
}
|
|
122
|
+
return await this.#loop(session, mode, run, runId, provider, model, offered, called, schema);
|
|
81
123
|
}
|
|
82
124
|
catch (error) {
|
|
83
|
-
|
|
125
|
+
// THE MODEL'S OWN WORDS DO NOT BELONG IN A RUN ROW OR AN EVENT. A schema
|
|
126
|
+
// violation names the values that missed, so its message carries pieces
|
|
127
|
+
// of the document — and an event carrying those would put model output in
|
|
128
|
+
// every listener's telemetry, which is the same reason tool arguments are
|
|
129
|
+
// names-only here. The document is already in the thread, in full, where
|
|
130
|
+
// it is read deliberately rather than shipped by default. An error that
|
|
131
|
+
// holds one records its CODE.
|
|
132
|
+
const failure = error instanceof HarnessError && error.document !== undefined
|
|
133
|
+
? error.code
|
|
134
|
+
: error instanceof Error
|
|
135
|
+
? error.message
|
|
136
|
+
: String(error);
|
|
84
137
|
await session.failRun(runId, failure);
|
|
85
138
|
this.#events?.emit({
|
|
86
139
|
type: 'run.failed',
|
|
@@ -93,13 +146,8 @@ export class AgentRuntime {
|
|
|
93
146
|
throw error;
|
|
94
147
|
}
|
|
95
148
|
}
|
|
96
|
-
async #loop(session, mode, run, runId, provider, model,
|
|
149
|
+
async #loop(session, mode, run, runId, provider, model, offered, called, schema) {
|
|
97
150
|
const thread = session.thread();
|
|
98
|
-
const resolved = await this.#tools.resolve(toolNames ?? mode.tools, session);
|
|
99
|
-
const offered = this.#authorizer
|
|
100
|
-
? await this.#authorizer.allowed(session, resolved)
|
|
101
|
-
: [...resolved.values()];
|
|
102
|
-
const called = [];
|
|
103
151
|
let text = '';
|
|
104
152
|
let finishReason = 'stop';
|
|
105
153
|
for (;;) {
|
|
@@ -111,79 +159,163 @@ export class AgentRuntime {
|
|
|
111
159
|
}
|
|
112
160
|
const response = await this.#client({
|
|
113
161
|
systemPrompt: mode.systemPrompt,
|
|
114
|
-
messages: (await thread.messages()).map((entry) => entry.message),
|
|
162
|
+
messages: threadView((await thread.messages()).map((entry) => entry.message)),
|
|
115
163
|
tools: offered,
|
|
116
164
|
provider,
|
|
117
165
|
model,
|
|
118
166
|
providerOptions: mode.providerOptions,
|
|
167
|
+
...(schema === undefined ? {} : { schema }),
|
|
119
168
|
});
|
|
120
169
|
run.ledger.recordSteps(1);
|
|
121
170
|
run.ledger.recordCost(response.costUsd ?? null);
|
|
122
171
|
text = response.text;
|
|
123
172
|
finishReason = response.finishReason;
|
|
124
173
|
const toolCalls = response.toolCalls ?? [];
|
|
174
|
+
const gated = toolCalls.filter((call) => mode.needsApproval(call.name));
|
|
175
|
+
const requests = gated.map((call) => ({
|
|
176
|
+
approval_id: `apr_${randomUUID().replaceAll('-', '')}`,
|
|
177
|
+
tool_call_id: call.id,
|
|
178
|
+
}));
|
|
125
179
|
// The next step's request is built from this row, so it keeps what a
|
|
126
|
-
// provider needs
|
|
127
|
-
//
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
additional_content: { ...(response.additionalContent ?? {}) },
|
|
134
|
-
},
|
|
135
|
-
], runId);
|
|
180
|
+
// provider needs sent back: each call's arguments and provider ids, the
|
|
181
|
+
// turn's provider state, and the approvals it is waiting on (G-58).
|
|
182
|
+
const answering = schema !== undefined && toolCalls.length === 0;
|
|
183
|
+
const metadata = answering
|
|
184
|
+
? { ...(response.additionalContent ?? {}), structured: response.structured ?? null }
|
|
185
|
+
: response.additionalContent;
|
|
186
|
+
await thread.record([assistantRow(response.text, toolCalls, metadata, requests)], runId);
|
|
136
187
|
if (toolCalls.length === 0) {
|
|
137
|
-
|
|
188
|
+
if (schema !== undefined) {
|
|
189
|
+
// Recorded first, then checked: the exchange happened either way, and
|
|
190
|
+
// a failure here fails the run through the caller's catch.
|
|
191
|
+
this.#assertDocument(schema, response);
|
|
192
|
+
}
|
|
193
|
+
const finished = await this.#finish(session, runId, called, finishReason, text, run, null);
|
|
194
|
+
if (schema === undefined)
|
|
195
|
+
return finished;
|
|
196
|
+
const structured = { ...finished, structured: response.structured };
|
|
197
|
+
return structured;
|
|
138
198
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
199
|
+
// The calls that need nobody run now, as in the reference, and their
|
|
200
|
+
// results are recorded even when the step then stops for a person.
|
|
201
|
+
const results = [];
|
|
202
|
+
for (const call of toolCalls.filter((candidate) => !gated.includes(candidate))) {
|
|
203
|
+
called.push(call.name);
|
|
204
|
+
results.push(await this.#invoke(offered, call));
|
|
205
|
+
}
|
|
206
|
+
if (results.length > 0) {
|
|
207
|
+
await thread.record([toolResultRow(results)], runId);
|
|
208
|
+
}
|
|
209
|
+
if (requests.length > 0) {
|
|
210
|
+
// FAILS CLOSED. The gated calls have not run, and the requests are in
|
|
211
|
+
// the thread, so a different process can resume after a person answers.
|
|
144
212
|
return {
|
|
145
213
|
runId,
|
|
146
214
|
text,
|
|
147
215
|
steps: run.ledger.steps,
|
|
148
216
|
toolCalls: called,
|
|
149
217
|
finishReason: 'awaiting_approval',
|
|
150
|
-
pendingApprovals:
|
|
218
|
+
pendingApprovals: gated.map((call, index) => ({
|
|
219
|
+
id: requests[index].approval_id,
|
|
220
|
+
toolCallId: call.id,
|
|
221
|
+
tool: call.name,
|
|
222
|
+
arguments: call.arguments,
|
|
223
|
+
})),
|
|
151
224
|
stoppedBecause: null,
|
|
152
225
|
};
|
|
153
226
|
}
|
|
154
|
-
for (const call of toolCalls) {
|
|
155
|
-
called.push(call.name);
|
|
156
|
-
await thread.record([await this.#invoke(offered, call)], runId);
|
|
157
|
-
}
|
|
158
227
|
}
|
|
159
228
|
}
|
|
160
229
|
/**
|
|
161
|
-
*
|
|
230
|
+
* Act on the decisions recorded for the last turn that stopped for a person.
|
|
162
231
|
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
232
|
+
* The model is NOT asked again. Asked again, a provider issues the call
|
|
233
|
+
* afresh under a new id, and a decision recorded against the old one never
|
|
234
|
+
* matches. The calls that stopped the run are answered where they are:
|
|
235
|
+
*
|
|
236
|
+
* - approved: run, once;
|
|
237
|
+
* - denied: the reason, as the result the model sees;
|
|
238
|
+
* - no decision: refused, "No approval response provided". Record every
|
|
239
|
+
* decision before resuming.
|
|
240
|
+
*
|
|
241
|
+
* A call that already has a result is done and never runs again, whatever
|
|
242
|
+
* its decision says. The results are recorded as one tool result row holding
|
|
243
|
+
* every result and decision for the turn, as the reference writes it.
|
|
244
|
+
*/
|
|
245
|
+
async #resolveApprovals(session, mode, offered, runId, called) {
|
|
246
|
+
// A read without the lock first: nearly every send() has nothing to resolve
|
|
247
|
+
// and should not wait on the session lock to find that out.
|
|
248
|
+
if ((await this.#unresolved(session, mode)) === null)
|
|
249
|
+
return;
|
|
250
|
+
// Then under the session lock, reading again inside it. Two workers resuming
|
|
251
|
+
// the same session at once would otherwise both find the approved call with
|
|
252
|
+
// no result, and both run it. The second now waits, finds the result the
|
|
253
|
+
// first recorded, and runs nothing. If it cannot get the lock it throws
|
|
254
|
+
// SessionLocked, and nothing runs.
|
|
255
|
+
await session.lock(async (live) => {
|
|
256
|
+
const work = await this.#unresolved(live, mode);
|
|
257
|
+
if (work === null)
|
|
258
|
+
return;
|
|
259
|
+
const resolved = [];
|
|
260
|
+
for (const call of work.calls) {
|
|
261
|
+
const input = { id: call.id, name: call.name, arguments: call.arguments, resultId: call.result_id };
|
|
262
|
+
const approvalId = work.approvalIds.get(call.id);
|
|
263
|
+
const decision = approvalId === undefined ? undefined : work.decisions.get(approvalId);
|
|
264
|
+
if (decision?.approved === true) {
|
|
265
|
+
called.push(call.name);
|
|
266
|
+
resolved.push(await this.#runApproved(offered, input));
|
|
267
|
+
}
|
|
268
|
+
else {
|
|
269
|
+
resolved.push(toolResultEntry(input, decision === undefined ? 'No approval response provided' : (decision.reason ?? 'User denied tool execution')));
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
await live.thread().record([toolResultRow([...work.results.values(), ...resolved], [...work.decisions.values()])], runId);
|
|
273
|
+
}, RESOLUTION_LOCK_SECONDS, RESOLUTION_WAIT_SECONDS);
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* The gated calls of the last tool-calling turn that have no result yet, with
|
|
277
|
+
* what is needed to answer them, or null when there are none.
|
|
166
278
|
*/
|
|
167
|
-
async #
|
|
168
|
-
const
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
279
|
+
async #unresolved(session, mode) {
|
|
280
|
+
const view = threadView((await session.thread().messages()).map((entry) => entry.message));
|
|
281
|
+
let index = view.length - 1;
|
|
282
|
+
while (index >= 0 && !(view[index].type === 'assistant' && view[index].tool_calls.length > 0)) {
|
|
283
|
+
index -= 1;
|
|
284
|
+
}
|
|
285
|
+
if (index === -1)
|
|
286
|
+
return null;
|
|
287
|
+
const assistant = view[index];
|
|
288
|
+
const answered = view.slice(index + 1).find((row) => row.type === 'tool_result');
|
|
289
|
+
const results = new Map((answered?.tool_results ?? []).map((entry) => [entry.tool_call_id, entry]));
|
|
290
|
+
const decisions = new Map((answered?.tool_approval_responses ?? []).map((entry) => [entry.approval_id, entry]));
|
|
291
|
+
const approvalIds = new Map(assistant.tool_approval_requests.map((request) => [request.tool_call_id, request.approval_id]));
|
|
292
|
+
// A call that has a result is DONE, whatever its decision says. A call that
|
|
293
|
+
// needs nobody is not this method's to run.
|
|
294
|
+
const calls = assistant.tool_calls.filter((call) => !results.has(call.id) && (approvalIds.has(call.id) || mode.needsApproval(call.name)));
|
|
295
|
+
return calls.length === 0 ? null : { calls, results, decisions, approvalIds };
|
|
175
296
|
}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
297
|
+
/**
|
|
298
|
+
* Run a call a person approved, or record why it could not run.
|
|
299
|
+
*
|
|
300
|
+
* Recorded rather than thrown. Resolution runs at the start of every send(),
|
|
301
|
+
* so a call that throws here, because its tool is no longer offered to this
|
|
302
|
+
* run or the authorizer now refuses it, would throw again on every later
|
|
303
|
+
* send(), and the session could never move on. The approval stands, but the
|
|
304
|
+
* call does not run.
|
|
305
|
+
*/
|
|
306
|
+
async #runApproved(offered, call) {
|
|
307
|
+
if (!offered.some((tool) => tool.name === call.name)) {
|
|
308
|
+
return toolResultEntry(call, `Not run: ${call.name} is not available to this run.`);
|
|
309
|
+
}
|
|
310
|
+
try {
|
|
311
|
+
return await this.#invoke(offered, call);
|
|
312
|
+
}
|
|
313
|
+
catch (error) {
|
|
314
|
+
if (error instanceof HarnessError && error.code === 'call_not_authorized') {
|
|
315
|
+
return toolResultEntry(call, `Not run: this call to ${call.name} is not authorized.`);
|
|
316
|
+
}
|
|
317
|
+
throw error;
|
|
185
318
|
}
|
|
186
|
-
return answered;
|
|
187
319
|
}
|
|
188
320
|
async #invoke(offered, call) {
|
|
189
321
|
const tool = offered.find((candidate) => candidate.name === call.name);
|
|
@@ -192,12 +324,7 @@ export class AgentRuntime {
|
|
|
192
324
|
}
|
|
193
325
|
try {
|
|
194
326
|
const result = await tool.handle(call.arguments);
|
|
195
|
-
return
|
|
196
|
-
type: 'tool_result',
|
|
197
|
-
tool_call_id: call.id,
|
|
198
|
-
name: call.name,
|
|
199
|
-
result: typeof result === 'string' ? result : JSON.stringify(result ?? null),
|
|
200
|
-
};
|
|
327
|
+
return toolResultEntry(call, typeof result === 'string' ? result : JSON.stringify(result ?? null));
|
|
201
328
|
}
|
|
202
329
|
catch (error) {
|
|
203
330
|
// A failed tool is a RESULT, not a crashed run: the model can often
|
|
@@ -205,13 +332,23 @@ export class AgentRuntime {
|
|
|
205
332
|
// call is different and is left to propagate — see `authorizedTool`.
|
|
206
333
|
if (error instanceof HarnessError && error.code === 'call_not_authorized')
|
|
207
334
|
throw error;
|
|
208
|
-
return {
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
335
|
+
return toolResultEntry(call, `The tool failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Refuse a document the caller cannot use, naming which of the two it is.
|
|
340
|
+
*
|
|
341
|
+
* Text holding no document at all and a document with the wrong shape are
|
|
342
|
+
* separate codes because the next move differs: the first is a prompting or
|
|
343
|
+
* budget problem, the second a schema one.
|
|
344
|
+
*/
|
|
345
|
+
#assertDocument(schema, response) {
|
|
346
|
+
if (response.structured === undefined || response.structured === null) {
|
|
347
|
+
throw HarnessError.structuredUnreadable(response.text);
|
|
348
|
+
}
|
|
349
|
+
const problems = schemaProblems(schema, response.structured, schemaName(schema));
|
|
350
|
+
if (problems.length > 0) {
|
|
351
|
+
throw HarnessError.structuredSchemaViolation(response.text, problems);
|
|
215
352
|
}
|
|
216
353
|
}
|
|
217
354
|
async #finish(session, runId, called, finishReason, text, run, stoppedBecause) {
|
|
@@ -245,15 +382,14 @@ export class AgentRuntime {
|
|
|
245
382
|
* already scoped to a participant, so nobody can answer another participant's
|
|
246
383
|
* approval through it, but "this user may approve THIS action" is a question
|
|
247
384
|
* only the host can answer. Authorize before calling.
|
|
385
|
+
*
|
|
386
|
+
* Nothing runs until the next `send()`, which acts on every decision recorded
|
|
387
|
+
* by then and refuses any pending call still without one. With several pending,
|
|
388
|
+
* record them all first.
|
|
389
|
+
*
|
|
390
|
+
* `approvalId` is `PendingApproval.id`, not the tool call id.
|
|
248
391
|
*/
|
|
249
392
|
export async function recordApproval(session, approvalId, approved, reason = null) {
|
|
250
393
|
const run = await session.run();
|
|
251
|
-
await session.thread().record([
|
|
252
|
-
{
|
|
253
|
-
type: 'tool_approval_response',
|
|
254
|
-
approval_id: approvalId,
|
|
255
|
-
approved,
|
|
256
|
-
reason,
|
|
257
|
-
},
|
|
258
|
-
], run?.id ?? null);
|
|
394
|
+
await session.thread().record([toolResultRow([], [{ approval_id: approvalId, approved, reason }])], run?.id ?? null);
|
|
259
395
|
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { JsonObject, JsonValue } from './json.js';
|
|
2
|
+
/**
|
|
3
|
+
* Does a document the model returned actually satisfy the schema it was given?
|
|
4
|
+
*
|
|
5
|
+
* A provider that supports strict structured output answers yes by
|
|
6
|
+
* construction. Most do not, and the ones that do can still be pointed at a
|
|
7
|
+
* model or an endpoint where the mode is unavailable — so the question has to
|
|
8
|
+
* be asked here, once, on the way back.
|
|
9
|
+
*
|
|
10
|
+
* WHY THIS IS NOT COERCION. The tempting alternative is to keep the fields that
|
|
11
|
+
* fit and drop the rest, which turns a wrong answer into a plausible one: a
|
|
12
|
+
* planning agent that returns a document missing its `nodes` array becomes a
|
|
13
|
+
* plan with no steps, and an empty plan settles a batch as done.
|
|
14
|
+
*
|
|
15
|
+
* WHAT IT CHECKS is what a JSON schema says without needing a resolver: the
|
|
16
|
+
* declared type, the required keys, the members of an enum, the items of an
|
|
17
|
+
* array, and, where a schema closes itself to additions, keys nobody declared.
|
|
18
|
+
*
|
|
19
|
+
* WHAT IT DOES NOT CHECK: `$ref`, `allOf`, `oneOf`, `not`, and the numeric and
|
|
20
|
+
* string facets (`minimum`, `pattern`, `minLength`). Each would be a partial
|
|
21
|
+
* implementation of a specification this package has no business owning, and a
|
|
22
|
+
* partial validator that reports nothing for a constraint it cannot read is
|
|
23
|
+
* worse than one whose limits are written down. What it cannot read, it passes.
|
|
24
|
+
*
|
|
25
|
+
* The same rules as `Prism\Harness\Structured\SchemaCheck` in the PHP
|
|
26
|
+
* reference, message for message, so a document refused in one language is
|
|
27
|
+
* refused in the others for the same stated reason.
|
|
28
|
+
*/
|
|
29
|
+
export declare function schemaProblems(schema: JsonObject, document: JsonValue, name?: string): string[];
|
|
30
|
+
/**
|
|
31
|
+
* What to call the document in a problem message.
|
|
32
|
+
*
|
|
33
|
+
* A schema that names itself — Prism's `ObjectSchema` writes `name` — gets its
|
|
34
|
+
* own name in the path, so `plan.steps[1].do` reads the way the schema's author
|
|
35
|
+
* wrote it. Anything else is just "document".
|
|
36
|
+
*/
|
|
37
|
+
export declare function schemaName(schema: Readonly<JsonObject>): string;
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does a document the model returned actually satisfy the schema it was given?
|
|
3
|
+
*
|
|
4
|
+
* A provider that supports strict structured output answers yes by
|
|
5
|
+
* construction. Most do not, and the ones that do can still be pointed at a
|
|
6
|
+
* model or an endpoint where the mode is unavailable — so the question has to
|
|
7
|
+
* be asked here, once, on the way back.
|
|
8
|
+
*
|
|
9
|
+
* WHY THIS IS NOT COERCION. The tempting alternative is to keep the fields that
|
|
10
|
+
* fit and drop the rest, which turns a wrong answer into a plausible one: a
|
|
11
|
+
* planning agent that returns a document missing its `nodes` array becomes a
|
|
12
|
+
* plan with no steps, and an empty plan settles a batch as done.
|
|
13
|
+
*
|
|
14
|
+
* WHAT IT CHECKS is what a JSON schema says without needing a resolver: the
|
|
15
|
+
* declared type, the required keys, the members of an enum, the items of an
|
|
16
|
+
* array, and, where a schema closes itself to additions, keys nobody declared.
|
|
17
|
+
*
|
|
18
|
+
* WHAT IT DOES NOT CHECK: `$ref`, `allOf`, `oneOf`, `not`, and the numeric and
|
|
19
|
+
* string facets (`minimum`, `pattern`, `minLength`). Each would be a partial
|
|
20
|
+
* implementation of a specification this package has no business owning, and a
|
|
21
|
+
* partial validator that reports nothing for a constraint it cannot read is
|
|
22
|
+
* worse than one whose limits are written down. What it cannot read, it passes.
|
|
23
|
+
*
|
|
24
|
+
* The same rules as `Prism\Harness\Structured\SchemaCheck` in the PHP
|
|
25
|
+
* reference, message for message, so a document refused in one language is
|
|
26
|
+
* refused in the others for the same stated reason.
|
|
27
|
+
*/
|
|
28
|
+
export function schemaProblems(schema, document, name = 'document') {
|
|
29
|
+
return check(schema, document, name);
|
|
30
|
+
}
|
|
31
|
+
function check(schema, value, path) {
|
|
32
|
+
if (Array.isArray(schema.anyOf)) {
|
|
33
|
+
return checkAnyOf(schema.anyOf, value, path);
|
|
34
|
+
}
|
|
35
|
+
if (Array.isArray(schema.enum)) {
|
|
36
|
+
return schema.enum.some((member) => member === value)
|
|
37
|
+
? []
|
|
38
|
+
: [`${path} is ${describe(value)}, which is not one of ${members(schema.enum)}.`];
|
|
39
|
+
}
|
|
40
|
+
const types = declaredTypes(schema);
|
|
41
|
+
if (types.length === 0) {
|
|
42
|
+
// Nothing declared to check against. A schema that says nothing about a
|
|
43
|
+
// value cannot be violated by it.
|
|
44
|
+
return [];
|
|
45
|
+
}
|
|
46
|
+
if (!types.some((type) => matches(type, value))) {
|
|
47
|
+
return [`${path} is ${describe(value)}, and the schema asks for ${members(types)}.`];
|
|
48
|
+
}
|
|
49
|
+
if (types.includes('object') && isObject(value)) {
|
|
50
|
+
return checkObject(schema, value, path);
|
|
51
|
+
}
|
|
52
|
+
if (types.includes('array') && Array.isArray(value)) {
|
|
53
|
+
return checkArray(schema, value, path);
|
|
54
|
+
}
|
|
55
|
+
return [];
|
|
56
|
+
}
|
|
57
|
+
function checkAnyOf(branches, value, path) {
|
|
58
|
+
for (const branch of branches) {
|
|
59
|
+
if (isObject(branch) && check(branch, value, path).length === 0)
|
|
60
|
+
return [];
|
|
61
|
+
}
|
|
62
|
+
return [`${path} is ${describe(value)}, which satisfies none of the alternatives the schema allows.`];
|
|
63
|
+
}
|
|
64
|
+
function checkObject(schema, value, path) {
|
|
65
|
+
const problems = [];
|
|
66
|
+
const properties = isObject(schema.properties) ? schema.properties : {};
|
|
67
|
+
const required = Array.isArray(schema.required) ? schema.required : [];
|
|
68
|
+
for (const name of required) {
|
|
69
|
+
if (typeof name === 'string' && !(name in value)) {
|
|
70
|
+
problems.push(`${path}.${name} is required and missing.`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
for (const [name, property] of Object.entries(properties)) {
|
|
74
|
+
if (!isObject(property) || !(name in value))
|
|
75
|
+
continue;
|
|
76
|
+
problems.push(...check(property, value[name], `${path}.${name}`));
|
|
77
|
+
}
|
|
78
|
+
// Only where the schema closed itself. An open object invites the extra key,
|
|
79
|
+
// so reporting it would be this package's opinion rather than the schema's.
|
|
80
|
+
if (schema.additionalProperties === false) {
|
|
81
|
+
for (const name of Object.keys(value)) {
|
|
82
|
+
if (!(name in properties)) {
|
|
83
|
+
problems.push(`${path}.${name} was returned, and the schema declares no such property.`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return problems;
|
|
88
|
+
}
|
|
89
|
+
function checkArray(schema, value, path) {
|
|
90
|
+
if (!isObject(schema.items))
|
|
91
|
+
return [];
|
|
92
|
+
const items = schema.items;
|
|
93
|
+
return value.flatMap((item, index) => check(items, item, `${path}[${index}]`));
|
|
94
|
+
}
|
|
95
|
+
/** The declared types, as a list — a nullable schema declares two. */
|
|
96
|
+
function declaredTypes(schema) {
|
|
97
|
+
if (typeof schema.type === 'string')
|
|
98
|
+
return [schema.type];
|
|
99
|
+
if (Array.isArray(schema.type))
|
|
100
|
+
return schema.type.filter((type) => typeof type === 'string');
|
|
101
|
+
return [];
|
|
102
|
+
}
|
|
103
|
+
function matches(type, value) {
|
|
104
|
+
switch (type) {
|
|
105
|
+
case 'string':
|
|
106
|
+
return typeof value === 'string';
|
|
107
|
+
// An integer satisfies `number`, as JSON Schema says it does. The other
|
|
108
|
+
// direction does not, and JavaScript is where that is easiest to get wrong:
|
|
109
|
+
// 2.0 parses as the same number as 2, so `Number.isInteger` is the test
|
|
110
|
+
// rather than the absence of a fractional part in the source text.
|
|
111
|
+
case 'number':
|
|
112
|
+
return typeof value === 'number' && Number.isFinite(value);
|
|
113
|
+
case 'integer':
|
|
114
|
+
return typeof value === 'number' && Number.isInteger(value);
|
|
115
|
+
case 'boolean':
|
|
116
|
+
return typeof value === 'boolean';
|
|
117
|
+
case 'null':
|
|
118
|
+
return value === null;
|
|
119
|
+
case 'array':
|
|
120
|
+
return Array.isArray(value);
|
|
121
|
+
case 'object':
|
|
122
|
+
return isObject(value);
|
|
123
|
+
default:
|
|
124
|
+
return true;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
function describe(value) {
|
|
128
|
+
if (value === null)
|
|
129
|
+
return 'null';
|
|
130
|
+
if (Array.isArray(value))
|
|
131
|
+
return 'an array';
|
|
132
|
+
if (typeof value === 'object')
|
|
133
|
+
return 'an object';
|
|
134
|
+
if (typeof value === 'boolean')
|
|
135
|
+
return value ? 'true' : 'false';
|
|
136
|
+
if (typeof value === 'string') {
|
|
137
|
+
return `the string "${value.length > 40 ? `${value.slice(0, 40)}…` : value}"`;
|
|
138
|
+
}
|
|
139
|
+
return `the number ${value}`;
|
|
140
|
+
}
|
|
141
|
+
function members(values) {
|
|
142
|
+
return values
|
|
143
|
+
.map((value) => (typeof value === 'string' ? `'${value}'` : JSON.stringify(value)))
|
|
144
|
+
.join(', ');
|
|
145
|
+
}
|
|
146
|
+
function isObject(value) {
|
|
147
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* What to call the document in a problem message.
|
|
151
|
+
*
|
|
152
|
+
* A schema that names itself — Prism's `ObjectSchema` writes `name` — gets its
|
|
153
|
+
* own name in the path, so `plan.steps[1].do` reads the way the schema's author
|
|
154
|
+
* wrote it. Anything else is just "document".
|
|
155
|
+
*/
|
|
156
|
+
export function schemaName(schema) {
|
|
157
|
+
return typeof schema.name === 'string' && schema.name !== '' ? schema.name : 'document';
|
|
158
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { JsonObject, JsonValue } from './json.js';
|
|
2
|
+
/**
|
|
3
|
+
* The rows a run writes to a thread, in the shape the PHP reference stores.
|
|
4
|
+
*
|
|
5
|
+
* prism-harness stores Prism's own messages: an assistant turn carries its tool
|
|
6
|
+
* calls (arguments and provider ids included), its provider state and its
|
|
7
|
+
* approval requests; a tool result turn carries every result of a step and any
|
|
8
|
+
* approval decisions. These builders write exactly that, with the row's `type`
|
|
9
|
+
* beside it, so a thread written here reads the same as one written there.
|
|
10
|
+
* prism-parity's `harness-thread-rows` corpus pins it.
|
|
11
|
+
*/
|
|
12
|
+
export type ToolCallRow = {
|
|
13
|
+
id: string;
|
|
14
|
+
name: string;
|
|
15
|
+
arguments: JsonObject;
|
|
16
|
+
result_id: string | null;
|
|
17
|
+
reasoning_id: string | null;
|
|
18
|
+
reasoning_summary: JsonValue[] | null;
|
|
19
|
+
};
|
|
20
|
+
export type ToolResultEntry = {
|
|
21
|
+
tool_call_id: string;
|
|
22
|
+
tool_name: string;
|
|
23
|
+
args: JsonObject;
|
|
24
|
+
result: string;
|
|
25
|
+
tool_call_result_id: string | null;
|
|
26
|
+
artifacts: JsonValue[];
|
|
27
|
+
};
|
|
28
|
+
export type ApprovalRequestEntry = {
|
|
29
|
+
approval_id: string;
|
|
30
|
+
tool_call_id: string;
|
|
31
|
+
};
|
|
32
|
+
export type ApprovalDecisionEntry = {
|
|
33
|
+
approval_id: string;
|
|
34
|
+
approved: boolean;
|
|
35
|
+
reason: string | null;
|
|
36
|
+
};
|
|
37
|
+
export type AssistantRow = {
|
|
38
|
+
type: 'assistant';
|
|
39
|
+
content: string;
|
|
40
|
+
tool_calls: ToolCallRow[];
|
|
41
|
+
additional_content: JsonObject;
|
|
42
|
+
tool_approval_requests: ApprovalRequestEntry[];
|
|
43
|
+
};
|
|
44
|
+
export type ToolResultRow = {
|
|
45
|
+
type: 'tool_result';
|
|
46
|
+
tool_results: ToolResultEntry[];
|
|
47
|
+
tool_approval_responses: ApprovalDecisionEntry[];
|
|
48
|
+
};
|
|
49
|
+
export interface ToolCallInput {
|
|
50
|
+
id: string;
|
|
51
|
+
name: string;
|
|
52
|
+
arguments: JsonObject;
|
|
53
|
+
resultId?: string | null;
|
|
54
|
+
reasoningId?: string | null;
|
|
55
|
+
reasoningSummary?: readonly JsonValue[] | null;
|
|
56
|
+
}
|
|
57
|
+
export declare function assistantRow(content: string, toolCalls: readonly ToolCallInput[], additionalContent?: Readonly<JsonObject>, approvalRequests?: readonly ApprovalRequestEntry[]): AssistantRow;
|
|
58
|
+
export declare function toolResultEntry(call: ToolCallInput, result: string): ToolResultEntry;
|
|
59
|
+
export declare function toolResultRow(results: readonly ToolResultEntry[], decisions?: readonly ApprovalDecisionEntry[]): ToolResultRow;
|
|
60
|
+
/**
|
|
61
|
+
* The thread as the model is shown it: rows written by 0.3.0 and earlier read
|
|
62
|
+
* in today's shape, and consecutive tool result rows folded into one.
|
|
63
|
+
*
|
|
64
|
+
* An approval leaves several tool result rows in a row: the results of the
|
|
65
|
+
* calls a stopped step did run, the decisions, and the results written when the
|
|
66
|
+
* run resumes. Rows are never rewritten, so all of them stay stored; shown to a
|
|
67
|
+
* provider separately, a tool output would go twice. Folded, a result is keyed
|
|
68
|
+
* by its tool call id and a decision by its approval id, and a later one
|
|
69
|
+
* replaces an earlier one in the earlier one's position.
|
|
70
|
+
*/
|
|
71
|
+
export declare function threadView(rows: readonly JsonObject[]): JsonObject[];
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
export function assistantRow(content, toolCalls, additionalContent = {}, approvalRequests = []) {
|
|
2
|
+
return {
|
|
3
|
+
type: 'assistant',
|
|
4
|
+
content,
|
|
5
|
+
tool_calls: toolCalls.map((call) => ({
|
|
6
|
+
id: call.id,
|
|
7
|
+
name: call.name,
|
|
8
|
+
arguments: { ...call.arguments },
|
|
9
|
+
result_id: call.resultId ?? null,
|
|
10
|
+
reasoning_id: call.reasoningId ?? null,
|
|
11
|
+
reasoning_summary: call.reasoningSummary ? [...call.reasoningSummary] : null,
|
|
12
|
+
})),
|
|
13
|
+
additional_content: { ...additionalContent },
|
|
14
|
+
tool_approval_requests: approvalRequests.map((request) => ({ ...request })),
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
export function toolResultEntry(call, result) {
|
|
18
|
+
return {
|
|
19
|
+
tool_call_id: call.id,
|
|
20
|
+
tool_name: call.name,
|
|
21
|
+
args: { ...call.arguments },
|
|
22
|
+
result,
|
|
23
|
+
tool_call_result_id: call.resultId ?? null,
|
|
24
|
+
artifacts: [],
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
export function toolResultRow(results, decisions = []) {
|
|
28
|
+
return {
|
|
29
|
+
type: 'tool_result',
|
|
30
|
+
tool_results: results.map((entry) => ({ ...entry })),
|
|
31
|
+
tool_approval_responses: decisions.map((decision) => ({ ...decision })),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The thread as the model is shown it: rows written by 0.3.0 and earlier read
|
|
36
|
+
* in today's shape, and consecutive tool result rows folded into one.
|
|
37
|
+
*
|
|
38
|
+
* An approval leaves several tool result rows in a row: the results of the
|
|
39
|
+
* calls a stopped step did run, the decisions, and the results written when the
|
|
40
|
+
* run resumes. Rows are never rewritten, so all of them stay stored; shown to a
|
|
41
|
+
* provider separately, a tool output would go twice. Folded, a result is keyed
|
|
42
|
+
* by its tool call id and a decision by its approval id, and a later one
|
|
43
|
+
* replaces an earlier one in the earlier one's position.
|
|
44
|
+
*/
|
|
45
|
+
export function threadView(rows) {
|
|
46
|
+
const view = [];
|
|
47
|
+
for (const row of rows) {
|
|
48
|
+
const normalised = normalise(row, view);
|
|
49
|
+
if (normalised === null)
|
|
50
|
+
continue;
|
|
51
|
+
const previous = view.at(-1);
|
|
52
|
+
if (normalised.type === 'tool_result' && previous?.type === 'tool_result') {
|
|
53
|
+
view[view.length - 1] = fold(previous, normalised);
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
view.push(normalised);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return view;
|
|
60
|
+
}
|
|
61
|
+
function normalise(row, view) {
|
|
62
|
+
switch (row.type) {
|
|
63
|
+
case 'assistant':
|
|
64
|
+
return assistantRow(typeof row.content === 'string' ? row.content : '', list(row.tool_calls).map((call) => ({
|
|
65
|
+
id: text(call.id),
|
|
66
|
+
name: text(call.name),
|
|
67
|
+
arguments: isObject(call.arguments) ? call.arguments : {},
|
|
68
|
+
resultId: nullableText(call.result_id),
|
|
69
|
+
reasoningId: nullableText(call.reasoning_id),
|
|
70
|
+
reasoningSummary: Array.isArray(call.reasoning_summary) ? call.reasoning_summary : null,
|
|
71
|
+
})), isObject(row.additional_content) ? row.additional_content : {}, list(row.tool_approval_requests).map((request) => ({
|
|
72
|
+
approval_id: text(request.approval_id),
|
|
73
|
+
tool_call_id: text(request.tool_call_id),
|
|
74
|
+
})));
|
|
75
|
+
case 'tool_result':
|
|
76
|
+
if (Array.isArray(row.tool_results) || Array.isArray(row.tool_approval_responses)) {
|
|
77
|
+
return toolResultRow(list(row.tool_results).map((entry) => ({
|
|
78
|
+
tool_call_id: text(entry.tool_call_id),
|
|
79
|
+
tool_name: text(entry.tool_name),
|
|
80
|
+
args: isObject(entry.args) ? entry.args : {},
|
|
81
|
+
result: text(entry.result),
|
|
82
|
+
tool_call_result_id: nullableText(entry.tool_call_result_id),
|
|
83
|
+
artifacts: Array.isArray(entry.artifacts) ? entry.artifacts : [],
|
|
84
|
+
})), list(row.tool_approval_responses).map(decision));
|
|
85
|
+
}
|
|
86
|
+
// 0.3.0 and earlier: one row per call, with no arguments on it.
|
|
87
|
+
return toolResultRow([
|
|
88
|
+
{
|
|
89
|
+
tool_call_id: text(row.tool_call_id),
|
|
90
|
+
tool_name: text(row.name),
|
|
91
|
+
args: {},
|
|
92
|
+
result: text(row.result),
|
|
93
|
+
tool_call_result_id: null,
|
|
94
|
+
artifacts: [],
|
|
95
|
+
},
|
|
96
|
+
]);
|
|
97
|
+
case 'tool_approval_request': {
|
|
98
|
+
// 0.3.0 and earlier kept the request in its own row, keyed by the CALL id.
|
|
99
|
+
// It belongs to the assistant turn before it.
|
|
100
|
+
const assistant = [...view].reverse().find((candidate) => candidate.type === 'assistant');
|
|
101
|
+
for (const approval of list(row.approvals)) {
|
|
102
|
+
assistant?.tool_approval_requests.push({ approval_id: text(approval.id), tool_call_id: text(approval.id) });
|
|
103
|
+
}
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
case 'tool_approval_response':
|
|
107
|
+
return toolResultRow([], [decision(row)]);
|
|
108
|
+
default:
|
|
109
|
+
return row;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function fold(earlier, later) {
|
|
113
|
+
const results = new Map();
|
|
114
|
+
const decisions = new Map();
|
|
115
|
+
for (const entry of [...earlier.tool_results, ...later.tool_results])
|
|
116
|
+
results.set(entry.tool_call_id, entry);
|
|
117
|
+
for (const entry of [...earlier.tool_approval_responses, ...later.tool_approval_responses])
|
|
118
|
+
decisions.set(entry.approval_id, entry);
|
|
119
|
+
return toolResultRow([...results.values()], [...decisions.values()]);
|
|
120
|
+
}
|
|
121
|
+
function decision(value) {
|
|
122
|
+
return { approval_id: text(value.approval_id), approved: value.approved === true, reason: nullableText(value.reason) };
|
|
123
|
+
}
|
|
124
|
+
function list(value) {
|
|
125
|
+
return Array.isArray(value) ? value.filter(isObject) : [];
|
|
126
|
+
}
|
|
127
|
+
function isObject(value) {
|
|
128
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
129
|
+
}
|
|
130
|
+
function text(value) {
|
|
131
|
+
return typeof value === 'string' ? value : value === undefined || value === null ? '' : JSON.stringify(value);
|
|
132
|
+
}
|
|
133
|
+
function nullableText(value) {
|
|
134
|
+
return typeof value === 'string' ? value : null;
|
|
135
|
+
}
|