@particle-academy/prism-harness 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -3
- package/dist/errors.d.ts +35 -1
- package/dist/errors.js +32 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -0
- package/dist/runtime.d.ts +50 -0
- package/dist/runtime.js +76 -5
- package/dist/structured.d.ts +37 -0
- package/dist/structured.js +158 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
Durable agent sessions — threads, session state and store drivers. The
|
|
4
4
|
TypeScript port of [`particle-academy/prism-harness`](https://github.com/Particle-Academy/prism-harness).
|
|
5
5
|
|
|
6
|
-
Zero runtime dependencies. Node
|
|
6
|
+
Zero runtime dependencies. Node 22+.
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
```
|
|
9
|
+
npm install @particle-academy/prism-harness
|
|
10
|
+
```
|
|
9
11
|
|
|
10
12
|
```ts
|
|
11
13
|
import { FileSessionStore, MemorySessionStore, PrismHarness } from '@particle-academy/prism-harness';
|
|
@@ -107,6 +109,8 @@ as `request.providerOptions`, on every step of every run in that mode. Pass them
|
|
|
107
109
|
to your provider call (with `prism-ts`, `withProviderOptions()`):
|
|
108
110
|
|
|
109
111
|
```ts
|
|
112
|
+
import { ModeRegistry } from '@particle-academy/prism-harness';
|
|
113
|
+
|
|
110
114
|
new ModeRegistry({
|
|
111
115
|
modes: { overseer: { system_prompt: '...', provider_options: { thinking: { type: 'adaptive' } } } },
|
|
112
116
|
});
|
|
@@ -131,12 +135,83 @@ sent to the provider once. Return a call's provider ids as `resultId`,
|
|
|
131
135
|
`reasoningId` and `reasoningSummary`, and provider state for the turn as
|
|
132
136
|
`additionalContent`: they are recorded and come back in `messages`.
|
|
133
137
|
|
|
138
|
+
## Structured turns
|
|
139
|
+
|
|
140
|
+
When the answer is a document rather than prose, hand the turn a schema:
|
|
141
|
+
|
|
142
|
+
```ts
|
|
143
|
+
const schema = {
|
|
144
|
+
name: 'plan',
|
|
145
|
+
type: 'object',
|
|
146
|
+
properties: {
|
|
147
|
+
title: { type: 'string' },
|
|
148
|
+
steps: { type: 'array', items: { type: 'string' } },
|
|
149
|
+
},
|
|
150
|
+
required: ['title', 'steps'],
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
const response = await runtime.sendStructured(session, 'Plan the release', schema);
|
|
154
|
+
|
|
155
|
+
response.structured; // { title: 'Ship it', steps: ['write', 'test'] }
|
|
156
|
+
response.text; // the document as the model wrote it
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
It is the same run as `send()` — the mode's system prompt, its tools, the step
|
|
160
|
+
budget, the events — asking the provider for structured output. Your client
|
|
161
|
+
receives the schema as `request.schema` and returns what it parsed as
|
|
162
|
+
`response.structured`; with `prism-ts` that is `Prism.structured().withSchema()`
|
|
163
|
+
and the response's `structured`.
|
|
164
|
+
|
|
165
|
+
**The thread keeps the text, with the parsed object beside it.** The assistant
|
|
166
|
+
message is the raw document, and `structured` rides along in the row's
|
|
167
|
+
`additional_content`. A later turn replays the conversation as messages and reads
|
|
168
|
+
the text, so a thread that contains a structured answer reads like any other.
|
|
169
|
+
|
|
170
|
+
**A document that misses the schema is refused, not repaired.**
|
|
171
|
+
|
|
172
|
+
```ts
|
|
173
|
+
import { HarnessError } from '@particle-academy/prism-harness';
|
|
174
|
+
|
|
175
|
+
try {
|
|
176
|
+
const plan = (await runtime.sendStructured(session, brief, schema)).structured;
|
|
177
|
+
} catch (error) {
|
|
178
|
+
if (error instanceof HarnessError) {
|
|
179
|
+
error.code; // structured_schema_violation, or structured_unreadable
|
|
180
|
+
error.problems; // every way it missed, not the first
|
|
181
|
+
error.document; // what the model actually said
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Nothing is coerced, nothing is trimmed to the fields that fit, and the result is
|
|
187
|
+
never an empty document. An empty plan settles a batch as `done`, which reads
|
|
188
|
+
exactly like a considered answer of "nothing to propose" — the failure this
|
|
189
|
+
refusal exists to prevent. The exchange is still recorded, and the run is marked
|
|
190
|
+
failed: a thread that omits the answer it did not like cannot explain the retry
|
|
191
|
+
sitting next to it. The run row and the `run.failed` event name the CODE, not the
|
|
192
|
+
message — the message quotes the values that missed, and an event carrying those
|
|
193
|
+
would ship model output to every listener, which is the same reason tool
|
|
194
|
+
arguments are names-only here.
|
|
195
|
+
|
|
196
|
+
The check reads the schema's own JSON Schema, so a hand-written schema is held to
|
|
197
|
+
the same terms. `schemaProblems()` is exported if you want it directly. It checks
|
|
198
|
+
declared types, required keys, enum members, array items, and — where a schema
|
|
199
|
+
closes itself — keys nobody declared. It does not read `$ref`, `allOf`, `oneOf`
|
|
200
|
+
or the numeric and string facets; what it cannot read, it passes, rather than
|
|
201
|
+
reporting a constraint it did not actually check. The same rules as the PHP
|
|
202
|
+
reference, message for message, so a document refused in one language is refused
|
|
203
|
+
in the other for the same stated reason.
|
|
204
|
+
|
|
134
205
|
## Approvals
|
|
135
206
|
|
|
136
207
|
A mode names the tools a person must approve:
|
|
137
208
|
|
|
138
209
|
```ts
|
|
139
|
-
|
|
210
|
+
new ModeRegistry({
|
|
211
|
+
modes: {
|
|
212
|
+
guarded: { system_prompt: '...', tools: ['read', 'delete'], requires_approval: ['delete'] },
|
|
213
|
+
},
|
|
214
|
+
});
|
|
140
215
|
```
|
|
141
216
|
|
|
142
217
|
A step that calls one stops with `finishReason: 'awaiting_approval'`. The calls
|
|
@@ -144,6 +219,8 @@ that need nobody have already run. Record a decision for every pending approval,
|
|
|
144
219
|
then resume with an empty prompt:
|
|
145
220
|
|
|
146
221
|
```ts
|
|
222
|
+
import { recordApproval } from '@particle-academy/prism-harness';
|
|
223
|
+
|
|
147
224
|
const response = await runtime.send(session, 'Clean up the failed run');
|
|
148
225
|
|
|
149
226
|
if (response.finishReason === 'awaiting_approval') {
|
package/dist/errors.d.ts
CHANGED
|
@@ -68,12 +68,30 @@ export type HarnessErrorCode =
|
|
|
68
68
|
/** A turn attachment carries no bytes, no file id and no chunks. */
|
|
69
69
|
| 'attachment_empty'
|
|
70
70
|
/** Attachments were offered with an empty prompt, which sends no user message. */
|
|
71
|
-
| 'attachment_without_prompt'
|
|
71
|
+
| 'attachment_without_prompt'
|
|
72
|
+
/** A structured turn came back with text holding no readable document. */
|
|
73
|
+
| 'structured_unreadable'
|
|
74
|
+
/** A structured turn returned a document its schema refuses. */
|
|
75
|
+
| 'structured_schema_violation';
|
|
72
76
|
export interface HarnessErrorOptions {
|
|
73
77
|
cause?: unknown;
|
|
78
|
+
/** The model's own text, on a structured failure. See `HarnessError.document`. */
|
|
79
|
+
document?: string;
|
|
80
|
+
/** Every way a document missed its schema, on a structured failure. */
|
|
81
|
+
problems?: readonly string[];
|
|
74
82
|
}
|
|
75
83
|
export declare class HarnessError extends Error {
|
|
76
84
|
readonly code: HarnessErrorCode;
|
|
85
|
+
/**
|
|
86
|
+
* The model's own text, on a structured failure.
|
|
87
|
+
*
|
|
88
|
+
* It travels on the error because the first question anyone asks is "what did
|
|
89
|
+
* it actually say", and an error that answers it turns a support thread into
|
|
90
|
+
* a log line. It is model output: log it where you log model output.
|
|
91
|
+
*/
|
|
92
|
+
readonly document?: string;
|
|
93
|
+
/** Every way a document missed its schema, rather than only the first. */
|
|
94
|
+
readonly problems: readonly string[];
|
|
77
95
|
constructor(code: HarnessErrorCode, message: string, options?: HarnessErrorOptions);
|
|
78
96
|
static sessionLocked(key: string, waitSeconds: number): HarnessError;
|
|
79
97
|
/**
|
|
@@ -216,4 +234,20 @@ export declare class HarnessError extends Error {
|
|
|
216
234
|
static attachmentEmpty(index: number): HarnessError;
|
|
217
235
|
/** An empty prompt resumes a paused run and sends no user message, so attachments would vanish. */
|
|
218
236
|
static attachmentWithoutPrompt(): HarnessError;
|
|
237
|
+
/**
|
|
238
|
+
* A structured turn whose text held no document at all.
|
|
239
|
+
*
|
|
240
|
+
* Separate from the schema violation because the caller's next move differs:
|
|
241
|
+
* an apology in prose or a truncated answer is a prompting or a budget
|
|
242
|
+
* problem, and a document with the wrong shape is a schema one.
|
|
243
|
+
*/
|
|
244
|
+
static structuredUnreadable(document: string): HarnessError;
|
|
245
|
+
/**
|
|
246
|
+
* Valid JSON, wrong shape.
|
|
247
|
+
*
|
|
248
|
+
* NOT coerced, not trimmed to the fields that fit, and never an empty
|
|
249
|
+
* document: one that fails the contract arriving as `{}` reads to the code
|
|
250
|
+
* receiving it exactly like a considered answer of "nothing".
|
|
251
|
+
*/
|
|
252
|
+
static structuredSchemaViolation(document: string, problems: readonly string[]): HarnessError;
|
|
219
253
|
}
|
package/dist/errors.js
CHANGED
|
@@ -1,9 +1,21 @@
|
|
|
1
1
|
export class HarnessError extends Error {
|
|
2
2
|
code;
|
|
3
|
+
/**
|
|
4
|
+
* The model's own text, on a structured failure.
|
|
5
|
+
*
|
|
6
|
+
* It travels on the error because the first question anyone asks is "what did
|
|
7
|
+
* it actually say", and an error that answers it turns a support thread into
|
|
8
|
+
* a log line. It is model output: log it where you log model output.
|
|
9
|
+
*/
|
|
10
|
+
document;
|
|
11
|
+
/** Every way a document missed its schema, rather than only the first. */
|
|
12
|
+
problems;
|
|
3
13
|
constructor(code, message, options = {}) {
|
|
4
14
|
super(message, options);
|
|
5
15
|
this.name = 'HarnessError';
|
|
6
16
|
this.code = code;
|
|
17
|
+
this.document = options.document;
|
|
18
|
+
this.problems = options.problems ?? [];
|
|
7
19
|
}
|
|
8
20
|
static sessionLocked(key, waitSeconds) {
|
|
9
21
|
return new HarnessError('session_locked', `Could not acquire the lock on session [${key}] within ${waitSeconds}s. ` +
|
|
@@ -220,4 +232,24 @@ export class HarnessError extends Error {
|
|
|
220
232
|
return new HarnessError('attachment_without_prompt', 'Attachments need a prompt to travel with. An empty prompt resumes a paused run and sends no user ' +
|
|
221
233
|
'message, so these attachments would be dropped without a word.');
|
|
222
234
|
}
|
|
235
|
+
/**
|
|
236
|
+
* A structured turn whose text held no document at all.
|
|
237
|
+
*
|
|
238
|
+
* Separate from the schema violation because the caller's next move differs:
|
|
239
|
+
* an apology in prose or a truncated answer is a prompting or a budget
|
|
240
|
+
* problem, and a document with the wrong shape is a schema one.
|
|
241
|
+
*/
|
|
242
|
+
static structuredUnreadable(document) {
|
|
243
|
+
return new HarnessError('structured_unreadable', 'The model returned no readable document for a structured turn. Its text is on this error as document.', { document });
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Valid JSON, wrong shape.
|
|
247
|
+
*
|
|
248
|
+
* NOT coerced, not trimmed to the fields that fit, and never an empty
|
|
249
|
+
* document: one that fails the contract arriving as `{}` reads to the code
|
|
250
|
+
* receiving it exactly like a considered answer of "nothing".
|
|
251
|
+
*/
|
|
252
|
+
static structuredSchemaViolation(document, problems) {
|
|
253
|
+
return new HarnessError('structured_schema_violation', `The model returned a document that does not satisfy the schema: ${problems.join(' ')}`, { document, problems });
|
|
254
|
+
}
|
|
223
255
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -26,7 +26,8 @@ export type { DoctorFinding, DoctorReport, DoctorSubjects } from './doctor.js';
|
|
|
26
26
|
export { AgentRuntime, recordApproval } from './runtime.js';
|
|
27
27
|
export { assistantRow, threadView, toolResultEntry, toolResultRow } from './thread-rows.js';
|
|
28
28
|
export type { ApprovalDecisionEntry, ApprovalRequestEntry, AssistantRow, ToolCallInput, ToolCallRow, ToolResultEntry, ToolResultRow, } from './thread-rows.js';
|
|
29
|
-
export type { AgentResponse, AgentRuntimeOptions, LlmClient, LlmRequest, LlmResponse, LlmToolCall, PendingApproval, } from './runtime.js';
|
|
29
|
+
export type { AgentResponse, AgentRuntimeOptions, LlmClient, LlmRequest, LlmResponse, LlmToolCall, PendingApproval, StructuredAgentResponse, } from './runtime.js';
|
|
30
|
+
export { schemaName, schemaProblems } from './structured.js';
|
|
30
31
|
export { DEFAULT_LEASE_SECONDS, TASK_STATES, asAgentTask, attributesToRecord, canonicalTaskJson, isTaskState, isTerminalState, recordToAttributes, taskRecordFrom, withAgentTask, } from './tasks.js';
|
|
31
32
|
export type { AgentTask, AgentTaskAttributeSource, AgentTaskAttributes, AgentTaskRecord, AgentTaskSource, LeasedAgentTask, TaskOutcome, TaskState, } from './tasks.js';
|
|
32
33
|
export { StoreTaskSource, StoredAgentTask } from './store-task-source.js';
|
package/dist/index.js
CHANGED
|
@@ -14,6 +14,7 @@ export { HarnessEvents } from './events.js';
|
|
|
14
14
|
export { diagnose } from './doctor.js';
|
|
15
15
|
export { AgentRuntime, recordApproval } from './runtime.js';
|
|
16
16
|
export { assistantRow, threadView, toolResultEntry, toolResultRow } from './thread-rows.js';
|
|
17
|
+
export { schemaName, schemaProblems } from './structured.js';
|
|
17
18
|
export { DEFAULT_LEASE_SECONDS, TASK_STATES, asAgentTask, attributesToRecord, canonicalTaskJson, isTaskState, isTerminalState, recordToAttributes, taskRecordFrom, withAgentTask, } from './tasks.js';
|
|
18
19
|
export { StoreTaskSource, StoredAgentTask } from './store-task-source.js';
|
|
19
20
|
export { agentCompletionTool } from './task-tools.js';
|
package/dist/runtime.d.ts
CHANGED
|
@@ -28,6 +28,15 @@ export interface LlmRequest {
|
|
|
28
28
|
* interpret them.
|
|
29
29
|
*/
|
|
30
30
|
providerOptions: Readonly<JsonObject>;
|
|
31
|
+
/**
|
|
32
|
+
* The JSON Schema a structured turn asks the answer to satisfy.
|
|
33
|
+
*
|
|
34
|
+
* Absent on an ordinary turn. A client passes it to its provider's structured
|
|
35
|
+
* mode (for prism-ts, `Prism.structured().withSchema()`) and returns what it
|
|
36
|
+
* parsed as `structured`; the harness checks that against this same schema
|
|
37
|
+
* before the caller sees it.
|
|
38
|
+
*/
|
|
39
|
+
schema?: Readonly<JsonObject>;
|
|
31
40
|
}
|
|
32
41
|
export interface LlmToolCall {
|
|
33
42
|
id: string;
|
|
@@ -58,6 +67,14 @@ export interface LlmResponse {
|
|
|
58
67
|
* straight through and read it back from `messages`.
|
|
59
68
|
*/
|
|
60
69
|
additionalContent?: Readonly<JsonObject>;
|
|
70
|
+
/**
|
|
71
|
+
* The document a structured turn parsed out of `text`.
|
|
72
|
+
*
|
|
73
|
+
* Null when the text held none — an apology in prose, a truncated answer, a
|
|
74
|
+
* fence that never closed. The harness tells that case apart from a document
|
|
75
|
+
* with the wrong shape, because the caller's next move differs.
|
|
76
|
+
*/
|
|
77
|
+
structured?: JsonValue | null;
|
|
61
78
|
}
|
|
62
79
|
export type LlmClient = (request: LlmRequest) => Promise<LlmResponse>;
|
|
63
80
|
export interface AgentResponse {
|
|
@@ -72,6 +89,17 @@ export interface AgentResponse {
|
|
|
72
89
|
/** Set when the run stopped because the tree ran out of budget, or was cancelled. */
|
|
73
90
|
stoppedBecause: string | null;
|
|
74
91
|
}
|
|
92
|
+
/**
|
|
93
|
+
* One structured run's result: the document, and the text it was read from.
|
|
94
|
+
*
|
|
95
|
+
* BOTH, not one. `structured` is what the caller asked for, and `text` is what
|
|
96
|
+
* the model actually sent — which is also exactly what the thread stored, so a
|
|
97
|
+
* transcript and a parse can be compared rather than trusted.
|
|
98
|
+
*/
|
|
99
|
+
export interface StructuredAgentResponse extends AgentResponse {
|
|
100
|
+
/** Checked against the schema before it got here; a run that could not produce one threw. */
|
|
101
|
+
structured: JsonValue;
|
|
102
|
+
}
|
|
75
103
|
export interface PendingApproval {
|
|
76
104
|
/** The APPROVAL id: what `recordApproval()` answers. Not the tool call id. */
|
|
77
105
|
id: string;
|
|
@@ -98,6 +126,28 @@ export declare class AgentRuntime {
|
|
|
98
126
|
* second instruction competing with the one the tool call came from.
|
|
99
127
|
*/
|
|
100
128
|
send(session: Session, prompt: string, toolNames?: readonly string[], context?: RunContext, additionalContent?: readonly unknown[]): Promise<AgentResponse>;
|
|
129
|
+
/**
|
|
130
|
+
* A turn whose answer is a document.
|
|
131
|
+
*
|
|
132
|
+
* The same run as `send()` — same mode, same tools, same budget, same events,
|
|
133
|
+
* same approvals — with a schema the answer has to satisfy. The schema
|
|
134
|
+
* travels on the request for the client to hand its provider, and what comes
|
|
135
|
+
* back is checked against it here before the caller sees it.
|
|
136
|
+
*
|
|
137
|
+
* WHAT THE THREAD KEEPS IS THE TEXT, with the parsed document beside it as
|
|
138
|
+
* `structured` in the assistant row's `additional_content`. Never instead of
|
|
139
|
+
* it: a later turn replays this conversation as messages, and a transcript
|
|
140
|
+
* that reads differently because of the SHAPE of the request that produced it
|
|
141
|
+
* is a difference nothing reports.
|
|
142
|
+
*
|
|
143
|
+
* A FAILED DOCUMENT IS STILL RECORDED, and then thrown. The exchange
|
|
144
|
+
* happened, and a thread that omits the answer it did not like cannot explain
|
|
145
|
+
* the retry sitting next to it. The run is marked failed and `run.failed` is
|
|
146
|
+
* emitted, as for any other failure.
|
|
147
|
+
*
|
|
148
|
+
* @throws HarnessError `structured_unreadable` or `structured_schema_violation`
|
|
149
|
+
*/
|
|
150
|
+
sendStructured(session: Session, prompt: string, schema: Readonly<JsonObject>, toolNames?: readonly string[], context?: RunContext, additionalContent?: readonly unknown[]): Promise<StructuredAgentResponse>;
|
|
101
151
|
}
|
|
102
152
|
/**
|
|
103
153
|
* Answer a pending approval, durably.
|
package/dist/runtime.js
CHANGED
|
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { assistantRow, threadView, toolResultEntry, toolResultRow, } from './thread-rows.js';
|
|
3
3
|
import { admitAttachments } from './attachments.js';
|
|
4
4
|
import { HarnessError } from './errors.js';
|
|
5
|
+
import { schemaName, schemaProblems } from './structured.js';
|
|
5
6
|
import { RunBudget, RunContext } from './subagents.js';
|
|
6
7
|
/**
|
|
7
8
|
* The loop: prompt in, turns out, everything recorded.
|
|
@@ -47,6 +48,35 @@ export class AgentRuntime {
|
|
|
47
48
|
* second instruction competing with the one the tool call came from.
|
|
48
49
|
*/
|
|
49
50
|
async send(session, prompt, toolNames, context, additionalContent = []) {
|
|
51
|
+
return await this.#turn(session, prompt, toolNames, context, additionalContent);
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* A turn whose answer is a document.
|
|
55
|
+
*
|
|
56
|
+
* The same run as `send()` — same mode, same tools, same budget, same events,
|
|
57
|
+
* same approvals — with a schema the answer has to satisfy. The schema
|
|
58
|
+
* travels on the request for the client to hand its provider, and what comes
|
|
59
|
+
* back is checked against it here before the caller sees it.
|
|
60
|
+
*
|
|
61
|
+
* WHAT THE THREAD KEEPS IS THE TEXT, with the parsed document beside it as
|
|
62
|
+
* `structured` in the assistant row's `additional_content`. Never instead of
|
|
63
|
+
* it: a later turn replays this conversation as messages, and a transcript
|
|
64
|
+
* that reads differently because of the SHAPE of the request that produced it
|
|
65
|
+
* is a difference nothing reports.
|
|
66
|
+
*
|
|
67
|
+
* A FAILED DOCUMENT IS STILL RECORDED, and then thrown. The exchange
|
|
68
|
+
* happened, and a thread that omits the answer it did not like cannot explain
|
|
69
|
+
* the retry sitting next to it. The run is marked failed and `run.failed` is
|
|
70
|
+
* emitted, as for any other failure.
|
|
71
|
+
*
|
|
72
|
+
* @throws HarnessError `structured_unreadable` or `structured_schema_violation`
|
|
73
|
+
*/
|
|
74
|
+
async sendStructured(session, prompt, schema, toolNames, context, additionalContent = []) {
|
|
75
|
+
const response = await this.#turn(session, prompt, toolNames, context, additionalContent, schema);
|
|
76
|
+
// Narrowed by #turn: with a schema it always returns the structured shape.
|
|
77
|
+
return response;
|
|
78
|
+
}
|
|
79
|
+
async #turn(session, prompt, toolNames, context, additionalContent = [], schema) {
|
|
50
80
|
// Refused before a run exists: a bad attachment is a mistake in the call,
|
|
51
81
|
// and it should not cost a run, events or budget.
|
|
52
82
|
const attachments = admitAttachments(prompt, additionalContent);
|
|
@@ -89,10 +119,21 @@ export class AgentRuntime {
|
|
|
89
119
|
: { type: 'user', content: prompt, additional_content: [...attachments, { text: prompt }], additional_attributes: {} };
|
|
90
120
|
await thread.record([turn], runId);
|
|
91
121
|
}
|
|
92
|
-
return await this.#loop(session, mode, run, runId, provider, model, offered, called);
|
|
122
|
+
return await this.#loop(session, mode, run, runId, provider, model, offered, called, schema);
|
|
93
123
|
}
|
|
94
124
|
catch (error) {
|
|
95
|
-
|
|
125
|
+
// THE MODEL'S OWN WORDS DO NOT BELONG IN A RUN ROW OR AN EVENT. A schema
|
|
126
|
+
// violation names the values that missed, so its message carries pieces
|
|
127
|
+
// of the document — and an event carrying those would put model output in
|
|
128
|
+
// every listener's telemetry, which is the same reason tool arguments are
|
|
129
|
+
// names-only here. The document is already in the thread, in full, where
|
|
130
|
+
// it is read deliberately rather than shipped by default. An error that
|
|
131
|
+
// holds one records its CODE.
|
|
132
|
+
const failure = error instanceof HarnessError && error.document !== undefined
|
|
133
|
+
? error.code
|
|
134
|
+
: error instanceof Error
|
|
135
|
+
? error.message
|
|
136
|
+
: String(error);
|
|
96
137
|
await session.failRun(runId, failure);
|
|
97
138
|
this.#events?.emit({
|
|
98
139
|
type: 'run.failed',
|
|
@@ -105,7 +146,7 @@ export class AgentRuntime {
|
|
|
105
146
|
throw error;
|
|
106
147
|
}
|
|
107
148
|
}
|
|
108
|
-
async #loop(session, mode, run, runId, provider, model, offered, called) {
|
|
149
|
+
async #loop(session, mode, run, runId, provider, model, offered, called, schema) {
|
|
109
150
|
const thread = session.thread();
|
|
110
151
|
let text = '';
|
|
111
152
|
let finishReason = 'stop';
|
|
@@ -123,6 +164,7 @@ export class AgentRuntime {
|
|
|
123
164
|
provider,
|
|
124
165
|
model,
|
|
125
166
|
providerOptions: mode.providerOptions,
|
|
167
|
+
...(schema === undefined ? {} : { schema }),
|
|
126
168
|
});
|
|
127
169
|
run.ledger.recordSteps(1);
|
|
128
170
|
run.ledger.recordCost(response.costUsd ?? null);
|
|
@@ -137,9 +179,22 @@ export class AgentRuntime {
|
|
|
137
179
|
// The next step's request is built from this row, so it keeps what a
|
|
138
180
|
// provider needs sent back: each call's arguments and provider ids, the
|
|
139
181
|
// turn's provider state, and the approvals it is waiting on (G-58).
|
|
140
|
-
|
|
182
|
+
const answering = schema !== undefined && toolCalls.length === 0;
|
|
183
|
+
const metadata = answering
|
|
184
|
+
? { ...(response.additionalContent ?? {}), structured: response.structured ?? null }
|
|
185
|
+
: response.additionalContent;
|
|
186
|
+
await thread.record([assistantRow(response.text, toolCalls, metadata, requests)], runId);
|
|
141
187
|
if (toolCalls.length === 0) {
|
|
142
|
-
|
|
188
|
+
if (schema !== undefined) {
|
|
189
|
+
// Recorded first, then checked: the exchange happened either way, and
|
|
190
|
+
// a failure here fails the run through the caller's catch.
|
|
191
|
+
this.#assertDocument(schema, response);
|
|
192
|
+
}
|
|
193
|
+
const finished = await this.#finish(session, runId, called, finishReason, text, run, null);
|
|
194
|
+
if (schema === undefined)
|
|
195
|
+
return finished;
|
|
196
|
+
const structured = { ...finished, structured: response.structured };
|
|
197
|
+
return structured;
|
|
143
198
|
}
|
|
144
199
|
// The calls that need nobody run now, as in the reference, and their
|
|
145
200
|
// results are recorded even when the step then stops for a person.
|
|
@@ -280,6 +335,22 @@ export class AgentRuntime {
|
|
|
280
335
|
return toolResultEntry(call, `The tool failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
281
336
|
}
|
|
282
337
|
}
|
|
338
|
+
/**
|
|
339
|
+
* Refuse a document the caller cannot use, naming which of the two it is.
|
|
340
|
+
*
|
|
341
|
+
* Text holding no document at all and a document with the wrong shape are
|
|
342
|
+
* separate codes because the next move differs: the first is a prompting or
|
|
343
|
+
* budget problem, the second a schema one.
|
|
344
|
+
*/
|
|
345
|
+
#assertDocument(schema, response) {
|
|
346
|
+
if (response.structured === undefined || response.structured === null) {
|
|
347
|
+
throw HarnessError.structuredUnreadable(response.text);
|
|
348
|
+
}
|
|
349
|
+
const problems = schemaProblems(schema, response.structured, schemaName(schema));
|
|
350
|
+
if (problems.length > 0) {
|
|
351
|
+
throw HarnessError.structuredSchemaViolation(response.text, problems);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
283
354
|
async #finish(session, runId, called, finishReason, text, run, stoppedBecause) {
|
|
284
355
|
await session.completeRun(runId, finishReason, called);
|
|
285
356
|
this.#events?.emit({
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { JsonObject, JsonValue } from './json.js';
|
|
2
|
+
/**
|
|
3
|
+
* Does a document the model returned actually satisfy the schema it was given?
|
|
4
|
+
*
|
|
5
|
+
* A provider that supports strict structured output answers yes by
|
|
6
|
+
* construction. Most do not, and the ones that do can still be pointed at a
|
|
7
|
+
* model or an endpoint where the mode is unavailable — so the question has to
|
|
8
|
+
* be asked here, once, on the way back.
|
|
9
|
+
*
|
|
10
|
+
* WHY THIS IS NOT COERCION. The tempting alternative is to keep the fields that
|
|
11
|
+
* fit and drop the rest, which turns a wrong answer into a plausible one: a
|
|
12
|
+
* planning agent that returns a document missing its `nodes` array becomes a
|
|
13
|
+
* plan with no steps, and an empty plan settles a batch as done.
|
|
14
|
+
*
|
|
15
|
+
* WHAT IT CHECKS is what a JSON schema says without needing a resolver: the
|
|
16
|
+
* declared type, the required keys, the members of an enum, the items of an
|
|
17
|
+
* array, and, where a schema closes itself to additions, keys nobody declared.
|
|
18
|
+
*
|
|
19
|
+
* WHAT IT DOES NOT CHECK: `$ref`, `allOf`, `oneOf`, `not`, and the numeric and
|
|
20
|
+
* string facets (`minimum`, `pattern`, `minLength`). Each would be a partial
|
|
21
|
+
* implementation of a specification this package has no business owning, and a
|
|
22
|
+
* partial validator that reports nothing for a constraint it cannot read is
|
|
23
|
+
* worse than one whose limits are written down. What it cannot read, it passes.
|
|
24
|
+
*
|
|
25
|
+
* The same rules as `Prism\Harness\Structured\SchemaCheck` in the PHP
|
|
26
|
+
* reference, message for message, so a document refused in one language is
|
|
27
|
+
* refused in the others for the same stated reason.
|
|
28
|
+
*/
|
|
29
|
+
export declare function schemaProblems(schema: JsonObject, document: JsonValue, name?: string): string[];
|
|
30
|
+
/**
|
|
31
|
+
* What to call the document in a problem message.
|
|
32
|
+
*
|
|
33
|
+
* A schema that names itself — Prism's `ObjectSchema` writes `name` — gets its
|
|
34
|
+
* own name in the path, so `plan.steps[1].do` reads the way the schema's author
|
|
35
|
+
* wrote it. Anything else is just "document".
|
|
36
|
+
*/
|
|
37
|
+
export declare function schemaName(schema: Readonly<JsonObject>): string;
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does a document the model returned actually satisfy the schema it was given?
|
|
3
|
+
*
|
|
4
|
+
* A provider that supports strict structured output answers yes by
|
|
5
|
+
* construction. Most do not, and the ones that do can still be pointed at a
|
|
6
|
+
* model or an endpoint where the mode is unavailable — so the question has to
|
|
7
|
+
* be asked here, once, on the way back.
|
|
8
|
+
*
|
|
9
|
+
* WHY THIS IS NOT COERCION. The tempting alternative is to keep the fields that
|
|
10
|
+
* fit and drop the rest, which turns a wrong answer into a plausible one: a
|
|
11
|
+
* planning agent that returns a document missing its `nodes` array becomes a
|
|
12
|
+
* plan with no steps, and an empty plan settles a batch as done.
|
|
13
|
+
*
|
|
14
|
+
* WHAT IT CHECKS is what a JSON schema says without needing a resolver: the
|
|
15
|
+
* declared type, the required keys, the members of an enum, the items of an
|
|
16
|
+
* array, and, where a schema closes itself to additions, keys nobody declared.
|
|
17
|
+
*
|
|
18
|
+
* WHAT IT DOES NOT CHECK: `$ref`, `allOf`, `oneOf`, `not`, and the numeric and
|
|
19
|
+
* string facets (`minimum`, `pattern`, `minLength`). Each would be a partial
|
|
20
|
+
* implementation of a specification this package has no business owning, and a
|
|
21
|
+
* partial validator that reports nothing for a constraint it cannot read is
|
|
22
|
+
* worse than one whose limits are written down. What it cannot read, it passes.
|
|
23
|
+
*
|
|
24
|
+
* The same rules as `Prism\Harness\Structured\SchemaCheck` in the PHP
|
|
25
|
+
* reference, message for message, so a document refused in one language is
|
|
26
|
+
* refused in the others for the same stated reason.
|
|
27
|
+
*/
|
|
28
|
+
export function schemaProblems(schema, document, name = 'document') {
|
|
29
|
+
return check(schema, document, name);
|
|
30
|
+
}
|
|
31
|
+
function check(schema, value, path) {
|
|
32
|
+
if (Array.isArray(schema.anyOf)) {
|
|
33
|
+
return checkAnyOf(schema.anyOf, value, path);
|
|
34
|
+
}
|
|
35
|
+
if (Array.isArray(schema.enum)) {
|
|
36
|
+
return schema.enum.some((member) => member === value)
|
|
37
|
+
? []
|
|
38
|
+
: [`${path} is ${describe(value)}, which is not one of ${members(schema.enum)}.`];
|
|
39
|
+
}
|
|
40
|
+
const types = declaredTypes(schema);
|
|
41
|
+
if (types.length === 0) {
|
|
42
|
+
// Nothing declared to check against. A schema that says nothing about a
|
|
43
|
+
// value cannot be violated by it.
|
|
44
|
+
return [];
|
|
45
|
+
}
|
|
46
|
+
if (!types.some((type) => matches(type, value))) {
|
|
47
|
+
return [`${path} is ${describe(value)}, and the schema asks for ${members(types)}.`];
|
|
48
|
+
}
|
|
49
|
+
if (types.includes('object') && isObject(value)) {
|
|
50
|
+
return checkObject(schema, value, path);
|
|
51
|
+
}
|
|
52
|
+
if (types.includes('array') && Array.isArray(value)) {
|
|
53
|
+
return checkArray(schema, value, path);
|
|
54
|
+
}
|
|
55
|
+
return [];
|
|
56
|
+
}
|
|
57
|
+
function checkAnyOf(branches, value, path) {
|
|
58
|
+
for (const branch of branches) {
|
|
59
|
+
if (isObject(branch) && check(branch, value, path).length === 0)
|
|
60
|
+
return [];
|
|
61
|
+
}
|
|
62
|
+
return [`${path} is ${describe(value)}, which satisfies none of the alternatives the schema allows.`];
|
|
63
|
+
}
|
|
64
|
+
function checkObject(schema, value, path) {
|
|
65
|
+
const problems = [];
|
|
66
|
+
const properties = isObject(schema.properties) ? schema.properties : {};
|
|
67
|
+
const required = Array.isArray(schema.required) ? schema.required : [];
|
|
68
|
+
for (const name of required) {
|
|
69
|
+
if (typeof name === 'string' && !(name in value)) {
|
|
70
|
+
problems.push(`${path}.${name} is required and missing.`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
for (const [name, property] of Object.entries(properties)) {
|
|
74
|
+
if (!isObject(property) || !(name in value))
|
|
75
|
+
continue;
|
|
76
|
+
problems.push(...check(property, value[name], `${path}.${name}`));
|
|
77
|
+
}
|
|
78
|
+
// Only where the schema closed itself. An open object invites the extra key,
|
|
79
|
+
// so reporting it would be this package's opinion rather than the schema's.
|
|
80
|
+
if (schema.additionalProperties === false) {
|
|
81
|
+
for (const name of Object.keys(value)) {
|
|
82
|
+
if (!(name in properties)) {
|
|
83
|
+
problems.push(`${path}.${name} was returned, and the schema declares no such property.`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return problems;
|
|
88
|
+
}
|
|
89
|
+
function checkArray(schema, value, path) {
|
|
90
|
+
if (!isObject(schema.items))
|
|
91
|
+
return [];
|
|
92
|
+
const items = schema.items;
|
|
93
|
+
return value.flatMap((item, index) => check(items, item, `${path}[${index}]`));
|
|
94
|
+
}
|
|
95
|
+
/** The declared types, as a list — a nullable schema declares two. */
|
|
96
|
+
function declaredTypes(schema) {
|
|
97
|
+
if (typeof schema.type === 'string')
|
|
98
|
+
return [schema.type];
|
|
99
|
+
if (Array.isArray(schema.type))
|
|
100
|
+
return schema.type.filter((type) => typeof type === 'string');
|
|
101
|
+
return [];
|
|
102
|
+
}
|
|
103
|
+
function matches(type, value) {
|
|
104
|
+
switch (type) {
|
|
105
|
+
case 'string':
|
|
106
|
+
return typeof value === 'string';
|
|
107
|
+
// An integer satisfies `number`, as JSON Schema says it does. The other
|
|
108
|
+
// direction does not, and JavaScript is where that is easiest to get wrong:
|
|
109
|
+
// 2.0 parses as the same number as 2, so `Number.isInteger` is the test
|
|
110
|
+
// rather than the absence of a fractional part in the source text.
|
|
111
|
+
case 'number':
|
|
112
|
+
return typeof value === 'number' && Number.isFinite(value);
|
|
113
|
+
case 'integer':
|
|
114
|
+
return typeof value === 'number' && Number.isInteger(value);
|
|
115
|
+
case 'boolean':
|
|
116
|
+
return typeof value === 'boolean';
|
|
117
|
+
case 'null':
|
|
118
|
+
return value === null;
|
|
119
|
+
case 'array':
|
|
120
|
+
return Array.isArray(value);
|
|
121
|
+
case 'object':
|
|
122
|
+
return isObject(value);
|
|
123
|
+
default:
|
|
124
|
+
return true;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
function describe(value) {
|
|
128
|
+
if (value === null)
|
|
129
|
+
return 'null';
|
|
130
|
+
if (Array.isArray(value))
|
|
131
|
+
return 'an array';
|
|
132
|
+
if (typeof value === 'object')
|
|
133
|
+
return 'an object';
|
|
134
|
+
if (typeof value === 'boolean')
|
|
135
|
+
return value ? 'true' : 'false';
|
|
136
|
+
if (typeof value === 'string') {
|
|
137
|
+
return `the string "${value.length > 40 ? `${value.slice(0, 40)}…` : value}"`;
|
|
138
|
+
}
|
|
139
|
+
return `the number ${value}`;
|
|
140
|
+
}
|
|
141
|
+
function members(values) {
|
|
142
|
+
return values
|
|
143
|
+
.map((value) => (typeof value === 'string' ? `'${value}'` : JSON.stringify(value)))
|
|
144
|
+
.join(', ');
|
|
145
|
+
}
|
|
146
|
+
function isObject(value) {
|
|
147
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* What to call the document in a problem message.
|
|
151
|
+
*
|
|
152
|
+
* A schema that names itself — Prism's `ObjectSchema` writes `name` — gets its
|
|
153
|
+
* own name in the path, so `plan.steps[1].do` reads the way the schema's author
|
|
154
|
+
* wrote it. Anything else is just "document".
|
|
155
|
+
*/
|
|
156
|
+
export function schemaName(schema) {
|
|
157
|
+
return typeof schema.name === 'string' && schema.name !== '' ? schema.name : 'document';
|
|
158
|
+
}
|
package/package.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@particle-academy/prism-harness",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Durable agent sessions for TypeScript — threads, session state and store drivers.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
7
7
|
"engines": {
|
|
8
|
-
"node": ">=
|
|
8
|
+
"node": ">=22"
|
|
9
9
|
},
|
|
10
10
|
"exports": {
|
|
11
11
|
".": {
|