@theoremjs/agents 0.3.5-next.896.g0fa6df9 → 0.3.5-next.898.gf800d9a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/src/guardrails/canary.js +2 -0
- package/esm/src/host/client-turn.js +5 -2
- package/esm/src/kernel/engine/background.js +4 -1
- package/esm/src/kernel/engine/runner/gates.js +11 -8
- package/esm/src/kernel/engine/runner/mod.d.ts +4 -1
- package/esm/src/kernel/engine/runner/mod.js +47 -11
- package/esm/src/kernel/engine/runner/nested.js +20 -12
- package/esm/src/kernel/engine/runner/schema-validation.d.ts +7 -3
- package/esm/src/kernel/engine/runner/schema-validation.js +31 -11
- package/esm/src/kernel/engine/session/mod.js +2 -2
- package/esm/src/kernel/engine/session/session-trace.d.ts +2 -1
- package/esm/src/kernel/engine/session/session-trace.js +4 -1
- package/esm/src/kernel/field-docs.js +9 -9
- package/esm/src/kernel/hook-failure.d.ts +22 -0
- package/esm/src/kernel/hook-failure.js +49 -0
- package/esm/src/kernel/provider-dispatch.js +3 -1
- package/esm/src/kernel/stages.d.ts +4 -0
- package/esm/src/kernel/stages.js +23 -4
- package/esm/src/kernel/system-lines.js +2 -0
- package/esm/src/kernel/tools/activity-label.d.ts +7 -1
- package/esm/src/kernel/tools/activity-label.js +47 -2
- package/esm/src/kernel/tools/background.d.ts +3 -0
- package/esm/src/kernel/tools/coordinate.js +104 -16
- package/esm/src/kernel/tools/invoke.js +16 -8
- package/esm/src/kernel/tools/resolve.d.ts +4 -0
- package/esm/src/kernel/tools/resolve.js +10 -0
- package/esm/src/kernel/tools/stage-run.js +12 -1
- package/esm/src/kernel/tools/types.d.ts +3 -3
- package/esm/src/kernel/turn-events.d.ts +20 -1
- package/esm/src/kernel/turn-events.js +19 -0
- package/esm/src/kernel/types.d.ts +17 -10
- package/esm/src/observability/trace-catalog.js +10 -0
- package/package.json +1 -1
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { describeError, errorKind } from '../guardrails/error.js';
|
|
2
|
+
/** The report of a host hook that threw: the one the host reads, with its copy on the span. */
|
|
3
|
+
export function hookFailed(hook, err, at = {}) {
|
|
4
|
+
const { span, ...where } = at;
|
|
5
|
+
const event = {
|
|
6
|
+
type: 'hook_failed',
|
|
7
|
+
hook,
|
|
8
|
+
...where,
|
|
9
|
+
errorKind: errorKind(err),
|
|
10
|
+
errorInternal: describeError(err),
|
|
11
|
+
};
|
|
12
|
+
span?.event('theorem.hook_failed', {
|
|
13
|
+
hook,
|
|
14
|
+
...(where.stage ? { stage: where.stage } : {}),
|
|
15
|
+
error_kind: event.errorKind,
|
|
16
|
+
reason: event.errorInternal ?? '',
|
|
17
|
+
});
|
|
18
|
+
return event;
|
|
19
|
+
}
|
|
20
|
+
/** A call a host hook could not approve, start or keep: it fails, and the model reads why. */
|
|
21
|
+
export function hookCrashFailure(err) {
|
|
22
|
+
return { code: 'hook_failed', kind: errorKind(err), message: describeError(err) };
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The crash reports of a call that runs past its turn, on its own trace. Its span ends when the
|
|
26
|
+
* call answers, so a crash after that, such as a failed late save, is kept and added to the span
|
|
27
|
+
* when the record is written.
|
|
28
|
+
*/
|
|
29
|
+
export function detachedCrashes(root) {
|
|
30
|
+
const late = [];
|
|
31
|
+
return {
|
|
32
|
+
report(hook, err) {
|
|
33
|
+
const span = root();
|
|
34
|
+
if (!span?.ended) {
|
|
35
|
+
hookFailed(hook, err, { span });
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const keep = (name, attributes = {}, timeUnixNano) => {
|
|
39
|
+
late.push({ name, timeUnixNano: timeUnixNano ?? span.nowUnixNano(), attributes });
|
|
40
|
+
};
|
|
41
|
+
hookFailed(hook, err, { span: { ...span, event: keep } });
|
|
42
|
+
},
|
|
43
|
+
/** The written spans, root first, with the late reports on the root. */
|
|
44
|
+
into(spans) {
|
|
45
|
+
spans[0]?.events.push(...late.splice(0));
|
|
46
|
+
return spans;
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
}
|
|
@@ -11,7 +11,9 @@ export function resolveRegisteredTurnModel(registry, profileId, modelId, options
|
|
|
11
11
|
if (!binding)
|
|
12
12
|
throw new TheoremError('config', labelText('provider.model_binding_missing'));
|
|
13
13
|
assertHistoryKept(profile.providerContinuation, request?.providerState, request?.input?.history);
|
|
14
|
-
|
|
14
|
+
// why: A compaction trigger is the host's function, which structuredClone cannot copy, and the provider never reads compaction.
|
|
15
|
+
const { compaction: _compaction, ...wire } = binding;
|
|
16
|
+
const turn = registeredTurnModel(registry.providers.require(binding.provider, binding.apiId), structuredClone(wire), profile.type, options, request?.providerState, profile.providerContinuation, request?.input?.history);
|
|
15
17
|
return {
|
|
16
18
|
complete: turn.complete,
|
|
17
19
|
forProfile: (id, model, req) => resolveRegisteredTurnModel(registry, id, model, options, req),
|
|
@@ -190,6 +190,10 @@ export interface RunStageArgs extends StageCallBag {
|
|
|
190
190
|
/** Applied stage output: every handler's inject, in call order, sanitized. */
|
|
191
191
|
export interface RunStageOutput extends Omit<StageApplyOutput, 'inject'> {
|
|
192
192
|
inject: InjectUnit[];
|
|
193
|
+
/** At `pre_tool`, what a handler threw: the call fails with it, and no other handler runs. */
|
|
194
|
+
crashed?: {
|
|
195
|
+
error: unknown;
|
|
196
|
+
};
|
|
193
197
|
}
|
|
194
198
|
/** Stage events always emit, even with no handlers. */
|
|
195
199
|
export declare function runStage(args: RunStageArgs): AsyncGenerator<TurnEvent, RunStageOutput>;
|
package/esm/src/kernel/stages.js
CHANGED
|
@@ -4,6 +4,7 @@ import { guardrailTurnEvent } from '../guardrails/events.js';
|
|
|
4
4
|
import { resolveGuardrailPolicy } from '../guardrails/policy.js';
|
|
5
5
|
import { requestRefused, sanitizeHistory } from '../guardrails/sanitize.js';
|
|
6
6
|
import { guardrailAttributes } from './engine/turn-trace.js';
|
|
7
|
+
import { hookFailed } from './hook-failure.js';
|
|
7
8
|
import { isRecord } from './util/record.js';
|
|
8
9
|
/**
|
|
9
10
|
* What a stage handler can ask for: inject messages, abort, deny a tool call, ask for
|
|
@@ -378,10 +379,13 @@ function mergeApplied(parts) {
|
|
|
378
379
|
/**
|
|
379
380
|
* Await each handler in order and apply its return on its own, so one handler's
|
|
380
381
|
* junk is that handler's warning and cannot erase another's affordance. Each
|
|
381
|
-
* handler gets its own shallow context; abort wins over any handler error.
|
|
382
|
+
* handler gets its own shallow context; abort wins over any handler error. A
|
|
383
|
+
* handler that throws adds nothing; at `pre_tool` it ends the run, since the
|
|
384
|
+
* call cannot go on unapproved.
|
|
382
385
|
*/
|
|
383
386
|
async function applyHandlers(handlers, ctx, apply, signal) {
|
|
384
387
|
const parts = [];
|
|
388
|
+
const crashes = [];
|
|
385
389
|
for (const handler of handlers) {
|
|
386
390
|
let raw;
|
|
387
391
|
try {
|
|
@@ -389,12 +393,15 @@ async function applyHandlers(handlers, ctx, apply, signal) {
|
|
|
389
393
|
}
|
|
390
394
|
catch (err) {
|
|
391
395
|
throwIfAborted(signal);
|
|
392
|
-
|
|
396
|
+
crashes.push(err);
|
|
397
|
+
if (apply.stage === 'pre_tool')
|
|
398
|
+
break;
|
|
399
|
+
continue;
|
|
393
400
|
}
|
|
394
401
|
throwIfAborted(signal);
|
|
395
402
|
parts.push(applyStageResult({ ...apply, result: raw }));
|
|
396
403
|
}
|
|
397
|
-
return parts;
|
|
404
|
+
return { parts, crashes };
|
|
398
405
|
}
|
|
399
406
|
/** Stage events always emit, even with no handlers. */
|
|
400
407
|
export async function* runStage(args) {
|
|
@@ -416,7 +423,19 @@ export async function* runStage(args) {
|
|
|
416
423
|
}
|
|
417
424
|
const ctx = { stage, step, history, host, ...bag };
|
|
418
425
|
const startedMs = span?.msSinceStart();
|
|
419
|
-
const
|
|
426
|
+
const { parts, crashes } = await applyHandlers(handlers, ctx, { stage, injectAllowed, injectWouldExceedMaxSteps, mutable }, signal);
|
|
427
|
+
if (stage === 'pre_tool' && crashes.length > 0) {
|
|
428
|
+
span?.event('theorem.stage', { stage, affordance: [] });
|
|
429
|
+
return { warnings: [], inject: [], crashed: { error: crashes[0] } };
|
|
430
|
+
}
|
|
431
|
+
for (const err of crashes) {
|
|
432
|
+
yield hookFailed('onStage', err, {
|
|
433
|
+
stage,
|
|
434
|
+
...(bag.callId === undefined ? {} : { callId: bag.callId }),
|
|
435
|
+
span,
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
const applied = mergeApplied(parts);
|
|
420
439
|
span?.event('theorem.stage', {
|
|
421
440
|
stage,
|
|
422
441
|
...(startedMs === undefined ? {} : { hook_ms: span.msSinceStart() - startedMs }),
|
|
@@ -32,6 +32,8 @@ function lineProblem(parts, schema) {
|
|
|
32
32
|
for (const check of line.when) {
|
|
33
33
|
if (check.path.includes('.'))
|
|
34
34
|
return `"${check.path}" isn't one option.`; // labels-exempt: developer contract error
|
|
35
|
+
if ('has' in check)
|
|
36
|
+
return 'An option is one value, so it has no fields.'; // labels-exempt: developer contract error
|
|
35
37
|
if (check.expected === null)
|
|
36
38
|
return 'An option has no null value, so "is null" never fits.'; // labels-exempt: developer contract error
|
|
37
39
|
const problem = checkProblem(check, schema, 'this profile’s options');
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
/** One line of wording, or a list of lines read top to bottom: the first that fits is shown. */
|
|
2
2
|
export type LabelTemplate = string | readonly string[];
|
|
3
|
-
/**
|
|
3
|
+
/**
|
|
4
|
+
* `when status is archived`, or `when action has erase`: whether the group at `path` holds the
|
|
5
|
+
* field `has`. `expected: null` is the check "is null".
|
|
6
|
+
*/
|
|
4
7
|
export type Check = {
|
|
5
8
|
path: string;
|
|
6
9
|
negated: boolean;
|
|
7
10
|
expected: string | null;
|
|
11
|
+
} | {
|
|
12
|
+
path: string;
|
|
13
|
+
has: string;
|
|
8
14
|
};
|
|
9
15
|
export type Line = {
|
|
10
16
|
joined: boolean;
|
|
@@ -18,6 +18,7 @@ const JOINED = /^\+\s+/;
|
|
|
18
18
|
/** The colon that ends a line's condition: the first one followed by a space or the line's end. */
|
|
19
19
|
const WHEN_ENDS = /:(?:\s|$)/;
|
|
20
20
|
const CHECK = /^(\S+) is (?:(not) )?(\S.*)$/;
|
|
21
|
+
const HAS = /^(\S+) has (\S+)$/;
|
|
21
22
|
function parsePlaceholder(raw) {
|
|
22
23
|
const text = raw.trim();
|
|
23
24
|
const end = text.search(/[\s|]/);
|
|
@@ -82,6 +83,11 @@ function parseChecks(raw) {
|
|
|
82
83
|
const checks = [];
|
|
83
84
|
// why: one space between words first, so no pattern below can try a run of spaces two ways.
|
|
84
85
|
for (const part of raw.replace(/\s+/g, ' ').split(' and ')) {
|
|
86
|
+
const has = HAS.exec(part.trim());
|
|
87
|
+
if (has?.[1] && has[2]) {
|
|
88
|
+
checks.push({ path: has[1], has: has[2] });
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
85
91
|
const match = CHECK.exec(part.trim());
|
|
86
92
|
const [, path, not, expected] = match ?? [];
|
|
87
93
|
if (!path || !expected)
|
|
@@ -131,6 +137,8 @@ export function checkHolds(check, input) {
|
|
|
131
137
|
if (!keys)
|
|
132
138
|
return false;
|
|
133
139
|
const value = valueAt(input, keys);
|
|
140
|
+
if ('has' in check)
|
|
141
|
+
return isRecord(value) && Object.hasOwn(value, check.has);
|
|
134
142
|
const comparable = typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean';
|
|
135
143
|
const is = check.expected === null ? value === null : comparable && `${value}` === check.expected;
|
|
136
144
|
return is !== check.negated;
|
|
@@ -375,6 +383,40 @@ function schemaLeavesAt(node, keys) {
|
|
|
375
383
|
}
|
|
376
384
|
return types.length === 0 && rawTypes(node).length === 0 ? [OPEN_LEAF] : [];
|
|
377
385
|
}
|
|
386
|
+
/** The field names a group at `keys` can hold, or `'any'` when the schema doesn't say. */
|
|
387
|
+
function schemaFieldsAt(node, keys) {
|
|
388
|
+
if (!isRecord(node) || '$ref' in node)
|
|
389
|
+
return 'any';
|
|
390
|
+
const branches = schemaBranches(node);
|
|
391
|
+
if (branches.length) {
|
|
392
|
+
const found = branches.map((branch) => schemaFieldsAt(branch, keys));
|
|
393
|
+
return found.includes('any')
|
|
394
|
+
? 'any'
|
|
395
|
+
: found.flatMap((fields) => (fields === 'any' ? [] : fields));
|
|
396
|
+
}
|
|
397
|
+
const [key, ...rest] = keys;
|
|
398
|
+
if (key === undefined) {
|
|
399
|
+
if (isRecord(node.properties))
|
|
400
|
+
return Object.keys(node.properties);
|
|
401
|
+
return schemaTypes(node).length === 0 ? 'any' : [];
|
|
402
|
+
}
|
|
403
|
+
if (schemaTypes(node).includes('array') && INDEX.test(key)) {
|
|
404
|
+
return schemaFieldsAt(node.items, rest);
|
|
405
|
+
}
|
|
406
|
+
if (isRecord(node.properties)) {
|
|
407
|
+
return Object.hasOwn(node.properties, key) ? schemaFieldsAt(node.properties[key], rest) : [];
|
|
408
|
+
}
|
|
409
|
+
return schemaTypes(node).length === 0 ? 'any' : [];
|
|
410
|
+
}
|
|
411
|
+
function hasProblem(path, has, holds, fields) {
|
|
412
|
+
if (holds !== 'group') {
|
|
413
|
+
return `"${path}" in the "when" is not a group, so it has no "${has}". "has" asks which field a group holds.`; // labels-exempt: builder diagnostic
|
|
414
|
+
}
|
|
415
|
+
if (fields === 'any' || fields.includes(has))
|
|
416
|
+
return undefined;
|
|
417
|
+
const itHas = fields.length ? ` It has ${oneOf(fields)}.` : ''; // labels-exempt: builder diagnostic
|
|
418
|
+
return `"${path}" never has "${has}".${itHas}`; // labels-exempt: builder diagnostic
|
|
419
|
+
}
|
|
378
420
|
/** The input's own field names, flags included: what a `when` can compare. */
|
|
379
421
|
function inputFields(node) {
|
|
380
422
|
if (!isRecord(node))
|
|
@@ -400,8 +442,11 @@ export function checkProblem(check, input, reads = "this tool's input") {
|
|
|
400
442
|
const tryThese = fields.length ? ` Try ${oneOf(fields)}.` : ''; // labels-exempt: builder diagnostic
|
|
401
443
|
return `"${check.path}" in the "when" is not a field of ${reads}.${tryThese}`; // labels-exempt: builder diagnostic
|
|
402
444
|
}
|
|
445
|
+
if ('has' in check) {
|
|
446
|
+
return hasProblem(check.path, check.has, found.holds, schemaFieldsAt(input, keys));
|
|
447
|
+
}
|
|
403
448
|
if (found.holds === 'group') {
|
|
404
|
-
return `"${check.path}" in the "when" is a list or group; a "when" compares text, a number, or true or false.`; // labels-exempt: builder diagnostic
|
|
449
|
+
return `"${check.path}" in the "when" is a list or group; a "when" compares text, a number, or true or false, or says which field a group has, like "when ${check.path} has <field>".`; // labels-exempt: builder diagnostic
|
|
405
450
|
}
|
|
406
451
|
const leaves = schemaLeavesAt(input, keys);
|
|
407
452
|
if (check.expected === null) {
|
|
@@ -420,7 +465,7 @@ export function checkProblem(check, input, reads = "this tool's input") {
|
|
|
420
465
|
}
|
|
421
466
|
function lineProblem(line, schemas, sides, mustShow) {
|
|
422
467
|
if (line.when === 'unreadable') {
|
|
423
|
-
return 'A line that opens with "when" needs a condition and a colon, like "when status is archived: archiving {name}". A condition uses "is", "is not", "is null", and "and".'; // labels-exempt: builder diagnostic
|
|
468
|
+
return 'A line that opens with "when" needs a condition and a colon, like "when status is archived: archiving {name}". A condition uses "is", "is not", "is null", "has", and "and".'; // labels-exempt: builder diagnostic
|
|
424
469
|
}
|
|
425
470
|
for (const check of line.when) {
|
|
426
471
|
const problem = checkProblem(check, schemas.input);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { labelText } from '../../guardrails/labels.js';
|
|
3
3
|
import type { SpanHandle, TraceAttributes } from '../../observability/trace-span.js';
|
|
4
|
+
import type { HookName } from '../turn-events.js';
|
|
4
5
|
import type { TurnEvent } from '../types.js';
|
|
5
6
|
import type { AgentCaller } from './agent.js';
|
|
6
7
|
import { type ToolCallBase } from './events.js';
|
|
@@ -62,6 +63,8 @@ export interface ToolBackgroundService {
|
|
|
62
63
|
/** A detached call's own trace record, joined to its turn through `traceparent`. */
|
|
63
64
|
export interface DetachedToolTrace {
|
|
64
65
|
open: (name: string, attributes: TraceAttributes) => SpanHandle;
|
|
66
|
+
/** Records a host hook's crash on the call's span, where no turn is left to read it. */
|
|
67
|
+
hookFailed: (hook: HookName, err: unknown) => void;
|
|
65
68
|
end: () => void | Promise<void>;
|
|
66
69
|
}
|
|
67
70
|
/** The service with what a request adds: its event projection and where a detached trace goes. */
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { errorKind, throwIfAborted } from '../../guardrails/error.js';
|
|
1
|
+
import { describeError, errorKind, TheoremError, throwIfAborted } from '../../guardrails/error.js';
|
|
2
2
|
import { settleable } from '../engine/runner/step-order.js';
|
|
3
|
+
import { hookCrashFailure, hookFailed } from '../hook-failure.js';
|
|
3
4
|
import { jsonObjectSchema } from '../json.js';
|
|
4
5
|
import { failureEvent } from './events.js';
|
|
5
6
|
import { formatToolFailureForModel } from './model-text.js';
|
|
@@ -17,6 +18,20 @@ class ReplayedReply {
|
|
|
17
18
|
this.reply = reply;
|
|
18
19
|
}
|
|
19
20
|
}
|
|
21
|
+
/** The host's operation store or background service threw. */
|
|
22
|
+
class StoreCrashed {
|
|
23
|
+
error;
|
|
24
|
+
constructor(error) {
|
|
25
|
+
this.error = error;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* A `TheoremError` from `lookup` or `begin` is the store's answer, such as a held operation or a full
|
|
30
|
+
* session, and keeps its own path; any other throw is a crash.
|
|
31
|
+
*/
|
|
32
|
+
function storeCrash(error) {
|
|
33
|
+
return error instanceof TheoremError ? error : new StoreCrashed(error);
|
|
34
|
+
}
|
|
20
35
|
const FIRED = Symbol('fired');
|
|
21
36
|
const CUT = Symbol('cut');
|
|
22
37
|
/**
|
|
@@ -85,6 +100,7 @@ class CoordinatedExecution {
|
|
|
85
100
|
claim;
|
|
86
101
|
committed = false;
|
|
87
102
|
writeFailed = false;
|
|
103
|
+
closed = false;
|
|
88
104
|
terminal;
|
|
89
105
|
stopped;
|
|
90
106
|
spent;
|
|
@@ -97,13 +113,23 @@ class CoordinatedExecution {
|
|
|
97
113
|
this.signal = signal;
|
|
98
114
|
this.trigger = trigger;
|
|
99
115
|
}
|
|
116
|
+
get callId() {
|
|
117
|
+
return this.operation.callId;
|
|
118
|
+
}
|
|
100
119
|
async beginBody(preparedInput) {
|
|
101
120
|
throwIfAborted(this.signal);
|
|
102
121
|
const operation = {
|
|
103
122
|
...this.operation,
|
|
104
123
|
invocation: { accepted: this.accepted, preparedInput: operationWireValue(preparedInput) },
|
|
105
124
|
};
|
|
106
|
-
|
|
125
|
+
let result;
|
|
126
|
+
try {
|
|
127
|
+
result = await this.coordinator.begin(operation);
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
throwIfAborted(this.signal);
|
|
131
|
+
throw storeCrash(error);
|
|
132
|
+
}
|
|
107
133
|
if (result.kind === 'settled')
|
|
108
134
|
throw new ReplayedOperation(result.outcome);
|
|
109
135
|
if (result.kind === 'background')
|
|
@@ -158,8 +184,15 @@ class CoordinatedExecution {
|
|
|
158
184
|
};
|
|
159
185
|
const whole = toolOperationOutcomeSchema.parse(operationWireValue(outcome));
|
|
160
186
|
const saved = this.withinLimit(whole);
|
|
161
|
-
|
|
162
|
-
|
|
187
|
+
const kept = { terminal: saved.event, settlement: saved === whole ? settlement : saved.replay };
|
|
188
|
+
try {
|
|
189
|
+
await this.save(saved);
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
throwIfAborted(this.signal);
|
|
193
|
+
return { ...kept, crashed: new StoreCrashed(error) };
|
|
194
|
+
}
|
|
195
|
+
return kept;
|
|
163
196
|
}
|
|
164
197
|
async save(outcome) {
|
|
165
198
|
this.writeFailed = true;
|
|
@@ -198,7 +231,7 @@ class CoordinatedExecution {
|
|
|
198
231
|
catch (error) {
|
|
199
232
|
this.writeFailed = !this.signal?.aborted;
|
|
200
233
|
trigger.abort();
|
|
201
|
-
throw error;
|
|
234
|
+
throw this.writeFailed ? new StoreCrashed(error) : error;
|
|
202
235
|
}
|
|
203
236
|
trigger.release();
|
|
204
237
|
this.goOn(detach, iterator, pending).finally(() => done.resolve({ unreadSteers: steers.close() }));
|
|
@@ -229,23 +262,26 @@ class CoordinatedExecution {
|
|
|
229
262
|
/** Reads the run past the reply, passing its events to the host, until the call settles. */
|
|
230
263
|
async goOn(detach, iterator, pending) {
|
|
231
264
|
const { service, project } = detach.support;
|
|
265
|
+
// why: No turn reads this run, so a crash of the host's own reporter goes to the call's trace.
|
|
232
266
|
const emit = async (event) => {
|
|
233
267
|
try {
|
|
234
268
|
await service.event(project(event), this.call());
|
|
235
269
|
}
|
|
236
|
-
catch {
|
|
237
|
-
|
|
270
|
+
catch (error) {
|
|
271
|
+
detach.trace.hookFailed('background', error);
|
|
238
272
|
}
|
|
239
273
|
};
|
|
240
274
|
try {
|
|
241
275
|
await this.drain(iterator, pending, emit);
|
|
242
276
|
}
|
|
243
|
-
catch {
|
|
244
|
-
|
|
277
|
+
catch (error) {
|
|
278
|
+
detach.trace.hookFailed('execution', error);
|
|
245
279
|
}
|
|
246
280
|
finally {
|
|
247
281
|
clearTimeout(this.graceTimer);
|
|
248
|
-
await this.close()
|
|
282
|
+
const crashed = await this.close();
|
|
283
|
+
if (crashed)
|
|
284
|
+
detach.trace.hookFailed('execution', crashed.error);
|
|
249
285
|
await Promise.resolve()
|
|
250
286
|
.then(() => detach.trace.end())
|
|
251
287
|
.catch(() => { });
|
|
@@ -271,8 +307,9 @@ class CoordinatedExecution {
|
|
|
271
307
|
catch (error) {
|
|
272
308
|
if (this.writeFailed)
|
|
273
309
|
throw error;
|
|
274
|
-
|
|
275
|
-
|
|
310
|
+
if (this.stopped)
|
|
311
|
+
return await this.settleStopped(emit);
|
|
312
|
+
await this.settleFailed({ code: 'run_failed', kind: errorKind(error), message: describeError(error) }, emit);
|
|
276
313
|
}
|
|
277
314
|
}
|
|
278
315
|
async settleAfterReply(settlement, emit) {
|
|
@@ -287,9 +324,11 @@ class CoordinatedExecution {
|
|
|
287
324
|
}
|
|
288
325
|
// why: No turn is open, so a post_tool inject or abort has nothing to reach.
|
|
289
326
|
const { pendingInject: _inject, aborted: _aborted, ...kept } = settlement;
|
|
290
|
-
const { terminal } = await this.finish(kept);
|
|
327
|
+
const { terminal, crashed } = await this.finish(kept);
|
|
291
328
|
if (terminal)
|
|
292
329
|
await emit(terminal);
|
|
330
|
+
if (crashed)
|
|
331
|
+
throw crashed.error;
|
|
293
332
|
}
|
|
294
333
|
/** A stop with a reason settles the call; a shutdown leaves it for `close` to mark. */
|
|
295
334
|
async settleStopped(emit) {
|
|
@@ -351,10 +390,25 @@ class CoordinatedExecution {
|
|
|
351
390
|
return unresolvedOperation(errorKind(error), 'The tool was sent and did not settle, so its effects are unknown', // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
352
391
|
error);
|
|
353
392
|
}
|
|
393
|
+
/** Marks a call that did not settle, once. A store that throws here gives its crash back. */
|
|
354
394
|
async close() {
|
|
355
|
-
if (this.
|
|
395
|
+
if (this.closed || !this.claim || this.committed)
|
|
396
|
+
return undefined;
|
|
397
|
+
this.closed = true;
|
|
398
|
+
try {
|
|
356
399
|
await this.coordinator.markUncertain(this.claim, this.writeFailed ? 'settlement_write_failed' : 'execution_interrupted');
|
|
357
400
|
}
|
|
401
|
+
catch (error) {
|
|
402
|
+
return new StoreCrashed(error);
|
|
403
|
+
}
|
|
404
|
+
return undefined;
|
|
405
|
+
}
|
|
406
|
+
/** A call the store could not start or keep: it fails, and the model reads that failure. */
|
|
407
|
+
failed(crashed) {
|
|
408
|
+
const effects = this.claim ? 'unknown' : 'not_started';
|
|
409
|
+
const outcome = failedOutcome(this.base(), effects, hookCrashFailure(crashed.error), this.spent);
|
|
410
|
+
const settlement = this.claim ? outcome.replay : { ...outcome.replay, callNotStarted: true };
|
|
411
|
+
return { event: outcome.event, settlement };
|
|
358
412
|
}
|
|
359
413
|
}
|
|
360
414
|
/** What a saved operation gives the turn, once the replay check has passed. */
|
|
@@ -387,6 +441,34 @@ function detaching(detach, signal) {
|
|
|
387
441
|
};
|
|
388
442
|
return { plan: detach, trigger, steers, hooks };
|
|
389
443
|
}
|
|
444
|
+
/** The store's saved operation, or its crash; a `TheoremError` is its answer and is thrown. */
|
|
445
|
+
async function lookup(coordinator, operation, signal) {
|
|
446
|
+
try {
|
|
447
|
+
return await coordinator.lookup(operation);
|
|
448
|
+
}
|
|
449
|
+
catch (error) {
|
|
450
|
+
throwIfAborted(signal);
|
|
451
|
+
const crash = storeCrash(error);
|
|
452
|
+
if (crash instanceof StoreCrashed)
|
|
453
|
+
return crash;
|
|
454
|
+
throw crash;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
/** Marks a call that did not settle, and reports each store crash once. */
|
|
458
|
+
async function* closeReporting(execution, crashed) {
|
|
459
|
+
const closeCrashed = await execution.close();
|
|
460
|
+
for (const crash of [crashed, closeCrashed]) {
|
|
461
|
+
if (crash)
|
|
462
|
+
yield hookFailed('execution', crash.error, { callId: execution.callId });
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
/** A call the store could not start or keep fails, and the model reads why. */
|
|
466
|
+
async function* failCall(execution, crash) {
|
|
467
|
+
const failed = execution.failed(crash);
|
|
468
|
+
yield failed.event;
|
|
469
|
+
yield* closeReporting(execution);
|
|
470
|
+
return failed.settlement;
|
|
471
|
+
}
|
|
390
472
|
export async function* coordinateTool(coordinator, identity, invocation, signal, run, authorizeReplay, detach) {
|
|
391
473
|
if (!coordinator)
|
|
392
474
|
return yield* run(undefined, { signal });
|
|
@@ -397,7 +479,9 @@ export async function* coordinateTool(coordinator, identity, invocation, signal,
|
|
|
397
479
|
const callSignal = trigger ? trigger.signal : signal;
|
|
398
480
|
const execution = new CoordinatedExecution(coordinator, operation, accepted, callSignal, trigger);
|
|
399
481
|
throwIfAborted(signal);
|
|
400
|
-
const existing = await
|
|
482
|
+
const existing = await lookup(coordinator, operation, signal);
|
|
483
|
+
if (existing instanceof StoreCrashed)
|
|
484
|
+
return yield* failCall(execution, existing);
|
|
401
485
|
throwIfAborted(signal);
|
|
402
486
|
if (existing?.kind === 'running' || existing?.kind === 'uncertain')
|
|
403
487
|
throw heldOperation(existing.kind);
|
|
@@ -427,9 +511,10 @@ export async function* coordinateTool(coordinator, identity, invocation, signal,
|
|
|
427
511
|
return { ...reply.replay };
|
|
428
512
|
}
|
|
429
513
|
if (next.done) {
|
|
430
|
-
const { terminal, settlement } = await execution.finish(next.value);
|
|
514
|
+
const { terminal, settlement, crashed } = await execution.finish(next.value);
|
|
431
515
|
if (terminal)
|
|
432
516
|
yield terminal;
|
|
517
|
+
yield* closeReporting(execution, crashed);
|
|
433
518
|
return settlement;
|
|
434
519
|
}
|
|
435
520
|
if (execution.observe(next.value))
|
|
@@ -443,12 +528,15 @@ export async function* coordinateTool(coordinator, identity, invocation, signal,
|
|
|
443
528
|
yield saved.event;
|
|
444
529
|
return saved.replay;
|
|
445
530
|
}
|
|
531
|
+
if (error instanceof StoreCrashed)
|
|
532
|
+
return yield* failCall(execution, error);
|
|
446
533
|
throw execution.failure(error);
|
|
447
534
|
}
|
|
448
535
|
finally {
|
|
449
536
|
if (!handedOff) {
|
|
450
537
|
trigger?.release();
|
|
451
538
|
try {
|
|
539
|
+
// why: A crash here has no reader: the turn already ends on another error, or was stopped.
|
|
452
540
|
await execution.close();
|
|
453
541
|
}
|
|
454
542
|
finally {
|
|
@@ -9,13 +9,14 @@ import { invokeAgentCaller } from '../engine/runner/mod.js';
|
|
|
9
9
|
import { inStep, runSideBySide, stepOrder } from '../engine/runner/step-order.js';
|
|
10
10
|
import { startToolTrace, toolSpanName } from '../engine/tool-trace.js';
|
|
11
11
|
import { endThrownSpan, runIdentity, traceLinks } from '../engine/turn-trace.js';
|
|
12
|
+
import { hookFailed } from '../hook-failure.js';
|
|
12
13
|
import { pickModel } from '../registry/resolve.js';
|
|
13
14
|
import { turnDoneOf } from '../turn-events.js';
|
|
14
15
|
import { assertBackgroundCoordinator, planDetach, } from './background.js';
|
|
15
16
|
import { coordinateTool } from './coordinate.js';
|
|
16
17
|
import { failedActivity, failureEvent, newCallId, toolCallArguments, toolCallRequestEvent, } from './events.js';
|
|
17
18
|
import { authorizeToolReplay, executeRegisteredTool, } from './execute.js';
|
|
18
|
-
import {
|
|
19
|
+
import { applyT1Policy, cloneTurnToolSnapshot, profileToolsParallel, promoteLoadedTools, resolveTurnTools, } from './resolve.js';
|
|
19
20
|
import { plainToolInput } from './schema.js';
|
|
20
21
|
function turnRequestFromInvoke(request) {
|
|
21
22
|
return {
|
|
@@ -33,7 +34,11 @@ async function prepareInvokeSnapshot(registry, request, profile) {
|
|
|
33
34
|
throw new TheoremError('request', `Profile ${profile.id}: type 'decision' cannot invoke tools`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
34
35
|
}
|
|
35
36
|
const model = profile.type === 'host' ? undefined : pickModel(profile, request.model);
|
|
36
|
-
|
|
37
|
+
const snapshot = resolveTurnTools(registry.tools, profile, req, model);
|
|
38
|
+
const t1Policy = await applyT1Policy(registry.tools, snapshot, profile, req);
|
|
39
|
+
if (!t1Policy)
|
|
40
|
+
return { snapshot };
|
|
41
|
+
return { snapshot, crashed: hookFailed('tools.t1Policy', t1Policy.crashed) };
|
|
37
42
|
}
|
|
38
43
|
/**
|
|
39
44
|
* One trace record around an invoke, written however it ends, including an early stop or a
|
|
@@ -105,10 +110,9 @@ function failCallBeforeTool(openSpan, invoke, callId) {
|
|
|
105
110
|
async function invokeSnapshot(registry, request) {
|
|
106
111
|
assertBackgroundCoordinator(request.background, request.execution);
|
|
107
112
|
const profile = registry.profiles.get(request.profile);
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
return { profile, snapshot };
|
|
113
|
+
if (request.snapshot)
|
|
114
|
+
return { profile, snapshot: cloneTurnToolSnapshot(request.snapshot) };
|
|
115
|
+
return { profile, ...(await prepareInvokeSnapshot(registry, request, profile)) };
|
|
112
116
|
}
|
|
113
117
|
function invokeTool(registry, request, sinkOverride) {
|
|
114
118
|
return startInvoke(registry, request, sinkOverride).events;
|
|
@@ -174,7 +178,9 @@ function startInvokeStep(registry, request, sinkOverride) {
|
|
|
174
178
|
};
|
|
175
179
|
}
|
|
176
180
|
async function* invokeStepTraced(registry, request, calls, openSpan, traceparent, canaries, background) {
|
|
177
|
-
const { profile, snapshot } = await invokeSnapshot(registry, request);
|
|
181
|
+
const { profile, snapshot, crashed } = await invokeSnapshot(registry, request);
|
|
182
|
+
if (crashed)
|
|
183
|
+
yield crashed;
|
|
178
184
|
const stepId = crypto.randomUUID();
|
|
179
185
|
for (const { callId, invoke, announce } of calls) {
|
|
180
186
|
if (!announce)
|
|
@@ -205,7 +211,9 @@ async function* invokeStepTraced(registry, request, calls, openSpan, traceparent
|
|
|
205
211
|
}, snapshot);
|
|
206
212
|
}
|
|
207
213
|
async function* invokeTraced(registry, request, callId, openSpan, failBeforeTool, traceparent, canaries, background) {
|
|
208
|
-
const { profile, snapshot } = await invokeSnapshot(registry, request);
|
|
214
|
+
const { profile, snapshot, crashed } = await invokeSnapshot(registry, request);
|
|
215
|
+
if (crashed)
|
|
216
|
+
yield crashed;
|
|
209
217
|
// why: A model's call was already announced by its turn.
|
|
210
218
|
if (request.callId === undefined) {
|
|
211
219
|
yield toolCallRequestEvent({ name: request.name, callId, operationId: request.operationId }, toolCallArguments(request.input));
|
|
@@ -21,6 +21,10 @@ export declare function resolveTurnTools(tools: ToolRegistry, profile: Profile,
|
|
|
21
21
|
export declare function prepareTurnToolSnapshot(tools: ToolRegistry, profile: Profile, req: TurnRequest, modelId: ModelId | undefined): Promise<TurnToolSnapshot>;
|
|
22
22
|
/** So concurrent host invokes do not share mutable state. */
|
|
23
23
|
export declare function cloneTurnToolSnapshot(state: TurnToolSnapshot): TurnToolSnapshot;
|
|
24
|
+
/** Applies the profile's `t1Policy`. A policy that throws or answers wrongly leaves the T0 tools, and its error comes back. */
|
|
25
|
+
export declare function applyT1Policy(tools: ToolRegistry, state: TurnToolSnapshot, profile: Profile, req: TurnRequest): Promise<{
|
|
26
|
+
crashed: unknown;
|
|
27
|
+
} | undefined>;
|
|
24
28
|
export declare function expandT1Policy(tools: ToolRegistry, state: TurnToolSnapshot, profile: Profile, req: TurnRequest): Promise<void>;
|
|
25
29
|
/** Puts the T2 tools among `ids` that the turn may call on its list; any other id is left out. */
|
|
26
30
|
export declare function promoteTurnTools(tools: ToolRegistry, state: TurnToolSnapshot, ids?: readonly ToolId[]): void;
|
|
@@ -182,6 +182,16 @@ export function cloneTurnToolSnapshot(state) {
|
|
|
182
182
|
...(state.modelBackground ? { modelBackground: state.modelBackground } : {}),
|
|
183
183
|
};
|
|
184
184
|
}
|
|
185
|
+
/** Applies the profile's `t1Policy`. A policy that throws or answers wrongly leaves the T0 tools, and its error comes back. */
|
|
186
|
+
export async function applyT1Policy(tools, state, profile, req) {
|
|
187
|
+
try {
|
|
188
|
+
await expandT1Policy(tools, state, profile, req);
|
|
189
|
+
}
|
|
190
|
+
catch (crashed) {
|
|
191
|
+
return { crashed };
|
|
192
|
+
}
|
|
193
|
+
return undefined;
|
|
194
|
+
}
|
|
185
195
|
export async function expandT1Policy(tools, state, profile, req) {
|
|
186
196
|
const t1Policy = profileToolsSpec(profile)?.t1Policy;
|
|
187
197
|
if (!t1Policy) {
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { throwIfAborted } from '../../guardrails/error.js';
|
|
1
2
|
import { labelText } from '../../guardrails/labels.js';
|
|
3
|
+
import { hookCrashFailure } from '../hook-failure.js';
|
|
2
4
|
import { jsonObjectSchema } from '../json.js';
|
|
3
5
|
import { runStage, stageEventFields, } from '../stages.js';
|
|
4
6
|
import { splitCallOrder, stepCallIds } from './call-order.js';
|
|
@@ -50,7 +52,13 @@ export async function* runPreToolPipeline(args) {
|
|
|
50
52
|
let toolPreTool;
|
|
51
53
|
const { confirmed, approved, approvedAsIs } = resumeApproval(ctx.resume);
|
|
52
54
|
if (tool.preTool && !approvedAsIs) {
|
|
53
|
-
|
|
55
|
+
try {
|
|
56
|
+
toolPreTool = (await tool.preTool(input, ctx)) ?? undefined;
|
|
57
|
+
}
|
|
58
|
+
catch (err) {
|
|
59
|
+
throwIfAborted(ctx.signal);
|
|
60
|
+
return { ok: false, kind: 'failed', failure: hookCrashFailure(err) };
|
|
61
|
+
}
|
|
54
62
|
}
|
|
55
63
|
if (!stages && toolPreTool === undefined && after.length === 0) {
|
|
56
64
|
return { ok: true, input };
|
|
@@ -114,6 +122,9 @@ async function* runPreToolStages(args) {
|
|
|
114
122
|
...(approved ? { approved: true } : {}),
|
|
115
123
|
...(stages.stepCalls ? { stepCalls: stages.stepCalls } : {}),
|
|
116
124
|
});
|
|
125
|
+
if (applied.crashed) {
|
|
126
|
+
return { ok: false, kind: 'failed', failure: hookCrashFailure(applied.crashed.error) };
|
|
127
|
+
}
|
|
117
128
|
if (applied.abort) {
|
|
118
129
|
return { ok: false, kind: 'aborted', aborted: applied.abort };
|
|
119
130
|
}
|