@theoremjs/agents 0.3.4-next.522.g9e684cd → 0.3.4-next.524.g5000023
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/src/guardrails/detect-at.d.ts +4 -2
- package/esm/src/guardrails/detect-at.js +41 -22
- package/esm/src/guardrails/detect-reply.d.ts +1 -1
- package/esm/src/guardrails/detect-reply.js +2 -1
- package/esm/src/guardrails/event-schemas.d.ts +5 -0
- package/esm/src/guardrails/event-schemas.js +1 -0
- package/esm/src/host/client-turn.js +3 -2
- package/esm/src/kernel/engine/turn-trace.js +2 -0
- package/esm/src/observability/trace-catalog.js +2 -0
- package/package.json +1 -1
|
@@ -15,6 +15,8 @@ interface Detection {
|
|
|
15
15
|
hits: GuardrailHit[];
|
|
16
16
|
/** Why a host's `find` failed, for the builder only: it reaches `guardrail.errorInternal`. */
|
|
17
17
|
errorInternal?: string;
|
|
18
|
+
/** The same failure in the kernel's words only, with no text from the host: what the trace stores. */
|
|
19
|
+
errorSummary?: string;
|
|
18
20
|
}
|
|
19
21
|
declare function stronger(left: DetectOutcome, right: DetectOutcome | 'ignore'): DetectOutcome;
|
|
20
22
|
/**
|
|
@@ -94,7 +96,7 @@ declare function storedHostSpans(text: string, detect?: ResolvedDetect): RedactS
|
|
|
94
96
|
interface BoundaryReader {
|
|
95
97
|
/** The text to let through, or `''` once a match blocks: the caller reads `found().action`. */
|
|
96
98
|
read(text: string): string;
|
|
97
|
-
found(): Pick<Detection, 'action' | 'hits' | 'errorInternal'>;
|
|
99
|
+
found(): Pick<Detection, 'action' | 'hits' | 'errorInternal' | 'errorSummary'>;
|
|
98
100
|
}
|
|
99
101
|
/** A reader for everything crossing `boundary` in one request. */
|
|
100
102
|
declare function boundaryReader(boundary: Boundary, detect: ResolvedDetect): BoundaryReader;
|
|
@@ -102,6 +104,6 @@ declare function boundaryReader(boundary: Boundary, detect: ResolvedDetect): Bou
|
|
|
102
104
|
* The guardrail event for what was found at a boundary, or `undefined` when nothing matched.
|
|
103
105
|
* `stage` is the boundary's own unless the text was read as it streamed.
|
|
104
106
|
*/
|
|
105
|
-
declare function detectEvent(boundary: Boundary, found: Pick<Detection, 'action' | 'hits' | 'errorInternal'>, provenance?: Provenance, stage?: GuardrailStage): GuardrailEvent | undefined;
|
|
107
|
+
declare function detectEvent(boundary: Boundary, found: Pick<Detection, 'action' | 'hits' | 'errorInternal' | 'errorSummary'>, provenance?: Provenance, stage?: GuardrailStage): GuardrailEvent | undefined;
|
|
106
108
|
export type { BoundaryReader, Detection, DetectOutcome, DetectScope, Release, StoredSides, StreamRead, Stretch, };
|
|
107
109
|
export { boundaryReader, detectAt, detectEvent, detectorsAt, detectReads, detectRelease, hostAt, isScoped, leakScopeOf, redactDetectors, scopeOf, storedHostSpans, storedSpans, stronger, };
|
|
@@ -16,6 +16,14 @@ import { sensitiveSpans } from './sensitive.js';
|
|
|
16
16
|
import { SENSITIVE_GROUPS } from './sensitive-groups.js';
|
|
17
17
|
import { directives, mergedRanges } from './tool-directives.js';
|
|
18
18
|
import { ownToolsWithout, toolLeakSpans } from './tool-leak.js';
|
|
19
|
+
function failureText(failures) {
|
|
20
|
+
if (failures.length === 0)
|
|
21
|
+
return {};
|
|
22
|
+
return {
|
|
23
|
+
errorInternal: failures.map(({ errorInternal }) => errorInternal).join('\n'),
|
|
24
|
+
errorSummary: failures.map(({ errorSummary }) => errorSummary).join('\n'),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
19
27
|
const STRENGTH = ['allow', 'flag', 'redact', 'block'];
|
|
20
28
|
function stronger(left, right) {
|
|
21
29
|
if (right === 'ignore')
|
|
@@ -139,15 +147,18 @@ function hitOf(detector, text, span) {
|
|
|
139
147
|
function leavesImages(detector, boundary, detect) {
|
|
140
148
|
return detector === 'ungiven_links' && detect.ungiven_images[boundary] !== 'ignore';
|
|
141
149
|
}
|
|
150
|
+
/** A thrown value's type: a name of letters only, else `Error`, so no host text gets through. */
|
|
151
|
+
function thrownType(err) {
|
|
152
|
+
const name = err instanceof Error ? err.name : typeof err;
|
|
153
|
+
return /^[A-Za-z]{1,64}$/.test(name) ? name : 'Error';
|
|
154
|
+
}
|
|
142
155
|
/** What a host's `find` matched in `text`, or how it failed. */
|
|
143
156
|
function foundBy(find, text, boundary) {
|
|
144
157
|
try {
|
|
145
158
|
const spans = find(text, boundary ? { boundary } : {});
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
reason: 'find returned no list of stretches inside the text',
|
|
150
|
-
};
|
|
159
|
+
// labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
|
|
160
|
+
const said = 'find returned no list of stretches inside the text';
|
|
161
|
+
const outside = { signal: 'find_result', reason: said, summary: said };
|
|
151
162
|
if (!Array.isArray(spans))
|
|
152
163
|
return outside;
|
|
153
164
|
const inside = spans.every(({ start, end }) => Number.isInteger(start) &&
|
|
@@ -159,7 +170,12 @@ function foundBy(find, text, boundary) {
|
|
|
159
170
|
}
|
|
160
171
|
catch (err) {
|
|
161
172
|
// why: A reading that failed found nothing it can vouch for, so the text does not cross.
|
|
162
|
-
|
|
173
|
+
const id = crypto.randomUUID().slice(0, 8);
|
|
174
|
+
return {
|
|
175
|
+
signal: 'find_threw',
|
|
176
|
+
reason: `find threw: ${describeError(err)} (error ${id})`,
|
|
177
|
+
summary: `find threw ${thrownType(err)} (error ${id})`,
|
|
178
|
+
};
|
|
163
179
|
}
|
|
164
180
|
}
|
|
165
181
|
function hostReader(host, boundary, chosen) {
|
|
@@ -174,8 +190,10 @@ function hostReader(host, boundary, chosen) {
|
|
|
174
190
|
if (!host.find)
|
|
175
191
|
return matched;
|
|
176
192
|
const found = foundBy(host.find, text, boundary);
|
|
177
|
-
if (!Array.isArray(found))
|
|
178
|
-
|
|
193
|
+
if (!Array.isArray(found)) {
|
|
194
|
+
const by = `Host detector '${host.id}'`;
|
|
195
|
+
return { ...found, reason: `${by} ${found.reason}`, summary: `${by} ${found.summary}` };
|
|
196
|
+
}
|
|
179
197
|
return [...matched, ...found];
|
|
180
198
|
},
|
|
181
199
|
hit: (text, span) => ({ ...hitFromSpan(text, span, rule, 'high'), label: host.label }),
|
|
@@ -219,13 +237,13 @@ function failedHit({ rule, label }, signal) {
|
|
|
219
237
|
function detectAt(text, boundary, detect, scope = NO_SCOPE) {
|
|
220
238
|
const hits = [];
|
|
221
239
|
const redact = [];
|
|
222
|
-
const
|
|
240
|
+
const failures = [];
|
|
223
241
|
let action = 'allow';
|
|
224
242
|
for (const reader of readersAt(boundary, detect, scope)) {
|
|
225
243
|
const spans = reader.spans(text);
|
|
226
244
|
if (!Array.isArray(spans)) {
|
|
227
245
|
hits.push(failedHit(reader, spans.signal));
|
|
228
|
-
|
|
246
|
+
failures.push({ errorInternal: spans.reason, errorSummary: spans.summary });
|
|
229
247
|
action = 'block';
|
|
230
248
|
continue;
|
|
231
249
|
}
|
|
@@ -239,7 +257,7 @@ function detectAt(text, boundary, detect, scope = NO_SCOPE) {
|
|
|
239
257
|
action = stronger(action, reader.chosen);
|
|
240
258
|
}
|
|
241
259
|
if (action === 'block') {
|
|
242
|
-
return { action, hits, ...(
|
|
260
|
+
return { action, hits, ...failureText(failures) };
|
|
243
261
|
}
|
|
244
262
|
if (redact.length === 0)
|
|
245
263
|
return { action, text, hits };
|
|
@@ -292,12 +310,12 @@ function detectReads(boundaries, detect, scope) {
|
|
|
292
310
|
*/
|
|
293
311
|
function detectRelease(window, { from, to, settled }, boundary, detect, { detectors, scope }) {
|
|
294
312
|
const failed = [];
|
|
295
|
-
const
|
|
313
|
+
const failures = [];
|
|
296
314
|
const found = readersAt(boundary, detect, scope, detectors).flatMap((reader) => {
|
|
297
315
|
const spans = reader.spans(window);
|
|
298
316
|
if (!Array.isArray(spans)) {
|
|
299
317
|
failed.push(failedHit(reader, spans.signal));
|
|
300
|
-
|
|
318
|
+
failures.push({ errorInternal: spans.reason, errorSummary: spans.summary });
|
|
301
319
|
return [];
|
|
302
320
|
}
|
|
303
321
|
return spans
|
|
@@ -305,7 +323,7 @@ function detectRelease(window, { from, to, settled }, boundary, detect, { detect
|
|
|
305
323
|
.map((span) => ({ span, rule: reader.rule, label: reader.label, chosen: reader.chosen }));
|
|
306
324
|
});
|
|
307
325
|
if (failed.length > 0) {
|
|
308
|
-
return { action: 'block', hits: failed, taken: 0,
|
|
326
|
+
return { action: 'block', hits: failed, taken: 0, ...failureText(failures) };
|
|
309
327
|
}
|
|
310
328
|
const hitOf = ({ span, rule, label }) => ({
|
|
311
329
|
...hitFromSpan(window, span, rule, 'high'),
|
|
@@ -377,23 +395,23 @@ function storedHostSpans(text, detect) {
|
|
|
377
395
|
/** A reader for everything crossing `boundary` in one request. */
|
|
378
396
|
function boundaryReader(boundary, detect) {
|
|
379
397
|
const hits = [];
|
|
380
|
-
const
|
|
398
|
+
const failures = [];
|
|
381
399
|
let action = 'allow';
|
|
382
400
|
return {
|
|
383
401
|
read(text) {
|
|
384
402
|
const detected = detectAt(text, boundary, detect);
|
|
385
403
|
for (const hit of detected.hits)
|
|
386
404
|
hits.push(hit);
|
|
387
|
-
if (detected.errorInternal)
|
|
388
|
-
|
|
405
|
+
if (detected.errorInternal && detected.errorSummary) {
|
|
406
|
+
failures.push({
|
|
407
|
+
errorInternal: detected.errorInternal,
|
|
408
|
+
errorSummary: detected.errorSummary,
|
|
409
|
+
});
|
|
410
|
+
}
|
|
389
411
|
action = stronger(action, detected.action);
|
|
390
412
|
return detected.text ?? '';
|
|
391
413
|
},
|
|
392
|
-
found: () => ({
|
|
393
|
-
action,
|
|
394
|
-
hits,
|
|
395
|
-
...(reasons.length ? { errorInternal: reasons.join('\n') } : {}),
|
|
396
|
-
}),
|
|
414
|
+
found: () => ({ action, hits, ...failureText(failures) }),
|
|
397
415
|
};
|
|
398
416
|
}
|
|
399
417
|
const TOOL_STAGE = Object.fromEntries(TOOL_BOUNDARIES.map((boundary) => [
|
|
@@ -439,6 +457,7 @@ function detectEvent(boundary, found, provenance, stage = BOUNDARY_STAGE[boundar
|
|
|
439
457
|
...(found.action === 'block' && found.errorInternal
|
|
440
458
|
? { errorInternal: found.errorInternal }
|
|
441
459
|
: {}),
|
|
460
|
+
...(found.action === 'block' && found.errorSummary ? { errorSummary: found.errorSummary } : {}),
|
|
442
461
|
};
|
|
443
462
|
}
|
|
444
463
|
export { boundaryReader, detectAt, detectEvent, detectorsAt, detectReads, detectRelease, hostAt, isScoped, leakScopeOf, redactDetectors, scopeOf, storedHostSpans, storedSpans, stronger, };
|
|
@@ -26,7 +26,7 @@ interface ReplyRead {
|
|
|
26
26
|
blocked?: GuardrailHit[];
|
|
27
27
|
}
|
|
28
28
|
/** A JSON value read at `boundary`, with each match in a string replaced, or what stops it crossing. */
|
|
29
|
-
declare function readValue(value: unknown, boundary: Boundary, detect: ResolvedDetect, scope?: DetectScope): Pick<Detection, 'action' | 'hits' | 'errorInternal'> & {
|
|
29
|
+
declare function readValue(value: unknown, boundary: Boundary, detect: ResolvedDetect, scope?: DetectScope): Pick<Detection, 'action' | 'hits' | 'errorInternal' | 'errorSummary'> & {
|
|
30
30
|
value?: unknown;
|
|
31
31
|
};
|
|
32
32
|
/**
|
|
@@ -24,7 +24,8 @@ function readValue(value, boundary, detect, scope = {}) {
|
|
|
24
24
|
}
|
|
25
25
|
const found = detectAt(scan.text, boundary, detect, scope);
|
|
26
26
|
if (found.action !== 'redact') {
|
|
27
|
-
const
|
|
27
|
+
const { errorInternal, errorSummary } = found;
|
|
28
|
+
const why = errorInternal && errorSummary ? { errorInternal, errorSummary } : {};
|
|
28
29
|
return { action: found.action, hits: found.hits, value, ...why };
|
|
29
30
|
}
|
|
30
31
|
const redacted = mapStrings(value, (leaf) => detectAt(leaf, boundary, detect, scope).text ?? '');
|
|
@@ -95,6 +95,11 @@ export interface GuardrailEvent {
|
|
|
95
95
|
provenance?: Provenance;
|
|
96
96
|
/** The block verdict's `errorInternal`: builder-only, stripped by `forClient`. */
|
|
97
97
|
errorInternal?: string;
|
|
98
|
+
/**
|
|
99
|
+
* A failed host detector's `errorInternal` in the kernel's words only, with the same error id:
|
|
100
|
+
* what the trace stores. Builder-only, stripped by `forClient`.
|
|
101
|
+
*/
|
|
102
|
+
errorSummary?: string;
|
|
98
103
|
}
|
|
99
104
|
/** The schema a guardrail event parses against. */
|
|
100
105
|
export declare const guardrailEventSchema: z.ZodType<GuardrailEvent>;
|
|
@@ -41,10 +41,11 @@ function stripCompactionInternal(event) {
|
|
|
41
41
|
return { ...event, failure };
|
|
42
42
|
}
|
|
43
43
|
function stripGuardrailInternal(event) {
|
|
44
|
-
if (event.type !== 'guardrail' ||
|
|
44
|
+
if (event.type !== 'guardrail' ||
|
|
45
|
+
!(event.guardrail?.errorInternal || event.guardrail?.errorSummary)) {
|
|
45
46
|
return event;
|
|
46
47
|
}
|
|
47
|
-
const { errorInternal: _internal, ...guardrail } = event.guardrail;
|
|
48
|
+
const { errorInternal: _internal, errorSummary: _summary, ...guardrail } = event.guardrail;
|
|
48
49
|
return { ...event, guardrail };
|
|
49
50
|
}
|
|
50
51
|
function stripEvidenceRaw(event) {
|
|
@@ -278,6 +278,7 @@ function guardrailAttributes(guardrail) {
|
|
|
278
278
|
hits: guardrail.hits.map((hit) => ({
|
|
279
279
|
rule: hit.rule,
|
|
280
280
|
severity: hit.severity,
|
|
281
|
+
...optional('signal', hit.signal),
|
|
281
282
|
...(hit.span ? { start: hit.span.start, end: hit.span.end } : {}),
|
|
282
283
|
...optional('match', hit.match),
|
|
283
284
|
...optional('label', hit.label),
|
|
@@ -285,6 +286,7 @@ function guardrailAttributes(guardrail) {
|
|
|
285
286
|
})),
|
|
286
287
|
...(guardrail.provenance ? { provenance: { ...guardrail.provenance } } : {}),
|
|
287
288
|
...(guardrail.errorInternal ? { error: traceContent(guardrail.errorInternal) } : {}),
|
|
289
|
+
...optional('error_summary', guardrail.errorSummary),
|
|
288
290
|
};
|
|
289
291
|
}
|
|
290
292
|
/** What a stream check looked at, for its pass record. */
|
|
@@ -646,6 +646,7 @@ const HITS = fields('error', 'Hits', 'Each rule that matched.', {
|
|
|
646
646
|
open: true,
|
|
647
647
|
},
|
|
648
648
|
severity: attr('error', 'Severity', 'text', 'How serious the rule says a match is.', SEVERITY_OPTIONS),
|
|
649
|
+
signal: attr('error', 'Signal', 'text', "What made a `tool_instructions` match one, or how a host detector's `find` failed: `find_threw` or `find_result`."),
|
|
649
650
|
start: attr('error', 'From', 'number', 'Where the match starts in the checked text.'),
|
|
650
651
|
end: attr('error', 'To', 'number', 'Where the match ends in the checked text.'),
|
|
651
652
|
match: attr('error', 'Match', 'text', 'The matched text, exact. Kept only when the profile records match previews.'),
|
|
@@ -680,6 +681,7 @@ const TRACE_EVENTS = {
|
|
|
680
681
|
depth: attr('tool', 'Hops', 'number', "Tool calls between the user's turn and this text: 1 for a direct call, more through another agent."),
|
|
681
682
|
}),
|
|
682
683
|
error: attr('error', 'Detail', 'content', "The guardrail's internal reason."),
|
|
684
|
+
error_summary: attr('error', 'Failure', 'text', "Which host detector failed, how, and its error id, in the kernel's words only. The host stream's reason ends with the same id."),
|
|
683
685
|
},
|
|
684
686
|
},
|
|
685
687
|
'theorem.attempt.retry': {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@theoremjs/agents",
|
|
3
|
-
"version": "0.3.4-next.
|
|
3
|
+
"version": "0.3.4-next.524.g5000023",
|
|
4
4
|
"description": "A flat TypeScript agent kernel for typed profiles, deterministic turn execution, registered tools, provider adapters, guardrails, and host-injected traces.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agent",
|