agent-inspect 6.19.0 → 6.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +3 -1
- package/SECURITY.md +2 -0
- package/docs/ARCHITECTURE.md +28 -16
- package/docs/KNOWN-ISSUES.md +3 -3
- package/docs/LIMITATIONS.md +5 -4
- package/docs/PERFORMANCE.md +1 -1
- package/docs/SUPPORT-LEVELS.md +2 -1
- package/docs/TRACE-CONTRACTS.md +64 -52
- package/package.json +3 -2
- package/packages/cli/dist/{chunk-R7Y5SGH5.mjs → chunk-QPRJUVLH.mjs} +57 -17
- package/packages/cli/dist/chunk-QPRJUVLH.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +69 -18
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +9 -6
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-T4EZERCX.mjs → src-6W5ZR3I7.mjs} +3 -3
- package/packages/cli/dist/{src-T4EZERCX.mjs.map → src-6W5ZR3I7.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +118 -18
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +2 -2
- package/packages/core/dist/advanced.d.ts +2 -2
- package/packages/core/dist/advanced.mjs +8 -8
- package/packages/core/dist/checks.cjs +443 -9
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +78 -16
- package/packages/core/dist/checks.d.ts +78 -16
- package/packages/core/dist/checks.mjs +3 -3
- package/packages/core/dist/{chunk-VFO76UH3.mjs → chunk-3XS5O4JI.mjs} +3 -3
- package/packages/core/dist/{chunk-VFO76UH3.mjs.map → chunk-3XS5O4JI.mjs.map} +1 -1
- package/packages/core/dist/{chunk-ZWA5RWMV.mjs → chunk-5YFJJFXU.mjs} +4 -4
- package/packages/core/dist/{chunk-ZWA5RWMV.mjs.map → chunk-5YFJJFXU.mjs.map} +1 -1
- package/packages/core/dist/{chunk-NABMW5DX.mjs → chunk-G47KQB42.mjs} +445 -13
- package/packages/core/dist/chunk-G47KQB42.mjs.map +1 -0
- package/packages/core/dist/{chunk-SGL4JGCD.mjs → chunk-JBNRJLIV.mjs} +58 -18
- package/packages/core/dist/chunk-JBNRJLIV.mjs.map +1 -0
- package/packages/core/dist/{chunk-ULCGIRTC.mjs → chunk-U25RHRSU.mjs} +65 -5
- package/packages/core/dist/chunk-U25RHRSU.mjs.map +1 -0
- package/packages/core/dist/{context-Ca5yK8Cs.d.cts → context-C5ye_xcS.d.cts} +0 -4
- package/packages/core/dist/{context-D6cS4XIL.d.ts → context-CNsNSgE2.d.ts} +0 -4
- package/packages/core/dist/exporters.cjs +55 -15
- package/packages/core/dist/exporters.cjs.map +1 -1
- package/packages/core/dist/exporters.mjs +3 -3
- package/packages/core/dist/{index-DKwuGSe-.d.ts → index-Bg9ZdsdS.d.ts} +10 -0
- package/packages/core/dist/{index-DCR816Yt.d.cts → index-rVpX5PPh.d.cts} +10 -0
- package/packages/core/dist/index.cjs +63 -3
- package/packages/core/dist/index.cjs.map +1 -1
- package/packages/core/dist/index.d.cts +1 -1
- package/packages/core/dist/index.d.ts +1 -1
- package/packages/core/dist/index.mjs +4 -4
- package/packages/core/dist/writers.cjs +33 -2
- package/packages/core/dist/writers.cjs.map +1 -1
- package/packages/core/dist/writers.mjs +1 -1
- package/packages/cli/dist/chunk-R7Y5SGH5.mjs.map +0 -1
- package/packages/core/dist/chunk-NABMW5DX.mjs.map +0 -1
- package/packages/core/dist/chunk-SGL4JGCD.mjs.map +0 -1
- package/packages/core/dist/chunk-ULCGIRTC.mjs.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { C as CreateInspectorOptions, D as DEFAULT_MAX_EVENT_BYTES, f as DEFAULT_MAX_METADATA_VALUE_LENGTH, h as DEFAULT_MAX_PREVIEW_LENGTH, I as Inspector, a as InspectorCaptureOptions, b as InspectorObserveOptions, c as InspectorRunOptions, i as InspectorRuntime, j as InspectorRuntimeContext, k as InspectorRuntimeDiagnostics, l as InspectorRuntimeOptions, d as InspectorStepOptions, T as TraceSafetyOptions, e as createInspector, n as createInspectorRuntime, o as getCurrentContext, g as getCurrentCorrelationMetadata, p as getCurrentDepth, q as getCurrentRunId, r as getCurrentRunName, s as getCurrentStepId, t as getParentStepId, u as getTraceDirFromContext, v as getTraceSafetyFromContext, w as hasActiveContext, x as isAgentInspectEnabled, y as isSilentContext, m as maybeInspectRun, z as prepareMetadataForDisk, A as prepareTraceEventForDisk, B as resolveTraceSafetyOptions, E as runWithContext, F as runWithStepContext } from './context-
|
|
1
|
+
export { C as CreateInspectorOptions, D as DEFAULT_MAX_EVENT_BYTES, f as DEFAULT_MAX_METADATA_VALUE_LENGTH, h as DEFAULT_MAX_PREVIEW_LENGTH, I as Inspector, a as InspectorCaptureOptions, b as InspectorObserveOptions, c as InspectorRunOptions, i as InspectorRuntime, j as InspectorRuntimeContext, k as InspectorRuntimeDiagnostics, l as InspectorRuntimeOptions, d as InspectorStepOptions, T as TraceSafetyOptions, e as createInspector, n as createInspectorRuntime, o as getCurrentContext, g as getCurrentCorrelationMetadata, p as getCurrentDepth, q as getCurrentRunId, r as getCurrentRunName, s as getCurrentStepId, t as getParentStepId, u as getTraceDirFromContext, v as getTraceSafetyFromContext, w as hasActiveContext, x as isAgentInspectEnabled, y as isSilentContext, m as maybeInspectRun, z as prepareMetadataForDisk, A as prepareTraceEventForDisk, B as resolveTraceSafetyOptions, E as runWithContext, F as runWithStepContext } from './context-C5ye_xcS.cjs';
|
|
2
2
|
import { R as RedactionRule } from './log-config-CFlIJHTf.cjs';
|
|
3
3
|
import { R as RedactionProfile, h as TraceEvent, E as ErrorInfo, b as RunStatus, e as StepStatus, j as RunSummary, k as TraceMetadata, l as TraceMetadataStatus, f as StepType, g as TraceCorrelationMetadata } from './types-UnrNaTo2.cjs';
|
|
4
4
|
export { A as ActiveStepContext, i as ExecutionContext, I as InspectRunOptions, O as ObserveOptions, a as Run, m as RunCompletedEvent, n as RunStartedEvent, c as Step, o as StepCompletedEvent, d as StepMetadata, S as StepOptions, p as StepStartedEvent, T as TokenMetadata, q as TraceEventBase, r as TraceSchemaVersion, s as isStepStatus, t as isStepType, u as isTraceEvent } from './types-UnrNaTo2.cjs';
|
|
@@ -8,7 +8,7 @@ import { P as PersistedInspectEvent } from './persisted-inspect-event-CVZWZldL.c
|
|
|
8
8
|
import { Stats } from 'node:fs';
|
|
9
9
|
import { b as ObservedOutcome } from './types-B562HgN_.cjs';
|
|
10
10
|
export { O as ObservedOutcomeStatus } from './types-B562HgN_.cjs';
|
|
11
|
-
import { b as TraceCheckResult, N as TraceCheckStatus } from './index-
|
|
11
|
+
import { b as TraceCheckResult, N as TraceCheckStatus } from './index-rVpX5PPh.cjs';
|
|
12
12
|
import './writers.cjs';
|
|
13
13
|
import './index-vqr-g6ev.cjs';
|
|
14
14
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { C as CreateInspectorOptions, D as DEFAULT_MAX_EVENT_BYTES, f as DEFAULT_MAX_METADATA_VALUE_LENGTH, h as DEFAULT_MAX_PREVIEW_LENGTH, I as Inspector, a as InspectorCaptureOptions, b as InspectorObserveOptions, c as InspectorRunOptions, i as InspectorRuntime, j as InspectorRuntimeContext, k as InspectorRuntimeDiagnostics, l as InspectorRuntimeOptions, d as InspectorStepOptions, T as TraceSafetyOptions, e as createInspector, n as createInspectorRuntime, o as getCurrentContext, g as getCurrentCorrelationMetadata, p as getCurrentDepth, q as getCurrentRunId, r as getCurrentRunName, s as getCurrentStepId, t as getParentStepId, u as getTraceDirFromContext, v as getTraceSafetyFromContext, w as hasActiveContext, x as isAgentInspectEnabled, y as isSilentContext, m as maybeInspectRun, z as prepareMetadataForDisk, A as prepareTraceEventForDisk, B as resolveTraceSafetyOptions, E as runWithContext, F as runWithStepContext } from './context-
|
|
1
|
+
export { C as CreateInspectorOptions, D as DEFAULT_MAX_EVENT_BYTES, f as DEFAULT_MAX_METADATA_VALUE_LENGTH, h as DEFAULT_MAX_PREVIEW_LENGTH, I as Inspector, a as InspectorCaptureOptions, b as InspectorObserveOptions, c as InspectorRunOptions, i as InspectorRuntime, j as InspectorRuntimeContext, k as InspectorRuntimeDiagnostics, l as InspectorRuntimeOptions, d as InspectorStepOptions, T as TraceSafetyOptions, e as createInspector, n as createInspectorRuntime, o as getCurrentContext, g as getCurrentCorrelationMetadata, p as getCurrentDepth, q as getCurrentRunId, r as getCurrentRunName, s as getCurrentStepId, t as getParentStepId, u as getTraceDirFromContext, v as getTraceSafetyFromContext, w as hasActiveContext, x as isAgentInspectEnabled, y as isSilentContext, m as maybeInspectRun, z as prepareMetadataForDisk, A as prepareTraceEventForDisk, B as resolveTraceSafetyOptions, E as runWithContext, F as runWithStepContext } from './context-CNsNSgE2.js';
|
|
2
2
|
import { R as RedactionRule } from './log-config-OZRfpHED.js';
|
|
3
3
|
import { R as RedactionProfile, h as TraceEvent, E as ErrorInfo, b as RunStatus, e as StepStatus, j as RunSummary, k as TraceMetadata, l as TraceMetadataStatus, f as StepType, g as TraceCorrelationMetadata } from './types--ainI31J.js';
|
|
4
4
|
export { A as ActiveStepContext, i as ExecutionContext, I as InspectRunOptions, O as ObserveOptions, a as Run, m as RunCompletedEvent, n as RunStartedEvent, c as Step, o as StepCompletedEvent, d as StepMetadata, S as StepOptions, p as StepStartedEvent, T as TokenMetadata, q as TraceEventBase, r as TraceSchemaVersion, s as isStepStatus, t as isStepType, u as isTraceEvent } from './types--ainI31J.js';
|
|
@@ -8,7 +8,7 @@ import { P as PersistedInspectEvent } from './persisted-inspect-event-BVucFVrP.j
|
|
|
8
8
|
import { Stats } from 'node:fs';
|
|
9
9
|
import { b as ObservedOutcome } from './types-B562HgN_.js';
|
|
10
10
|
export { O as ObservedOutcomeStatus } from './types-B562HgN_.js';
|
|
11
|
-
import { b as TraceCheckResult, N as TraceCheckStatus } from './index-
|
|
11
|
+
import { b as TraceCheckResult, N as TraceCheckStatus } from './index-Bg9ZdsdS.js';
|
|
12
12
|
import './writers.js';
|
|
13
13
|
import './index-AMAGuO_i.js';
|
|
14
14
|
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-
|
|
2
|
-
export { MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, TERMINAL_INDENT, TraceDirectory, createInspector, createInspectorRuntime, formatTerminalName, getCurrentContext, getCurrentCorrelationMetadata, getCurrentDepth, getCurrentRunId, getCurrentRunName, getCurrentStepId, getIndent, getParentStepId, getTraceDirFromContext, getTraceSafetyFromContext, hasActiveContext, isAgentInspectEnabled, isSilentContext, maybeInspectRun, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, renderErrorLine, renderRunSummary, renderStepLine, resolveTraceDir, runWithContext, runWithStepContext } from './chunk-
|
|
3
|
-
import { buildRunTimeline, filterTraces, extractMetadata, buildRunSummary } from './chunk-
|
|
4
|
-
export { buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildTraceStats, extractMetadata, filterTraces, formatStepLabel, renderRunWhat, renderTimeline, renderTraceStats } from './chunk-
|
|
5
|
-
import { extractOutcomesFromTraceEvents, parseObservationFilter, summarizeObservedOutcomes, extractOutcomesFromPersistedEvents, renderObservedOutcomesHtml } from './chunk-
|
|
6
|
-
export { extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents } from './chunk-
|
|
1
|
+
import { runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-G47KQB42.mjs';
|
|
2
|
+
export { MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, TERMINAL_INDENT, TraceDirectory, createInspector, createInspectorRuntime, formatTerminalName, getCurrentContext, getCurrentCorrelationMetadata, getCurrentDepth, getCurrentRunId, getCurrentRunName, getCurrentStepId, getIndent, getParentStepId, getTraceDirFromContext, getTraceSafetyFromContext, hasActiveContext, isAgentInspectEnabled, isSilentContext, maybeInspectRun, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, renderErrorLine, renderRunSummary, renderStepLine, resolveTraceDir, runWithContext, runWithStepContext } from './chunk-5YFJJFXU.mjs';
|
|
3
|
+
import { buildRunTimeline, filterTraces, extractMetadata, buildRunSummary } from './chunk-JBNRJLIV.mjs';
|
|
4
|
+
export { buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildTraceStats, extractMetadata, filterTraces, formatStepLabel, renderRunWhat, renderTimeline, renderTraceStats } from './chunk-JBNRJLIV.mjs';
|
|
5
|
+
import { extractOutcomesFromTraceEvents, parseObservationFilter, summarizeObservedOutcomes, extractOutcomesFromPersistedEvents, renderObservedOutcomesHtml } from './chunk-3XS5O4JI.mjs';
|
|
6
|
+
export { extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents } from './chunk-3XS5O4JI.mjs';
|
|
7
7
|
import { diffTraceEvents, renderRunDiff } from './chunk-F7STQ5JF.mjs';
|
|
8
8
|
import './chunk-BT7CATSD.mjs';
|
|
9
9
|
import { escapeHtml, flattenTree } from './chunk-CAWPF22J.mjs';
|
|
10
|
-
import { resolveRedactionProfile, readTraceEventsFromFile } from './chunk-
|
|
11
|
-
export { DEFAULT_MAX_EVENT_BYTES, DEFAULT_MAX_METADATA_VALUE_LENGTH, DEFAULT_MAX_PREVIEW_LENGTH, getRunIdFromTraceFileName, initializeTraceFile, listTraceFiles, prepareMetadataForDisk, prepareTraceEventForDisk, readTraceEvents, readTraceFile, resolveRedactionProfile, resolveTraceSafetyOptions, serializeEvent, validateEvent, writeTraceEvent } from './chunk-
|
|
10
|
+
import { resolveRedactionProfile, readTraceEventsFromFile } from './chunk-U25RHRSU.mjs';
|
|
11
|
+
export { DEFAULT_MAX_EVENT_BYTES, DEFAULT_MAX_METADATA_VALUE_LENGTH, DEFAULT_MAX_PREVIEW_LENGTH, getRunIdFromTraceFileName, initializeTraceFile, listTraceFiles, prepareMetadataForDisk, prepareTraceEventForDisk, readTraceEvents, readTraceFile, resolveRedactionProfile, resolveTraceSafetyOptions, serializeEvent, validateEvent, writeTraceEvent } from './chunk-U25RHRSU.mjs';
|
|
12
12
|
import { Redactor } from './chunk-VU6O5QAH.mjs';
|
|
13
13
|
import { openTrace } from './chunk-HT54OLPE.mjs';
|
|
14
14
|
import './chunk-FWK3Q3OJ.mjs';
|
|
@@ -928,6 +928,17 @@ function normalizeStatus(status) {
|
|
|
928
928
|
if (status === "failed") return "error";
|
|
929
929
|
return "error";
|
|
930
930
|
}
|
|
931
|
+
function cloneBody(body) {
|
|
932
|
+
return {
|
|
933
|
+
...body.run ? { run: { ...body.run } } : {},
|
|
934
|
+
...body.tools ? { tools: { ...body.tools } } : {},
|
|
935
|
+
...body.llm ? { llm: { ...body.llm } } : {},
|
|
936
|
+
...body.observations ? { observations: { ...body.observations } } : {}
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
function bodyHasRules(body) {
|
|
940
|
+
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0;
|
|
941
|
+
}
|
|
931
942
|
function contractToRules(contract) {
|
|
932
943
|
const rules = [];
|
|
933
944
|
const allowedStatuses = contract.run?.allowedStatuses ?? [];
|
|
@@ -1002,6 +1013,7 @@ function contractToRules(contract) {
|
|
|
1002
1013
|
}
|
|
1003
1014
|
if (contract.tools) {
|
|
1004
1015
|
const order = contract.tools.requiredOrder ?? [];
|
|
1016
|
+
const requiredOrderMode = contract.tools.requiredOrderMode ?? "first-occurrence";
|
|
1005
1017
|
const required = [
|
|
1006
1018
|
.../* @__PURE__ */ new Set([
|
|
1007
1019
|
...contract.tools.required ?? [],
|
|
@@ -1026,7 +1038,8 @@ function contractToRules(contract) {
|
|
|
1026
1038
|
createToolOrderingRule({
|
|
1027
1039
|
before: order[i],
|
|
1028
1040
|
after: order[i + 1],
|
|
1029
|
-
id: `contract.tool.order.${i}
|
|
1041
|
+
id: `contract.tool.order.${i}`,
|
|
1042
|
+
mode: requiredOrderMode
|
|
1030
1043
|
})
|
|
1031
1044
|
);
|
|
1032
1045
|
}
|
|
@@ -1076,24 +1089,318 @@ function contractToRules(contract) {
|
|
|
1076
1089
|
}
|
|
1077
1090
|
return rules;
|
|
1078
1091
|
}
|
|
1079
|
-
function
|
|
1092
|
+
function validateAlternativesShape(alternatives) {
|
|
1093
|
+
const diagnostics = [];
|
|
1094
|
+
if (alternatives === void 0) return diagnostics;
|
|
1095
|
+
const branches = alternatives.anyOf ?? [];
|
|
1096
|
+
if (branches.length === 0) {
|
|
1097
|
+
diagnostics.push({
|
|
1098
|
+
code: "contract.alternatives.empty",
|
|
1099
|
+
severity: "error",
|
|
1100
|
+
message: "alternatives.anyOf must contain at least one branch.",
|
|
1101
|
+
path: "alternatives.anyOf"
|
|
1102
|
+
});
|
|
1103
|
+
return diagnostics;
|
|
1104
|
+
}
|
|
1105
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1106
|
+
for (const [index, branch] of branches.entries()) {
|
|
1107
|
+
const path3 = `alternatives.anyOf[${index}]`;
|
|
1108
|
+
if (!branch?.id || branch.id.trim() === "") {
|
|
1109
|
+
diagnostics.push({
|
|
1110
|
+
code: "contract.alternatives.missing-id",
|
|
1111
|
+
severity: "error",
|
|
1112
|
+
message: "Each alternatives.anyOf branch requires a non-empty id.",
|
|
1113
|
+
path: path3
|
|
1114
|
+
});
|
|
1115
|
+
continue;
|
|
1116
|
+
}
|
|
1117
|
+
if (seen.has(branch.id)) {
|
|
1118
|
+
diagnostics.push({
|
|
1119
|
+
code: "contract.alternatives.duplicate-id",
|
|
1120
|
+
severity: "error",
|
|
1121
|
+
message: `Duplicate alternatives.anyOf branch id: ${branch.id}`,
|
|
1122
|
+
path: `${path3}.id`
|
|
1123
|
+
});
|
|
1124
|
+
}
|
|
1125
|
+
seen.add(branch.id);
|
|
1126
|
+
if (!bodyHasRules(branch.contract ?? {})) {
|
|
1127
|
+
diagnostics.push({
|
|
1128
|
+
code: "contract.alternatives.empty-branch",
|
|
1129
|
+
severity: "error",
|
|
1130
|
+
message: `Branch ${branch.id} must declare at least one run/tools/llm/observations rule.`,
|
|
1131
|
+
path: `${path3}.contract`
|
|
1132
|
+
});
|
|
1133
|
+
}
|
|
1134
|
+
const nested = branch.contract?.alternatives;
|
|
1135
|
+
if (nested !== void 0) {
|
|
1136
|
+
diagnostics.push({
|
|
1137
|
+
code: "contract.alternatives.nested",
|
|
1138
|
+
severity: "error",
|
|
1139
|
+
message: "Nested alternatives.anyOf is not supported. Keep one alternatives level only.",
|
|
1140
|
+
path: `${path3}.contract.alternatives`
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
return diagnostics;
|
|
1145
|
+
}
|
|
1146
|
+
function emptyPassResult(input, runId) {
|
|
1080
1147
|
return {
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
...
|
|
1148
|
+
ok: true,
|
|
1149
|
+
status: "pass",
|
|
1150
|
+
format: input.read.format,
|
|
1151
|
+
...runId !== void 0 ? { runId } : {},
|
|
1152
|
+
summary: { passed: 0, failed: 0, warnings: 0, errors: 0, rulesEvaluated: 0 },
|
|
1153
|
+
findings: [],
|
|
1154
|
+
diagnostics: [],
|
|
1155
|
+
ruleExecutions: []
|
|
1085
1156
|
};
|
|
1086
1157
|
}
|
|
1087
|
-
function
|
|
1088
|
-
const
|
|
1158
|
+
function mergeContractResults(input, parts, extraFindings = []) {
|
|
1159
|
+
const findings = [...parts.flatMap((part) => part.findings), ...extraFindings];
|
|
1160
|
+
const diagnostics = parts.flatMap((part) => part.diagnostics);
|
|
1161
|
+
const ruleExecutions = parts.flatMap((part) => part.ruleExecutions);
|
|
1162
|
+
const failed = findings.filter(
|
|
1163
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
1164
|
+
).length;
|
|
1165
|
+
const errors = diagnostics.filter((item) => item.severity === "error").length;
|
|
1166
|
+
const status = errors > 0 ? "error" : failed > 0 ? "fail" : "pass";
|
|
1167
|
+
const runId = parts.find((part) => part.runId !== void 0)?.runId;
|
|
1168
|
+
return {
|
|
1169
|
+
ok: status === "pass",
|
|
1170
|
+
status,
|
|
1171
|
+
format: input.read.format,
|
|
1172
|
+
...runId !== void 0 ? { runId } : {},
|
|
1173
|
+
summary: {
|
|
1174
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
1175
|
+
failed,
|
|
1176
|
+
warnings: findings.filter(
|
|
1177
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
1178
|
+
).length,
|
|
1179
|
+
errors,
|
|
1180
|
+
rulesEvaluated: ruleExecutions.length
|
|
1181
|
+
},
|
|
1182
|
+
findings,
|
|
1183
|
+
diagnostics,
|
|
1184
|
+
ruleExecutions
|
|
1185
|
+
};
|
|
1186
|
+
}
|
|
1187
|
+
function evaluateBody(input, body, options = {}) {
|
|
1188
|
+
const rules = contractToRules(body);
|
|
1189
|
+
if (rules.length === 0) {
|
|
1190
|
+
return emptyPassResult(input, options.runId);
|
|
1191
|
+
}
|
|
1089
1192
|
return runTraceChecks(input, {
|
|
1090
1193
|
rules,
|
|
1091
1194
|
...options.runId !== void 0 ? { runId: options.runId } : {}
|
|
1092
1195
|
});
|
|
1093
1196
|
}
|
|
1197
|
+
function defineTraceContract(input) {
|
|
1198
|
+
const shapeErrors = validateAlternativesShape(input.alternatives).filter(
|
|
1199
|
+
(item) => item.severity === "error"
|
|
1200
|
+
);
|
|
1201
|
+
if (shapeErrors.length > 0) {
|
|
1202
|
+
throw new TypeError(shapeErrors.map((item) => item.message).join(" "));
|
|
1203
|
+
}
|
|
1204
|
+
const body = cloneBody(input);
|
|
1205
|
+
const alternatives = input.alternatives ? {
|
|
1206
|
+
anyOf: input.alternatives.anyOf.map((branch) => ({
|
|
1207
|
+
id: branch.id,
|
|
1208
|
+
...branch.description !== void 0 ? { description: branch.description } : {},
|
|
1209
|
+
contract: cloneBody(branch.contract)
|
|
1210
|
+
}))
|
|
1211
|
+
} : void 0;
|
|
1212
|
+
return {
|
|
1213
|
+
...body,
|
|
1214
|
+
...alternatives ? { alternatives } : {}
|
|
1215
|
+
};
|
|
1216
|
+
}
|
|
1217
|
+
function evaluateTraceContract(input, contract, options = {}) {
|
|
1218
|
+
const shapeErrors = validateAlternativesShape(contract.alternatives).filter(
|
|
1219
|
+
(item) => item.severity === "error"
|
|
1220
|
+
);
|
|
1221
|
+
if (shapeErrors.length > 0) {
|
|
1222
|
+
return {
|
|
1223
|
+
ok: false,
|
|
1224
|
+
status: "error",
|
|
1225
|
+
format: input.read.format,
|
|
1226
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
1227
|
+
summary: {
|
|
1228
|
+
passed: 0,
|
|
1229
|
+
failed: 0,
|
|
1230
|
+
warnings: 0,
|
|
1231
|
+
errors: shapeErrors.length,
|
|
1232
|
+
rulesEvaluated: 0
|
|
1233
|
+
},
|
|
1234
|
+
findings: [],
|
|
1235
|
+
diagnostics: shapeErrors.map((item) => ({
|
|
1236
|
+
code: "AI_CHECK_INVALID_CONFIG",
|
|
1237
|
+
message: item.message,
|
|
1238
|
+
severity: "error",
|
|
1239
|
+
ruleId: item.code
|
|
1240
|
+
})),
|
|
1241
|
+
ruleExecutions: []
|
|
1242
|
+
};
|
|
1243
|
+
}
|
|
1244
|
+
const base = evaluateBody(input, contract, options);
|
|
1245
|
+
const branches = contract.alternatives?.anyOf ?? [];
|
|
1246
|
+
if (branches.length === 0) {
|
|
1247
|
+
return base;
|
|
1248
|
+
}
|
|
1249
|
+
const branchEvaluations = branches.map((branch) => {
|
|
1250
|
+
const result = evaluateBody(input, branch.contract, options);
|
|
1251
|
+
return { branch, result };
|
|
1252
|
+
});
|
|
1253
|
+
const satisfied = branchEvaluations.filter((item) => item.result.ok && item.result.status === "pass").map((item) => item.branch.id);
|
|
1254
|
+
const failedBranches = branchEvaluations.filter(
|
|
1255
|
+
(item) => !(item.result.ok && item.result.status === "pass")
|
|
1256
|
+
);
|
|
1257
|
+
if (satisfied.length > 0) {
|
|
1258
|
+
const passFinding = {
|
|
1259
|
+
ruleId: "contract.alternatives.anyOf",
|
|
1260
|
+
severity: "info",
|
|
1261
|
+
status: "pass",
|
|
1262
|
+
message: `Satisfied alternative branch(es): ${satisfied.join(", ")}.`,
|
|
1263
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
1264
|
+
actual: {
|
|
1265
|
+
satisfied,
|
|
1266
|
+
failed: failedBranches.map((item) => item.branch.id)
|
|
1267
|
+
},
|
|
1268
|
+
evidence: []
|
|
1269
|
+
};
|
|
1270
|
+
return mergeContractResults(input, [base], [passFinding]);
|
|
1271
|
+
}
|
|
1272
|
+
const noneFinding = {
|
|
1273
|
+
ruleId: "contract.alternatives.none-satisfied",
|
|
1274
|
+
severity: "error",
|
|
1275
|
+
status: "fail",
|
|
1276
|
+
message: "No alternatives.anyOf branch fully satisfied its contract.",
|
|
1277
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
1278
|
+
actual: {
|
|
1279
|
+
branchFailures: Object.fromEntries(
|
|
1280
|
+
failedBranches.map((item) => [
|
|
1281
|
+
item.branch.id,
|
|
1282
|
+
item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
1283
|
+
ruleId: finding.ruleId,
|
|
1284
|
+
message: finding.message
|
|
1285
|
+
}))
|
|
1286
|
+
])
|
|
1287
|
+
)
|
|
1288
|
+
},
|
|
1289
|
+
evidence: []
|
|
1290
|
+
};
|
|
1291
|
+
const prefixedBranchFindings = failedBranches.flatMap(
|
|
1292
|
+
(item) => item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
1293
|
+
...finding,
|
|
1294
|
+
ruleId: `contract.alternatives.${item.branch.id}.${finding.ruleId}`
|
|
1295
|
+
}))
|
|
1296
|
+
);
|
|
1297
|
+
return mergeContractResults(input, [base], [noneFinding, ...prefixedBranchFindings]);
|
|
1298
|
+
}
|
|
1094
1299
|
function evaluateTraceContractRead(read, contract, options = {}) {
|
|
1095
1300
|
return evaluateTraceContract({ read }, contract, options);
|
|
1096
1301
|
}
|
|
1302
|
+
function lintTraceContract(contract) {
|
|
1303
|
+
const diagnostics = validateAlternativesShape(contract.alternatives);
|
|
1304
|
+
const branches = contract.alternatives?.anyOf ?? [];
|
|
1305
|
+
if (branches.length === 1) {
|
|
1306
|
+
diagnostics.push({
|
|
1307
|
+
code: "contract.alternatives.single-branch",
|
|
1308
|
+
severity: "warning",
|
|
1309
|
+
message: "alternatives.anyOf has a single branch. Prefer base rules unless a second valid path exists.",
|
|
1310
|
+
path: "alternatives.anyOf"
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1313
|
+
const required = [
|
|
1314
|
+
...contract.tools?.required ?? [],
|
|
1315
|
+
...contract.tools?.requiredTools ?? []
|
|
1316
|
+
];
|
|
1317
|
+
if (required.length > 0 && branches.length === 0) {
|
|
1318
|
+
diagnostics.push({
|
|
1319
|
+
code: "contract.brittle.unconditional-required",
|
|
1320
|
+
severity: "info",
|
|
1321
|
+
message: "tools.required is unconditional. Use alternatives.anyOf when a legitimate shortcut may skip a tool.",
|
|
1322
|
+
path: "tools.required"
|
|
1323
|
+
});
|
|
1324
|
+
}
|
|
1325
|
+
const order = contract.tools?.requiredOrder ?? [];
|
|
1326
|
+
if (order.length >= 2 && contract.tools?.requiredOrderMode === void 0) {
|
|
1327
|
+
diagnostics.push({
|
|
1328
|
+
code: "contract.brittle.first-occurrence-default",
|
|
1329
|
+
severity: "info",
|
|
1330
|
+
message: "requiredOrder defaults to first-occurrence. Set requiredOrderMode when repeated calls must fail.",
|
|
1331
|
+
path: "tools.requiredOrderMode"
|
|
1332
|
+
});
|
|
1333
|
+
}
|
|
1334
|
+
return diagnostics;
|
|
1335
|
+
}
|
|
1336
|
+
function explainTraceContract(contract) {
|
|
1337
|
+
const lines = [];
|
|
1338
|
+
if (contract.run?.requireCompleted !== false) {
|
|
1339
|
+
lines.push("Base: run must complete (no running events).");
|
|
1340
|
+
}
|
|
1341
|
+
if (contract.run?.allowedStatuses?.length) {
|
|
1342
|
+
lines.push(`Base: run status in [${contract.run.allowedStatuses.join(", ")}].`);
|
|
1343
|
+
}
|
|
1344
|
+
if (contract.run?.maxDurationMs !== void 0) {
|
|
1345
|
+
lines.push(`Base: run duration <= ${contract.run.maxDurationMs}ms.`);
|
|
1346
|
+
}
|
|
1347
|
+
if (contract.tools) {
|
|
1348
|
+
const required = [
|
|
1349
|
+
.../* @__PURE__ */ new Set([
|
|
1350
|
+
...contract.tools.required ?? [],
|
|
1351
|
+
...contract.tools.requiredTools ?? [],
|
|
1352
|
+
...contract.tools.requiredOrder ?? []
|
|
1353
|
+
])
|
|
1354
|
+
];
|
|
1355
|
+
if (required.length > 0) {
|
|
1356
|
+
lines.push(`Base: required tools [${required.join(", ")}].`);
|
|
1357
|
+
}
|
|
1358
|
+
const forbidden = [
|
|
1359
|
+
...contract.tools.forbidden ?? [],
|
|
1360
|
+
...contract.tools.forbiddenTools ?? []
|
|
1361
|
+
];
|
|
1362
|
+
if (forbidden.length > 0) {
|
|
1363
|
+
lines.push(`Base: forbidden tools [${forbidden.join(", ")}].`);
|
|
1364
|
+
}
|
|
1365
|
+
if ((contract.tools.requiredOrder?.length ?? 0) >= 2) {
|
|
1366
|
+
const mode = contract.tools.requiredOrderMode ?? "first-occurrence";
|
|
1367
|
+
lines.push(
|
|
1368
|
+
`Base: requiredOrder [${contract.tools.requiredOrder.join(" \u2192 ")}] mode=${mode}.`
|
|
1369
|
+
);
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
if (contract.llm) {
|
|
1373
|
+
if (contract.llm.maxCalls !== void 0) {
|
|
1374
|
+
lines.push(`Base: LLM maxCalls=${contract.llm.maxCalls}.`);
|
|
1375
|
+
}
|
|
1376
|
+
if (contract.llm.maxTotalTokens !== void 0) {
|
|
1377
|
+
lines.push(`Base: LLM maxTotalTokens=${contract.llm.maxTotalTokens}.`);
|
|
1378
|
+
}
|
|
1379
|
+
if (contract.llm.allowedModels?.length) {
|
|
1380
|
+
lines.push(`Base: LLM allowedModels [${contract.llm.allowedModels.join(", ")}].`);
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
if (contract.observations?.required?.length) {
|
|
1384
|
+
lines.push(`Base: required observations [${contract.observations.required.join(", ")}].`);
|
|
1385
|
+
}
|
|
1386
|
+
if (contract.observations?.failOn?.length) {
|
|
1387
|
+
lines.push(`Base: fail on observations [${contract.observations.failOn.join(", ")}].`);
|
|
1388
|
+
}
|
|
1389
|
+
const branches = contract.alternatives?.anyOf ?? [];
|
|
1390
|
+
if (branches.length > 0) {
|
|
1391
|
+
lines.push(
|
|
1392
|
+
`Alternatives: at least one of [${branches.map((branch) => branch.id).join(", ")}] must fully pass.`
|
|
1393
|
+
);
|
|
1394
|
+
for (const branch of branches) {
|
|
1395
|
+
const detail = branch.description ? ` \u2014 ${branch.description}` : "";
|
|
1396
|
+
lines.push(` branch ${branch.id}${detail}`);
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
if (lines.length === 0) {
|
|
1400
|
+
lines.push("Empty contract: no base rules and no alternatives.");
|
|
1401
|
+
}
|
|
1402
|
+
return lines;
|
|
1403
|
+
}
|
|
1097
1404
|
|
|
1098
1405
|
// packages/core/src/checks/index.ts
|
|
1099
1406
|
var SEVERITY_RANK = {
|
|
@@ -1977,6 +2284,7 @@ function createToolUsageRule(options) {
|
|
|
1977
2284
|
}
|
|
1978
2285
|
function createToolOrderingRule(options) {
|
|
1979
2286
|
const ruleId = options.id ?? "tool.order";
|
|
2287
|
+
const mode = options.mode ?? "first-occurrence";
|
|
1980
2288
|
return {
|
|
1981
2289
|
id: ruleId,
|
|
1982
2290
|
category: "tool",
|
|
@@ -1989,7 +2297,7 @@ function createToolOrderingRule(options) {
|
|
|
1989
2297
|
if (beforeIndex === void 0 || afterIndex === void 0) {
|
|
1990
2298
|
return [];
|
|
1991
2299
|
}
|
|
1992
|
-
if (beforeIndex >= afterIndex) {
|
|
2300
|
+
if (mode === "first-occurrence" && beforeIndex >= afterIndex) {
|
|
1993
2301
|
return [
|
|
1994
2302
|
failFinding(
|
|
1995
2303
|
ruleId,
|
|
@@ -2004,6 +2312,130 @@ function createToolOrderingRule(options) {
|
|
|
2004
2312
|
const afterEvent = tools[afterIndex];
|
|
2005
2313
|
const beforeEnd = eventEndMs(beforeEvent);
|
|
2006
2314
|
const afterStart = eventStartMs(afterEvent);
|
|
2315
|
+
if (mode === "happens-before") {
|
|
2316
|
+
const expected = {
|
|
2317
|
+
before: options.before,
|
|
2318
|
+
after: options.after,
|
|
2319
|
+
mode,
|
|
2320
|
+
relation: "before.end <= after.start"
|
|
2321
|
+
};
|
|
2322
|
+
if (options.before === options.after) {
|
|
2323
|
+
return [
|
|
2324
|
+
failFinding(
|
|
2325
|
+
ruleId,
|
|
2326
|
+
`Tool ${options.before} cannot causally happen before itself.`,
|
|
2327
|
+
[eventEvidence(beforeEvent)],
|
|
2328
|
+
expected,
|
|
2329
|
+
{ code: "tool.order.same-tool" }
|
|
2330
|
+
)
|
|
2331
|
+
];
|
|
2332
|
+
}
|
|
2333
|
+
if (beforeEnd === void 0 || afterStart === void 0) {
|
|
2334
|
+
return [
|
|
2335
|
+
failFinding(
|
|
2336
|
+
ruleId,
|
|
2337
|
+
`Tool order ${options.before} before ${options.after} could not establish causal timing.`,
|
|
2338
|
+
[eventEvidence(beforeEvent), eventEvidence(afterEvent)],
|
|
2339
|
+
expected,
|
|
2340
|
+
{
|
|
2341
|
+
code: "tool.order.interval-unresolved",
|
|
2342
|
+
beforeEndResolved: beforeEnd !== void 0,
|
|
2343
|
+
afterStartResolved: afterStart !== void 0
|
|
2344
|
+
}
|
|
2345
|
+
)
|
|
2346
|
+
];
|
|
2347
|
+
}
|
|
2348
|
+
if (beforeEnd > afterStart) {
|
|
2349
|
+
return [
|
|
2350
|
+
failFinding(
|
|
2351
|
+
ruleId,
|
|
2352
|
+
`Tool ${options.before} must finish before ${options.after} starts.`,
|
|
2353
|
+
[eventEvidence(beforeEvent), eventEvidence(afterEvent)],
|
|
2354
|
+
expected,
|
|
2355
|
+
{
|
|
2356
|
+
beforeEndedAt: new Date(beforeEnd).toISOString(),
|
|
2357
|
+
afterStartedAt: new Date(afterStart).toISOString()
|
|
2358
|
+
}
|
|
2359
|
+
)
|
|
2360
|
+
];
|
|
2361
|
+
}
|
|
2362
|
+
return [];
|
|
2363
|
+
}
|
|
2364
|
+
if (mode === "all-occurrences") {
|
|
2365
|
+
const beforeEvents = tools.filter((event) => toolName(event) === options.before);
|
|
2366
|
+
const afterEvents = tools.filter((event) => toolName(event) === options.after);
|
|
2367
|
+
const expected = {
|
|
2368
|
+
before: options.before,
|
|
2369
|
+
after: options.after,
|
|
2370
|
+
mode,
|
|
2371
|
+
relation: "max(before.end) <= min(after.start)"
|
|
2372
|
+
};
|
|
2373
|
+
if (options.before === options.after) {
|
|
2374
|
+
return [
|
|
2375
|
+
failFinding(
|
|
2376
|
+
ruleId,
|
|
2377
|
+
`Tool ${options.before} cannot causally happen before itself.`,
|
|
2378
|
+
[eventEvidence(beforeEvent)],
|
|
2379
|
+
expected,
|
|
2380
|
+
{ code: "tool.order.same-tool" }
|
|
2381
|
+
)
|
|
2382
|
+
];
|
|
2383
|
+
}
|
|
2384
|
+
let latestBefore;
|
|
2385
|
+
let earliestAfter;
|
|
2386
|
+
let missingBeforeEnds = 0;
|
|
2387
|
+
let missingAfterStarts = 0;
|
|
2388
|
+
let firstMissingBefore;
|
|
2389
|
+
let firstMissingAfter;
|
|
2390
|
+
for (const event of beforeEvents) {
|
|
2391
|
+
const end = eventEndMs(event);
|
|
2392
|
+
if (end === void 0) {
|
|
2393
|
+
missingBeforeEnds += 1;
|
|
2394
|
+
firstMissingBefore ??= event;
|
|
2395
|
+
} else if (latestBefore === void 0 || end > latestBefore.end) {
|
|
2396
|
+
latestBefore = { event, end };
|
|
2397
|
+
}
|
|
2398
|
+
}
|
|
2399
|
+
for (const event of afterEvents) {
|
|
2400
|
+
const start = eventStartMs(event);
|
|
2401
|
+
if (start === void 0) {
|
|
2402
|
+
missingAfterStarts += 1;
|
|
2403
|
+
firstMissingAfter ??= event;
|
|
2404
|
+
} else if (earliestAfter === void 0 || start < earliestAfter.start) {
|
|
2405
|
+
earliestAfter = { event, start };
|
|
2406
|
+
}
|
|
2407
|
+
}
|
|
2408
|
+
if (missingBeforeEnds > 0 || missingAfterStarts > 0 || latestBefore === void 0 || earliestAfter === void 0) {
|
|
2409
|
+
return [
|
|
2410
|
+
failFinding(
|
|
2411
|
+
ruleId,
|
|
2412
|
+
`Tool order ${options.before} before ${options.after} could not establish all causal intervals.`,
|
|
2413
|
+
[firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence(event)),
|
|
2414
|
+
expected,
|
|
2415
|
+
{
|
|
2416
|
+
code: "tool.order.interval-unresolved",
|
|
2417
|
+
missingBeforeEnds,
|
|
2418
|
+
missingAfterStarts
|
|
2419
|
+
}
|
|
2420
|
+
)
|
|
2421
|
+
];
|
|
2422
|
+
}
|
|
2423
|
+
if (latestBefore.end > earliestAfter.start) {
|
|
2424
|
+
return [
|
|
2425
|
+
failFinding(
|
|
2426
|
+
ruleId,
|
|
2427
|
+
`Every ${options.before} tool call must finish before any ${options.after} tool call starts.`,
|
|
2428
|
+
[eventEvidence(latestBefore.event), eventEvidence(earliestAfter.event)],
|
|
2429
|
+
expected,
|
|
2430
|
+
{
|
|
2431
|
+
latestBeforeEndedAt: new Date(latestBefore.end).toISOString(),
|
|
2432
|
+
earliestAfterStartedAt: new Date(earliestAfter.start).toISOString()
|
|
2433
|
+
}
|
|
2434
|
+
)
|
|
2435
|
+
];
|
|
2436
|
+
}
|
|
2437
|
+
return [];
|
|
2438
|
+
}
|
|
2007
2439
|
if (beforeEnd !== void 0 && afterStart !== void 0 && beforeEnd > afterStart) {
|
|
2008
2440
|
return [
|
|
2009
2441
|
{
|
|
@@ -2967,7 +3399,9 @@ exports.defineTraceContract = defineTraceContract;
|
|
|
2967
3399
|
exports.deriveFailureFacts = deriveFailureFacts;
|
|
2968
3400
|
exports.evaluateTraceContract = evaluateTraceContract;
|
|
2969
3401
|
exports.evaluateTraceContractRead = evaluateTraceContractRead;
|
|
3402
|
+
exports.explainTraceContract = explainTraceContract;
|
|
2970
3403
|
exports.formatProgrammaticDiagnostic = formatProgrammaticDiagnostic;
|
|
3404
|
+
exports.lintTraceContract = lintTraceContract;
|
|
2971
3405
|
exports.projectLogicalEvents = projectLogicalEvents;
|
|
2972
3406
|
exports.resolveCanonicalToolName = resolveCanonicalToolName;
|
|
2973
3407
|
exports.runTraceChecks = runTraceChecks;
|