agent-inspect 6.20.0 → 6.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/docs/TRACE-CONTRACTS.md +39 -0
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-QPRJUVLH.mjs → chunk-CHBYF4LE.mjs} +25 -2
- package/packages/cli/dist/chunk-CHBYF4LE.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +50 -3
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +4 -4
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-6W5ZR3I7.mjs → src-MY5WIBPF.mjs} +3 -3
- package/packages/cli/dist/{src-6W5ZR3I7.mjs.map → src-MY5WIBPF.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +41 -0
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +39 -157
- package/packages/core/dist/advanced.d.ts +39 -157
- package/packages/core/dist/advanced.mjs +35 -61
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +736 -45
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +132 -5
- package/packages/core/dist/checks.d.ts +132 -5
- package/packages/core/dist/checks.mjs +2 -2
- package/packages/core/dist/{chunk-HT54OLPE.mjs → chunk-NDRMIL7Z.mjs} +103 -3
- package/packages/core/dist/chunk-NDRMIL7Z.mjs.map +1 -0
- package/packages/core/dist/{chunk-G47KQB42.mjs → chunk-OG3GAWCX.mjs} +727 -43
- package/packages/core/dist/chunk-OG3GAWCX.mjs.map +1 -0
- package/packages/core/dist/{chunk-3XS5O4JI.mjs → chunk-R4SSYU6D.mjs} +3 -3
- package/packages/core/dist/chunk-R4SSYU6D.mjs.map +1 -0
- package/packages/core/dist/{chunk-5YFJJFXU.mjs → chunk-WGDSM37E.mjs} +3 -3
- package/packages/core/dist/{chunk-5YFJJFXU.mjs.map → chunk-WGDSM37E.mjs.map} +1 -1
- package/packages/core/dist/{context-C5ye_xcS.d.cts → context-DOOT9GZ-.d.cts} +1 -1
- package/packages/core/dist/{context-CNsNSgE2.d.ts → context-DOX2-RjH.d.ts} +1 -1
- package/packages/core/dist/diff.d.cts +1 -1
- package/packages/core/dist/diff.d.ts +1 -1
- package/packages/core/dist/exporters.d.cts +1 -1
- package/packages/core/dist/exporters.d.ts +1 -1
- package/packages/core/dist/exporters.mjs +1 -1
- package/packages/core/dist/{index-rVpX5PPh.d.cts → index-CR4CCRqP.d.cts} +159 -2
- package/packages/core/dist/{index-Bg9ZdsdS.d.ts → index-CtK_p-ps.d.ts} +159 -2
- package/packages/core/dist/{index-AMAGuO_i.d.ts → index-D72DPqAF.d.ts} +2 -1
- package/packages/core/dist/{index-vqr-g6ev.d.cts → index-DLLz9sce.d.cts} +2 -1
- package/packages/core/dist/index.d.cts +3 -3
- package/packages/core/dist/index.d.ts +3 -3
- package/packages/core/dist/index.mjs +3 -3
- package/packages/core/dist/persisted.d.cts +1 -1
- package/packages/core/dist/persisted.d.ts +1 -1
- package/packages/core/dist/programmatic-gZTWLObB.d.cts +25 -0
- package/packages/core/dist/programmatic-gZTWLObB.d.ts +25 -0
- package/packages/core/dist/readers.cjs +105 -0
- package/packages/core/dist/readers.cjs.map +1 -1
- package/packages/core/dist/readers.d.cts +43 -18
- package/packages/core/dist/readers.d.ts +43 -18
- package/packages/core/dist/readers.mjs +1 -1
- package/packages/core/dist/{types--ainI31J.d.ts → types-Bx3mEZ04.d.ts} +1 -1
- package/packages/core/dist/{types-UnrNaTo2.d.cts → types-VzKihKQo.d.cts} +1 -1
- package/packages/cli/dist/chunk-QPRJUVLH.mjs.map +0 -1
- package/packages/core/dist/chunk-3XS5O4JI.mjs.map +0 -1
- package/packages/core/dist/chunk-G47KQB42.mjs.map +0 -1
- package/packages/core/dist/chunk-HT54OLPE.mjs.map +0 -1
|
@@ -1,6 +1,70 @@
|
|
|
1
|
-
import { extractOutcomesFromPersistedEvents, outcomesMatchingStatus } from './chunk-
|
|
1
|
+
import { OBSERVED_OUTCOME_METHODS, extractOutcomesFromPersistedEvents, outcomesMatchingStatus } from './chunk-R4SSYU6D.mjs';
|
|
2
2
|
import { formatProgrammaticDiagnostic } from './chunk-FWK3Q3OJ.mjs';
|
|
3
3
|
|
|
4
|
+
// packages/core/src/sessions/metadata.ts
|
|
5
|
+
function isNonEmptyString(value) {
|
|
6
|
+
return typeof value === "string" && value.trim() !== "";
|
|
7
|
+
}
|
|
8
|
+
function finitePositiveInt(value) {
|
|
9
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
10
|
+
return void 0;
|
|
11
|
+
}
|
|
12
|
+
return Math.trunc(value);
|
|
13
|
+
}
|
|
14
|
+
function extractSessionWorkflowMetadata(record) {
|
|
15
|
+
if (!record) return void 0;
|
|
16
|
+
const out = {};
|
|
17
|
+
let found = false;
|
|
18
|
+
const assignString = (key, value) => {
|
|
19
|
+
if (isNonEmptyString(value)) {
|
|
20
|
+
out[key] = value.trim();
|
|
21
|
+
found = true;
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
assignString("sessionId", record.sessionId);
|
|
25
|
+
assignString("conversationId", record.conversationId);
|
|
26
|
+
assignString("groupId", record.groupId);
|
|
27
|
+
assignString("parentGroupId", record.parentGroupId);
|
|
28
|
+
assignString("retryOf", record.retryOf);
|
|
29
|
+
assignString("retryReason", record.retryReason);
|
|
30
|
+
assignString("handoffFrom", record.handoffFrom);
|
|
31
|
+
assignString("handoffTo", record.handoffTo);
|
|
32
|
+
assignString("subAgentId", record.subAgentId);
|
|
33
|
+
assignString("subAgentName", record.subAgentName);
|
|
34
|
+
assignString("jobId", record.jobId);
|
|
35
|
+
assignString("queueName", record.queueName);
|
|
36
|
+
assignString("workflowName", record.workflowName);
|
|
37
|
+
assignString("workflowStep", record.workflowStep);
|
|
38
|
+
assignString("toolCallId", record.toolCallId);
|
|
39
|
+
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
40
|
+
assignString("linkedStepId", record.linkedStepId);
|
|
41
|
+
assignString("operationId", record.operationId);
|
|
42
|
+
assignString("attemptId", record.attemptId);
|
|
43
|
+
assignString("fallbackOf", record.fallbackOf);
|
|
44
|
+
assignString("idempotencyKey", record.idempotencyKey);
|
|
45
|
+
assignString("correlationId", record.correlationId);
|
|
46
|
+
assignString("requestId", record.requestId);
|
|
47
|
+
assignString("decisionId", record.decisionId);
|
|
48
|
+
const attempt = finitePositiveInt(record.attempt);
|
|
49
|
+
if (attempt !== void 0) {
|
|
50
|
+
out.attempt = attempt;
|
|
51
|
+
found = true;
|
|
52
|
+
}
|
|
53
|
+
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
54
|
+
if (attemptNumber !== void 0) {
|
|
55
|
+
out.attemptNumber = attemptNumber;
|
|
56
|
+
found = true;
|
|
57
|
+
}
|
|
58
|
+
return found ? out : void 0;
|
|
59
|
+
}
|
|
60
|
+
function sessionKeyForRun(meta, options) {
|
|
61
|
+
if (meta?.sessionId) return meta.sessionId;
|
|
62
|
+
if (options?.correlateByGroupId && meta?.groupId) {
|
|
63
|
+
return `group:${meta.groupId}`;
|
|
64
|
+
}
|
|
65
|
+
return void 0;
|
|
66
|
+
}
|
|
67
|
+
|
|
4
68
|
// packages/core/src/checks/logical-events.ts
|
|
5
69
|
function isRecord(value) {
|
|
6
70
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -278,10 +342,10 @@ function projectLogicalEvents(events) {
|
|
|
278
342
|
};
|
|
279
343
|
}
|
|
280
344
|
function resolveCanonicalToolName(event) {
|
|
281
|
-
const
|
|
282
|
-
const direct = pickString(
|
|
345
|
+
const attrs2 = event.attributes;
|
|
346
|
+
const direct = pickString(attrs2, ["toolName", "tool"]);
|
|
283
347
|
if (direct) return direct;
|
|
284
|
-
const metadata =
|
|
348
|
+
const metadata = attrs2?.metadata;
|
|
285
349
|
if (isRecord(metadata)) {
|
|
286
350
|
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
287
351
|
if (nested) return nested;
|
|
@@ -323,10 +387,10 @@ function pickNumber(record, keys) {
|
|
|
323
387
|
return void 0;
|
|
324
388
|
}
|
|
325
389
|
function eventMetadata(event) {
|
|
326
|
-
const
|
|
327
|
-
const nested =
|
|
390
|
+
const attrs2 = isRecord2(event.attributes) ? event.attributes : void 0;
|
|
391
|
+
const nested = attrs2 !== void 0 && isRecord2(attrs2.metadata) ? attrs2.metadata : void 0;
|
|
328
392
|
return {
|
|
329
|
-
...
|
|
393
|
+
...attrs2 ?? {},
|
|
330
394
|
...nested ?? {}
|
|
331
395
|
};
|
|
332
396
|
}
|
|
@@ -619,6 +683,151 @@ function deriveFailureFacts(logicalEvents) {
|
|
|
619
683
|
};
|
|
620
684
|
}
|
|
621
685
|
|
|
686
|
+
// packages/core/src/checks/relationship-facts.ts
|
|
687
|
+
function attrs(event) {
|
|
688
|
+
return event.attributes && typeof event.attributes === "object" ? event.attributes : {};
|
|
689
|
+
}
|
|
690
|
+
function stringAttr(record, key) {
|
|
691
|
+
const value = record[key];
|
|
692
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
693
|
+
}
|
|
694
|
+
function deriveRelationshipFacts(events) {
|
|
695
|
+
const relationships = [];
|
|
696
|
+
const diagnostics = [];
|
|
697
|
+
const byId = new Map(events.map((event) => [event.eventId, event]));
|
|
698
|
+
const seen = /* @__PURE__ */ new Set();
|
|
699
|
+
const push = (edge) => {
|
|
700
|
+
const key = `${edge.type}:${edge.fromEventId}:${edge.toEventId ?? ""}:${edge.externalRef ?? ""}`;
|
|
701
|
+
if (seen.has(key)) return;
|
|
702
|
+
seen.add(key);
|
|
703
|
+
relationships.push(edge);
|
|
704
|
+
};
|
|
705
|
+
for (const event of events) {
|
|
706
|
+
if (event.parentId) {
|
|
707
|
+
push({
|
|
708
|
+
type: "parent-child",
|
|
709
|
+
fromEventId: event.parentId,
|
|
710
|
+
toEventId: event.eventId,
|
|
711
|
+
confidence: byId.has(event.parentId) ? "explicit" : "unknown",
|
|
712
|
+
basis: ["persisted.parentId"]
|
|
713
|
+
});
|
|
714
|
+
if (!byId.has(event.parentId)) {
|
|
715
|
+
diagnostics.push({
|
|
716
|
+
code: "AI_RELATIONSHIP_PARENT_MISSING",
|
|
717
|
+
message: `parentId ${event.parentId} is not present in the event set.`,
|
|
718
|
+
eventId: event.eventId
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
const bag = attrs(event);
|
|
723
|
+
const meta = extractSessionWorkflowMetadata(bag) ?? {};
|
|
724
|
+
const nested = bag.metadata && typeof bag.metadata === "object" ? extractSessionWorkflowMetadata(bag.metadata) : void 0;
|
|
725
|
+
const workflow = { ...meta, ...nested };
|
|
726
|
+
if (workflow.retryOf) {
|
|
727
|
+
const target = events.find((candidate) => candidate.runId === workflow.retryOf);
|
|
728
|
+
push({
|
|
729
|
+
type: "retry-of",
|
|
730
|
+
fromEventId: event.eventId,
|
|
731
|
+
...target ? { toEventId: target.eventId } : {},
|
|
732
|
+
externalRef: workflow.retryOf,
|
|
733
|
+
confidence: target ? "explicit" : "correlated",
|
|
734
|
+
basis: ["attributes.retryOf"]
|
|
735
|
+
});
|
|
736
|
+
}
|
|
737
|
+
const attemptOf = stringAttr(bag, "attemptOf") ?? stringAttr(bag, "operationId");
|
|
738
|
+
if (attemptOf && workflow.attempt !== void 0) {
|
|
739
|
+
push({
|
|
740
|
+
type: "attempt-of",
|
|
741
|
+
fromEventId: event.eventId,
|
|
742
|
+
externalRef: attemptOf,
|
|
743
|
+
confidence: "explicit",
|
|
744
|
+
basis: workflow.attempt !== void 0 ? ["attributes.attempt", "attributes.operationId"] : ["attributes.operationId"]
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
const fallbackOf = stringAttr(bag, "fallbackOf");
|
|
748
|
+
if (fallbackOf) {
|
|
749
|
+
push({
|
|
750
|
+
type: "fallback-of",
|
|
751
|
+
fromEventId: event.eventId,
|
|
752
|
+
externalRef: fallbackOf,
|
|
753
|
+
confidence: "explicit",
|
|
754
|
+
basis: ["attributes.fallbackOf"]
|
|
755
|
+
});
|
|
756
|
+
}
|
|
757
|
+
const remediationOf = stringAttr(bag, "remediationOf");
|
|
758
|
+
if (remediationOf) {
|
|
759
|
+
push({
|
|
760
|
+
type: "remediation-of",
|
|
761
|
+
fromEventId: event.eventId,
|
|
762
|
+
externalRef: remediationOf,
|
|
763
|
+
confidence: "explicit",
|
|
764
|
+
basis: ["attributes.remediationOf"]
|
|
765
|
+
});
|
|
766
|
+
}
|
|
767
|
+
const evidenceFor = stringAttr(bag, "evidenceFor");
|
|
768
|
+
if (evidenceFor) {
|
|
769
|
+
push({
|
|
770
|
+
type: "evidence-for",
|
|
771
|
+
fromEventId: event.eventId,
|
|
772
|
+
...byId.has(evidenceFor) ? { toEventId: evidenceFor } : { externalRef: evidenceFor },
|
|
773
|
+
confidence: byId.has(evidenceFor) ? "explicit" : "correlated",
|
|
774
|
+
basis: ["attributes.evidenceFor"]
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
const acceptedBy = stringAttr(bag, "acceptedBy");
|
|
778
|
+
if (acceptedBy) {
|
|
779
|
+
push({
|
|
780
|
+
type: "accepted-by",
|
|
781
|
+
fromEventId: event.eventId,
|
|
782
|
+
...byId.has(acceptedBy) ? { toEventId: acceptedBy } : { externalRef: acceptedBy },
|
|
783
|
+
confidence: byId.has(acceptedBy) ? "explicit" : "correlated",
|
|
784
|
+
basis: ["attributes.acceptedBy"]
|
|
785
|
+
});
|
|
786
|
+
}
|
|
787
|
+
const supersedes = stringAttr(bag, "supersedes");
|
|
788
|
+
if (supersedes) {
|
|
789
|
+
push({
|
|
790
|
+
type: "supersedes",
|
|
791
|
+
fromEventId: event.eventId,
|
|
792
|
+
...byId.has(supersedes) ? { toEventId: supersedes } : { externalRef: supersedes },
|
|
793
|
+
confidence: byId.has(supersedes) ? "explicit" : "correlated",
|
|
794
|
+
basis: ["attributes.supersedes"]
|
|
795
|
+
});
|
|
796
|
+
}
|
|
797
|
+
const sourceLineage = stringAttr(bag, "sourceLineage") ?? stringAttr(bag, "sourceEventId");
|
|
798
|
+
if (sourceLineage) {
|
|
799
|
+
push({
|
|
800
|
+
type: "source-lineage",
|
|
801
|
+
fromEventId: event.eventId,
|
|
802
|
+
externalRef: sourceLineage,
|
|
803
|
+
confidence: "explicit",
|
|
804
|
+
basis: ["attributes.sourceLineage"]
|
|
805
|
+
});
|
|
806
|
+
}
|
|
807
|
+
const unsupported = stringAttr(bag, "relationshipType");
|
|
808
|
+
if (unsupported && unsupported !== "parent-child" && ![
|
|
809
|
+
"source-lineage",
|
|
810
|
+
"attempt-of",
|
|
811
|
+
"retry-of",
|
|
812
|
+
"fallback-of",
|
|
813
|
+
"remediation-of",
|
|
814
|
+
"evidence-for",
|
|
815
|
+
"accepted-by",
|
|
816
|
+
"supersedes"
|
|
817
|
+
].includes(unsupported)) {
|
|
818
|
+
diagnostics.push({
|
|
819
|
+
code: "AI_RELATIONSHIP_UNSUPPORTED_TYPE",
|
|
820
|
+
message: `Unsupported relationshipType ${unsupported} was reported without flattening.`,
|
|
821
|
+
eventId: event.eventId
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
return {
|
|
826
|
+
relationships: Object.freeze(relationships),
|
|
827
|
+
diagnostics: Object.freeze(diagnostics)
|
|
828
|
+
};
|
|
829
|
+
}
|
|
830
|
+
|
|
622
831
|
// packages/core/src/checks/trace-facts.ts
|
|
623
832
|
function summarizeSemanticParity(events) {
|
|
624
833
|
const projection = projectLogicalEvents(events);
|
|
@@ -705,6 +914,7 @@ function buildTraceFacts(input) {
|
|
|
705
914
|
toolsByName.set(name, Object.freeze([...list]));
|
|
706
915
|
}
|
|
707
916
|
const derived = deriveFailureFacts(projection.logicalEvents);
|
|
917
|
+
const relationship = deriveRelationshipFacts(events);
|
|
708
918
|
const summary = summarizeSemanticParity(events);
|
|
709
919
|
return {
|
|
710
920
|
rawEvents: Object.freeze([...events]),
|
|
@@ -718,7 +928,203 @@ function buildTraceFacts(input) {
|
|
|
718
928
|
failureRoleCounts: derived.failureRoleCounts
|
|
719
929
|
},
|
|
720
930
|
failureFacts: derived.failureFacts,
|
|
721
|
-
failuresByRole: derived.failuresByRole
|
|
931
|
+
failuresByRole: derived.failuresByRole,
|
|
932
|
+
relationships: relationship.relationships,
|
|
933
|
+
relationshipDiagnostics: relationship.diagnostics
|
|
934
|
+
};
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
// packages/core/src/checks/contract-scope.ts
|
|
938
|
+
function diagnostic(code, message, ruleId) {
|
|
939
|
+
return {
|
|
940
|
+
code,
|
|
941
|
+
message,
|
|
942
|
+
severity: "error",
|
|
943
|
+
...ruleId !== void 0 ? { ruleId } : {}
|
|
944
|
+
};
|
|
945
|
+
}
|
|
946
|
+
function isNonEmptyString2(value) {
|
|
947
|
+
return typeof value === "string" && value.trim() !== "";
|
|
948
|
+
}
|
|
949
|
+
function scopeEntries(scope) {
|
|
950
|
+
const out = [];
|
|
951
|
+
if (isNonEmptyString2(scope.runId)) out.push(["runId", scope.runId.trim()]);
|
|
952
|
+
if (isNonEmptyString2(scope.subAgentId)) out.push(["subAgentId", scope.subAgentId.trim()]);
|
|
953
|
+
if (isNonEmptyString2(scope.groupId)) out.push(["groupId", scope.groupId.trim()]);
|
|
954
|
+
if (isNonEmptyString2(scope.workflowStep)) {
|
|
955
|
+
out.push(["workflowStep", scope.workflowStep.trim()]);
|
|
956
|
+
}
|
|
957
|
+
if (isNonEmptyString2(scope.rootEventId)) {
|
|
958
|
+
out.push(["rootEventId", scope.rootEventId.trim()]);
|
|
959
|
+
}
|
|
960
|
+
return out;
|
|
961
|
+
}
|
|
962
|
+
function mergeWorkflowMeta(into, from) {
|
|
963
|
+
if (!from) return;
|
|
964
|
+
for (const [key, value] of Object.entries(from)) {
|
|
965
|
+
if (value === void 0) continue;
|
|
966
|
+
if (into[key] === void 0) {
|
|
967
|
+
into[key] = value;
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
function workflowMetadataForRun(events, runId) {
|
|
972
|
+
const preferred = {};
|
|
973
|
+
const fallback = {};
|
|
974
|
+
for (const event of events) {
|
|
975
|
+
if (event.runId !== runId) continue;
|
|
976
|
+
const attrs2 = event.attributes;
|
|
977
|
+
if (!attrs2 || typeof attrs2 !== "object") continue;
|
|
978
|
+
const direct = extractSessionWorkflowMetadata(attrs2);
|
|
979
|
+
const nested = attrs2.metadata && typeof attrs2.metadata === "object" ? extractSessionWorkflowMetadata(attrs2.metadata) : void 0;
|
|
980
|
+
const bag = {};
|
|
981
|
+
mergeWorkflowMeta(bag, direct);
|
|
982
|
+
mergeWorkflowMeta(bag, nested);
|
|
983
|
+
if (event.kind === "RUN") {
|
|
984
|
+
mergeWorkflowMeta(preferred, bag);
|
|
985
|
+
} else {
|
|
986
|
+
mergeWorkflowMeta(fallback, bag);
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
mergeWorkflowMeta(preferred, fallback);
|
|
990
|
+
return preferred;
|
|
991
|
+
}
|
|
992
|
+
function flattenNodes(nodes) {
|
|
993
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
994
|
+
}
|
|
995
|
+
function collectSubtreeEventIds(run, rootEventId) {
|
|
996
|
+
const nodes = flattenNodes(run.children);
|
|
997
|
+
const byId = new Map(nodes.map((node) => [node.event.eventId, node]));
|
|
998
|
+
const root = byId.get(rootEventId);
|
|
999
|
+
if (!root) return void 0;
|
|
1000
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1001
|
+
const visit = (node) => {
|
|
1002
|
+
ids.add(node.event.eventId);
|
|
1003
|
+
for (const child of node.children) visit(child);
|
|
1004
|
+
};
|
|
1005
|
+
visit(root);
|
|
1006
|
+
return ids;
|
|
1007
|
+
}
|
|
1008
|
+
function filterTree(nodes, keep) {
|
|
1009
|
+
const out = [];
|
|
1010
|
+
for (const node of nodes) {
|
|
1011
|
+
if (!keep.has(node.event.eventId)) continue;
|
|
1012
|
+
out.push({
|
|
1013
|
+
...node,
|
|
1014
|
+
children: filterTree(node.children, keep)
|
|
1015
|
+
});
|
|
1016
|
+
}
|
|
1017
|
+
return out;
|
|
1018
|
+
}
|
|
1019
|
+
function projectInput(input, run, eventIds) {
|
|
1020
|
+
const runEvents = input.read.events.filter((event) => event.runId === run.runId);
|
|
1021
|
+
const projectedEvents = eventIds === void 0 ? runEvents : runEvents.filter((event) => eventIds.has(event.eventId) || event.kind === "RUN");
|
|
1022
|
+
const projectedChildren = eventIds === void 0 ? run.children : filterTree(run.children, eventIds);
|
|
1023
|
+
const projectedRun = {
|
|
1024
|
+
...run,
|
|
1025
|
+
children: projectedChildren,
|
|
1026
|
+
metadata: {
|
|
1027
|
+
...run.metadata,
|
|
1028
|
+
totalEvents: flattenNodes(projectedChildren).length
|
|
1029
|
+
}
|
|
1030
|
+
};
|
|
1031
|
+
return {
|
|
1032
|
+
evidenceEventCount: projectedEvents.length,
|
|
1033
|
+
input: {
|
|
1034
|
+
...input,
|
|
1035
|
+
selectedRun: projectedRun,
|
|
1036
|
+
read: {
|
|
1037
|
+
...input.read,
|
|
1038
|
+
runs: [projectedRun],
|
|
1039
|
+
events: projectedEvents
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
};
|
|
1043
|
+
}
|
|
1044
|
+
function resolveTraceContractScope(input, scope) {
|
|
1045
|
+
if (scope === void 0) {
|
|
1046
|
+
return { diagnostics: [] };
|
|
1047
|
+
}
|
|
1048
|
+
const entries = scopeEntries(scope);
|
|
1049
|
+
if (entries.length === 0) {
|
|
1050
|
+
return {
|
|
1051
|
+
diagnostics: [
|
|
1052
|
+
diagnostic(
|
|
1053
|
+
"AI_CHECK_INVALID_CONFIG",
|
|
1054
|
+
"TraceContract scope requires at least one explicit selector (runId, subAgentId, groupId, workflowStep, or rootEventId).",
|
|
1055
|
+
"contract.scope.empty"
|
|
1056
|
+
)
|
|
1057
|
+
]
|
|
1058
|
+
};
|
|
1059
|
+
}
|
|
1060
|
+
const matchedSelectors = Object.fromEntries(entries);
|
|
1061
|
+
const actorKeys = entries.filter(([key]) => key !== "rootEventId");
|
|
1062
|
+
const rootEventId = matchedSelectors.rootEventId;
|
|
1063
|
+
let candidates = [...input.read.runs];
|
|
1064
|
+
if (matchedSelectors.runId) {
|
|
1065
|
+
candidates = candidates.filter((run2) => run2.runId === matchedSelectors.runId);
|
|
1066
|
+
}
|
|
1067
|
+
for (const [key, value] of actorKeys) {
|
|
1068
|
+
if (key === "runId") continue;
|
|
1069
|
+
candidates = candidates.filter((run2) => {
|
|
1070
|
+
const meta = workflowMetadataForRun(input.read.events, run2.runId);
|
|
1071
|
+
const actual = meta[key];
|
|
1072
|
+
return actual === value;
|
|
1073
|
+
});
|
|
1074
|
+
}
|
|
1075
|
+
if (rootEventId && actorKeys.length === 0) {
|
|
1076
|
+
candidates = candidates.filter(
|
|
1077
|
+
(run2) => flattenNodes(run2.children).some((node) => node.event.eventId === rootEventId)
|
|
1078
|
+
);
|
|
1079
|
+
}
|
|
1080
|
+
if (candidates.length === 0) {
|
|
1081
|
+
return {
|
|
1082
|
+
diagnostics: [
|
|
1083
|
+
diagnostic(
|
|
1084
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
1085
|
+
`TraceContract scope matched zero actors for ${JSON.stringify(matchedSelectors)}. Missing actor metadata does not fall back to the whole session.`,
|
|
1086
|
+
"contract.scope.zero-match"
|
|
1087
|
+
)
|
|
1088
|
+
]
|
|
1089
|
+
};
|
|
1090
|
+
}
|
|
1091
|
+
if (candidates.length > 1) {
|
|
1092
|
+
return {
|
|
1093
|
+
diagnostics: [
|
|
1094
|
+
diagnostic(
|
|
1095
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
1096
|
+
`TraceContract scope matched ${candidates.length} actors for ${JSON.stringify(matchedSelectors)} (runIds: ${candidates.map((run2) => run2.runId).sort().join(", ")}). Singular actor selectors must resolve to exactly one run.`,
|
|
1097
|
+
"contract.scope.ambiguous"
|
|
1098
|
+
)
|
|
1099
|
+
]
|
|
1100
|
+
};
|
|
1101
|
+
}
|
|
1102
|
+
const run = candidates[0];
|
|
1103
|
+
let eventIds;
|
|
1104
|
+
if (rootEventId) {
|
|
1105
|
+
eventIds = collectSubtreeEventIds(run, rootEventId);
|
|
1106
|
+
if (eventIds === void 0) {
|
|
1107
|
+
return {
|
|
1108
|
+
diagnostics: [
|
|
1109
|
+
diagnostic(
|
|
1110
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
1111
|
+
`TraceContract scope rootEventId ${rootEventId} was not found in run ${run.runId}.`,
|
|
1112
|
+
"contract.scope.root-missing"
|
|
1113
|
+
)
|
|
1114
|
+
]
|
|
1115
|
+
};
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
const projected = projectInput(input, run, eventIds);
|
|
1119
|
+
return {
|
|
1120
|
+
resolved: {
|
|
1121
|
+
runId: run.runId,
|
|
1122
|
+
...rootEventId !== void 0 ? { rootEventId } : {},
|
|
1123
|
+
matchedSelectors,
|
|
1124
|
+
evidenceEventCount: projected.evidenceEventCount,
|
|
1125
|
+
input: projected.input
|
|
1126
|
+
},
|
|
1127
|
+
diagnostics: []
|
|
722
1128
|
};
|
|
723
1129
|
}
|
|
724
1130
|
|
|
@@ -878,7 +1284,7 @@ var DEFAULT_SECRET_PATTERNS = [
|
|
|
878
1284
|
function compareStrings(a, b) {
|
|
879
1285
|
return (a ?? "").localeCompare(b ?? "");
|
|
880
1286
|
}
|
|
881
|
-
function
|
|
1287
|
+
function diagnostic2(code, message, ruleId) {
|
|
882
1288
|
return {
|
|
883
1289
|
code,
|
|
884
1290
|
message,
|
|
@@ -911,14 +1317,14 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
911
1317
|
ruleExecutions: [...ruleExecutions]
|
|
912
1318
|
};
|
|
913
1319
|
}
|
|
914
|
-
function
|
|
915
|
-
return nodes.flatMap((node) => [node, ...
|
|
1320
|
+
function flattenNodes2(nodes) {
|
|
1321
|
+
return nodes.flatMap((node) => [node, ...flattenNodes2(node.children)]);
|
|
916
1322
|
}
|
|
917
1323
|
function buildFacts(input, selectedRun) {
|
|
918
1324
|
const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
|
|
919
1325
|
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
920
1326
|
const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
921
|
-
const nodes =
|
|
1327
|
+
const nodes = flattenNodes2(scopedRuns.flatMap((run) => run.children));
|
|
922
1328
|
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
923
1329
|
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
924
1330
|
for (const node of nodes) {
|
|
@@ -950,7 +1356,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
950
1356
|
if (runId && input.selectedRun.runId !== runId) {
|
|
951
1357
|
return {
|
|
952
1358
|
diagnostics: [
|
|
953
|
-
|
|
1359
|
+
diagnostic2(
|
|
954
1360
|
"AI_CHECK_INVALID_ARGUMENTS",
|
|
955
1361
|
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
956
1362
|
)
|
|
@@ -964,7 +1370,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
964
1370
|
if (!run) {
|
|
965
1371
|
return {
|
|
966
1372
|
diagnostics: [
|
|
967
|
-
|
|
1373
|
+
diagnostic2("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
968
1374
|
]
|
|
969
1375
|
};
|
|
970
1376
|
}
|
|
@@ -976,7 +1382,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
976
1382
|
if (input.read.runs.length === 0) {
|
|
977
1383
|
return {
|
|
978
1384
|
diagnostics: [
|
|
979
|
-
|
|
1385
|
+
diagnostic2(
|
|
980
1386
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
981
1387
|
formatProgrammaticDiagnostic(
|
|
982
1388
|
"AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
@@ -988,7 +1394,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
988
1394
|
}
|
|
989
1395
|
return {
|
|
990
1396
|
diagnostics: [
|
|
991
|
-
|
|
1397
|
+
diagnostic2(
|
|
992
1398
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
993
1399
|
formatProgrammaticDiagnostic("AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED")
|
|
994
1400
|
)
|
|
@@ -1001,7 +1407,7 @@ function selectRules(rules, selectedIds) {
|
|
|
1001
1407
|
for (const rule of rules) {
|
|
1002
1408
|
if (byId.has(rule.id)) {
|
|
1003
1409
|
diagnostics.push(
|
|
1004
|
-
|
|
1410
|
+
diagnostic2("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
1005
1411
|
);
|
|
1006
1412
|
continue;
|
|
1007
1413
|
}
|
|
@@ -1012,7 +1418,7 @@ function selectRules(rules, selectedIds) {
|
|
|
1012
1418
|
for (const id of selected) {
|
|
1013
1419
|
if (!byId.has(id)) {
|
|
1014
1420
|
diagnostics.push(
|
|
1015
|
-
|
|
1421
|
+
diagnostic2("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
1016
1422
|
);
|
|
1017
1423
|
}
|
|
1018
1424
|
}
|
|
@@ -1088,7 +1494,7 @@ function classifyRuleExecution(findings, threw) {
|
|
|
1088
1494
|
}
|
|
1089
1495
|
return "pass";
|
|
1090
1496
|
}
|
|
1091
|
-
function
|
|
1497
|
+
function stringAttr2(event, keys) {
|
|
1092
1498
|
for (const key of keys) {
|
|
1093
1499
|
const value = event.attributes?.[key];
|
|
1094
1500
|
if (typeof value === "string" && value.trim() !== "") return value;
|
|
@@ -1166,15 +1572,15 @@ function semanticEvents(context) {
|
|
|
1166
1572
|
}
|
|
1167
1573
|
function llmModel(event) {
|
|
1168
1574
|
const keys = ["model", "modelId", "responseModelId", "modelName", "model_name"];
|
|
1169
|
-
return
|
|
1575
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
1170
1576
|
}
|
|
1171
1577
|
function llmProvider(event) {
|
|
1172
1578
|
const keys = ["provider", "providerName", "provider_name"];
|
|
1173
|
-
return
|
|
1579
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys);
|
|
1174
1580
|
}
|
|
1175
1581
|
function llmFinishReason(event) {
|
|
1176
1582
|
const keys = ["finishReason", "rawFinishReason", "finish_reason"];
|
|
1177
|
-
return
|
|
1583
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys);
|
|
1178
1584
|
}
|
|
1179
1585
|
function retryCount(event) {
|
|
1180
1586
|
return numericAttr(event, ["retryCount", "retryAttempt", "retry_attempt", "attempt"]);
|
|
@@ -1316,7 +1722,7 @@ function parentMarkedUnresolved(event) {
|
|
|
1316
1722
|
]) === true) {
|
|
1317
1723
|
return true;
|
|
1318
1724
|
}
|
|
1319
|
-
const resolution =
|
|
1725
|
+
const resolution = stringAttr2(event, [
|
|
1320
1726
|
"parentResolution",
|
|
1321
1727
|
"relationshipResolution",
|
|
1322
1728
|
"relationshipStatus"
|
|
@@ -1324,13 +1730,13 @@ function parentMarkedUnresolved(event) {
|
|
|
1324
1730
|
return resolution === "unresolved" || resolution === "missing-parent";
|
|
1325
1731
|
}
|
|
1326
1732
|
function signalName(event, attributeKeys, prefixes) {
|
|
1327
|
-
return
|
|
1733
|
+
return stringAttr2(event, attributeKeys) ?? stripPrefix(event.name, prefixes);
|
|
1328
1734
|
}
|
|
1329
1735
|
function guardrailEvents(context) {
|
|
1330
1736
|
return finishedEvents(context).filter((event) => {
|
|
1331
1737
|
const name = event.name.toLowerCase();
|
|
1332
1738
|
if (name.startsWith("guardrail:") || name.includes(".guardrail.")) return true;
|
|
1333
|
-
return
|
|
1739
|
+
return stringAttr2(event, ["guardrailName", "guardrail", "guardrailId"]) !== void 0;
|
|
1334
1740
|
});
|
|
1335
1741
|
}
|
|
1336
1742
|
function retryValue(event) {
|
|
@@ -2699,7 +3105,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
2699
3105
|
return errorResult(
|
|
2700
3106
|
input,
|
|
2701
3107
|
[
|
|
2702
|
-
|
|
3108
|
+
diagnostic2(
|
|
2703
3109
|
"AI_CHECK_NO_RULES_EVALUATED",
|
|
2704
3110
|
"No trace check rules were evaluated. Configure at least one rule, contract, or CLI check option."
|
|
2705
3111
|
)
|
|
@@ -2730,7 +3136,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
2730
3136
|
} catch (error) {
|
|
2731
3137
|
const message = error instanceof Error ? error.message : String(error);
|
|
2732
3138
|
diagnostics.push(
|
|
2733
|
-
|
|
3139
|
+
diagnostic2("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
2734
3140
|
);
|
|
2735
3141
|
ruleExecutions.push({
|
|
2736
3142
|
ruleId: rule.id,
|
|
@@ -2783,12 +3189,141 @@ function cloneBody(body) {
|
|
|
2783
3189
|
...body.run ? { run: { ...body.run } } : {},
|
|
2784
3190
|
...body.tools ? { tools: { ...body.tools } } : {},
|
|
2785
3191
|
...body.llm ? { llm: { ...body.llm } } : {},
|
|
2786
|
-
...body.observations ? {
|
|
3192
|
+
...body.observations ? {
|
|
3193
|
+
observations: {
|
|
3194
|
+
...body.observations,
|
|
3195
|
+
...body.observations.requireProvenance ? { requireProvenance: { ...body.observations.requireProvenance } } : {}
|
|
3196
|
+
}
|
|
3197
|
+
} : {}
|
|
2787
3198
|
};
|
|
2788
3199
|
}
|
|
2789
3200
|
function bodyHasRules(body) {
|
|
2790
3201
|
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0;
|
|
2791
3202
|
}
|
|
3203
|
+
var MAX_EVIDENCE_EVENT_IDS = 16;
|
|
3204
|
+
var METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
3205
|
+
function parseEvidenceEventIds(evidence) {
|
|
3206
|
+
if (evidence === void 0 || evidence === null) {
|
|
3207
|
+
return { status: "missing" };
|
|
3208
|
+
}
|
|
3209
|
+
if (typeof evidence === "string") {
|
|
3210
|
+
const trimmed = evidence.trim();
|
|
3211
|
+
if (trimmed === "") {
|
|
3212
|
+
return { status: "invalid", reason: "empty evidence string" };
|
|
3213
|
+
}
|
|
3214
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
3215
|
+
}
|
|
3216
|
+
if (typeof evidence !== "object" || Array.isArray(evidence)) {
|
|
3217
|
+
return {
|
|
3218
|
+
status: "invalid",
|
|
3219
|
+
reason: "evidence must be a string event id or { eventId } / { eventIds }"
|
|
3220
|
+
};
|
|
3221
|
+
}
|
|
3222
|
+
const record = evidence;
|
|
3223
|
+
if (typeof record.eventId === "string") {
|
|
3224
|
+
const trimmed = record.eventId.trim();
|
|
3225
|
+
if (trimmed === "") {
|
|
3226
|
+
return { status: "invalid", reason: "empty evidence.eventId" };
|
|
3227
|
+
}
|
|
3228
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
3229
|
+
}
|
|
3230
|
+
if (Array.isArray(record.eventIds)) {
|
|
3231
|
+
if (record.eventIds.length === 0) {
|
|
3232
|
+
return { status: "invalid", reason: "empty evidence.eventIds" };
|
|
3233
|
+
}
|
|
3234
|
+
if (record.eventIds.length > MAX_EVIDENCE_EVENT_IDS) {
|
|
3235
|
+
return {
|
|
3236
|
+
status: "invalid",
|
|
3237
|
+
reason: `evidence.eventIds exceeds bound of ${MAX_EVIDENCE_EVENT_IDS}`
|
|
3238
|
+
};
|
|
3239
|
+
}
|
|
3240
|
+
const eventIds = [];
|
|
3241
|
+
for (const item of record.eventIds) {
|
|
3242
|
+
if (typeof item !== "string" || item.trim() === "") {
|
|
3243
|
+
return { status: "invalid", reason: "evidence.eventIds contains a non-string id" };
|
|
3244
|
+
}
|
|
3245
|
+
eventIds.push(item.trim());
|
|
3246
|
+
}
|
|
3247
|
+
return { status: "ok", eventIds };
|
|
3248
|
+
}
|
|
3249
|
+
return {
|
|
3250
|
+
status: "invalid",
|
|
3251
|
+
reason: "evidence must include eventId or eventIds"
|
|
3252
|
+
};
|
|
3253
|
+
}
|
|
3254
|
+
function provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds) {
|
|
3255
|
+
const findings = [];
|
|
3256
|
+
const evidenceBase = [
|
|
3257
|
+
{
|
|
3258
|
+
runId: outcome.runId,
|
|
3259
|
+
eventId: outcome.outcomeId,
|
|
3260
|
+
kind: "OUTCOME",
|
|
3261
|
+
name: outcome.name,
|
|
3262
|
+
path: `outcome.${outcome.name}`
|
|
3263
|
+
}
|
|
3264
|
+
];
|
|
3265
|
+
if (requireProvenance.method) {
|
|
3266
|
+
if (outcome.method === void 0 || outcome.method.trim() === "") {
|
|
3267
|
+
findings.push(
|
|
3268
|
+
contractFailFinding(
|
|
3269
|
+
"contract.observation.provenance.method",
|
|
3270
|
+
`Observation ${outcome.name} is missing a structural method.`,
|
|
3271
|
+
evidenceBase,
|
|
3272
|
+
{ method: true },
|
|
3273
|
+
{ name: outcome.name, method: outcome.method ?? null }
|
|
3274
|
+
)
|
|
3275
|
+
);
|
|
3276
|
+
} else if (!METHOD_VOCABULARY.has(outcome.method)) {
|
|
3277
|
+
findings.push(
|
|
3278
|
+
contractFailFinding(
|
|
3279
|
+
"contract.observation.provenance.method",
|
|
3280
|
+
`Observation ${outcome.name} method ${outcome.method} is outside the bounded ObservedOutcomeMethod vocabulary.`,
|
|
3281
|
+
evidenceBase,
|
|
3282
|
+
{ methodVocabulary: [...OBSERVED_OUTCOME_METHODS] },
|
|
3283
|
+
{ name: outcome.name, method: outcome.method }
|
|
3284
|
+
)
|
|
3285
|
+
);
|
|
3286
|
+
}
|
|
3287
|
+
}
|
|
3288
|
+
if (requireProvenance.evidence || requireProvenance.sameRunEventReference) {
|
|
3289
|
+
const parsed = parseEvidenceEventIds(outcome.evidence);
|
|
3290
|
+
if (parsed.status === "missing") {
|
|
3291
|
+
findings.push(
|
|
3292
|
+
contractFailFinding(
|
|
3293
|
+
"contract.observation.provenance.evidence",
|
|
3294
|
+
`Observation ${outcome.name} is missing bounded evidence references.`,
|
|
3295
|
+
evidenceBase,
|
|
3296
|
+
{ evidence: true },
|
|
3297
|
+
{ name: outcome.name, evidence: null }
|
|
3298
|
+
)
|
|
3299
|
+
);
|
|
3300
|
+
} else if (parsed.status === "invalid") {
|
|
3301
|
+
findings.push(
|
|
3302
|
+
contractFailFinding(
|
|
3303
|
+
"contract.observation.provenance.evidence",
|
|
3304
|
+
`Observation ${outcome.name} evidence is not a bounded event reference (${parsed.reason}).`,
|
|
3305
|
+
evidenceBase,
|
|
3306
|
+
{ evidence: true },
|
|
3307
|
+
{ name: outcome.name, reason: parsed.reason }
|
|
3308
|
+
)
|
|
3309
|
+
);
|
|
3310
|
+
} else if (requireProvenance.sameRunEventReference) {
|
|
3311
|
+
const missing = parsed.eventIds.filter((eventId) => !knownEventIds.has(eventId));
|
|
3312
|
+
if (missing.length > 0) {
|
|
3313
|
+
findings.push(
|
|
3314
|
+
contractFailFinding(
|
|
3315
|
+
"contract.observation.provenance.same-run",
|
|
3316
|
+
`Observation ${outcome.name} references event ids missing from the same run: ${missing.join(", ")}.`,
|
|
3317
|
+
evidenceBase,
|
|
3318
|
+
{ sameRunEventReference: true, eventIds: parsed.eventIds },
|
|
3319
|
+
{ name: outcome.name, missing }
|
|
3320
|
+
)
|
|
3321
|
+
);
|
|
3322
|
+
}
|
|
3323
|
+
}
|
|
3324
|
+
}
|
|
3325
|
+
return findings;
|
|
3326
|
+
}
|
|
2792
3327
|
function contractToRules(contract) {
|
|
2793
3328
|
const rules = [];
|
|
2794
3329
|
const allowedStatuses = contract.run?.allowedStatuses ?? [];
|
|
@@ -2936,9 +3471,71 @@ function contractToRules(contract) {
|
|
|
2936
3471
|
})
|
|
2937
3472
|
);
|
|
2938
3473
|
}
|
|
3474
|
+
const requireProvenance = contract.observations.requireProvenance;
|
|
3475
|
+
if (requireProvenance && (requireProvenance.method || requireProvenance.evidence || requireProvenance.sameRunEventReference)) {
|
|
3476
|
+
rules.push({
|
|
3477
|
+
id: "contract.observation.provenance",
|
|
3478
|
+
category: "run",
|
|
3479
|
+
defaultSeverity: "error",
|
|
3480
|
+
evaluate(context) {
|
|
3481
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
3482
|
+
const knownEventIds = new Set(context.events.map((event) => event.eventId));
|
|
3483
|
+
const targets = required.length > 0 ? outcomes.filter((outcome) => required.includes(outcome.name)) : outcomes;
|
|
3484
|
+
return targets.flatMap(
|
|
3485
|
+
(outcome) => provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds)
|
|
3486
|
+
);
|
|
3487
|
+
}
|
|
3488
|
+
});
|
|
3489
|
+
}
|
|
2939
3490
|
}
|
|
2940
3491
|
return rules;
|
|
2941
3492
|
}
|
|
3493
|
+
function validateScopeShape(scope) {
|
|
3494
|
+
if (scope === void 0) return [];
|
|
3495
|
+
const hasSelector = typeof scope.runId === "string" && scope.runId.trim() !== "" || typeof scope.subAgentId === "string" && scope.subAgentId.trim() !== "" || typeof scope.groupId === "string" && scope.groupId.trim() !== "" || typeof scope.workflowStep === "string" && scope.workflowStep.trim() !== "" || typeof scope.rootEventId === "string" && scope.rootEventId.trim() !== "";
|
|
3496
|
+
if (!hasSelector) {
|
|
3497
|
+
return [
|
|
3498
|
+
{
|
|
3499
|
+
code: "contract.scope.empty",
|
|
3500
|
+
severity: "error",
|
|
3501
|
+
message: "scope requires at least one explicit selector (runId, subAgentId, groupId, workflowStep, or rootEventId).",
|
|
3502
|
+
path: "scope"
|
|
3503
|
+
}
|
|
3504
|
+
];
|
|
3505
|
+
}
|
|
3506
|
+
return [];
|
|
3507
|
+
}
|
|
3508
|
+
function validateProvenanceShape(observations) {
|
|
3509
|
+
const diagnostics = [];
|
|
3510
|
+
const provenance = observations?.requireProvenance;
|
|
3511
|
+
if (!provenance) return diagnostics;
|
|
3512
|
+
const enabled = provenance.method === true || provenance.evidence === true || provenance.sameRunEventReference === true;
|
|
3513
|
+
if (!enabled) {
|
|
3514
|
+
diagnostics.push({
|
|
3515
|
+
code: "contract.observation.provenance.empty",
|
|
3516
|
+
severity: "warning",
|
|
3517
|
+
message: "observations.requireProvenance is set but no flags are enabled.",
|
|
3518
|
+
path: "observations.requireProvenance"
|
|
3519
|
+
});
|
|
3520
|
+
}
|
|
3521
|
+
if (provenance.sameRunEventReference === true && provenance.evidence !== true) {
|
|
3522
|
+
diagnostics.push({
|
|
3523
|
+
code: "contract.observation.provenance.same-run-without-evidence",
|
|
3524
|
+
severity: "info",
|
|
3525
|
+
message: "sameRunEventReference implies evidence checking; set requireProvenance.evidence: true for clarity.",
|
|
3526
|
+
path: "observations.requireProvenance.sameRunEventReference"
|
|
3527
|
+
});
|
|
3528
|
+
}
|
|
3529
|
+
if (enabled && (observations?.required?.length ?? 0) === 0) {
|
|
3530
|
+
diagnostics.push({
|
|
3531
|
+
code: "contract.observation.provenance.without-required",
|
|
3532
|
+
severity: "warning",
|
|
3533
|
+
message: "requireProvenance without observations.required applies to every observed outcome in scope.",
|
|
3534
|
+
path: "observations.requireProvenance"
|
|
3535
|
+
});
|
|
3536
|
+
}
|
|
3537
|
+
return diagnostics;
|
|
3538
|
+
}
|
|
2942
3539
|
function validateAlternativesShape(alternatives) {
|
|
2943
3540
|
const diagnostics = [];
|
|
2944
3541
|
if (alternatives === void 0) return diagnostics;
|
|
@@ -3045,9 +3642,11 @@ function evaluateBody(input, body, options = {}) {
|
|
|
3045
3642
|
});
|
|
3046
3643
|
}
|
|
3047
3644
|
function defineTraceContract(input) {
|
|
3048
|
-
const shapeErrors =
|
|
3049
|
-
(
|
|
3050
|
-
|
|
3645
|
+
const shapeErrors = [
|
|
3646
|
+
...validateAlternativesShape(input.alternatives),
|
|
3647
|
+
...validateScopeShape(input.scope),
|
|
3648
|
+
...validateProvenanceShape(input.observations)
|
|
3649
|
+
].filter((item) => item.severity === "error");
|
|
3051
3650
|
if (shapeErrors.length > 0) {
|
|
3052
3651
|
throw new TypeError(shapeErrors.map((item) => item.message).join(" "));
|
|
3053
3652
|
}
|
|
@@ -3059,15 +3658,25 @@ function defineTraceContract(input) {
|
|
|
3059
3658
|
contract: cloneBody(branch.contract)
|
|
3060
3659
|
}))
|
|
3061
3660
|
} : void 0;
|
|
3661
|
+
const scope = input.scope ? {
|
|
3662
|
+
...input.scope.runId !== void 0 ? { runId: input.scope.runId } : {},
|
|
3663
|
+
...input.scope.subAgentId !== void 0 ? { subAgentId: input.scope.subAgentId } : {},
|
|
3664
|
+
...input.scope.groupId !== void 0 ? { groupId: input.scope.groupId } : {},
|
|
3665
|
+
...input.scope.workflowStep !== void 0 ? { workflowStep: input.scope.workflowStep } : {},
|
|
3666
|
+
...input.scope.rootEventId !== void 0 ? { rootEventId: input.scope.rootEventId } : {}
|
|
3667
|
+
} : void 0;
|
|
3062
3668
|
return {
|
|
3063
3669
|
...body,
|
|
3670
|
+
...scope ? { scope } : {},
|
|
3064
3671
|
...alternatives ? { alternatives } : {}
|
|
3065
3672
|
};
|
|
3066
3673
|
}
|
|
3067
3674
|
function evaluateTraceContract(input, contract, options = {}) {
|
|
3068
|
-
const shapeErrors =
|
|
3069
|
-
(
|
|
3070
|
-
|
|
3675
|
+
const shapeErrors = [
|
|
3676
|
+
...validateAlternativesShape(contract.alternatives),
|
|
3677
|
+
...validateScopeShape(contract.scope),
|
|
3678
|
+
...validateProvenanceShape(contract.observations)
|
|
3679
|
+
].filter((item) => item.severity === "error");
|
|
3071
3680
|
if (shapeErrors.length > 0) {
|
|
3072
3681
|
return {
|
|
3073
3682
|
ok: false,
|
|
@@ -3091,13 +3700,58 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
3091
3700
|
ruleExecutions: []
|
|
3092
3701
|
};
|
|
3093
3702
|
}
|
|
3094
|
-
const
|
|
3703
|
+
const scoped = resolveTraceContractScope(input, contract.scope);
|
|
3704
|
+
if (scoped.diagnostics.length > 0) {
|
|
3705
|
+
return {
|
|
3706
|
+
ok: false,
|
|
3707
|
+
status: "error",
|
|
3708
|
+
format: input.read.format,
|
|
3709
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
3710
|
+
...scoped.resolved?.runId !== void 0 ? { runId: scoped.resolved.runId } : {},
|
|
3711
|
+
summary: {
|
|
3712
|
+
passed: 0,
|
|
3713
|
+
failed: 0,
|
|
3714
|
+
warnings: 0,
|
|
3715
|
+
errors: scoped.diagnostics.length,
|
|
3716
|
+
rulesEvaluated: 0
|
|
3717
|
+
},
|
|
3718
|
+
findings: [],
|
|
3719
|
+
diagnostics: scoped.diagnostics,
|
|
3720
|
+
ruleExecutions: []
|
|
3721
|
+
};
|
|
3722
|
+
}
|
|
3723
|
+
const effectiveInput = scoped.resolved?.input ?? input;
|
|
3724
|
+
const effectiveRunId = scoped.resolved?.runId ?? options.runId;
|
|
3725
|
+
const base = evaluateBody(effectiveInput, contract, {
|
|
3726
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
3727
|
+
});
|
|
3728
|
+
const scopeFinding = scoped.resolved ? {
|
|
3729
|
+
ruleId: "contract.scope.selected",
|
|
3730
|
+
severity: "info",
|
|
3731
|
+
status: "pass",
|
|
3732
|
+
message: `Selected actor ${scoped.resolved.runId} with ${scoped.resolved.evidenceEventCount} evidence event(s).`,
|
|
3733
|
+
expected: scoped.resolved.matchedSelectors,
|
|
3734
|
+
actual: {
|
|
3735
|
+
runId: scoped.resolved.runId,
|
|
3736
|
+
evidenceEventCount: scoped.resolved.evidenceEventCount,
|
|
3737
|
+
...scoped.resolved.rootEventId !== void 0 ? { rootEventId: scoped.resolved.rootEventId } : {}
|
|
3738
|
+
},
|
|
3739
|
+
evidence: [
|
|
3740
|
+
{
|
|
3741
|
+
runId: scoped.resolved.runId,
|
|
3742
|
+
kind: "RUN",
|
|
3743
|
+
path: "scope"
|
|
3744
|
+
}
|
|
3745
|
+
]
|
|
3746
|
+
} : void 0;
|
|
3095
3747
|
const branches = contract.alternatives?.anyOf ?? [];
|
|
3096
3748
|
if (branches.length === 0) {
|
|
3097
|
-
return base;
|
|
3749
|
+
return scopeFinding ? mergeContractResults(effectiveInput, [base], [scopeFinding]) : base;
|
|
3098
3750
|
}
|
|
3099
3751
|
const branchEvaluations = branches.map((branch) => {
|
|
3100
|
-
const result = evaluateBody(
|
|
3752
|
+
const result = evaluateBody(effectiveInput, branch.contract, {
|
|
3753
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
3754
|
+
});
|
|
3101
3755
|
return { branch, result };
|
|
3102
3756
|
});
|
|
3103
3757
|
const satisfied = branchEvaluations.filter((item) => item.result.ok && item.result.status === "pass").map((item) => item.branch.id);
|
|
@@ -3117,7 +3771,11 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
3117
3771
|
},
|
|
3118
3772
|
evidence: []
|
|
3119
3773
|
};
|
|
3120
|
-
return mergeContractResults(
|
|
3774
|
+
return mergeContractResults(
|
|
3775
|
+
effectiveInput,
|
|
3776
|
+
[base],
|
|
3777
|
+
[...scopeFinding ? [scopeFinding] : [], passFinding]
|
|
3778
|
+
);
|
|
3121
3779
|
}
|
|
3122
3780
|
const noneFinding = {
|
|
3123
3781
|
ruleId: "contract.alternatives.none-satisfied",
|
|
@@ -3144,13 +3802,21 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
3144
3802
|
ruleId: `contract.alternatives.${item.branch.id}.${finding.ruleId}`
|
|
3145
3803
|
}))
|
|
3146
3804
|
);
|
|
3147
|
-
return mergeContractResults(
|
|
3805
|
+
return mergeContractResults(
|
|
3806
|
+
effectiveInput,
|
|
3807
|
+
[base],
|
|
3808
|
+
[...scopeFinding ? [scopeFinding] : [], noneFinding, ...prefixedBranchFindings]
|
|
3809
|
+
);
|
|
3148
3810
|
}
|
|
3149
3811
|
function evaluateTraceContractRead(read, contract, options = {}) {
|
|
3150
3812
|
return evaluateTraceContract({ read }, contract, options);
|
|
3151
3813
|
}
|
|
3152
3814
|
function lintTraceContract(contract) {
|
|
3153
|
-
const diagnostics =
|
|
3815
|
+
const diagnostics = [
|
|
3816
|
+
...validateAlternativesShape(contract.alternatives),
|
|
3817
|
+
...validateScopeShape(contract.scope),
|
|
3818
|
+
...validateProvenanceShape(contract.observations)
|
|
3819
|
+
];
|
|
3154
3820
|
const branches = contract.alternatives?.anyOf ?? [];
|
|
3155
3821
|
if (branches.length === 1) {
|
|
3156
3822
|
diagnostics.push({
|
|
@@ -3185,6 +3851,12 @@ function lintTraceContract(contract) {
|
|
|
3185
3851
|
}
|
|
3186
3852
|
function explainTraceContract(contract) {
|
|
3187
3853
|
const lines = [];
|
|
3854
|
+
if (contract.scope) {
|
|
3855
|
+
const parts = Object.entries(contract.scope).filter(([, value]) => typeof value === "string" && value.trim() !== "").map(([key, value]) => `${key}=${value}`);
|
|
3856
|
+
if (parts.length > 0) {
|
|
3857
|
+
lines.push(`Scope: select actor by [${parts.join(", ")}] before evaluation.`);
|
|
3858
|
+
}
|
|
3859
|
+
}
|
|
3188
3860
|
if (contract.run?.requireCompleted !== false) {
|
|
3189
3861
|
lines.push("Base: run must complete (no running events).");
|
|
3190
3862
|
}
|
|
@@ -3236,6 +3908,18 @@ function explainTraceContract(contract) {
|
|
|
3236
3908
|
if (contract.observations?.failOn?.length) {
|
|
3237
3909
|
lines.push(`Base: fail on observations [${contract.observations.failOn.join(", ")}].`);
|
|
3238
3910
|
}
|
|
3911
|
+
if (contract.observations?.requireProvenance) {
|
|
3912
|
+
const flags = [
|
|
3913
|
+
contract.observations.requireProvenance.method ? "method" : void 0,
|
|
3914
|
+
contract.observations.requireProvenance.evidence ? "evidence" : void 0,
|
|
3915
|
+
contract.observations.requireProvenance.sameRunEventReference ? "sameRunEventReference" : void 0
|
|
3916
|
+
].filter((item) => item !== void 0);
|
|
3917
|
+
if (flags.length > 0) {
|
|
3918
|
+
lines.push(
|
|
3919
|
+
`Base: require structural observation provenance [${flags.join(", ")}] (not semantic truth).`
|
|
3920
|
+
);
|
|
3921
|
+
}
|
|
3922
|
+
}
|
|
3239
3923
|
const branches = contract.alternatives?.anyOf ?? [];
|
|
3240
3924
|
if (branches.length > 0) {
|
|
3241
3925
|
lines.push(
|
|
@@ -3252,6 +3936,6 @@ function explainTraceContract(contract) {
|
|
|
3252
3936
|
return lines;
|
|
3253
3937
|
}
|
|
3254
3938
|
|
|
3255
|
-
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, runTraceChecks, summarizeSemanticParity };
|
|
3256
|
-
//# sourceMappingURL=chunk-
|
|
3257
|
-
//# sourceMappingURL=chunk-
|
|
3939
|
+
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractSessionWorkflowMetadata, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveTraceContractScope, runTraceChecks, sessionKeyForRun, summarizeSemanticParity, workflowMetadataForRun };
|
|
3940
|
+
//# sourceMappingURL=chunk-OG3GAWCX.mjs.map
|
|
3941
|
+
//# sourceMappingURL=chunk-OG3GAWCX.mjs.map
|