agent-inspect 6.20.0 → 6.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/docs/TRACE-CONTRACTS.md +39 -0
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-QPRJUVLH.mjs → chunk-CHBYF4LE.mjs} +25 -2
- package/packages/cli/dist/chunk-CHBYF4LE.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +50 -3
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +4 -4
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-6W5ZR3I7.mjs → src-MY5WIBPF.mjs} +3 -3
- package/packages/cli/dist/{src-6W5ZR3I7.mjs.map → src-MY5WIBPF.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +41 -0
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +39 -157
- package/packages/core/dist/advanced.d.ts +39 -157
- package/packages/core/dist/advanced.mjs +35 -61
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +736 -45
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +132 -5
- package/packages/core/dist/checks.d.ts +132 -5
- package/packages/core/dist/checks.mjs +2 -2
- package/packages/core/dist/{chunk-HT54OLPE.mjs → chunk-NDRMIL7Z.mjs} +103 -3
- package/packages/core/dist/chunk-NDRMIL7Z.mjs.map +1 -0
- package/packages/core/dist/{chunk-G47KQB42.mjs → chunk-OG3GAWCX.mjs} +727 -43
- package/packages/core/dist/chunk-OG3GAWCX.mjs.map +1 -0
- package/packages/core/dist/{chunk-3XS5O4JI.mjs → chunk-R4SSYU6D.mjs} +3 -3
- package/packages/core/dist/chunk-R4SSYU6D.mjs.map +1 -0
- package/packages/core/dist/{chunk-5YFJJFXU.mjs → chunk-WGDSM37E.mjs} +3 -3
- package/packages/core/dist/{chunk-5YFJJFXU.mjs.map → chunk-WGDSM37E.mjs.map} +1 -1
- package/packages/core/dist/{context-C5ye_xcS.d.cts → context-DOOT9GZ-.d.cts} +1 -1
- package/packages/core/dist/{context-CNsNSgE2.d.ts → context-DOX2-RjH.d.ts} +1 -1
- package/packages/core/dist/diff.d.cts +1 -1
- package/packages/core/dist/diff.d.ts +1 -1
- package/packages/core/dist/exporters.d.cts +1 -1
- package/packages/core/dist/exporters.d.ts +1 -1
- package/packages/core/dist/exporters.mjs +1 -1
- package/packages/core/dist/{index-rVpX5PPh.d.cts → index-CR4CCRqP.d.cts} +159 -2
- package/packages/core/dist/{index-Bg9ZdsdS.d.ts → index-CtK_p-ps.d.ts} +159 -2
- package/packages/core/dist/{index-AMAGuO_i.d.ts → index-D72DPqAF.d.ts} +2 -1
- package/packages/core/dist/{index-vqr-g6ev.d.cts → index-DLLz9sce.d.cts} +2 -1
- package/packages/core/dist/index.d.cts +3 -3
- package/packages/core/dist/index.d.ts +3 -3
- package/packages/core/dist/index.mjs +3 -3
- package/packages/core/dist/persisted.d.cts +1 -1
- package/packages/core/dist/persisted.d.ts +1 -1
- package/packages/core/dist/programmatic-gZTWLObB.d.cts +25 -0
- package/packages/core/dist/programmatic-gZTWLObB.d.ts +25 -0
- package/packages/core/dist/readers.cjs +105 -0
- package/packages/core/dist/readers.cjs.map +1 -1
- package/packages/core/dist/readers.d.cts +43 -18
- package/packages/core/dist/readers.d.ts +43 -18
- package/packages/core/dist/readers.mjs +1 -1
- package/packages/core/dist/{types--ainI31J.d.ts → types-Bx3mEZ04.d.ts} +1 -1
- package/packages/core/dist/{types-UnrNaTo2.d.cts → types-VzKihKQo.d.cts} +1 -1
- package/packages/cli/dist/chunk-QPRJUVLH.mjs.map +0 -1
- package/packages/core/dist/chunk-3XS5O4JI.mjs.map +0 -1
- package/packages/core/dist/chunk-G47KQB42.mjs.map +0 -1
- package/packages/core/dist/chunk-HT54OLPE.mjs.map +0 -1
|
@@ -64,6 +64,17 @@ function formatProgrammaticDiagnostic(code, detail) {
|
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
// packages/core/src/outcomes/types.ts
|
|
67
|
+
var OBSERVED_OUTCOME_METHODS = [
|
|
68
|
+
"dom",
|
|
69
|
+
"accessibility",
|
|
70
|
+
"snapshot",
|
|
71
|
+
"network",
|
|
72
|
+
"storage",
|
|
73
|
+
"filesystem",
|
|
74
|
+
"database",
|
|
75
|
+
"queue",
|
|
76
|
+
"custom"
|
|
77
|
+
];
|
|
67
78
|
var OUTCOME_ATTRIBUTE_STATUS_KEY = "outcomeStatus";
|
|
68
79
|
var OUTCOME_ATTRIBUTE_EXPECTATION_KEY = "expectation";
|
|
69
80
|
var OUTCOME_ATTRIBUTE_METHOD_KEY = "method";
|
|
@@ -85,11 +96,11 @@ function parseMethod(value) {
|
|
|
85
96
|
}
|
|
86
97
|
function fromPersistedOutcome(event) {
|
|
87
98
|
if (event.kind !== "OUTCOME") return void 0;
|
|
88
|
-
const
|
|
89
|
-
const statusRaw =
|
|
99
|
+
const attrs2 = event.attributes ?? {};
|
|
100
|
+
const statusRaw = attrs2[OUTCOME_ATTRIBUTE_STATUS_KEY];
|
|
90
101
|
if (!isOutcomeStatus(statusRaw)) return void 0;
|
|
91
|
-
const expectation = typeof
|
|
92
|
-
const observedAtRaw =
|
|
102
|
+
const expectation = typeof attrs2[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] === "string" ? attrs2[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] : event.name;
|
|
103
|
+
const observedAtRaw = attrs2[OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY];
|
|
93
104
|
const observedAt = typeof observedAtRaw === "string" ? Date.parse(observedAtRaw) : typeof observedAtRaw === "number" && Number.isFinite(observedAtRaw) ? observedAtRaw : Date.parse(event.timestamp);
|
|
94
105
|
return {
|
|
95
106
|
outcomeId: event.eventId,
|
|
@@ -98,9 +109,9 @@ function fromPersistedOutcome(event) {
|
|
|
98
109
|
name: event.name,
|
|
99
110
|
expectation,
|
|
100
111
|
status: statusRaw,
|
|
101
|
-
...parseMethod(
|
|
112
|
+
...parseMethod(attrs2[OUTCOME_ATTRIBUTE_METHOD_KEY]) !== void 0 ? { method: parseMethod(attrs2[OUTCOME_ATTRIBUTE_METHOD_KEY]) } : {},
|
|
102
113
|
...event.outputSummary !== void 0 ? { actual: event.outputSummary } : {},
|
|
103
|
-
...
|
|
114
|
+
...attrs2.evidence !== void 0 ? { evidence: attrs2.evidence } : {},
|
|
104
115
|
observedAt: Number.isFinite(observedAt) ? observedAt : Date.parse(event.timestamp)
|
|
105
116
|
};
|
|
106
117
|
}
|
|
@@ -466,10 +477,10 @@ function projectLogicalEvents(events) {
|
|
|
466
477
|
};
|
|
467
478
|
}
|
|
468
479
|
function resolveCanonicalToolName(event) {
|
|
469
|
-
const
|
|
470
|
-
const direct = pickString(
|
|
480
|
+
const attrs2 = event.attributes;
|
|
481
|
+
const direct = pickString(attrs2, ["toolName", "tool"]);
|
|
471
482
|
if (direct) return direct;
|
|
472
|
-
const metadata =
|
|
483
|
+
const metadata = attrs2?.metadata;
|
|
473
484
|
if (isRecord(metadata)) {
|
|
474
485
|
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
475
486
|
if (nested) return nested;
|
|
@@ -511,10 +522,10 @@ function pickNumber(record, keys) {
|
|
|
511
522
|
return void 0;
|
|
512
523
|
}
|
|
513
524
|
function eventMetadata(event) {
|
|
514
|
-
const
|
|
515
|
-
const nested =
|
|
525
|
+
const attrs2 = isRecord2(event.attributes) ? event.attributes : void 0;
|
|
526
|
+
const nested = attrs2 !== void 0 && isRecord2(attrs2.metadata) ? attrs2.metadata : void 0;
|
|
516
527
|
return {
|
|
517
|
-
...
|
|
528
|
+
...attrs2 ?? {},
|
|
518
529
|
...nested ?? {}
|
|
519
530
|
};
|
|
520
531
|
}
|
|
@@ -807,6 +818,208 @@ function deriveFailureFacts(logicalEvents) {
|
|
|
807
818
|
};
|
|
808
819
|
}
|
|
809
820
|
|
|
821
|
+
// packages/core/src/sessions/metadata.ts
|
|
822
|
+
function isNonEmptyString(value) {
|
|
823
|
+
return typeof value === "string" && value.trim() !== "";
|
|
824
|
+
}
|
|
825
|
+
function finitePositiveInt(value) {
|
|
826
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
827
|
+
return void 0;
|
|
828
|
+
}
|
|
829
|
+
return Math.trunc(value);
|
|
830
|
+
}
|
|
831
|
+
function extractSessionWorkflowMetadata(record) {
|
|
832
|
+
if (!record) return void 0;
|
|
833
|
+
const out = {};
|
|
834
|
+
let found = false;
|
|
835
|
+
const assignString = (key, value) => {
|
|
836
|
+
if (isNonEmptyString(value)) {
|
|
837
|
+
out[key] = value.trim();
|
|
838
|
+
found = true;
|
|
839
|
+
}
|
|
840
|
+
};
|
|
841
|
+
assignString("sessionId", record.sessionId);
|
|
842
|
+
assignString("conversationId", record.conversationId);
|
|
843
|
+
assignString("groupId", record.groupId);
|
|
844
|
+
assignString("parentGroupId", record.parentGroupId);
|
|
845
|
+
assignString("retryOf", record.retryOf);
|
|
846
|
+
assignString("retryReason", record.retryReason);
|
|
847
|
+
assignString("handoffFrom", record.handoffFrom);
|
|
848
|
+
assignString("handoffTo", record.handoffTo);
|
|
849
|
+
assignString("subAgentId", record.subAgentId);
|
|
850
|
+
assignString("subAgentName", record.subAgentName);
|
|
851
|
+
assignString("jobId", record.jobId);
|
|
852
|
+
assignString("queueName", record.queueName);
|
|
853
|
+
assignString("workflowName", record.workflowName);
|
|
854
|
+
assignString("workflowStep", record.workflowStep);
|
|
855
|
+
assignString("toolCallId", record.toolCallId);
|
|
856
|
+
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
857
|
+
assignString("linkedStepId", record.linkedStepId);
|
|
858
|
+
assignString("operationId", record.operationId);
|
|
859
|
+
assignString("attemptId", record.attemptId);
|
|
860
|
+
assignString("fallbackOf", record.fallbackOf);
|
|
861
|
+
assignString("idempotencyKey", record.idempotencyKey);
|
|
862
|
+
assignString("correlationId", record.correlationId);
|
|
863
|
+
assignString("requestId", record.requestId);
|
|
864
|
+
assignString("decisionId", record.decisionId);
|
|
865
|
+
const attempt = finitePositiveInt(record.attempt);
|
|
866
|
+
if (attempt !== void 0) {
|
|
867
|
+
out.attempt = attempt;
|
|
868
|
+
found = true;
|
|
869
|
+
}
|
|
870
|
+
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
871
|
+
if (attemptNumber !== void 0) {
|
|
872
|
+
out.attemptNumber = attemptNumber;
|
|
873
|
+
found = true;
|
|
874
|
+
}
|
|
875
|
+
return found ? out : void 0;
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
// packages/core/src/checks/relationship-facts.ts
|
|
879
|
+
function attrs(event) {
|
|
880
|
+
return event.attributes && typeof event.attributes === "object" ? event.attributes : {};
|
|
881
|
+
}
|
|
882
|
+
function stringAttr(record, key) {
|
|
883
|
+
const value = record[key];
|
|
884
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
885
|
+
}
|
|
886
|
+
function deriveRelationshipFacts(events) {
|
|
887
|
+
const relationships = [];
|
|
888
|
+
const diagnostics = [];
|
|
889
|
+
const byId = new Map(events.map((event) => [event.eventId, event]));
|
|
890
|
+
const seen = /* @__PURE__ */ new Set();
|
|
891
|
+
const push = (edge) => {
|
|
892
|
+
const key = `${edge.type}:${edge.fromEventId}:${edge.toEventId ?? ""}:${edge.externalRef ?? ""}`;
|
|
893
|
+
if (seen.has(key)) return;
|
|
894
|
+
seen.add(key);
|
|
895
|
+
relationships.push(edge);
|
|
896
|
+
};
|
|
897
|
+
for (const event of events) {
|
|
898
|
+
if (event.parentId) {
|
|
899
|
+
push({
|
|
900
|
+
type: "parent-child",
|
|
901
|
+
fromEventId: event.parentId,
|
|
902
|
+
toEventId: event.eventId,
|
|
903
|
+
confidence: byId.has(event.parentId) ? "explicit" : "unknown",
|
|
904
|
+
basis: ["persisted.parentId"]
|
|
905
|
+
});
|
|
906
|
+
if (!byId.has(event.parentId)) {
|
|
907
|
+
diagnostics.push({
|
|
908
|
+
code: "AI_RELATIONSHIP_PARENT_MISSING",
|
|
909
|
+
message: `parentId ${event.parentId} is not present in the event set.`,
|
|
910
|
+
eventId: event.eventId
|
|
911
|
+
});
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
const bag = attrs(event);
|
|
915
|
+
const meta = extractSessionWorkflowMetadata(bag) ?? {};
|
|
916
|
+
const nested = bag.metadata && typeof bag.metadata === "object" ? extractSessionWorkflowMetadata(bag.metadata) : void 0;
|
|
917
|
+
const workflow = { ...meta, ...nested };
|
|
918
|
+
if (workflow.retryOf) {
|
|
919
|
+
const target = events.find((candidate) => candidate.runId === workflow.retryOf);
|
|
920
|
+
push({
|
|
921
|
+
type: "retry-of",
|
|
922
|
+
fromEventId: event.eventId,
|
|
923
|
+
...target ? { toEventId: target.eventId } : {},
|
|
924
|
+
externalRef: workflow.retryOf,
|
|
925
|
+
confidence: target ? "explicit" : "correlated",
|
|
926
|
+
basis: ["attributes.retryOf"]
|
|
927
|
+
});
|
|
928
|
+
}
|
|
929
|
+
const attemptOf = stringAttr(bag, "attemptOf") ?? stringAttr(bag, "operationId");
|
|
930
|
+
if (attemptOf && workflow.attempt !== void 0) {
|
|
931
|
+
push({
|
|
932
|
+
type: "attempt-of",
|
|
933
|
+
fromEventId: event.eventId,
|
|
934
|
+
externalRef: attemptOf,
|
|
935
|
+
confidence: "explicit",
|
|
936
|
+
basis: workflow.attempt !== void 0 ? ["attributes.attempt", "attributes.operationId"] : ["attributes.operationId"]
|
|
937
|
+
});
|
|
938
|
+
}
|
|
939
|
+
const fallbackOf = stringAttr(bag, "fallbackOf");
|
|
940
|
+
if (fallbackOf) {
|
|
941
|
+
push({
|
|
942
|
+
type: "fallback-of",
|
|
943
|
+
fromEventId: event.eventId,
|
|
944
|
+
externalRef: fallbackOf,
|
|
945
|
+
confidence: "explicit",
|
|
946
|
+
basis: ["attributes.fallbackOf"]
|
|
947
|
+
});
|
|
948
|
+
}
|
|
949
|
+
const remediationOf = stringAttr(bag, "remediationOf");
|
|
950
|
+
if (remediationOf) {
|
|
951
|
+
push({
|
|
952
|
+
type: "remediation-of",
|
|
953
|
+
fromEventId: event.eventId,
|
|
954
|
+
externalRef: remediationOf,
|
|
955
|
+
confidence: "explicit",
|
|
956
|
+
basis: ["attributes.remediationOf"]
|
|
957
|
+
});
|
|
958
|
+
}
|
|
959
|
+
const evidenceFor = stringAttr(bag, "evidenceFor");
|
|
960
|
+
if (evidenceFor) {
|
|
961
|
+
push({
|
|
962
|
+
type: "evidence-for",
|
|
963
|
+
fromEventId: event.eventId,
|
|
964
|
+
...byId.has(evidenceFor) ? { toEventId: evidenceFor } : { externalRef: evidenceFor },
|
|
965
|
+
confidence: byId.has(evidenceFor) ? "explicit" : "correlated",
|
|
966
|
+
basis: ["attributes.evidenceFor"]
|
|
967
|
+
});
|
|
968
|
+
}
|
|
969
|
+
const acceptedBy = stringAttr(bag, "acceptedBy");
|
|
970
|
+
if (acceptedBy) {
|
|
971
|
+
push({
|
|
972
|
+
type: "accepted-by",
|
|
973
|
+
fromEventId: event.eventId,
|
|
974
|
+
...byId.has(acceptedBy) ? { toEventId: acceptedBy } : { externalRef: acceptedBy },
|
|
975
|
+
confidence: byId.has(acceptedBy) ? "explicit" : "correlated",
|
|
976
|
+
basis: ["attributes.acceptedBy"]
|
|
977
|
+
});
|
|
978
|
+
}
|
|
979
|
+
const supersedes = stringAttr(bag, "supersedes");
|
|
980
|
+
if (supersedes) {
|
|
981
|
+
push({
|
|
982
|
+
type: "supersedes",
|
|
983
|
+
fromEventId: event.eventId,
|
|
984
|
+
...byId.has(supersedes) ? { toEventId: supersedes } : { externalRef: supersedes },
|
|
985
|
+
confidence: byId.has(supersedes) ? "explicit" : "correlated",
|
|
986
|
+
basis: ["attributes.supersedes"]
|
|
987
|
+
});
|
|
988
|
+
}
|
|
989
|
+
const sourceLineage = stringAttr(bag, "sourceLineage") ?? stringAttr(bag, "sourceEventId");
|
|
990
|
+
if (sourceLineage) {
|
|
991
|
+
push({
|
|
992
|
+
type: "source-lineage",
|
|
993
|
+
fromEventId: event.eventId,
|
|
994
|
+
externalRef: sourceLineage,
|
|
995
|
+
confidence: "explicit",
|
|
996
|
+
basis: ["attributes.sourceLineage"]
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
const unsupported = stringAttr(bag, "relationshipType");
|
|
1000
|
+
if (unsupported && unsupported !== "parent-child" && ![
|
|
1001
|
+
"source-lineage",
|
|
1002
|
+
"attempt-of",
|
|
1003
|
+
"retry-of",
|
|
1004
|
+
"fallback-of",
|
|
1005
|
+
"remediation-of",
|
|
1006
|
+
"evidence-for",
|
|
1007
|
+
"accepted-by",
|
|
1008
|
+
"supersedes"
|
|
1009
|
+
].includes(unsupported)) {
|
|
1010
|
+
diagnostics.push({
|
|
1011
|
+
code: "AI_RELATIONSHIP_UNSUPPORTED_TYPE",
|
|
1012
|
+
message: `Unsupported relationshipType ${unsupported} was reported without flattening.`,
|
|
1013
|
+
eventId: event.eventId
|
|
1014
|
+
});
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
return {
|
|
1018
|
+
relationships: Object.freeze(relationships),
|
|
1019
|
+
diagnostics: Object.freeze(diagnostics)
|
|
1020
|
+
};
|
|
1021
|
+
}
|
|
1022
|
+
|
|
810
1023
|
// packages/core/src/checks/trace-facts.ts
|
|
811
1024
|
function summarizeSemanticParity(events) {
|
|
812
1025
|
const projection = projectLogicalEvents(events);
|
|
@@ -893,6 +1106,7 @@ function buildTraceFacts(input) {
|
|
|
893
1106
|
toolsByName.set(name, Object.freeze([...list]));
|
|
894
1107
|
}
|
|
895
1108
|
const derived = deriveFailureFacts(projection.logicalEvents);
|
|
1109
|
+
const relationship = deriveRelationshipFacts(events);
|
|
896
1110
|
const summary = summarizeSemanticParity(events);
|
|
897
1111
|
return {
|
|
898
1112
|
rawEvents: Object.freeze([...events]),
|
|
@@ -906,7 +1120,203 @@ function buildTraceFacts(input) {
|
|
|
906
1120
|
failureRoleCounts: derived.failureRoleCounts
|
|
907
1121
|
},
|
|
908
1122
|
failureFacts: derived.failureFacts,
|
|
909
|
-
failuresByRole: derived.failuresByRole
|
|
1123
|
+
failuresByRole: derived.failuresByRole,
|
|
1124
|
+
relationships: relationship.relationships,
|
|
1125
|
+
relationshipDiagnostics: relationship.diagnostics
|
|
1126
|
+
};
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
// packages/core/src/checks/contract-scope.ts
|
|
1130
|
+
function diagnostic(code, message, ruleId) {
|
|
1131
|
+
return {
|
|
1132
|
+
code,
|
|
1133
|
+
message,
|
|
1134
|
+
severity: "error",
|
|
1135
|
+
...ruleId !== void 0 ? { ruleId } : {}
|
|
1136
|
+
};
|
|
1137
|
+
}
|
|
1138
|
+
function isNonEmptyString2(value) {
|
|
1139
|
+
return typeof value === "string" && value.trim() !== "";
|
|
1140
|
+
}
|
|
1141
|
+
function scopeEntries(scope) {
|
|
1142
|
+
const out = [];
|
|
1143
|
+
if (isNonEmptyString2(scope.runId)) out.push(["runId", scope.runId.trim()]);
|
|
1144
|
+
if (isNonEmptyString2(scope.subAgentId)) out.push(["subAgentId", scope.subAgentId.trim()]);
|
|
1145
|
+
if (isNonEmptyString2(scope.groupId)) out.push(["groupId", scope.groupId.trim()]);
|
|
1146
|
+
if (isNonEmptyString2(scope.workflowStep)) {
|
|
1147
|
+
out.push(["workflowStep", scope.workflowStep.trim()]);
|
|
1148
|
+
}
|
|
1149
|
+
if (isNonEmptyString2(scope.rootEventId)) {
|
|
1150
|
+
out.push(["rootEventId", scope.rootEventId.trim()]);
|
|
1151
|
+
}
|
|
1152
|
+
return out;
|
|
1153
|
+
}
|
|
1154
|
+
function mergeWorkflowMeta(into, from) {
|
|
1155
|
+
if (!from) return;
|
|
1156
|
+
for (const [key, value] of Object.entries(from)) {
|
|
1157
|
+
if (value === void 0) continue;
|
|
1158
|
+
if (into[key] === void 0) {
|
|
1159
|
+
into[key] = value;
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
function workflowMetadataForRun(events, runId) {
|
|
1164
|
+
const preferred = {};
|
|
1165
|
+
const fallback = {};
|
|
1166
|
+
for (const event of events) {
|
|
1167
|
+
if (event.runId !== runId) continue;
|
|
1168
|
+
const attrs2 = event.attributes;
|
|
1169
|
+
if (!attrs2 || typeof attrs2 !== "object") continue;
|
|
1170
|
+
const direct = extractSessionWorkflowMetadata(attrs2);
|
|
1171
|
+
const nested = attrs2.metadata && typeof attrs2.metadata === "object" ? extractSessionWorkflowMetadata(attrs2.metadata) : void 0;
|
|
1172
|
+
const bag = {};
|
|
1173
|
+
mergeWorkflowMeta(bag, direct);
|
|
1174
|
+
mergeWorkflowMeta(bag, nested);
|
|
1175
|
+
if (event.kind === "RUN") {
|
|
1176
|
+
mergeWorkflowMeta(preferred, bag);
|
|
1177
|
+
} else {
|
|
1178
|
+
mergeWorkflowMeta(fallback, bag);
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
mergeWorkflowMeta(preferred, fallback);
|
|
1182
|
+
return preferred;
|
|
1183
|
+
}
|
|
1184
|
+
function flattenNodes(nodes) {
|
|
1185
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
1186
|
+
}
|
|
1187
|
+
function collectSubtreeEventIds(run, rootEventId) {
|
|
1188
|
+
const nodes = flattenNodes(run.children);
|
|
1189
|
+
const byId = new Map(nodes.map((node) => [node.event.eventId, node]));
|
|
1190
|
+
const root = byId.get(rootEventId);
|
|
1191
|
+
if (!root) return void 0;
|
|
1192
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1193
|
+
const visit = (node) => {
|
|
1194
|
+
ids.add(node.event.eventId);
|
|
1195
|
+
for (const child of node.children) visit(child);
|
|
1196
|
+
};
|
|
1197
|
+
visit(root);
|
|
1198
|
+
return ids;
|
|
1199
|
+
}
|
|
1200
|
+
function filterTree(nodes, keep) {
|
|
1201
|
+
const out = [];
|
|
1202
|
+
for (const node of nodes) {
|
|
1203
|
+
if (!keep.has(node.event.eventId)) continue;
|
|
1204
|
+
out.push({
|
|
1205
|
+
...node,
|
|
1206
|
+
children: filterTree(node.children, keep)
|
|
1207
|
+
});
|
|
1208
|
+
}
|
|
1209
|
+
return out;
|
|
1210
|
+
}
|
|
1211
|
+
function projectInput(input, run, eventIds) {
|
|
1212
|
+
const runEvents = input.read.events.filter((event) => event.runId === run.runId);
|
|
1213
|
+
const projectedEvents = eventIds === void 0 ? runEvents : runEvents.filter((event) => eventIds.has(event.eventId) || event.kind === "RUN");
|
|
1214
|
+
const projectedChildren = eventIds === void 0 ? run.children : filterTree(run.children, eventIds);
|
|
1215
|
+
const projectedRun = {
|
|
1216
|
+
...run,
|
|
1217
|
+
children: projectedChildren,
|
|
1218
|
+
metadata: {
|
|
1219
|
+
...run.metadata,
|
|
1220
|
+
totalEvents: flattenNodes(projectedChildren).length
|
|
1221
|
+
}
|
|
1222
|
+
};
|
|
1223
|
+
return {
|
|
1224
|
+
evidenceEventCount: projectedEvents.length,
|
|
1225
|
+
input: {
|
|
1226
|
+
...input,
|
|
1227
|
+
selectedRun: projectedRun,
|
|
1228
|
+
read: {
|
|
1229
|
+
...input.read,
|
|
1230
|
+
runs: [projectedRun],
|
|
1231
|
+
events: projectedEvents
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
};
|
|
1235
|
+
}
|
|
1236
|
+
function resolveTraceContractScope(input, scope) {
|
|
1237
|
+
if (scope === void 0) {
|
|
1238
|
+
return { diagnostics: [] };
|
|
1239
|
+
}
|
|
1240
|
+
const entries = scopeEntries(scope);
|
|
1241
|
+
if (entries.length === 0) {
|
|
1242
|
+
return {
|
|
1243
|
+
diagnostics: [
|
|
1244
|
+
diagnostic(
|
|
1245
|
+
"AI_CHECK_INVALID_CONFIG",
|
|
1246
|
+
"TraceContract scope requires at least one explicit selector (runId, subAgentId, groupId, workflowStep, or rootEventId).",
|
|
1247
|
+
"contract.scope.empty"
|
|
1248
|
+
)
|
|
1249
|
+
]
|
|
1250
|
+
};
|
|
1251
|
+
}
|
|
1252
|
+
const matchedSelectors = Object.fromEntries(entries);
|
|
1253
|
+
const actorKeys = entries.filter(([key]) => key !== "rootEventId");
|
|
1254
|
+
const rootEventId = matchedSelectors.rootEventId;
|
|
1255
|
+
let candidates = [...input.read.runs];
|
|
1256
|
+
if (matchedSelectors.runId) {
|
|
1257
|
+
candidates = candidates.filter((run2) => run2.runId === matchedSelectors.runId);
|
|
1258
|
+
}
|
|
1259
|
+
for (const [key, value] of actorKeys) {
|
|
1260
|
+
if (key === "runId") continue;
|
|
1261
|
+
candidates = candidates.filter((run2) => {
|
|
1262
|
+
const meta = workflowMetadataForRun(input.read.events, run2.runId);
|
|
1263
|
+
const actual = meta[key];
|
|
1264
|
+
return actual === value;
|
|
1265
|
+
});
|
|
1266
|
+
}
|
|
1267
|
+
if (rootEventId && actorKeys.length === 0) {
|
|
1268
|
+
candidates = candidates.filter(
|
|
1269
|
+
(run2) => flattenNodes(run2.children).some((node) => node.event.eventId === rootEventId)
|
|
1270
|
+
);
|
|
1271
|
+
}
|
|
1272
|
+
if (candidates.length === 0) {
|
|
1273
|
+
return {
|
|
1274
|
+
diagnostics: [
|
|
1275
|
+
diagnostic(
|
|
1276
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
1277
|
+
`TraceContract scope matched zero actors for ${JSON.stringify(matchedSelectors)}. Missing actor metadata does not fall back to the whole session.`,
|
|
1278
|
+
"contract.scope.zero-match"
|
|
1279
|
+
)
|
|
1280
|
+
]
|
|
1281
|
+
};
|
|
1282
|
+
}
|
|
1283
|
+
if (candidates.length > 1) {
|
|
1284
|
+
return {
|
|
1285
|
+
diagnostics: [
|
|
1286
|
+
diagnostic(
|
|
1287
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
1288
|
+
`TraceContract scope matched ${candidates.length} actors for ${JSON.stringify(matchedSelectors)} (runIds: ${candidates.map((run2) => run2.runId).sort().join(", ")}). Singular actor selectors must resolve to exactly one run.`,
|
|
1289
|
+
"contract.scope.ambiguous"
|
|
1290
|
+
)
|
|
1291
|
+
]
|
|
1292
|
+
};
|
|
1293
|
+
}
|
|
1294
|
+
const run = candidates[0];
|
|
1295
|
+
let eventIds;
|
|
1296
|
+
if (rootEventId) {
|
|
1297
|
+
eventIds = collectSubtreeEventIds(run, rootEventId);
|
|
1298
|
+
if (eventIds === void 0) {
|
|
1299
|
+
return {
|
|
1300
|
+
diagnostics: [
|
|
1301
|
+
diagnostic(
|
|
1302
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
1303
|
+
`TraceContract scope rootEventId ${rootEventId} was not found in run ${run.runId}.`,
|
|
1304
|
+
"contract.scope.root-missing"
|
|
1305
|
+
)
|
|
1306
|
+
]
|
|
1307
|
+
};
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
const projected = projectInput(input, run, eventIds);
|
|
1311
|
+
return {
|
|
1312
|
+
resolved: {
|
|
1313
|
+
runId: run.runId,
|
|
1314
|
+
...rootEventId !== void 0 ? { rootEventId } : {},
|
|
1315
|
+
matchedSelectors,
|
|
1316
|
+
evidenceEventCount: projected.evidenceEventCount,
|
|
1317
|
+
input: projected.input
|
|
1318
|
+
},
|
|
1319
|
+
diagnostics: []
|
|
910
1320
|
};
|
|
911
1321
|
}
|
|
912
1322
|
|
|
@@ -933,12 +1343,141 @@ function cloneBody(body) {
|
|
|
933
1343
|
...body.run ? { run: { ...body.run } } : {},
|
|
934
1344
|
...body.tools ? { tools: { ...body.tools } } : {},
|
|
935
1345
|
...body.llm ? { llm: { ...body.llm } } : {},
|
|
936
|
-
...body.observations ? {
|
|
1346
|
+
...body.observations ? {
|
|
1347
|
+
observations: {
|
|
1348
|
+
...body.observations,
|
|
1349
|
+
...body.observations.requireProvenance ? { requireProvenance: { ...body.observations.requireProvenance } } : {}
|
|
1350
|
+
}
|
|
1351
|
+
} : {}
|
|
937
1352
|
};
|
|
938
1353
|
}
|
|
939
1354
|
function bodyHasRules(body) {
|
|
940
1355
|
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0;
|
|
941
1356
|
}
|
|
1357
|
+
var MAX_EVIDENCE_EVENT_IDS = 16;
|
|
1358
|
+
var METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
1359
|
+
function parseEvidenceEventIds(evidence) {
|
|
1360
|
+
if (evidence === void 0 || evidence === null) {
|
|
1361
|
+
return { status: "missing" };
|
|
1362
|
+
}
|
|
1363
|
+
if (typeof evidence === "string") {
|
|
1364
|
+
const trimmed = evidence.trim();
|
|
1365
|
+
if (trimmed === "") {
|
|
1366
|
+
return { status: "invalid", reason: "empty evidence string" };
|
|
1367
|
+
}
|
|
1368
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
1369
|
+
}
|
|
1370
|
+
if (typeof evidence !== "object" || Array.isArray(evidence)) {
|
|
1371
|
+
return {
|
|
1372
|
+
status: "invalid",
|
|
1373
|
+
reason: "evidence must be a string event id or { eventId } / { eventIds }"
|
|
1374
|
+
};
|
|
1375
|
+
}
|
|
1376
|
+
const record = evidence;
|
|
1377
|
+
if (typeof record.eventId === "string") {
|
|
1378
|
+
const trimmed = record.eventId.trim();
|
|
1379
|
+
if (trimmed === "") {
|
|
1380
|
+
return { status: "invalid", reason: "empty evidence.eventId" };
|
|
1381
|
+
}
|
|
1382
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
1383
|
+
}
|
|
1384
|
+
if (Array.isArray(record.eventIds)) {
|
|
1385
|
+
if (record.eventIds.length === 0) {
|
|
1386
|
+
return { status: "invalid", reason: "empty evidence.eventIds" };
|
|
1387
|
+
}
|
|
1388
|
+
if (record.eventIds.length > MAX_EVIDENCE_EVENT_IDS) {
|
|
1389
|
+
return {
|
|
1390
|
+
status: "invalid",
|
|
1391
|
+
reason: `evidence.eventIds exceeds bound of ${MAX_EVIDENCE_EVENT_IDS}`
|
|
1392
|
+
};
|
|
1393
|
+
}
|
|
1394
|
+
const eventIds = [];
|
|
1395
|
+
for (const item of record.eventIds) {
|
|
1396
|
+
if (typeof item !== "string" || item.trim() === "") {
|
|
1397
|
+
return { status: "invalid", reason: "evidence.eventIds contains a non-string id" };
|
|
1398
|
+
}
|
|
1399
|
+
eventIds.push(item.trim());
|
|
1400
|
+
}
|
|
1401
|
+
return { status: "ok", eventIds };
|
|
1402
|
+
}
|
|
1403
|
+
return {
|
|
1404
|
+
status: "invalid",
|
|
1405
|
+
reason: "evidence must include eventId or eventIds"
|
|
1406
|
+
};
|
|
1407
|
+
}
|
|
1408
|
+
function provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds) {
|
|
1409
|
+
const findings = [];
|
|
1410
|
+
const evidenceBase = [
|
|
1411
|
+
{
|
|
1412
|
+
runId: outcome.runId,
|
|
1413
|
+
eventId: outcome.outcomeId,
|
|
1414
|
+
kind: "OUTCOME",
|
|
1415
|
+
name: outcome.name,
|
|
1416
|
+
path: `outcome.${outcome.name}`
|
|
1417
|
+
}
|
|
1418
|
+
];
|
|
1419
|
+
if (requireProvenance.method) {
|
|
1420
|
+
if (outcome.method === void 0 || outcome.method.trim() === "") {
|
|
1421
|
+
findings.push(
|
|
1422
|
+
contractFailFinding(
|
|
1423
|
+
"contract.observation.provenance.method",
|
|
1424
|
+
`Observation ${outcome.name} is missing a structural method.`,
|
|
1425
|
+
evidenceBase,
|
|
1426
|
+
{ method: true },
|
|
1427
|
+
{ name: outcome.name, method: outcome.method ?? null }
|
|
1428
|
+
)
|
|
1429
|
+
);
|
|
1430
|
+
} else if (!METHOD_VOCABULARY.has(outcome.method)) {
|
|
1431
|
+
findings.push(
|
|
1432
|
+
contractFailFinding(
|
|
1433
|
+
"contract.observation.provenance.method",
|
|
1434
|
+
`Observation ${outcome.name} method ${outcome.method} is outside the bounded ObservedOutcomeMethod vocabulary.`,
|
|
1435
|
+
evidenceBase,
|
|
1436
|
+
{ methodVocabulary: [...OBSERVED_OUTCOME_METHODS] },
|
|
1437
|
+
{ name: outcome.name, method: outcome.method }
|
|
1438
|
+
)
|
|
1439
|
+
);
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
if (requireProvenance.evidence || requireProvenance.sameRunEventReference) {
|
|
1443
|
+
const parsed = parseEvidenceEventIds(outcome.evidence);
|
|
1444
|
+
if (parsed.status === "missing") {
|
|
1445
|
+
findings.push(
|
|
1446
|
+
contractFailFinding(
|
|
1447
|
+
"contract.observation.provenance.evidence",
|
|
1448
|
+
`Observation ${outcome.name} is missing bounded evidence references.`,
|
|
1449
|
+
evidenceBase,
|
|
1450
|
+
{ evidence: true },
|
|
1451
|
+
{ name: outcome.name, evidence: null }
|
|
1452
|
+
)
|
|
1453
|
+
);
|
|
1454
|
+
} else if (parsed.status === "invalid") {
|
|
1455
|
+
findings.push(
|
|
1456
|
+
contractFailFinding(
|
|
1457
|
+
"contract.observation.provenance.evidence",
|
|
1458
|
+
`Observation ${outcome.name} evidence is not a bounded event reference (${parsed.reason}).`,
|
|
1459
|
+
evidenceBase,
|
|
1460
|
+
{ evidence: true },
|
|
1461
|
+
{ name: outcome.name, reason: parsed.reason }
|
|
1462
|
+
)
|
|
1463
|
+
);
|
|
1464
|
+
} else if (requireProvenance.sameRunEventReference) {
|
|
1465
|
+
const missing = parsed.eventIds.filter((eventId) => !knownEventIds.has(eventId));
|
|
1466
|
+
if (missing.length > 0) {
|
|
1467
|
+
findings.push(
|
|
1468
|
+
contractFailFinding(
|
|
1469
|
+
"contract.observation.provenance.same-run",
|
|
1470
|
+
`Observation ${outcome.name} references event ids missing from the same run: ${missing.join(", ")}.`,
|
|
1471
|
+
evidenceBase,
|
|
1472
|
+
{ sameRunEventReference: true, eventIds: parsed.eventIds },
|
|
1473
|
+
{ name: outcome.name, missing }
|
|
1474
|
+
)
|
|
1475
|
+
);
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
return findings;
|
|
1480
|
+
}
|
|
942
1481
|
function contractToRules(contract) {
|
|
943
1482
|
const rules = [];
|
|
944
1483
|
const allowedStatuses = contract.run?.allowedStatuses ?? [];
|
|
@@ -1086,9 +1625,71 @@ function contractToRules(contract) {
|
|
|
1086
1625
|
})
|
|
1087
1626
|
);
|
|
1088
1627
|
}
|
|
1628
|
+
const requireProvenance = contract.observations.requireProvenance;
|
|
1629
|
+
if (requireProvenance && (requireProvenance.method || requireProvenance.evidence || requireProvenance.sameRunEventReference)) {
|
|
1630
|
+
rules.push({
|
|
1631
|
+
id: "contract.observation.provenance",
|
|
1632
|
+
category: "run",
|
|
1633
|
+
defaultSeverity: "error",
|
|
1634
|
+
evaluate(context) {
|
|
1635
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
1636
|
+
const knownEventIds = new Set(context.events.map((event) => event.eventId));
|
|
1637
|
+
const targets = required.length > 0 ? outcomes.filter((outcome) => required.includes(outcome.name)) : outcomes;
|
|
1638
|
+
return targets.flatMap(
|
|
1639
|
+
(outcome) => provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds)
|
|
1640
|
+
);
|
|
1641
|
+
}
|
|
1642
|
+
});
|
|
1643
|
+
}
|
|
1089
1644
|
}
|
|
1090
1645
|
return rules;
|
|
1091
1646
|
}
|
|
1647
|
+
function validateScopeShape(scope) {
|
|
1648
|
+
if (scope === void 0) return [];
|
|
1649
|
+
const hasSelector = typeof scope.runId === "string" && scope.runId.trim() !== "" || typeof scope.subAgentId === "string" && scope.subAgentId.trim() !== "" || typeof scope.groupId === "string" && scope.groupId.trim() !== "" || typeof scope.workflowStep === "string" && scope.workflowStep.trim() !== "" || typeof scope.rootEventId === "string" && scope.rootEventId.trim() !== "";
|
|
1650
|
+
if (!hasSelector) {
|
|
1651
|
+
return [
|
|
1652
|
+
{
|
|
1653
|
+
code: "contract.scope.empty",
|
|
1654
|
+
severity: "error",
|
|
1655
|
+
message: "scope requires at least one explicit selector (runId, subAgentId, groupId, workflowStep, or rootEventId).",
|
|
1656
|
+
path: "scope"
|
|
1657
|
+
}
|
|
1658
|
+
];
|
|
1659
|
+
}
|
|
1660
|
+
return [];
|
|
1661
|
+
}
|
|
1662
|
+
function validateProvenanceShape(observations) {
|
|
1663
|
+
const diagnostics = [];
|
|
1664
|
+
const provenance = observations?.requireProvenance;
|
|
1665
|
+
if (!provenance) return diagnostics;
|
|
1666
|
+
const enabled = provenance.method === true || provenance.evidence === true || provenance.sameRunEventReference === true;
|
|
1667
|
+
if (!enabled) {
|
|
1668
|
+
diagnostics.push({
|
|
1669
|
+
code: "contract.observation.provenance.empty",
|
|
1670
|
+
severity: "warning",
|
|
1671
|
+
message: "observations.requireProvenance is set but no flags are enabled.",
|
|
1672
|
+
path: "observations.requireProvenance"
|
|
1673
|
+
});
|
|
1674
|
+
}
|
|
1675
|
+
if (provenance.sameRunEventReference === true && provenance.evidence !== true) {
|
|
1676
|
+
diagnostics.push({
|
|
1677
|
+
code: "contract.observation.provenance.same-run-without-evidence",
|
|
1678
|
+
severity: "info",
|
|
1679
|
+
message: "sameRunEventReference implies evidence checking; set requireProvenance.evidence: true for clarity.",
|
|
1680
|
+
path: "observations.requireProvenance.sameRunEventReference"
|
|
1681
|
+
});
|
|
1682
|
+
}
|
|
1683
|
+
if (enabled && (observations?.required?.length ?? 0) === 0) {
|
|
1684
|
+
diagnostics.push({
|
|
1685
|
+
code: "contract.observation.provenance.without-required",
|
|
1686
|
+
severity: "warning",
|
|
1687
|
+
message: "requireProvenance without observations.required applies to every observed outcome in scope.",
|
|
1688
|
+
path: "observations.requireProvenance"
|
|
1689
|
+
});
|
|
1690
|
+
}
|
|
1691
|
+
return diagnostics;
|
|
1692
|
+
}
|
|
1092
1693
|
function validateAlternativesShape(alternatives) {
|
|
1093
1694
|
const diagnostics = [];
|
|
1094
1695
|
if (alternatives === void 0) return diagnostics;
|
|
@@ -1195,9 +1796,11 @@ function evaluateBody(input, body, options = {}) {
|
|
|
1195
1796
|
});
|
|
1196
1797
|
}
|
|
1197
1798
|
function defineTraceContract(input) {
|
|
1198
|
-
const shapeErrors =
|
|
1199
|
-
(
|
|
1200
|
-
|
|
1799
|
+
const shapeErrors = [
|
|
1800
|
+
...validateAlternativesShape(input.alternatives),
|
|
1801
|
+
...validateScopeShape(input.scope),
|
|
1802
|
+
...validateProvenanceShape(input.observations)
|
|
1803
|
+
].filter((item) => item.severity === "error");
|
|
1201
1804
|
if (shapeErrors.length > 0) {
|
|
1202
1805
|
throw new TypeError(shapeErrors.map((item) => item.message).join(" "));
|
|
1203
1806
|
}
|
|
@@ -1209,15 +1812,25 @@ function defineTraceContract(input) {
|
|
|
1209
1812
|
contract: cloneBody(branch.contract)
|
|
1210
1813
|
}))
|
|
1211
1814
|
} : void 0;
|
|
1815
|
+
const scope = input.scope ? {
|
|
1816
|
+
...input.scope.runId !== void 0 ? { runId: input.scope.runId } : {},
|
|
1817
|
+
...input.scope.subAgentId !== void 0 ? { subAgentId: input.scope.subAgentId } : {},
|
|
1818
|
+
...input.scope.groupId !== void 0 ? { groupId: input.scope.groupId } : {},
|
|
1819
|
+
...input.scope.workflowStep !== void 0 ? { workflowStep: input.scope.workflowStep } : {},
|
|
1820
|
+
...input.scope.rootEventId !== void 0 ? { rootEventId: input.scope.rootEventId } : {}
|
|
1821
|
+
} : void 0;
|
|
1212
1822
|
return {
|
|
1213
1823
|
...body,
|
|
1824
|
+
...scope ? { scope } : {},
|
|
1214
1825
|
...alternatives ? { alternatives } : {}
|
|
1215
1826
|
};
|
|
1216
1827
|
}
|
|
1217
1828
|
function evaluateTraceContract(input, contract, options = {}) {
|
|
1218
|
-
const shapeErrors =
|
|
1219
|
-
(
|
|
1220
|
-
|
|
1829
|
+
const shapeErrors = [
|
|
1830
|
+
...validateAlternativesShape(contract.alternatives),
|
|
1831
|
+
...validateScopeShape(contract.scope),
|
|
1832
|
+
...validateProvenanceShape(contract.observations)
|
|
1833
|
+
].filter((item) => item.severity === "error");
|
|
1221
1834
|
if (shapeErrors.length > 0) {
|
|
1222
1835
|
return {
|
|
1223
1836
|
ok: false,
|
|
@@ -1241,13 +1854,58 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
1241
1854
|
ruleExecutions: []
|
|
1242
1855
|
};
|
|
1243
1856
|
}
|
|
1244
|
-
const
|
|
1857
|
+
const scoped = resolveTraceContractScope(input, contract.scope);
|
|
1858
|
+
if (scoped.diagnostics.length > 0) {
|
|
1859
|
+
return {
|
|
1860
|
+
ok: false,
|
|
1861
|
+
status: "error",
|
|
1862
|
+
format: input.read.format,
|
|
1863
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
1864
|
+
...scoped.resolved?.runId !== void 0 ? { runId: scoped.resolved.runId } : {},
|
|
1865
|
+
summary: {
|
|
1866
|
+
passed: 0,
|
|
1867
|
+
failed: 0,
|
|
1868
|
+
warnings: 0,
|
|
1869
|
+
errors: scoped.diagnostics.length,
|
|
1870
|
+
rulesEvaluated: 0
|
|
1871
|
+
},
|
|
1872
|
+
findings: [],
|
|
1873
|
+
diagnostics: scoped.diagnostics,
|
|
1874
|
+
ruleExecutions: []
|
|
1875
|
+
};
|
|
1876
|
+
}
|
|
1877
|
+
const effectiveInput = scoped.resolved?.input ?? input;
|
|
1878
|
+
const effectiveRunId = scoped.resolved?.runId ?? options.runId;
|
|
1879
|
+
const base = evaluateBody(effectiveInput, contract, {
|
|
1880
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
1881
|
+
});
|
|
1882
|
+
const scopeFinding = scoped.resolved ? {
|
|
1883
|
+
ruleId: "contract.scope.selected",
|
|
1884
|
+
severity: "info",
|
|
1885
|
+
status: "pass",
|
|
1886
|
+
message: `Selected actor ${scoped.resolved.runId} with ${scoped.resolved.evidenceEventCount} evidence event(s).`,
|
|
1887
|
+
expected: scoped.resolved.matchedSelectors,
|
|
1888
|
+
actual: {
|
|
1889
|
+
runId: scoped.resolved.runId,
|
|
1890
|
+
evidenceEventCount: scoped.resolved.evidenceEventCount,
|
|
1891
|
+
...scoped.resolved.rootEventId !== void 0 ? { rootEventId: scoped.resolved.rootEventId } : {}
|
|
1892
|
+
},
|
|
1893
|
+
evidence: [
|
|
1894
|
+
{
|
|
1895
|
+
runId: scoped.resolved.runId,
|
|
1896
|
+
kind: "RUN",
|
|
1897
|
+
path: "scope"
|
|
1898
|
+
}
|
|
1899
|
+
]
|
|
1900
|
+
} : void 0;
|
|
1245
1901
|
const branches = contract.alternatives?.anyOf ?? [];
|
|
1246
1902
|
if (branches.length === 0) {
|
|
1247
|
-
return base;
|
|
1903
|
+
return scopeFinding ? mergeContractResults(effectiveInput, [base], [scopeFinding]) : base;
|
|
1248
1904
|
}
|
|
1249
1905
|
const branchEvaluations = branches.map((branch) => {
|
|
1250
|
-
const result = evaluateBody(
|
|
1906
|
+
const result = evaluateBody(effectiveInput, branch.contract, {
|
|
1907
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
1908
|
+
});
|
|
1251
1909
|
return { branch, result };
|
|
1252
1910
|
});
|
|
1253
1911
|
const satisfied = branchEvaluations.filter((item) => item.result.ok && item.result.status === "pass").map((item) => item.branch.id);
|
|
@@ -1267,7 +1925,11 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
1267
1925
|
},
|
|
1268
1926
|
evidence: []
|
|
1269
1927
|
};
|
|
1270
|
-
return mergeContractResults(
|
|
1928
|
+
return mergeContractResults(
|
|
1929
|
+
effectiveInput,
|
|
1930
|
+
[base],
|
|
1931
|
+
[...scopeFinding ? [scopeFinding] : [], passFinding]
|
|
1932
|
+
);
|
|
1271
1933
|
}
|
|
1272
1934
|
const noneFinding = {
|
|
1273
1935
|
ruleId: "contract.alternatives.none-satisfied",
|
|
@@ -1294,13 +1956,21 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
1294
1956
|
ruleId: `contract.alternatives.${item.branch.id}.${finding.ruleId}`
|
|
1295
1957
|
}))
|
|
1296
1958
|
);
|
|
1297
|
-
return mergeContractResults(
|
|
1959
|
+
return mergeContractResults(
|
|
1960
|
+
effectiveInput,
|
|
1961
|
+
[base],
|
|
1962
|
+
[...scopeFinding ? [scopeFinding] : [], noneFinding, ...prefixedBranchFindings]
|
|
1963
|
+
);
|
|
1298
1964
|
}
|
|
1299
1965
|
function evaluateTraceContractRead(read, contract, options = {}) {
|
|
1300
1966
|
return evaluateTraceContract({ read }, contract, options);
|
|
1301
1967
|
}
|
|
1302
1968
|
function lintTraceContract(contract) {
|
|
1303
|
-
const diagnostics =
|
|
1969
|
+
const diagnostics = [
|
|
1970
|
+
...validateAlternativesShape(contract.alternatives),
|
|
1971
|
+
...validateScopeShape(contract.scope),
|
|
1972
|
+
...validateProvenanceShape(contract.observations)
|
|
1973
|
+
];
|
|
1304
1974
|
const branches = contract.alternatives?.anyOf ?? [];
|
|
1305
1975
|
if (branches.length === 1) {
|
|
1306
1976
|
diagnostics.push({
|
|
@@ -1335,6 +2005,12 @@ function lintTraceContract(contract) {
|
|
|
1335
2005
|
}
|
|
1336
2006
|
function explainTraceContract(contract) {
|
|
1337
2007
|
const lines = [];
|
|
2008
|
+
if (contract.scope) {
|
|
2009
|
+
const parts = Object.entries(contract.scope).filter(([, value]) => typeof value === "string" && value.trim() !== "").map(([key, value]) => `${key}=${value}`);
|
|
2010
|
+
if (parts.length > 0) {
|
|
2011
|
+
lines.push(`Scope: select actor by [${parts.join(", ")}] before evaluation.`);
|
|
2012
|
+
}
|
|
2013
|
+
}
|
|
1338
2014
|
if (contract.run?.requireCompleted !== false) {
|
|
1339
2015
|
lines.push("Base: run must complete (no running events).");
|
|
1340
2016
|
}
|
|
@@ -1386,6 +2062,18 @@ function explainTraceContract(contract) {
|
|
|
1386
2062
|
if (contract.observations?.failOn?.length) {
|
|
1387
2063
|
lines.push(`Base: fail on observations [${contract.observations.failOn.join(", ")}].`);
|
|
1388
2064
|
}
|
|
2065
|
+
if (contract.observations?.requireProvenance) {
|
|
2066
|
+
const flags = [
|
|
2067
|
+
contract.observations.requireProvenance.method ? "method" : void 0,
|
|
2068
|
+
contract.observations.requireProvenance.evidence ? "evidence" : void 0,
|
|
2069
|
+
contract.observations.requireProvenance.sameRunEventReference ? "sameRunEventReference" : void 0
|
|
2070
|
+
].filter((item) => item !== void 0);
|
|
2071
|
+
if (flags.length > 0) {
|
|
2072
|
+
lines.push(
|
|
2073
|
+
`Base: require structural observation provenance [${flags.join(", ")}] (not semantic truth).`
|
|
2074
|
+
);
|
|
2075
|
+
}
|
|
2076
|
+
}
|
|
1389
2077
|
const branches = contract.alternatives?.anyOf ?? [];
|
|
1390
2078
|
if (branches.length > 0) {
|
|
1391
2079
|
lines.push(
|
|
@@ -1486,7 +2174,7 @@ var DEFAULT_SECRET_PATTERNS = [
|
|
|
1486
2174
|
function compareStrings(a, b) {
|
|
1487
2175
|
return (a ?? "").localeCompare(b ?? "");
|
|
1488
2176
|
}
|
|
1489
|
-
function
|
|
2177
|
+
function diagnostic2(code, message, ruleId) {
|
|
1490
2178
|
return {
|
|
1491
2179
|
code,
|
|
1492
2180
|
message,
|
|
@@ -1519,14 +2207,14 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
1519
2207
|
ruleExecutions: [...ruleExecutions]
|
|
1520
2208
|
};
|
|
1521
2209
|
}
|
|
1522
|
-
function
|
|
1523
|
-
return nodes.flatMap((node) => [node, ...
|
|
2210
|
+
function flattenNodes2(nodes) {
|
|
2211
|
+
return nodes.flatMap((node) => [node, ...flattenNodes2(node.children)]);
|
|
1524
2212
|
}
|
|
1525
2213
|
function buildFacts(input, selectedRun) {
|
|
1526
2214
|
const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
|
|
1527
2215
|
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
1528
2216
|
const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
1529
|
-
const nodes =
|
|
2217
|
+
const nodes = flattenNodes2(scopedRuns.flatMap((run) => run.children));
|
|
1530
2218
|
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
1531
2219
|
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
1532
2220
|
for (const node of nodes) {
|
|
@@ -1558,7 +2246,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
1558
2246
|
if (runId && input.selectedRun.runId !== runId) {
|
|
1559
2247
|
return {
|
|
1560
2248
|
diagnostics: [
|
|
1561
|
-
|
|
2249
|
+
diagnostic2(
|
|
1562
2250
|
"AI_CHECK_INVALID_ARGUMENTS",
|
|
1563
2251
|
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
1564
2252
|
)
|
|
@@ -1572,7 +2260,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
1572
2260
|
if (!run) {
|
|
1573
2261
|
return {
|
|
1574
2262
|
diagnostics: [
|
|
1575
|
-
|
|
2263
|
+
diagnostic2("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
1576
2264
|
]
|
|
1577
2265
|
};
|
|
1578
2266
|
}
|
|
@@ -1584,7 +2272,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
1584
2272
|
if (input.read.runs.length === 0) {
|
|
1585
2273
|
return {
|
|
1586
2274
|
diagnostics: [
|
|
1587
|
-
|
|
2275
|
+
diagnostic2(
|
|
1588
2276
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
1589
2277
|
formatProgrammaticDiagnostic(
|
|
1590
2278
|
"AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
@@ -1596,7 +2284,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
1596
2284
|
}
|
|
1597
2285
|
return {
|
|
1598
2286
|
diagnostics: [
|
|
1599
|
-
|
|
2287
|
+
diagnostic2(
|
|
1600
2288
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
1601
2289
|
formatProgrammaticDiagnostic("AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED")
|
|
1602
2290
|
)
|
|
@@ -1609,7 +2297,7 @@ function selectRules(rules, selectedIds) {
|
|
|
1609
2297
|
for (const rule of rules) {
|
|
1610
2298
|
if (byId.has(rule.id)) {
|
|
1611
2299
|
diagnostics.push(
|
|
1612
|
-
|
|
2300
|
+
diagnostic2("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
1613
2301
|
);
|
|
1614
2302
|
continue;
|
|
1615
2303
|
}
|
|
@@ -1620,7 +2308,7 @@ function selectRules(rules, selectedIds) {
|
|
|
1620
2308
|
for (const id of selected) {
|
|
1621
2309
|
if (!byId.has(id)) {
|
|
1622
2310
|
diagnostics.push(
|
|
1623
|
-
|
|
2311
|
+
diagnostic2("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
1624
2312
|
);
|
|
1625
2313
|
}
|
|
1626
2314
|
}
|
|
@@ -1696,7 +2384,7 @@ function classifyRuleExecution(findings, threw) {
|
|
|
1696
2384
|
}
|
|
1697
2385
|
return "pass";
|
|
1698
2386
|
}
|
|
1699
|
-
function
|
|
2387
|
+
function stringAttr2(event, keys) {
|
|
1700
2388
|
for (const key of keys) {
|
|
1701
2389
|
const value = event.attributes?.[key];
|
|
1702
2390
|
if (typeof value === "string" && value.trim() !== "") return value;
|
|
@@ -1774,15 +2462,15 @@ function semanticEvents(context) {
|
|
|
1774
2462
|
}
|
|
1775
2463
|
function llmModel(event) {
|
|
1776
2464
|
const keys = ["model", "modelId", "responseModelId", "modelName", "model_name"];
|
|
1777
|
-
return
|
|
2465
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
1778
2466
|
}
|
|
1779
2467
|
function llmProvider(event) {
|
|
1780
2468
|
const keys = ["provider", "providerName", "provider_name"];
|
|
1781
|
-
return
|
|
2469
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys);
|
|
1782
2470
|
}
|
|
1783
2471
|
function llmFinishReason(event) {
|
|
1784
2472
|
const keys = ["finishReason", "rawFinishReason", "finish_reason"];
|
|
1785
|
-
return
|
|
2473
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys);
|
|
1786
2474
|
}
|
|
1787
2475
|
function retryCount(event) {
|
|
1788
2476
|
return numericAttr(event, ["retryCount", "retryAttempt", "retry_attempt", "attempt"]);
|
|
@@ -1924,7 +2612,7 @@ function parentMarkedUnresolved(event) {
|
|
|
1924
2612
|
]) === true) {
|
|
1925
2613
|
return true;
|
|
1926
2614
|
}
|
|
1927
|
-
const resolution =
|
|
2615
|
+
const resolution = stringAttr2(event, [
|
|
1928
2616
|
"parentResolution",
|
|
1929
2617
|
"relationshipResolution",
|
|
1930
2618
|
"relationshipStatus"
|
|
@@ -1932,13 +2620,13 @@ function parentMarkedUnresolved(event) {
|
|
|
1932
2620
|
return resolution === "unresolved" || resolution === "missing-parent";
|
|
1933
2621
|
}
|
|
1934
2622
|
function signalName(event, attributeKeys, prefixes) {
|
|
1935
|
-
return
|
|
2623
|
+
return stringAttr2(event, attributeKeys) ?? stripPrefix(event.name, prefixes);
|
|
1936
2624
|
}
|
|
1937
2625
|
function guardrailEvents(context) {
|
|
1938
2626
|
return finishedEvents(context).filter((event) => {
|
|
1939
2627
|
const name = event.name.toLowerCase();
|
|
1940
2628
|
if (name.startsWith("guardrail:") || name.includes(".guardrail.")) return true;
|
|
1941
|
-
return
|
|
2629
|
+
return stringAttr2(event, ["guardrailName", "guardrail", "guardrailId"]) !== void 0;
|
|
1942
2630
|
});
|
|
1943
2631
|
}
|
|
1944
2632
|
function retryValue(event) {
|
|
@@ -3307,7 +3995,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
3307
3995
|
return errorResult(
|
|
3308
3996
|
input,
|
|
3309
3997
|
[
|
|
3310
|
-
|
|
3998
|
+
diagnostic2(
|
|
3311
3999
|
"AI_CHECK_NO_RULES_EVALUATED",
|
|
3312
4000
|
"No trace check rules were evaluated. Configure at least one rule, contract, or CLI check option."
|
|
3313
4001
|
)
|
|
@@ -3338,7 +4026,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
3338
4026
|
} catch (error) {
|
|
3339
4027
|
const message = error instanceof Error ? error.message : String(error);
|
|
3340
4028
|
diagnostics.push(
|
|
3341
|
-
|
|
4029
|
+
diagnostic2("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
3342
4030
|
);
|
|
3343
4031
|
ruleExecutions.push({
|
|
3344
4032
|
ruleId: rule.id,
|
|
@@ -3397,6 +4085,7 @@ exports.createToolOrderingRule = createToolOrderingRule;
|
|
|
3397
4085
|
exports.createToolUsageRule = createToolUsageRule;
|
|
3398
4086
|
exports.defineTraceContract = defineTraceContract;
|
|
3399
4087
|
exports.deriveFailureFacts = deriveFailureFacts;
|
|
4088
|
+
exports.deriveRelationshipFacts = deriveRelationshipFacts;
|
|
3400
4089
|
exports.evaluateTraceContract = evaluateTraceContract;
|
|
3401
4090
|
exports.evaluateTraceContractRead = evaluateTraceContractRead;
|
|
3402
4091
|
exports.explainTraceContract = explainTraceContract;
|
|
@@ -3404,7 +4093,9 @@ exports.formatProgrammaticDiagnostic = formatProgrammaticDiagnostic;
|
|
|
3404
4093
|
exports.lintTraceContract = lintTraceContract;
|
|
3405
4094
|
exports.projectLogicalEvents = projectLogicalEvents;
|
|
3406
4095
|
exports.resolveCanonicalToolName = resolveCanonicalToolName;
|
|
4096
|
+
exports.resolveTraceContractScope = resolveTraceContractScope;
|
|
3407
4097
|
exports.runTraceChecks = runTraceChecks;
|
|
3408
4098
|
exports.summarizeSemanticParity = summarizeSemanticParity;
|
|
4099
|
+
exports.workflowMetadataForRun = workflowMetadataForRun;
|
|
3409
4100
|
//# sourceMappingURL=checks.cjs.map
|
|
3410
4101
|
//# sourceMappingURL=checks.cjs.map
|