agent-inspect 6.21.0 → 6.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/docs/TRACE-CONTRACTS.md +56 -0
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-CRUHIL5J.mjs → chunk-BVQO5CWA.mjs} +11 -2
- package/packages/cli/dist/chunk-BVQO5CWA.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +55 -1
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +4 -4
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-MNJFYW3W.mjs → src-Y4NXOO2K.mjs} +3 -3
- package/packages/cli/dist/{src-MNJFYW3W.mjs.map → src-Y4NXOO2K.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +38 -0
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +36 -4
- package/packages/core/dist/advanced.d.ts +36 -4
- package/packages/core/dist/advanced.mjs +32 -4
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +956 -122
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +227 -5
- package/packages/core/dist/checks.d.ts +227 -5
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-HT54OLPE.mjs → chunk-NDRMIL7Z.mjs} +103 -3
- package/packages/core/dist/chunk-NDRMIL7Z.mjs.map +1 -0
- package/packages/core/dist/{chunk-43SHIPZM.mjs → chunk-VMDCDWBE.mjs} +855 -25
- package/packages/core/dist/chunk-VMDCDWBE.mjs.map +1 -0
- package/packages/core/dist/{index-BG-ZBESI.d.cts → index-CR4CCRqP.d.cts} +7 -2
- package/packages/core/dist/{index-xevIlqdd.d.ts → index-CtK_p-ps.d.ts} +7 -2
- package/packages/core/dist/{index-AMAGuO_i.d.ts → index-D72DPqAF.d.ts} +2 -1
- package/packages/core/dist/{index-vqr-g6ev.d.cts → index-DLLz9sce.d.cts} +2 -1
- package/packages/core/dist/programmatic-gZTWLObB.d.cts +25 -0
- package/packages/core/dist/programmatic-gZTWLObB.d.ts +25 -0
- package/packages/core/dist/readers.cjs +105 -0
- package/packages/core/dist/readers.cjs.map +1 -1
- package/packages/core/dist/readers.d.cts +43 -18
- package/packages/core/dist/readers.d.ts +43 -18
- package/packages/core/dist/readers.mjs +1 -1
- package/packages/cli/dist/chunk-CRUHIL5J.mjs.map +0 -1
- package/packages/core/dist/chunk-43SHIPZM.mjs.map +0 -1
- package/packages/core/dist/chunk-HT54OLPE.mjs.map +0 -1
|
@@ -96,11 +96,11 @@ function parseMethod(value) {
|
|
|
96
96
|
}
|
|
97
97
|
function fromPersistedOutcome(event) {
|
|
98
98
|
if (event.kind !== "OUTCOME") return void 0;
|
|
99
|
-
const
|
|
100
|
-
const statusRaw =
|
|
99
|
+
const attrs2 = event.attributes ?? {};
|
|
100
|
+
const statusRaw = attrs2[OUTCOME_ATTRIBUTE_STATUS_KEY];
|
|
101
101
|
if (!isOutcomeStatus(statusRaw)) return void 0;
|
|
102
|
-
const expectation = typeof
|
|
103
|
-
const observedAtRaw =
|
|
102
|
+
const expectation = typeof attrs2[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] === "string" ? attrs2[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] : event.name;
|
|
103
|
+
const observedAtRaw = attrs2[OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY];
|
|
104
104
|
const observedAt = typeof observedAtRaw === "string" ? Date.parse(observedAtRaw) : typeof observedAtRaw === "number" && Number.isFinite(observedAtRaw) ? observedAtRaw : Date.parse(event.timestamp);
|
|
105
105
|
return {
|
|
106
106
|
outcomeId: event.eventId,
|
|
@@ -109,9 +109,9 @@ function fromPersistedOutcome(event) {
|
|
|
109
109
|
name: event.name,
|
|
110
110
|
expectation,
|
|
111
111
|
status: statusRaw,
|
|
112
|
-
...parseMethod(
|
|
112
|
+
...parseMethod(attrs2[OUTCOME_ATTRIBUTE_METHOD_KEY]) !== void 0 ? { method: parseMethod(attrs2[OUTCOME_ATTRIBUTE_METHOD_KEY]) } : {},
|
|
113
113
|
...event.outputSummary !== void 0 ? { actual: event.outputSummary } : {},
|
|
114
|
-
...
|
|
114
|
+
...attrs2.evidence !== void 0 ? { evidence: attrs2.evidence } : {},
|
|
115
115
|
observedAt: Number.isFinite(observedAt) ? observedAt : Date.parse(event.timestamp)
|
|
116
116
|
};
|
|
117
117
|
}
|
|
@@ -477,10 +477,10 @@ function projectLogicalEvents(events) {
|
|
|
477
477
|
};
|
|
478
478
|
}
|
|
479
479
|
function resolveCanonicalToolName(event) {
|
|
480
|
-
const
|
|
481
|
-
const direct = pickString(
|
|
480
|
+
const attrs2 = event.attributes;
|
|
481
|
+
const direct = pickString(attrs2, ["toolName", "tool"]);
|
|
482
482
|
if (direct) return direct;
|
|
483
|
-
const metadata =
|
|
483
|
+
const metadata = attrs2?.metadata;
|
|
484
484
|
if (isRecord(metadata)) {
|
|
485
485
|
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
486
486
|
if (nested) return nested;
|
|
@@ -522,10 +522,10 @@ function pickNumber(record, keys) {
|
|
|
522
522
|
return void 0;
|
|
523
523
|
}
|
|
524
524
|
function eventMetadata(event) {
|
|
525
|
-
const
|
|
526
|
-
const nested =
|
|
525
|
+
const attrs2 = isRecord2(event.attributes) ? event.attributes : void 0;
|
|
526
|
+
const nested = attrs2 !== void 0 && isRecord2(attrs2.metadata) ? attrs2.metadata : void 0;
|
|
527
527
|
return {
|
|
528
|
-
...
|
|
528
|
+
...attrs2 ?? {},
|
|
529
529
|
...nested ?? {}
|
|
530
530
|
};
|
|
531
531
|
}
|
|
@@ -818,6 +818,208 @@ function deriveFailureFacts(logicalEvents) {
|
|
|
818
818
|
};
|
|
819
819
|
}
|
|
820
820
|
|
|
821
|
+
// packages/core/src/sessions/metadata.ts
|
|
822
|
+
function isNonEmptyString(value) {
|
|
823
|
+
return typeof value === "string" && value.trim() !== "";
|
|
824
|
+
}
|
|
825
|
+
function finitePositiveInt(value) {
|
|
826
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
827
|
+
return void 0;
|
|
828
|
+
}
|
|
829
|
+
return Math.trunc(value);
|
|
830
|
+
}
|
|
831
|
+
function extractSessionWorkflowMetadata(record) {
|
|
832
|
+
if (!record) return void 0;
|
|
833
|
+
const out = {};
|
|
834
|
+
let found = false;
|
|
835
|
+
const assignString = (key, value) => {
|
|
836
|
+
if (isNonEmptyString(value)) {
|
|
837
|
+
out[key] = value.trim();
|
|
838
|
+
found = true;
|
|
839
|
+
}
|
|
840
|
+
};
|
|
841
|
+
assignString("sessionId", record.sessionId);
|
|
842
|
+
assignString("conversationId", record.conversationId);
|
|
843
|
+
assignString("groupId", record.groupId);
|
|
844
|
+
assignString("parentGroupId", record.parentGroupId);
|
|
845
|
+
assignString("retryOf", record.retryOf);
|
|
846
|
+
assignString("retryReason", record.retryReason);
|
|
847
|
+
assignString("handoffFrom", record.handoffFrom);
|
|
848
|
+
assignString("handoffTo", record.handoffTo);
|
|
849
|
+
assignString("subAgentId", record.subAgentId);
|
|
850
|
+
assignString("subAgentName", record.subAgentName);
|
|
851
|
+
assignString("jobId", record.jobId);
|
|
852
|
+
assignString("queueName", record.queueName);
|
|
853
|
+
assignString("workflowName", record.workflowName);
|
|
854
|
+
assignString("workflowStep", record.workflowStep);
|
|
855
|
+
assignString("toolCallId", record.toolCallId);
|
|
856
|
+
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
857
|
+
assignString("linkedStepId", record.linkedStepId);
|
|
858
|
+
assignString("operationId", record.operationId);
|
|
859
|
+
assignString("attemptId", record.attemptId);
|
|
860
|
+
assignString("fallbackOf", record.fallbackOf);
|
|
861
|
+
assignString("idempotencyKey", record.idempotencyKey);
|
|
862
|
+
assignString("correlationId", record.correlationId);
|
|
863
|
+
assignString("requestId", record.requestId);
|
|
864
|
+
assignString("decisionId", record.decisionId);
|
|
865
|
+
const attempt = finitePositiveInt(record.attempt);
|
|
866
|
+
if (attempt !== void 0) {
|
|
867
|
+
out.attempt = attempt;
|
|
868
|
+
found = true;
|
|
869
|
+
}
|
|
870
|
+
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
871
|
+
if (attemptNumber !== void 0) {
|
|
872
|
+
out.attemptNumber = attemptNumber;
|
|
873
|
+
found = true;
|
|
874
|
+
}
|
|
875
|
+
return found ? out : void 0;
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
// packages/core/src/checks/relationship-facts.ts
|
|
879
|
+
function attrs(event) {
|
|
880
|
+
return event.attributes && typeof event.attributes === "object" ? event.attributes : {};
|
|
881
|
+
}
|
|
882
|
+
function stringAttr(record, key) {
|
|
883
|
+
const value = record[key];
|
|
884
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
885
|
+
}
|
|
886
|
+
function deriveRelationshipFacts(events) {
|
|
887
|
+
const relationships = [];
|
|
888
|
+
const diagnostics = [];
|
|
889
|
+
const byId = new Map(events.map((event) => [event.eventId, event]));
|
|
890
|
+
const seen = /* @__PURE__ */ new Set();
|
|
891
|
+
const push = (edge) => {
|
|
892
|
+
const key = `${edge.type}:${edge.fromEventId}:${edge.toEventId ?? ""}:${edge.externalRef ?? ""}`;
|
|
893
|
+
if (seen.has(key)) return;
|
|
894
|
+
seen.add(key);
|
|
895
|
+
relationships.push(edge);
|
|
896
|
+
};
|
|
897
|
+
for (const event of events) {
|
|
898
|
+
if (event.parentId) {
|
|
899
|
+
push({
|
|
900
|
+
type: "parent-child",
|
|
901
|
+
fromEventId: event.parentId,
|
|
902
|
+
toEventId: event.eventId,
|
|
903
|
+
confidence: byId.has(event.parentId) ? "explicit" : "unknown",
|
|
904
|
+
basis: ["persisted.parentId"]
|
|
905
|
+
});
|
|
906
|
+
if (!byId.has(event.parentId)) {
|
|
907
|
+
diagnostics.push({
|
|
908
|
+
code: "AI_RELATIONSHIP_PARENT_MISSING",
|
|
909
|
+
message: `parentId ${event.parentId} is not present in the event set.`,
|
|
910
|
+
eventId: event.eventId
|
|
911
|
+
});
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
const bag = attrs(event);
|
|
915
|
+
const meta = extractSessionWorkflowMetadata(bag) ?? {};
|
|
916
|
+
const nested = bag.metadata && typeof bag.metadata === "object" ? extractSessionWorkflowMetadata(bag.metadata) : void 0;
|
|
917
|
+
const workflow = { ...meta, ...nested };
|
|
918
|
+
if (workflow.retryOf) {
|
|
919
|
+
const target = events.find((candidate) => candidate.runId === workflow.retryOf);
|
|
920
|
+
push({
|
|
921
|
+
type: "retry-of",
|
|
922
|
+
fromEventId: event.eventId,
|
|
923
|
+
...target ? { toEventId: target.eventId } : {},
|
|
924
|
+
externalRef: workflow.retryOf,
|
|
925
|
+
confidence: target ? "explicit" : "correlated",
|
|
926
|
+
basis: ["attributes.retryOf"]
|
|
927
|
+
});
|
|
928
|
+
}
|
|
929
|
+
const attemptOf = stringAttr(bag, "attemptOf") ?? stringAttr(bag, "operationId");
|
|
930
|
+
if (attemptOf && workflow.attempt !== void 0) {
|
|
931
|
+
push({
|
|
932
|
+
type: "attempt-of",
|
|
933
|
+
fromEventId: event.eventId,
|
|
934
|
+
externalRef: attemptOf,
|
|
935
|
+
confidence: "explicit",
|
|
936
|
+
basis: workflow.attempt !== void 0 ? ["attributes.attempt", "attributes.operationId"] : ["attributes.operationId"]
|
|
937
|
+
});
|
|
938
|
+
}
|
|
939
|
+
const fallbackOf = stringAttr(bag, "fallbackOf");
|
|
940
|
+
if (fallbackOf) {
|
|
941
|
+
push({
|
|
942
|
+
type: "fallback-of",
|
|
943
|
+
fromEventId: event.eventId,
|
|
944
|
+
externalRef: fallbackOf,
|
|
945
|
+
confidence: "explicit",
|
|
946
|
+
basis: ["attributes.fallbackOf"]
|
|
947
|
+
});
|
|
948
|
+
}
|
|
949
|
+
const remediationOf = stringAttr(bag, "remediationOf");
|
|
950
|
+
if (remediationOf) {
|
|
951
|
+
push({
|
|
952
|
+
type: "remediation-of",
|
|
953
|
+
fromEventId: event.eventId,
|
|
954
|
+
externalRef: remediationOf,
|
|
955
|
+
confidence: "explicit",
|
|
956
|
+
basis: ["attributes.remediationOf"]
|
|
957
|
+
});
|
|
958
|
+
}
|
|
959
|
+
const evidenceFor = stringAttr(bag, "evidenceFor");
|
|
960
|
+
if (evidenceFor) {
|
|
961
|
+
push({
|
|
962
|
+
type: "evidence-for",
|
|
963
|
+
fromEventId: event.eventId,
|
|
964
|
+
...byId.has(evidenceFor) ? { toEventId: evidenceFor } : { externalRef: evidenceFor },
|
|
965
|
+
confidence: byId.has(evidenceFor) ? "explicit" : "correlated",
|
|
966
|
+
basis: ["attributes.evidenceFor"]
|
|
967
|
+
});
|
|
968
|
+
}
|
|
969
|
+
const acceptedBy = stringAttr(bag, "acceptedBy");
|
|
970
|
+
if (acceptedBy) {
|
|
971
|
+
push({
|
|
972
|
+
type: "accepted-by",
|
|
973
|
+
fromEventId: event.eventId,
|
|
974
|
+
...byId.has(acceptedBy) ? { toEventId: acceptedBy } : { externalRef: acceptedBy },
|
|
975
|
+
confidence: byId.has(acceptedBy) ? "explicit" : "correlated",
|
|
976
|
+
basis: ["attributes.acceptedBy"]
|
|
977
|
+
});
|
|
978
|
+
}
|
|
979
|
+
const supersedes = stringAttr(bag, "supersedes");
|
|
980
|
+
if (supersedes) {
|
|
981
|
+
push({
|
|
982
|
+
type: "supersedes",
|
|
983
|
+
fromEventId: event.eventId,
|
|
984
|
+
...byId.has(supersedes) ? { toEventId: supersedes } : { externalRef: supersedes },
|
|
985
|
+
confidence: byId.has(supersedes) ? "explicit" : "correlated",
|
|
986
|
+
basis: ["attributes.supersedes"]
|
|
987
|
+
});
|
|
988
|
+
}
|
|
989
|
+
const sourceLineage = stringAttr(bag, "sourceLineage") ?? stringAttr(bag, "sourceEventId");
|
|
990
|
+
if (sourceLineage) {
|
|
991
|
+
push({
|
|
992
|
+
type: "source-lineage",
|
|
993
|
+
fromEventId: event.eventId,
|
|
994
|
+
externalRef: sourceLineage,
|
|
995
|
+
confidence: "explicit",
|
|
996
|
+
basis: ["attributes.sourceLineage"]
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
const unsupported = stringAttr(bag, "relationshipType");
|
|
1000
|
+
if (unsupported && unsupported !== "parent-child" && ![
|
|
1001
|
+
"source-lineage",
|
|
1002
|
+
"attempt-of",
|
|
1003
|
+
"retry-of",
|
|
1004
|
+
"fallback-of",
|
|
1005
|
+
"remediation-of",
|
|
1006
|
+
"evidence-for",
|
|
1007
|
+
"accepted-by",
|
|
1008
|
+
"supersedes"
|
|
1009
|
+
].includes(unsupported)) {
|
|
1010
|
+
diagnostics.push({
|
|
1011
|
+
code: "AI_RELATIONSHIP_UNSUPPORTED_TYPE",
|
|
1012
|
+
message: `Unsupported relationshipType ${unsupported} was reported without flattening.`,
|
|
1013
|
+
eventId: event.eventId
|
|
1014
|
+
});
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
return {
|
|
1018
|
+
relationships: Object.freeze(relationships),
|
|
1019
|
+
diagnostics: Object.freeze(diagnostics)
|
|
1020
|
+
};
|
|
1021
|
+
}
|
|
1022
|
+
|
|
821
1023
|
// packages/core/src/checks/trace-facts.ts
|
|
822
1024
|
function summarizeSemanticParity(events) {
|
|
823
1025
|
const projection = projectLogicalEvents(events);
|
|
@@ -904,6 +1106,7 @@ function buildTraceFacts(input) {
|
|
|
904
1106
|
toolsByName.set(name, Object.freeze([...list]));
|
|
905
1107
|
}
|
|
906
1108
|
const derived = deriveFailureFacts(projection.logicalEvents);
|
|
1109
|
+
const relationship = deriveRelationshipFacts(events);
|
|
907
1110
|
const summary = summarizeSemanticParity(events);
|
|
908
1111
|
return {
|
|
909
1112
|
rawEvents: Object.freeze([...events]),
|
|
@@ -917,56 +1120,10 @@ function buildTraceFacts(input) {
|
|
|
917
1120
|
failureRoleCounts: derived.failureRoleCounts
|
|
918
1121
|
},
|
|
919
1122
|
failureFacts: derived.failureFacts,
|
|
920
|
-
failuresByRole: derived.failuresByRole
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
// packages/core/src/sessions/metadata.ts
|
|
925
|
-
function isNonEmptyString(value) {
|
|
926
|
-
return typeof value === "string" && value.trim() !== "";
|
|
927
|
-
}
|
|
928
|
-
function finitePositiveInt(value) {
|
|
929
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
930
|
-
return void 0;
|
|
931
|
-
}
|
|
932
|
-
return Math.trunc(value);
|
|
933
|
-
}
|
|
934
|
-
function extractSessionWorkflowMetadata(record) {
|
|
935
|
-
if (!record) return void 0;
|
|
936
|
-
const out = {};
|
|
937
|
-
let found = false;
|
|
938
|
-
const assignString = (key, value) => {
|
|
939
|
-
if (isNonEmptyString(value)) {
|
|
940
|
-
out[key] = value.trim();
|
|
941
|
-
found = true;
|
|
942
|
-
}
|
|
1123
|
+
failuresByRole: derived.failuresByRole,
|
|
1124
|
+
relationships: relationship.relationships,
|
|
1125
|
+
relationshipDiagnostics: relationship.diagnostics
|
|
943
1126
|
};
|
|
944
|
-
assignString("sessionId", record.sessionId);
|
|
945
|
-
assignString("conversationId", record.conversationId);
|
|
946
|
-
assignString("groupId", record.groupId);
|
|
947
|
-
assignString("parentGroupId", record.parentGroupId);
|
|
948
|
-
assignString("retryOf", record.retryOf);
|
|
949
|
-
assignString("retryReason", record.retryReason);
|
|
950
|
-
assignString("handoffFrom", record.handoffFrom);
|
|
951
|
-
assignString("handoffTo", record.handoffTo);
|
|
952
|
-
assignString("subAgentId", record.subAgentId);
|
|
953
|
-
assignString("subAgentName", record.subAgentName);
|
|
954
|
-
assignString("jobId", record.jobId);
|
|
955
|
-
assignString("queueName", record.queueName);
|
|
956
|
-
assignString("workflowName", record.workflowName);
|
|
957
|
-
assignString("workflowStep", record.workflowStep);
|
|
958
|
-
assignString("toolCallId", record.toolCallId);
|
|
959
|
-
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
960
|
-
assignString("linkedStepId", record.linkedStepId);
|
|
961
|
-
assignString("correlationId", record.correlationId);
|
|
962
|
-
assignString("requestId", record.requestId);
|
|
963
|
-
assignString("decisionId", record.decisionId);
|
|
964
|
-
const attempt = finitePositiveInt(record.attempt);
|
|
965
|
-
if (attempt !== void 0) {
|
|
966
|
-
out.attempt = attempt;
|
|
967
|
-
found = true;
|
|
968
|
-
}
|
|
969
|
-
return found ? out : void 0;
|
|
970
1127
|
}
|
|
971
1128
|
|
|
972
1129
|
// packages/core/src/checks/contract-scope.ts
|
|
@@ -1008,10 +1165,10 @@ function workflowMetadataForRun(events, runId) {
|
|
|
1008
1165
|
const fallback = {};
|
|
1009
1166
|
for (const event of events) {
|
|
1010
1167
|
if (event.runId !== runId) continue;
|
|
1011
|
-
const
|
|
1012
|
-
if (!
|
|
1013
|
-
const direct = extractSessionWorkflowMetadata(
|
|
1014
|
-
const nested =
|
|
1168
|
+
const attrs2 = event.attributes;
|
|
1169
|
+
if (!attrs2 || typeof attrs2 !== "object") continue;
|
|
1170
|
+
const direct = extractSessionWorkflowMetadata(attrs2);
|
|
1171
|
+
const nested = attrs2.metadata && typeof attrs2.metadata === "object" ? extractSessionWorkflowMetadata(attrs2.metadata) : void 0;
|
|
1015
1172
|
const bag = {};
|
|
1016
1173
|
mergeWorkflowMeta(bag, direct);
|
|
1017
1174
|
mergeWorkflowMeta(bag, nested);
|
|
@@ -1163,6 +1320,465 @@ function resolveTraceContractScope(input, scope) {
|
|
|
1163
1320
|
};
|
|
1164
1321
|
}
|
|
1165
1322
|
|
|
1323
|
+
// packages/core/src/checks/control-rules.ts
|
|
1324
|
+
function asStringList(value) {
|
|
1325
|
+
if (!Array.isArray(value)) return void 0;
|
|
1326
|
+
const out = [];
|
|
1327
|
+
for (const item of value) {
|
|
1328
|
+
if (typeof item !== "string" || item.trim() === "") return void 0;
|
|
1329
|
+
out.push(item.trim());
|
|
1330
|
+
}
|
|
1331
|
+
return out;
|
|
1332
|
+
}
|
|
1333
|
+
function attributeList(events, attribute) {
|
|
1334
|
+
if (!attribute) return void 0;
|
|
1335
|
+
for (const event of events) {
|
|
1336
|
+
const attrs2 = event.attributes;
|
|
1337
|
+
if (!attrs2 || typeof attrs2 !== "object") continue;
|
|
1338
|
+
const direct = asStringList(attrs2[attribute]);
|
|
1339
|
+
if (direct) return direct;
|
|
1340
|
+
const nested = attrs2.metadata && typeof attrs2.metadata === "object" ? asStringList(attrs2.metadata[attribute]) : void 0;
|
|
1341
|
+
if (nested) return nested;
|
|
1342
|
+
}
|
|
1343
|
+
return void 0;
|
|
1344
|
+
}
|
|
1345
|
+
function fail(ruleId, message, evidence, expected, actual) {
|
|
1346
|
+
return {
|
|
1347
|
+
ruleId,
|
|
1348
|
+
severity: "error",
|
|
1349
|
+
status: "fail",
|
|
1350
|
+
message,
|
|
1351
|
+
...expected !== void 0 ? { expected } : {},
|
|
1352
|
+
...actual !== void 0 ? { actual } : {},
|
|
1353
|
+
evidence: [...evidence]
|
|
1354
|
+
};
|
|
1355
|
+
}
|
|
1356
|
+
function sortedUnique(values) {
|
|
1357
|
+
return [...new Set(values)].sort((a, b) => a.localeCompare(b));
|
|
1358
|
+
}
|
|
1359
|
+
function setEqual(left, right) {
|
|
1360
|
+
const a = sortedUnique(left);
|
|
1361
|
+
const b = sortedUnique(right);
|
|
1362
|
+
return a.length === b.length && a.every((value, index) => value === b[index]);
|
|
1363
|
+
}
|
|
1364
|
+
function evaluateControlRules(events, rules, runEvidence2, observationNames) {
|
|
1365
|
+
const findings = [];
|
|
1366
|
+
const declared = rules.declaredTools ?? attributeList(events, rules.declaredToolsAttribute);
|
|
1367
|
+
const enforced = rules.enforcedTools ?? attributeList(events, rules.enforcedToolsAttribute);
|
|
1368
|
+
if (rules.requireDeclaredMatchesEnforced) {
|
|
1369
|
+
if (declared === void 0 || enforced === void 0) {
|
|
1370
|
+
findings.push(
|
|
1371
|
+
fail(
|
|
1372
|
+
"contract.controls.declared-matches-enforced",
|
|
1373
|
+
"Declared and enforced tool sets could not both be resolved.",
|
|
1374
|
+
runEvidence2,
|
|
1375
|
+
{ declaredDefined: declared !== void 0, enforcedDefined: enforced !== void 0 },
|
|
1376
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
1377
|
+
)
|
|
1378
|
+
);
|
|
1379
|
+
} else if (!setEqual(declared, enforced)) {
|
|
1380
|
+
findings.push(
|
|
1381
|
+
fail(
|
|
1382
|
+
"contract.controls.declared-matches-enforced",
|
|
1383
|
+
"Declared tools differ from enforced tools.",
|
|
1384
|
+
runEvidence2,
|
|
1385
|
+
sortedUnique(declared),
|
|
1386
|
+
sortedUnique(enforced)
|
|
1387
|
+
)
|
|
1388
|
+
);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
const observedTools = sortedUnique(
|
|
1392
|
+
events.filter((event) => event.kind === "TOOL" && event.status !== "running").map((event) => resolveCanonicalToolName(event))
|
|
1393
|
+
);
|
|
1394
|
+
if (rules.requireObservedWithinEnforced) {
|
|
1395
|
+
if (enforced === void 0) {
|
|
1396
|
+
findings.push(
|
|
1397
|
+
fail(
|
|
1398
|
+
"contract.controls.observed-within-enforced",
|
|
1399
|
+
"Enforced tool set unavailable for observed-within-enforced check.",
|
|
1400
|
+
runEvidence2,
|
|
1401
|
+
void 0,
|
|
1402
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
1403
|
+
)
|
|
1404
|
+
);
|
|
1405
|
+
} else {
|
|
1406
|
+
const enforcedSet = new Set(enforced);
|
|
1407
|
+
const outside = observedTools.filter((name) => !enforcedSet.has(name));
|
|
1408
|
+
if (outside.length > 0) {
|
|
1409
|
+
findings.push(
|
|
1410
|
+
fail(
|
|
1411
|
+
"contract.controls.observed-within-enforced",
|
|
1412
|
+
`Observed tools outside enforced allowlist: ${outside.join(", ")}.`,
|
|
1413
|
+
runEvidence2,
|
|
1414
|
+
sortedUnique(enforced),
|
|
1415
|
+
outside
|
|
1416
|
+
)
|
|
1417
|
+
);
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
if (rules.requireObservedWithinDeclared) {
|
|
1422
|
+
if (declared === void 0) {
|
|
1423
|
+
findings.push(
|
|
1424
|
+
fail(
|
|
1425
|
+
"contract.controls.observed-within-declared",
|
|
1426
|
+
"Declared tool set unavailable for observed-within-declared check.",
|
|
1427
|
+
runEvidence2,
|
|
1428
|
+
void 0,
|
|
1429
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
1430
|
+
)
|
|
1431
|
+
);
|
|
1432
|
+
} else {
|
|
1433
|
+
const declaredSet = new Set(declared);
|
|
1434
|
+
const outside = observedTools.filter((name) => !declaredSet.has(name));
|
|
1435
|
+
if (outside.length > 0) {
|
|
1436
|
+
findings.push(
|
|
1437
|
+
fail(
|
|
1438
|
+
"contract.controls.observed-within-declared",
|
|
1439
|
+
`Observed tools outside declared set: ${outside.join(", ")}.`,
|
|
1440
|
+
runEvidence2,
|
|
1441
|
+
sortedUnique(declared),
|
|
1442
|
+
outside
|
|
1443
|
+
)
|
|
1444
|
+
);
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
for (const stage of rules.requiredStages ?? []) {
|
|
1449
|
+
const observation = stage.observation ?? `control.${stage.stage}`;
|
|
1450
|
+
if (!observationNames.has(observation)) {
|
|
1451
|
+
findings.push(
|
|
1452
|
+
fail(
|
|
1453
|
+
`contract.controls.stage.${stage.stage}`,
|
|
1454
|
+
`Required control stage observation missing: ${observation}.`,
|
|
1455
|
+
runEvidence2,
|
|
1456
|
+
observation,
|
|
1457
|
+
[...observationNames]
|
|
1458
|
+
)
|
|
1459
|
+
);
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1462
|
+
return findings;
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
// packages/core/src/checks/retry-safety.ts
|
|
1466
|
+
function fail2(ruleId, message, evidence, expected, actual) {
|
|
1467
|
+
return {
|
|
1468
|
+
ruleId,
|
|
1469
|
+
severity: "error",
|
|
1470
|
+
status: "fail",
|
|
1471
|
+
message,
|
|
1472
|
+
...expected !== void 0 ? { expected } : {},
|
|
1473
|
+
...actual !== void 0 ? { actual } : {},
|
|
1474
|
+
evidence: [...evidence]
|
|
1475
|
+
};
|
|
1476
|
+
}
|
|
1477
|
+
function workflowFor(event) {
|
|
1478
|
+
const attrs2 = event.attributes;
|
|
1479
|
+
if (!attrs2 || typeof attrs2 !== "object") return {};
|
|
1480
|
+
const direct = extractSessionWorkflowMetadata(attrs2);
|
|
1481
|
+
const nested = attrs2.metadata && typeof attrs2.metadata === "object" ? extractSessionWorkflowMetadata(attrs2.metadata) : void 0;
|
|
1482
|
+
return { ...nested, ...direct };
|
|
1483
|
+
}
|
|
1484
|
+
function hasIdempotencyEvidence(event) {
|
|
1485
|
+
const workflow = workflowFor(event);
|
|
1486
|
+
if (typeof workflow.idempotencyKey === "string" && workflow.idempotencyKey.trim() !== "") {
|
|
1487
|
+
return true;
|
|
1488
|
+
}
|
|
1489
|
+
const attrs2 = event.attributes ?? {};
|
|
1490
|
+
if (attrs2.noSideEffect === true) return true;
|
|
1491
|
+
if (attrs2.sideEffect === false) return true;
|
|
1492
|
+
return false;
|
|
1493
|
+
}
|
|
1494
|
+
function eventEvidence(event) {
|
|
1495
|
+
return {
|
|
1496
|
+
runId: event.runId,
|
|
1497
|
+
eventId: event.eventId,
|
|
1498
|
+
kind: event.kind,
|
|
1499
|
+
name: event.name,
|
|
1500
|
+
status: event.status
|
|
1501
|
+
};
|
|
1502
|
+
}
|
|
1503
|
+
function evaluateRetrySafetyRules(events, rules, _runEvidence) {
|
|
1504
|
+
const findings = [];
|
|
1505
|
+
const byOperation = /* @__PURE__ */ new Map();
|
|
1506
|
+
for (const event of events) {
|
|
1507
|
+
const workflow = workflowFor(event);
|
|
1508
|
+
const operationId = workflow.operationId;
|
|
1509
|
+
if (!operationId) continue;
|
|
1510
|
+
const list = byOperation.get(operationId) ?? [];
|
|
1511
|
+
list.push(event);
|
|
1512
|
+
byOperation.set(operationId, list);
|
|
1513
|
+
}
|
|
1514
|
+
if (rules.maxAttempts !== void 0) {
|
|
1515
|
+
for (const [operationId, members] of byOperation) {
|
|
1516
|
+
const attemptIds = new Set(
|
|
1517
|
+
members.map((event) => workflowFor(event).attemptId).filter((value) => typeof value === "string" && value.trim() !== "")
|
|
1518
|
+
);
|
|
1519
|
+
const attemptNumbers = members.map((event) => workflowFor(event).attemptNumber ?? workflowFor(event).attempt).filter((value) => typeof value === "number" && Number.isFinite(value));
|
|
1520
|
+
const count = attemptIds.size > 0 ? attemptIds.size : attemptNumbers.length > 0 ? Math.max(...attemptNumbers) : members.filter((event) => event.kind === "TOOL" || event.kind === "LLM").length;
|
|
1521
|
+
if (count > rules.maxAttempts) {
|
|
1522
|
+
findings.push(
|
|
1523
|
+
fail2(
|
|
1524
|
+
"contract.retry.max-attempts",
|
|
1525
|
+
`Operation ${operationId} exceeded maxAttempts ${rules.maxAttempts}.`,
|
|
1526
|
+
members.slice(0, 4).map(eventEvidence),
|
|
1527
|
+
rules.maxAttempts,
|
|
1528
|
+
count
|
|
1529
|
+
)
|
|
1530
|
+
);
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
if (rules.requireTerminalResult) {
|
|
1535
|
+
for (const [operationId, members] of byOperation) {
|
|
1536
|
+
const hasTerminal = members.some((event) => event.status === "ok" || event.status === "error");
|
|
1537
|
+
if (!hasTerminal) {
|
|
1538
|
+
findings.push(
|
|
1539
|
+
fail2(
|
|
1540
|
+
"contract.retry.terminal-result",
|
|
1541
|
+
`Operation ${operationId} has no terminal ok/error result.`,
|
|
1542
|
+
members.slice(0, 4).map(eventEvidence),
|
|
1543
|
+
"ok|error",
|
|
1544
|
+
members.map((event) => event.status)
|
|
1545
|
+
)
|
|
1546
|
+
);
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
if (rules.fallbackOnlyAfterFailure) {
|
|
1551
|
+
for (const event of events) {
|
|
1552
|
+
const workflow = workflowFor(event);
|
|
1553
|
+
const fallbackOf = workflow.fallbackOf;
|
|
1554
|
+
if (!fallbackOf) continue;
|
|
1555
|
+
const prior = byOperation.get(fallbackOf) ?? events.filter((candidate) => {
|
|
1556
|
+
const meta = workflowFor(candidate);
|
|
1557
|
+
return meta.operationId === fallbackOf || candidate.runId === fallbackOf;
|
|
1558
|
+
});
|
|
1559
|
+
const priorFailure = prior.some((candidate) => candidate.status === "error");
|
|
1560
|
+
if (!priorFailure) {
|
|
1561
|
+
findings.push(
|
|
1562
|
+
fail2(
|
|
1563
|
+
"contract.retry.fallback-after-failure",
|
|
1564
|
+
`Fallback for ${fallbackOf} appeared without a prior failure.`,
|
|
1565
|
+
[eventEvidence(event)],
|
|
1566
|
+
"prior error attempt",
|
|
1567
|
+
{ fallbackOf }
|
|
1568
|
+
)
|
|
1569
|
+
);
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
const nonIdempotent = new Set(rules.nonIdempotentTools ?? []);
|
|
1574
|
+
if (nonIdempotent.size > 0 || rules.requireIdempotencyEvidenceForRetry) {
|
|
1575
|
+
const toolEvents = events.filter(
|
|
1576
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
1577
|
+
);
|
|
1578
|
+
const byToolOp = /* @__PURE__ */ new Map();
|
|
1579
|
+
for (const event of toolEvents) {
|
|
1580
|
+
const name = resolveCanonicalToolName(event);
|
|
1581
|
+
const workflow = workflowFor(event);
|
|
1582
|
+
const key = `${workflow.operationId ?? name}::${name}`;
|
|
1583
|
+
const list = byToolOp.get(key) ?? [];
|
|
1584
|
+
list.push(event);
|
|
1585
|
+
byToolOp.set(key, list);
|
|
1586
|
+
}
|
|
1587
|
+
for (const [, members] of byToolOp) {
|
|
1588
|
+
const ordered = [...members].sort((a, b) => {
|
|
1589
|
+
const aTime = a.startedAt ?? a.timestamp ?? "";
|
|
1590
|
+
const bTime = b.startedAt ?? b.timestamp ?? "";
|
|
1591
|
+
return aTime.localeCompare(bTime);
|
|
1592
|
+
});
|
|
1593
|
+
let sawOk = false;
|
|
1594
|
+
let sawSideEffectOk = false;
|
|
1595
|
+
for (const event of ordered) {
|
|
1596
|
+
const name = resolveCanonicalToolName(event);
|
|
1597
|
+
const isRetry = sawOk;
|
|
1598
|
+
if (isRetry) {
|
|
1599
|
+
if (rules.requireIdempotencyEvidenceForRetry && !hasIdempotencyEvidence(event)) {
|
|
1600
|
+
findings.push(
|
|
1601
|
+
fail2(
|
|
1602
|
+
"contract.retry.idempotency-evidence",
|
|
1603
|
+
`Retry of tool ${name} lacks idempotencyKey / noSideEffect evidence.`,
|
|
1604
|
+
[eventEvidence(event)],
|
|
1605
|
+
"idempotencyKey|noSideEffect",
|
|
1606
|
+
{ code: "AI_CHECK_RETRY_EVIDENCE_UNAVAILABLE" }
|
|
1607
|
+
)
|
|
1608
|
+
);
|
|
1609
|
+
}
|
|
1610
|
+
if (nonIdempotent.has(name) && sawSideEffectOk && !hasIdempotencyEvidence(event)) {
|
|
1611
|
+
findings.push(
|
|
1612
|
+
fail2(
|
|
1613
|
+
"contract.retry.non-idempotent-side-effect",
|
|
1614
|
+
`Retry of non-idempotent tool ${name} after a confirmed ok side effect.`,
|
|
1615
|
+
[eventEvidence(event)],
|
|
1616
|
+
"no retry after side effect",
|
|
1617
|
+
name
|
|
1618
|
+
)
|
|
1619
|
+
);
|
|
1620
|
+
}
|
|
1621
|
+
}
|
|
1622
|
+
if (event.status === "ok") {
|
|
1623
|
+
sawOk = true;
|
|
1624
|
+
if (nonIdempotent.has(name) && !hasIdempotencyEvidence(event)) {
|
|
1625
|
+
sawSideEffectOk = true;
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
if (rules.requireRecoveredFailureVisible) {
|
|
1632
|
+
for (const [operationId, members] of byOperation) {
|
|
1633
|
+
const hasOk = members.some((event) => event.status === "ok");
|
|
1634
|
+
const hasError = members.some((event) => event.status === "error");
|
|
1635
|
+
const attemptish = members.some((event) => {
|
|
1636
|
+
const meta = workflowFor(event);
|
|
1637
|
+
return meta.attemptId !== void 0 || meta.attemptNumber !== void 0 || meta.attempt !== void 0;
|
|
1638
|
+
}) || members.length > 1;
|
|
1639
|
+
if (hasOk && attemptish && !hasError) {
|
|
1640
|
+
findings.push(
|
|
1641
|
+
fail2(
|
|
1642
|
+
"contract.retry.recovered-failure-visible",
|
|
1643
|
+
`Operation ${operationId} recovered without retaining a visible failure attempt.`,
|
|
1644
|
+
members.slice(0, 4).map(eventEvidence),
|
|
1645
|
+
"error attempt retained",
|
|
1646
|
+
{ hasOk, hasError }
|
|
1647
|
+
)
|
|
1648
|
+
);
|
|
1649
|
+
}
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
return findings;
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
// packages/core/src/checks/tool-arguments.ts
|
|
1656
|
+
function decodeToken(token) {
|
|
1657
|
+
return token.replace(/~1/g, "/").replace(/~0/g, "~");
|
|
1658
|
+
}
|
|
1659
|
+
function resolveJsonPointer(document, pointer) {
|
|
1660
|
+
if (pointer === "") {
|
|
1661
|
+
return { found: true, value: document };
|
|
1662
|
+
}
|
|
1663
|
+
if (!pointer.startsWith("/")) {
|
|
1664
|
+
return { found: false };
|
|
1665
|
+
}
|
|
1666
|
+
const tokens = pointer.slice(1).split("/").map(decodeToken);
|
|
1667
|
+
let current = document;
|
|
1668
|
+
for (const token of tokens) {
|
|
1669
|
+
if (current === null || current === void 0) {
|
|
1670
|
+
return { found: false };
|
|
1671
|
+
}
|
|
1672
|
+
if (Array.isArray(current)) {
|
|
1673
|
+
if (!/^(0|[1-9][0-9]*)$/.test(token)) {
|
|
1674
|
+
return { found: false };
|
|
1675
|
+
}
|
|
1676
|
+
const index = Number(token);
|
|
1677
|
+
if (index >= current.length) {
|
|
1678
|
+
return { found: false };
|
|
1679
|
+
}
|
|
1680
|
+
current = current[index];
|
|
1681
|
+
continue;
|
|
1682
|
+
}
|
|
1683
|
+
if (typeof current !== "object") {
|
|
1684
|
+
return { found: false };
|
|
1685
|
+
}
|
|
1686
|
+
const record = current;
|
|
1687
|
+
if (!Object.prototype.hasOwnProperty.call(record, token)) {
|
|
1688
|
+
return { found: false };
|
|
1689
|
+
}
|
|
1690
|
+
current = record[token];
|
|
1691
|
+
}
|
|
1692
|
+
return { found: true, value: current };
|
|
1693
|
+
}
|
|
1694
|
+
function jsonType(value) {
|
|
1695
|
+
if (value === null) return "null";
|
|
1696
|
+
if (Array.isArray(value)) return "array";
|
|
1697
|
+
return typeof value;
|
|
1698
|
+
}
|
|
1699
|
+
function valuesEqual(left, right) {
|
|
1700
|
+
return JSON.stringify(left) === JSON.stringify(right);
|
|
1701
|
+
}
|
|
1702
|
+
function extractToolArgumentPayload(event) {
|
|
1703
|
+
const attrs2 = event.attributes ?? {};
|
|
1704
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
1705
|
+
const candidate = attrs2[key];
|
|
1706
|
+
if (candidate !== void 0 && candidate !== null && typeof candidate === "object") {
|
|
1707
|
+
return { present: true, value: candidate };
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
if (event.inputSummary !== void 0 && event.inputSummary !== null && typeof event.inputSummary === "object") {
|
|
1711
|
+
return { present: true, value: event.inputSummary };
|
|
1712
|
+
}
|
|
1713
|
+
return { present: false, value: void 0 };
|
|
1714
|
+
}
|
|
1715
|
+
function evaluateToolArgumentValue(value, check, evidencePresent) {
|
|
1716
|
+
if (!evidencePresent) {
|
|
1717
|
+
return {
|
|
1718
|
+
status: "unavailable",
|
|
1719
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
1720
|
+
message: `Structured argument evidence unavailable for tool ${check.tool} at ${check.path}.`
|
|
1721
|
+
};
|
|
1722
|
+
}
|
|
1723
|
+
const resolved = resolveJsonPointer(value, check.path);
|
|
1724
|
+
if (check.operator === "exists") {
|
|
1725
|
+
return resolved.found ? { status: "pass" } : {
|
|
1726
|
+
status: "fail",
|
|
1727
|
+
code: "AI_CHECK_TOOL_ARGUMENT_MISSING",
|
|
1728
|
+
message: `Expected path ${check.path} to exist on tool ${check.tool}.`
|
|
1729
|
+
};
|
|
1730
|
+
}
|
|
1731
|
+
if (!resolved.found) {
|
|
1732
|
+
return {
|
|
1733
|
+
status: "unavailable",
|
|
1734
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
1735
|
+
message: `Path ${check.path} missing for tool ${check.tool}.`
|
|
1736
|
+
};
|
|
1737
|
+
}
|
|
1738
|
+
if (check.operator === "type") {
|
|
1739
|
+
const actual = jsonType(resolved.value);
|
|
1740
|
+
if (check.type === void 0) {
|
|
1741
|
+
return {
|
|
1742
|
+
status: "fail",
|
|
1743
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
1744
|
+
message: "type operator requires check.type"
|
|
1745
|
+
};
|
|
1746
|
+
}
|
|
1747
|
+
return actual === check.type ? { status: "pass" } : {
|
|
1748
|
+
status: "fail",
|
|
1749
|
+
code: "AI_CHECK_TOOL_ARGUMENT_TYPE",
|
|
1750
|
+
message: `Expected type ${check.type} at ${check.path} for tool ${check.tool}; found ${actual}.`
|
|
1751
|
+
};
|
|
1752
|
+
}
|
|
1753
|
+
if (check.operator === "equals") {
|
|
1754
|
+
return valuesEqual(resolved.value, check.expected) ? { status: "pass" } : {
|
|
1755
|
+
status: "fail",
|
|
1756
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EQUALS",
|
|
1757
|
+
message: `Value at ${check.path} for tool ${check.tool} did not equal expected (bounded comparison).`
|
|
1758
|
+
};
|
|
1759
|
+
}
|
|
1760
|
+
if (check.operator === "oneOf") {
|
|
1761
|
+
const options = check.oneOf ?? [];
|
|
1762
|
+
if (options.length === 0) {
|
|
1763
|
+
return {
|
|
1764
|
+
status: "fail",
|
|
1765
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
1766
|
+
message: "oneOf operator requires a non-empty oneOf array"
|
|
1767
|
+
};
|
|
1768
|
+
}
|
|
1769
|
+
return options.some((option) => valuesEqual(resolved.value, option)) ? { status: "pass" } : {
|
|
1770
|
+
status: "fail",
|
|
1771
|
+
code: "AI_CHECK_TOOL_ARGUMENT_ONE_OF",
|
|
1772
|
+
message: `Value at ${check.path} for tool ${check.tool} was not in the allowed oneOf set.`
|
|
1773
|
+
};
|
|
1774
|
+
}
|
|
1775
|
+
return {
|
|
1776
|
+
status: "fail",
|
|
1777
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
1778
|
+
message: `Unsupported operator.`
|
|
1779
|
+
};
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1166
1782
|
// packages/core/src/checks/contract.ts
|
|
1167
1783
|
function contractFailFinding(ruleId, message, evidence, expected, actual) {
|
|
1168
1784
|
return {
|
|
@@ -1184,18 +1800,38 @@ function normalizeStatus(status) {
|
|
|
1184
1800
|
function cloneBody(body) {
|
|
1185
1801
|
return {
|
|
1186
1802
|
...body.run ? { run: { ...body.run } } : {},
|
|
1187
|
-
...body.tools ? {
|
|
1803
|
+
...body.tools ? {
|
|
1804
|
+
tools: {
|
|
1805
|
+
...body.tools,
|
|
1806
|
+
...body.tools.arguments ? { arguments: body.tools.arguments.map((item) => ({ ...item })) } : {},
|
|
1807
|
+
...body.tools.orderRules ? { orderRules: body.tools.orderRules.map((item) => ({ ...item })) } : {}
|
|
1808
|
+
}
|
|
1809
|
+
} : {},
|
|
1188
1810
|
...body.llm ? { llm: { ...body.llm } } : {},
|
|
1189
1811
|
...body.observations ? {
|
|
1190
1812
|
observations: {
|
|
1191
1813
|
...body.observations,
|
|
1192
1814
|
...body.observations.requireProvenance ? { requireProvenance: { ...body.observations.requireProvenance } } : {}
|
|
1193
1815
|
}
|
|
1816
|
+
} : {},
|
|
1817
|
+
...body.controls ? {
|
|
1818
|
+
controls: {
|
|
1819
|
+
...body.controls,
|
|
1820
|
+
...body.controls.declaredTools ? { declaredTools: [...body.controls.declaredTools] } : {},
|
|
1821
|
+
...body.controls.enforcedTools ? { enforcedTools: [...body.controls.enforcedTools] } : {},
|
|
1822
|
+
...body.controls.requiredStages ? { requiredStages: body.controls.requiredStages.map((item) => ({ ...item })) } : {}
|
|
1823
|
+
}
|
|
1824
|
+
} : {},
|
|
1825
|
+
...body.retry ? {
|
|
1826
|
+
retry: {
|
|
1827
|
+
...body.retry,
|
|
1828
|
+
...body.retry.nonIdempotentTools ? { nonIdempotentTools: [...body.retry.nonIdempotentTools] } : {}
|
|
1829
|
+
}
|
|
1194
1830
|
} : {}
|
|
1195
1831
|
};
|
|
1196
1832
|
}
|
|
1197
1833
|
function bodyHasRules(body) {
|
|
1198
|
-
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0;
|
|
1834
|
+
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
1199
1835
|
}
|
|
1200
1836
|
var MAX_EVIDENCE_EVENT_IDS = 16;
|
|
1201
1837
|
var METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
@@ -1396,11 +2032,14 @@ function contractToRules(contract) {
|
|
|
1396
2032
|
if (contract.tools) {
|
|
1397
2033
|
const order = contract.tools.requiredOrder ?? [];
|
|
1398
2034
|
const requiredOrderMode = contract.tools.requiredOrderMode ?? "first-occurrence";
|
|
2035
|
+
const orderRules = contract.tools.orderRules ?? [];
|
|
2036
|
+
const endpointRequired = orderRules.filter((rule) => rule.requireEndpoints !== false).flatMap((rule) => [rule.before, rule.after]);
|
|
1399
2037
|
const required = [
|
|
1400
2038
|
.../* @__PURE__ */ new Set([
|
|
1401
2039
|
...contract.tools.required ?? [],
|
|
1402
2040
|
...contract.tools.requiredTools ?? [],
|
|
1403
|
-
...order
|
|
2041
|
+
...order,
|
|
2042
|
+
...endpointRequired
|
|
1404
2043
|
])
|
|
1405
2044
|
];
|
|
1406
2045
|
const forbidden = [
|
|
@@ -1425,6 +2064,106 @@ function contractToRules(contract) {
|
|
|
1425
2064
|
})
|
|
1426
2065
|
);
|
|
1427
2066
|
}
|
|
2067
|
+
const defaultOccurrenceMode = contract.tools.defaultOccurrenceMode ?? requiredOrderMode;
|
|
2068
|
+
for (const [index, rule] of orderRules.entries()) {
|
|
2069
|
+
rules.push(
|
|
2070
|
+
createToolOrderingRule({
|
|
2071
|
+
before: rule.before,
|
|
2072
|
+
after: rule.after,
|
|
2073
|
+
id: `contract.tool.orderRule.${index}`,
|
|
2074
|
+
mode: rule.occurrenceMode ?? defaultOccurrenceMode
|
|
2075
|
+
})
|
|
2076
|
+
);
|
|
2077
|
+
}
|
|
2078
|
+
const argumentChecks = contract.tools.arguments ?? [];
|
|
2079
|
+
if (argumentChecks.length > 0) {
|
|
2080
|
+
rules.push({
|
|
2081
|
+
id: "contract.tool.arguments",
|
|
2082
|
+
category: "tool",
|
|
2083
|
+
defaultSeverity: "error",
|
|
2084
|
+
evaluate(context) {
|
|
2085
|
+
const tools = (context.logicalEvents ?? context.events).filter(
|
|
2086
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
2087
|
+
);
|
|
2088
|
+
const findings = [];
|
|
2089
|
+
for (const [index, check] of argumentChecks.entries()) {
|
|
2090
|
+
const matches = tools.filter(
|
|
2091
|
+
(event) => resolveCanonicalToolName(event) === check.tool
|
|
2092
|
+
);
|
|
2093
|
+
if (matches.length === 0) {
|
|
2094
|
+
findings.push(
|
|
2095
|
+
contractFailFinding(
|
|
2096
|
+
`contract.tool.arguments.${index}`,
|
|
2097
|
+
`No finished tool named ${check.tool} for argument check.`,
|
|
2098
|
+
context.selectedRun ? [
|
|
2099
|
+
{
|
|
2100
|
+
runId: context.selectedRun.runId,
|
|
2101
|
+
kind: "RUN",
|
|
2102
|
+
name: context.selectedRun.name
|
|
2103
|
+
}
|
|
2104
|
+
] : [],
|
|
2105
|
+
{ tool: check.tool, path: check.path },
|
|
2106
|
+
{ toolCount: 0 }
|
|
2107
|
+
)
|
|
2108
|
+
);
|
|
2109
|
+
continue;
|
|
2110
|
+
}
|
|
2111
|
+
const occurrence = check.occurrence ?? "all";
|
|
2112
|
+
const selected = occurrence === "first" ? [matches[0]] : occurrence === "last" ? [matches[matches.length - 1]] : matches;
|
|
2113
|
+
const results = selected.map((event) => {
|
|
2114
|
+
const payload = extractToolArgumentPayload(event);
|
|
2115
|
+
return evaluateToolArgumentValue(
|
|
2116
|
+
payload.value,
|
|
2117
|
+
check,
|
|
2118
|
+
payload.present
|
|
2119
|
+
);
|
|
2120
|
+
});
|
|
2121
|
+
if (occurrence === "any") {
|
|
2122
|
+
if (results.some((result) => result.status === "pass")) continue;
|
|
2123
|
+
const firstFail = results.find((result) => result.status !== "pass");
|
|
2124
|
+
findings.push(
|
|
2125
|
+
contractFailFinding(
|
|
2126
|
+
`contract.tool.arguments.${index}`,
|
|
2127
|
+
firstFail.message,
|
|
2128
|
+
selected.slice(0, 1).map((event) => ({
|
|
2129
|
+
runId: event.runId,
|
|
2130
|
+
eventId: event.eventId,
|
|
2131
|
+
kind: event.kind,
|
|
2132
|
+
name: event.name,
|
|
2133
|
+
path: `tool.${check.tool}${check.path}`
|
|
2134
|
+
})),
|
|
2135
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
2136
|
+
{ code: firstFail.code }
|
|
2137
|
+
)
|
|
2138
|
+
);
|
|
2139
|
+
continue;
|
|
2140
|
+
}
|
|
2141
|
+
for (const [selIndex, result] of results.entries()) {
|
|
2142
|
+
if (result.status === "pass") continue;
|
|
2143
|
+
const event = selected[selIndex];
|
|
2144
|
+
findings.push(
|
|
2145
|
+
contractFailFinding(
|
|
2146
|
+
`contract.tool.arguments.${index}`,
|
|
2147
|
+
result.message,
|
|
2148
|
+
[
|
|
2149
|
+
{
|
|
2150
|
+
runId: event.runId,
|
|
2151
|
+
eventId: event.eventId,
|
|
2152
|
+
kind: event.kind,
|
|
2153
|
+
name: event.name,
|
|
2154
|
+
path: `tool.${check.tool}${check.path}`
|
|
2155
|
+
}
|
|
2156
|
+
],
|
|
2157
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
2158
|
+
{ code: result.code }
|
|
2159
|
+
)
|
|
2160
|
+
);
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
return findings;
|
|
2164
|
+
}
|
|
2165
|
+
});
|
|
2166
|
+
}
|
|
1428
2167
|
}
|
|
1429
2168
|
if (contract.llm) {
|
|
1430
2169
|
rules.push(
|
|
@@ -1485,6 +2224,46 @@ function contractToRules(contract) {
|
|
|
1485
2224
|
});
|
|
1486
2225
|
}
|
|
1487
2226
|
}
|
|
2227
|
+
if (contract.controls) {
|
|
2228
|
+
const controls = contract.controls;
|
|
2229
|
+
rules.push({
|
|
2230
|
+
id: "contract.controls",
|
|
2231
|
+
category: "run",
|
|
2232
|
+
defaultSeverity: "error",
|
|
2233
|
+
evaluate(context) {
|
|
2234
|
+
const events = context.logicalEvents ?? context.events;
|
|
2235
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
2236
|
+
const observationNames = new Set(outcomes.map((item) => item.name));
|
|
2237
|
+
const runEvidence2 = context.selectedRun ? [
|
|
2238
|
+
{
|
|
2239
|
+
runId: context.selectedRun.runId,
|
|
2240
|
+
kind: "RUN",
|
|
2241
|
+
name: context.selectedRun.name
|
|
2242
|
+
}
|
|
2243
|
+
] : [];
|
|
2244
|
+
return evaluateControlRules(events, controls, runEvidence2, observationNames);
|
|
2245
|
+
}
|
|
2246
|
+
});
|
|
2247
|
+
}
|
|
2248
|
+
if (contract.retry) {
|
|
2249
|
+
const retry = contract.retry;
|
|
2250
|
+
rules.push({
|
|
2251
|
+
id: "contract.retry",
|
|
2252
|
+
category: "run",
|
|
2253
|
+
defaultSeverity: "error",
|
|
2254
|
+
evaluate(context) {
|
|
2255
|
+
const events = context.logicalEvents ?? context.events;
|
|
2256
|
+
context.selectedRun ? [
|
|
2257
|
+
{
|
|
2258
|
+
runId: context.selectedRun.runId,
|
|
2259
|
+
kind: "RUN",
|
|
2260
|
+
name: context.selectedRun.name
|
|
2261
|
+
}
|
|
2262
|
+
] : [];
|
|
2263
|
+
return evaluateRetrySafetyRules(events, retry);
|
|
2264
|
+
}
|
|
2265
|
+
});
|
|
2266
|
+
}
|
|
1488
2267
|
return rules;
|
|
1489
2268
|
}
|
|
1490
2269
|
function validateScopeShape(scope) {
|
|
@@ -1571,7 +2350,7 @@ function validateAlternativesShape(alternatives) {
|
|
|
1571
2350
|
diagnostics.push({
|
|
1572
2351
|
code: "contract.alternatives.empty-branch",
|
|
1573
2352
|
severity: "error",
|
|
1574
|
-
message: `Branch ${branch.id} must declare at least one run/tools/llm/observations rule.`,
|
|
2353
|
+
message: `Branch ${branch.id} must declare at least one run/tools/llm/observations/controls/retry rule.`,
|
|
1575
2354
|
path: `${path3}.contract`
|
|
1576
2355
|
});
|
|
1577
2356
|
}
|
|
@@ -1844,6 +2623,40 @@ function lintTraceContract(contract) {
|
|
|
1844
2623
|
path: "tools.requiredOrderMode"
|
|
1845
2624
|
});
|
|
1846
2625
|
}
|
|
2626
|
+
const orderRules = contract.tools?.orderRules ?? [];
|
|
2627
|
+
const seenPairs = /* @__PURE__ */ new Set();
|
|
2628
|
+
for (const [index, rule] of orderRules.entries()) {
|
|
2629
|
+
const key = `${rule.before}\0${rule.after}`;
|
|
2630
|
+
if (seenPairs.has(key)) {
|
|
2631
|
+
diagnostics.push({
|
|
2632
|
+
code: "contract.tools.orderRules.duplicate",
|
|
2633
|
+
severity: "warning",
|
|
2634
|
+
message: `Duplicate orderRules pair ${rule.before} \u2192 ${rule.after}.`,
|
|
2635
|
+
path: `tools.orderRules[${index}]`
|
|
2636
|
+
});
|
|
2637
|
+
}
|
|
2638
|
+
seenPairs.add(key);
|
|
2639
|
+
if (rule.before === rule.after) {
|
|
2640
|
+
diagnostics.push({
|
|
2641
|
+
code: "contract.tools.orderRules.self",
|
|
2642
|
+
severity: "error",
|
|
2643
|
+
message: "orderRules before and after must differ.",
|
|
2644
|
+
path: `tools.orderRules[${index}]`
|
|
2645
|
+
});
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
if ((contract.tools?.arguments?.length ?? 0) > 0) {
|
|
2649
|
+
for (const [index, check] of (contract.tools?.arguments ?? []).entries()) {
|
|
2650
|
+
if (!check.path.startsWith("/") && check.path !== "") {
|
|
2651
|
+
diagnostics.push({
|
|
2652
|
+
code: "contract.tools.arguments.path",
|
|
2653
|
+
severity: "error",
|
|
2654
|
+
message: 'tools.arguments path must be a JSON Pointer ("" or start with "/").',
|
|
2655
|
+
path: `tools.arguments[${index}].path`
|
|
2656
|
+
});
|
|
2657
|
+
}
|
|
2658
|
+
}
|
|
2659
|
+
}
|
|
1847
2660
|
return diagnostics;
|
|
1848
2661
|
}
|
|
1849
2662
|
function explainTraceContract(contract) {
|
|
@@ -1887,6 +2700,17 @@ function explainTraceContract(contract) {
|
|
|
1887
2700
|
`Base: requiredOrder [${contract.tools.requiredOrder.join(" \u2192 ")}] mode=${mode}.`
|
|
1888
2701
|
);
|
|
1889
2702
|
}
|
|
2703
|
+
if ((contract.tools.orderRules?.length ?? 0) > 0) {
|
|
2704
|
+
const defaultMode = contract.tools.defaultOccurrenceMode ?? "first-occurrence";
|
|
2705
|
+
lines.push(
|
|
2706
|
+
`Base: ${contract.tools.orderRules.length} orderRules (defaultOccurrenceMode=${defaultMode}).`
|
|
2707
|
+
);
|
|
2708
|
+
}
|
|
2709
|
+
if ((contract.tools.arguments?.length ?? 0) > 0) {
|
|
2710
|
+
lines.push(
|
|
2711
|
+
`Base: ${contract.tools.arguments.length} structured tool-argument check(s).`
|
|
2712
|
+
);
|
|
2713
|
+
}
|
|
1890
2714
|
}
|
|
1891
2715
|
if (contract.llm) {
|
|
1892
2716
|
if (contract.llm.maxCalls !== void 0) {
|
|
@@ -1917,6 +2741,12 @@ function explainTraceContract(contract) {
|
|
|
1917
2741
|
);
|
|
1918
2742
|
}
|
|
1919
2743
|
}
|
|
2744
|
+
if (contract.controls) {
|
|
2745
|
+
lines.push("Base: declared-versus-enforced control checks enabled.");
|
|
2746
|
+
}
|
|
2747
|
+
if (contract.retry) {
|
|
2748
|
+
lines.push("Base: retry/side-effect safety checks enabled.");
|
|
2749
|
+
}
|
|
1920
2750
|
const branches = contract.alternatives?.anyOf ?? [];
|
|
1921
2751
|
if (branches.length > 0) {
|
|
1922
2752
|
lines.push(
|
|
@@ -2227,7 +3057,7 @@ function classifyRuleExecution(findings, threw) {
|
|
|
2227
3057
|
}
|
|
2228
3058
|
return "pass";
|
|
2229
3059
|
}
|
|
2230
|
-
function
|
|
3060
|
+
function stringAttr2(event, keys) {
|
|
2231
3061
|
for (const key of keys) {
|
|
2232
3062
|
const value = event.attributes?.[key];
|
|
2233
3063
|
if (typeof value === "string" && value.trim() !== "") return value;
|
|
@@ -2266,7 +3096,7 @@ function stripPrefix(name, prefixes) {
|
|
|
2266
3096
|
}
|
|
2267
3097
|
return name;
|
|
2268
3098
|
}
|
|
2269
|
-
function
|
|
3099
|
+
function eventEvidence2(event, path3) {
|
|
2270
3100
|
return {
|
|
2271
3101
|
runId: event.runId,
|
|
2272
3102
|
eventId: event.eventId,
|
|
@@ -2305,15 +3135,15 @@ function semanticEvents(context) {
|
|
|
2305
3135
|
}
|
|
2306
3136
|
function llmModel(event) {
|
|
2307
3137
|
const keys = ["model", "modelId", "responseModelId", "modelName", "model_name"];
|
|
2308
|
-
return
|
|
3138
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
2309
3139
|
}
|
|
2310
3140
|
function llmProvider(event) {
|
|
2311
3141
|
const keys = ["provider", "providerName", "provider_name"];
|
|
2312
|
-
return
|
|
3142
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys);
|
|
2313
3143
|
}
|
|
2314
3144
|
function llmFinishReason(event) {
|
|
2315
3145
|
const keys = ["finishReason", "rawFinishReason", "finish_reason"];
|
|
2316
|
-
return
|
|
3146
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys);
|
|
2317
3147
|
}
|
|
2318
3148
|
function retryCount(event) {
|
|
2319
3149
|
return numericAttr(event, ["retryCount", "retryAttempt", "retry_attempt", "attempt"]);
|
|
@@ -2455,7 +3285,7 @@ function parentMarkedUnresolved(event) {
|
|
|
2455
3285
|
]) === true) {
|
|
2456
3286
|
return true;
|
|
2457
3287
|
}
|
|
2458
|
-
const resolution =
|
|
3288
|
+
const resolution = stringAttr2(event, [
|
|
2459
3289
|
"parentResolution",
|
|
2460
3290
|
"relationshipResolution",
|
|
2461
3291
|
"relationshipStatus"
|
|
@@ -2463,13 +3293,13 @@ function parentMarkedUnresolved(event) {
|
|
|
2463
3293
|
return resolution === "unresolved" || resolution === "missing-parent";
|
|
2464
3294
|
}
|
|
2465
3295
|
function signalName(event, attributeKeys, prefixes) {
|
|
2466
|
-
return
|
|
3296
|
+
return stringAttr2(event, attributeKeys) ?? stripPrefix(event.name, prefixes);
|
|
2467
3297
|
}
|
|
2468
3298
|
function guardrailEvents(context) {
|
|
2469
3299
|
return finishedEvents(context).filter((event) => {
|
|
2470
3300
|
const name = event.name.toLowerCase();
|
|
2471
3301
|
if (name.startsWith("guardrail:") || name.includes(".guardrail.")) return true;
|
|
2472
|
-
return
|
|
3302
|
+
return stringAttr2(event, ["guardrailName", "guardrail", "guardrailId"]) !== void 0;
|
|
2473
3303
|
});
|
|
2474
3304
|
}
|
|
2475
3305
|
function retryValue(event) {
|
|
@@ -2536,7 +3366,7 @@ function guardrailShape(context) {
|
|
|
2536
3366
|
}
|
|
2537
3367
|
function firstEvidenceForKind(context, kind, path3) {
|
|
2538
3368
|
const event = semanticEvents(context).find((candidate) => candidate.kind === kind);
|
|
2539
|
-
return event ? [
|
|
3369
|
+
return event ? [eventEvidence2(event, path3)] : runEvidence(context.selectedRun);
|
|
2540
3370
|
}
|
|
2541
3371
|
function baselineDiffFinding(message, evidence, expected, actual) {
|
|
2542
3372
|
return failFinding("baseline.regression", message, evidence, expected, actual);
|
|
@@ -2569,7 +3399,7 @@ function createRunStatusRule(options = {}) {
|
|
|
2569
3399
|
failFinding(
|
|
2570
3400
|
"run.status",
|
|
2571
3401
|
"Run contains incomplete running events.",
|
|
2572
|
-
running.map((event) =>
|
|
3402
|
+
running.map((event) => eventEvidence2(event)),
|
|
2573
3403
|
"no running events",
|
|
2574
3404
|
running.length
|
|
2575
3405
|
)
|
|
@@ -2615,7 +3445,7 @@ function createMaxStepDurationRule(options) {
|
|
|
2615
3445
|
failFinding(
|
|
2616
3446
|
"run.maxStepDuration",
|
|
2617
3447
|
`${over.length} step(s) exceeded max duration ${options.maxDurationMs}ms.`,
|
|
2618
|
-
over.map((event) =>
|
|
3448
|
+
over.map((event) => eventEvidence2(event, "durationMs")),
|
|
2619
3449
|
{ maxDurationMs: options.maxDurationMs },
|
|
2620
3450
|
over.map((event) => ({
|
|
2621
3451
|
eventId: event.eventId,
|
|
@@ -2641,7 +3471,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
2641
3471
|
failFinding(
|
|
2642
3472
|
"run.stall",
|
|
2643
3473
|
`Found ${running.length} event(s) still running (possible stall).`,
|
|
2644
|
-
running.map((event) =>
|
|
3474
|
+
running.map((event) => eventEvidence2(event, "status")),
|
|
2645
3475
|
"no running events",
|
|
2646
3476
|
running.length
|
|
2647
3477
|
)
|
|
@@ -2656,7 +3486,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
2656
3486
|
failFinding(
|
|
2657
3487
|
"run.stall",
|
|
2658
3488
|
`Found ${incomplete.length} started event(s) without endedAt.`,
|
|
2659
|
-
incomplete.map((event) =>
|
|
3489
|
+
incomplete.map((event) => eventEvidence2(event, "endedAt")),
|
|
2660
3490
|
"endedAt for started events",
|
|
2661
3491
|
incomplete.length
|
|
2662
3492
|
)
|
|
@@ -2692,7 +3522,7 @@ function createRequireCompletedRule() {
|
|
|
2692
3522
|
failFinding(
|
|
2693
3523
|
"run.requireCompleted",
|
|
2694
3524
|
`Run has ${running.length} incomplete running event(s).`,
|
|
2695
|
-
running.map((event) =>
|
|
3525
|
+
running.map((event) => eventEvidence2(event, "status")),
|
|
2696
3526
|
"no running events",
|
|
2697
3527
|
running.length
|
|
2698
3528
|
)
|
|
@@ -2790,12 +3620,12 @@ function createToolUsageRule(options) {
|
|
|
2790
3620
|
const name = toolName(event);
|
|
2791
3621
|
if (forbidden.has(name)) {
|
|
2792
3622
|
findings.push(
|
|
2793
|
-
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [
|
|
3623
|
+
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence2(event)], "tool absent", name)
|
|
2794
3624
|
);
|
|
2795
3625
|
}
|
|
2796
3626
|
if (allowed && !allowed.has(name)) {
|
|
2797
3627
|
findings.push(
|
|
2798
|
-
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [
|
|
3628
|
+
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence2(event)], [...allowed].sort(), name)
|
|
2799
3629
|
);
|
|
2800
3630
|
}
|
|
2801
3631
|
}
|
|
@@ -2806,7 +3636,7 @@ function createToolUsageRule(options) {
|
|
|
2806
3636
|
}
|
|
2807
3637
|
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
2808
3638
|
findings.push(
|
|
2809
|
-
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) =>
|
|
3639
|
+
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence2(event)), { maxCount: options.maxCount }, tools.length)
|
|
2810
3640
|
);
|
|
2811
3641
|
}
|
|
2812
3642
|
return findings;
|
|
@@ -2833,7 +3663,7 @@ function createToolOrderingRule(options) {
|
|
|
2833
3663
|
failFinding(
|
|
2834
3664
|
ruleId,
|
|
2835
3665
|
`Tool ${options.before} must appear before ${options.after}.`,
|
|
2836
|
-
[
|
|
3666
|
+
[eventEvidence2(tools[beforeIndex]), eventEvidence2(tools[afterIndex])],
|
|
2837
3667
|
{ before: options.before, after: options.after },
|
|
2838
3668
|
tools.map(toolName)
|
|
2839
3669
|
)
|
|
@@ -2855,7 +3685,7 @@ function createToolOrderingRule(options) {
|
|
|
2855
3685
|
failFinding(
|
|
2856
3686
|
ruleId,
|
|
2857
3687
|
`Tool ${options.before} cannot causally happen before itself.`,
|
|
2858
|
-
[
|
|
3688
|
+
[eventEvidence2(beforeEvent)],
|
|
2859
3689
|
expected,
|
|
2860
3690
|
{ code: "tool.order.same-tool" }
|
|
2861
3691
|
)
|
|
@@ -2866,7 +3696,7 @@ function createToolOrderingRule(options) {
|
|
|
2866
3696
|
failFinding(
|
|
2867
3697
|
ruleId,
|
|
2868
3698
|
`Tool order ${options.before} before ${options.after} could not establish causal timing.`,
|
|
2869
|
-
[
|
|
3699
|
+
[eventEvidence2(beforeEvent), eventEvidence2(afterEvent)],
|
|
2870
3700
|
expected,
|
|
2871
3701
|
{
|
|
2872
3702
|
code: "tool.order.interval-unresolved",
|
|
@@ -2881,7 +3711,7 @@ function createToolOrderingRule(options) {
|
|
|
2881
3711
|
failFinding(
|
|
2882
3712
|
ruleId,
|
|
2883
3713
|
`Tool ${options.before} must finish before ${options.after} starts.`,
|
|
2884
|
-
[
|
|
3714
|
+
[eventEvidence2(beforeEvent), eventEvidence2(afterEvent)],
|
|
2885
3715
|
expected,
|
|
2886
3716
|
{
|
|
2887
3717
|
beforeEndedAt: new Date(beforeEnd).toISOString(),
|
|
@@ -2906,7 +3736,7 @@ function createToolOrderingRule(options) {
|
|
|
2906
3736
|
failFinding(
|
|
2907
3737
|
ruleId,
|
|
2908
3738
|
`Tool ${options.before} cannot causally happen before itself.`,
|
|
2909
|
-
[
|
|
3739
|
+
[eventEvidence2(beforeEvent)],
|
|
2910
3740
|
expected,
|
|
2911
3741
|
{ code: "tool.order.same-tool" }
|
|
2912
3742
|
)
|
|
@@ -2941,7 +3771,7 @@ function createToolOrderingRule(options) {
|
|
|
2941
3771
|
failFinding(
|
|
2942
3772
|
ruleId,
|
|
2943
3773
|
`Tool order ${options.before} before ${options.after} could not establish all causal intervals.`,
|
|
2944
|
-
[firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) =>
|
|
3774
|
+
[firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence2(event)),
|
|
2945
3775
|
expected,
|
|
2946
3776
|
{
|
|
2947
3777
|
code: "tool.order.interval-unresolved",
|
|
@@ -2956,7 +3786,7 @@ function createToolOrderingRule(options) {
|
|
|
2956
3786
|
failFinding(
|
|
2957
3787
|
ruleId,
|
|
2958
3788
|
`Every ${options.before} tool call must finish before any ${options.after} tool call starts.`,
|
|
2959
|
-
[
|
|
3789
|
+
[eventEvidence2(latestBefore.event), eventEvidence2(earliestAfter.event)],
|
|
2960
3790
|
expected,
|
|
2961
3791
|
{
|
|
2962
3792
|
latestBeforeEndedAt: new Date(latestBefore.end).toISOString(),
|
|
@@ -2980,7 +3810,7 @@ function createToolOrderingRule(options) {
|
|
|
2980
3810
|
beforeEndedAt: beforeEvent.endedAt,
|
|
2981
3811
|
afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
|
|
2982
3812
|
},
|
|
2983
|
-
evidence: [
|
|
3813
|
+
evidence: [eventEvidence2(beforeEvent), eventEvidence2(afterEvent)]
|
|
2984
3814
|
}
|
|
2985
3815
|
];
|
|
2986
3816
|
}
|
|
@@ -3003,7 +3833,7 @@ function createToolFailureRule(options) {
|
|
|
3003
3833
|
failFinding(
|
|
3004
3834
|
"tool.failures",
|
|
3005
3835
|
`Tool failure count ${failures.length} exceeded ${options.maxFailures}.`,
|
|
3006
|
-
failures.map((event) =>
|
|
3836
|
+
failures.map((event) => eventEvidence2(event)),
|
|
3007
3837
|
{ maxFailures: options.maxFailures },
|
|
3008
3838
|
failures.length
|
|
3009
3839
|
)
|
|
@@ -3016,7 +3846,7 @@ function createToolFailureRule(options) {
|
|
|
3016
3846
|
failFinding(
|
|
3017
3847
|
"tool.failures",
|
|
3018
3848
|
`Tool retry count exceeded ${options.maxRetries}.`,
|
|
3019
|
-
excessiveRetries.map((item) =>
|
|
3849
|
+
excessiveRetries.map((item) => eventEvidence2(item.event, "attributes.retryCount")),
|
|
3020
3850
|
{ maxRetries: options.maxRetries },
|
|
3021
3851
|
excessiveRetries.map((item) => ({ tool: toolName(item.event), retries: item.count }))
|
|
3022
3852
|
)
|
|
@@ -3043,7 +3873,7 @@ function createLlmUsageRule(options) {
|
|
|
3043
3873
|
failFinding(
|
|
3044
3874
|
"llm.usage",
|
|
3045
3875
|
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
3046
|
-
llms.map((event) =>
|
|
3876
|
+
llms.map((event) => eventEvidence2(event)),
|
|
3047
3877
|
{ maxCalls: options.maxCalls },
|
|
3048
3878
|
llms.length
|
|
3049
3879
|
)
|
|
@@ -3055,17 +3885,17 @@ function createLlmUsageRule(options) {
|
|
|
3055
3885
|
const finishReason = llmFinishReason(event);
|
|
3056
3886
|
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
3057
3887
|
findings.push(
|
|
3058
|
-
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [
|
|
3888
|
+
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence2(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
3059
3889
|
);
|
|
3060
3890
|
}
|
|
3061
3891
|
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
3062
3892
|
findings.push(
|
|
3063
|
-
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [
|
|
3893
|
+
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence2(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
3064
3894
|
);
|
|
3065
3895
|
}
|
|
3066
3896
|
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
3067
3897
|
findings.push(
|
|
3068
|
-
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [
|
|
3898
|
+
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence2(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
3069
3899
|
);
|
|
3070
3900
|
}
|
|
3071
3901
|
}
|
|
@@ -3090,7 +3920,7 @@ function createLlmUsageRule(options) {
|
|
|
3090
3920
|
failFinding(
|
|
3091
3921
|
"llm.usage",
|
|
3092
3922
|
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
3093
|
-
llms.map((event) =>
|
|
3923
|
+
llms.map((event) => eventEvidence2(event, `tokenUsage.${key}`)),
|
|
3094
3924
|
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
3095
3925
|
tokenTotals[key]
|
|
3096
3926
|
)
|
|
@@ -3115,7 +3945,7 @@ function createStructureIncompleteRule(options = {}) {
|
|
|
3115
3945
|
failFinding(
|
|
3116
3946
|
"structure.incomplete",
|
|
3117
3947
|
"Trace contains incomplete running events.",
|
|
3118
|
-
running.map((event) =>
|
|
3948
|
+
running.map((event) => eventEvidence2(event, "status")),
|
|
3119
3949
|
"no running events",
|
|
3120
3950
|
running.length
|
|
3121
3951
|
)
|
|
@@ -3131,7 +3961,7 @@ function createStructureIncompleteRule(options = {}) {
|
|
|
3131
3961
|
failFinding(
|
|
3132
3962
|
"structure.incomplete",
|
|
3133
3963
|
"Trace contains events with startedAt but no endedAt.",
|
|
3134
|
-
missingEndedAt.map((event) =>
|
|
3964
|
+
missingEndedAt.map((event) => eventEvidence2(event, "endedAt")),
|
|
3135
3965
|
"endedAt for started events",
|
|
3136
3966
|
missingEndedAt.length
|
|
3137
3967
|
)
|
|
@@ -3159,7 +3989,7 @@ function createStructureOrphanRule(options = {}) {
|
|
|
3159
3989
|
failFinding(
|
|
3160
3990
|
"structure.orphan",
|
|
3161
3991
|
"Trace contains events whose parentId is not present in the selected run.",
|
|
3162
|
-
orphans.map((event) =>
|
|
3992
|
+
orphans.map((event) => eventEvidence2(event, "parentId")),
|
|
3163
3993
|
"parentId resolves to an event in the selected run",
|
|
3164
3994
|
orphans.length
|
|
3165
3995
|
)
|
|
@@ -3191,7 +4021,7 @@ function createStructureCycleRule() {
|
|
|
3191
4021
|
failFinding(
|
|
3192
4022
|
"structure.cycle",
|
|
3193
4023
|
formatProgrammaticDiagnostic("AI_TRACE_RELATIONSHIP_CYCLE"),
|
|
3194
|
-
cycle.map((item) =>
|
|
4024
|
+
cycle.map((item) => eventEvidence2(item, "parentId")),
|
|
3195
4025
|
"acyclic parentId graph",
|
|
3196
4026
|
cycle.map((item) => item.eventId).sort()
|
|
3197
4027
|
)
|
|
@@ -3223,7 +4053,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
3223
4053
|
failFinding(
|
|
3224
4054
|
"structure.relationship",
|
|
3225
4055
|
`Event confidence ${event.confidence} is below ${minConfidence}.`,
|
|
3226
|
-
[
|
|
4056
|
+
[eventEvidence2(event, "confidence")],
|
|
3227
4057
|
{ minConfidence },
|
|
3228
4058
|
event.confidence
|
|
3229
4059
|
)
|
|
@@ -3235,7 +4065,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
3235
4065
|
failFinding(
|
|
3236
4066
|
"structure.relationship",
|
|
3237
4067
|
"Event parentId points to itself.",
|
|
3238
|
-
[
|
|
4068
|
+
[eventEvidence2(event, "parentId")],
|
|
3239
4069
|
"parentId references a distinct event",
|
|
3240
4070
|
"self"
|
|
3241
4071
|
)
|
|
@@ -3252,7 +4082,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
3252
4082
|
failFinding(
|
|
3253
4083
|
"structure.relationship",
|
|
3254
4084
|
"Parent event starts after child event.",
|
|
3255
|
-
[
|
|
4085
|
+
[eventEvidence2(parent), eventEvidence2(event, "parentId")],
|
|
3256
4086
|
"parent start <= child start",
|
|
3257
4087
|
{ parentEventId: parent.eventId, childEventId: event.eventId }
|
|
3258
4088
|
)
|
|
@@ -3266,7 +4096,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
3266
4096
|
failFinding(
|
|
3267
4097
|
"structure.relationship",
|
|
3268
4098
|
"Trace parentSpanId does not match parent spanId.",
|
|
3269
|
-
[
|
|
4099
|
+
[eventEvidence2(event, "trace.parentSpanId")],
|
|
3270
4100
|
{ parentSpanId: parent.trace.spanId },
|
|
3271
4101
|
actual ?? "missing"
|
|
3272
4102
|
)
|
|
@@ -3295,7 +4125,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
3295
4125
|
"structure.parallelWidth",
|
|
3296
4126
|
`Parent ${parentId} has ${children.length} children, exceeding ${options.maxChildren}.`,
|
|
3297
4127
|
[
|
|
3298
|
-
...parent ? [
|
|
4128
|
+
...parent ? [eventEvidence2(parent)] : [{ runId: context.selectedRun?.runId, eventId: parentId }],
|
|
3299
4129
|
...children.map((child) => ({
|
|
3300
4130
|
runId: child.event.runId,
|
|
3301
4131
|
eventId: child.event.eventId,
|
|
@@ -3343,7 +4173,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
3343
4173
|
failFinding(
|
|
3344
4174
|
"structure.parallelWidth",
|
|
3345
4175
|
`Concurrent event width ${maxActive.length} exceeded ${options.maxConcurrent}.`,
|
|
3346
|
-
maxActive.map((event) =>
|
|
4176
|
+
maxActive.map((event) => eventEvidence2(event)),
|
|
3347
4177
|
{ maxConcurrent: options.maxConcurrent },
|
|
3348
4178
|
maxActive.length
|
|
3349
4179
|
)
|
|
@@ -3386,7 +4216,7 @@ function createSignalRule(ruleId, label, options, selectEvents, nameForEvent) {
|
|
|
3386
4216
|
failFinding(
|
|
3387
4217
|
ruleId,
|
|
3388
4218
|
`Forbidden ${label} ${name} appeared.`,
|
|
3389
|
-
[
|
|
4219
|
+
[eventEvidence2(event)],
|
|
3390
4220
|
`${label} absent`,
|
|
3391
4221
|
name
|
|
3392
4222
|
)
|
|
@@ -3397,7 +4227,7 @@ function createSignalRule(ruleId, label, options, selectEvents, nameForEvent) {
|
|
|
3397
4227
|
failFinding(
|
|
3398
4228
|
ruleId,
|
|
3399
4229
|
`${label[0].toUpperCase()}${label.slice(1)} ${name} is not in the allowed set.`,
|
|
3400
|
-
[
|
|
4230
|
+
[eventEvidence2(event)],
|
|
3401
4231
|
[...allowed].sort(),
|
|
3402
4232
|
name
|
|
3403
4233
|
)
|
|
@@ -3420,7 +4250,7 @@ function createSignalRule(ruleId, label, options, selectEvents, nameForEvent) {
|
|
|
3420
4250
|
failFinding(
|
|
3421
4251
|
ruleId,
|
|
3422
4252
|
`${label[0].toUpperCase()}${label.slice(1)} count ${events.length} exceeded maximum ${options.maxCount}.`,
|
|
3423
|
-
events.map((event) =>
|
|
4253
|
+
events.map((event) => eventEvidence2(event)),
|
|
3424
4254
|
{ maxCount: options.maxCount },
|
|
3425
4255
|
events.length
|
|
3426
4256
|
)
|
|
@@ -3474,7 +4304,7 @@ function createSafetyRedactionRule(options = {}) {
|
|
|
3474
4304
|
failFinding(
|
|
3475
4305
|
"safety.redaction",
|
|
3476
4306
|
`Sensitive-looking field at ${entry.path} is not redacted.`,
|
|
3477
|
-
[
|
|
4307
|
+
[eventEvidence2(event, entry.path)],
|
|
3478
4308
|
"redaction marker",
|
|
3479
4309
|
{ path: entry.path, valueType: valueType(entry.value) },
|
|
3480
4310
|
{
|
|
@@ -3508,7 +4338,7 @@ function createSafetyRawContentRule(options = {}) {
|
|
|
3508
4338
|
failFinding(
|
|
3509
4339
|
"safety.rawPrompt",
|
|
3510
4340
|
`Raw content-like field ${entry.path} is present.`,
|
|
3511
|
-
[
|
|
4341
|
+
[eventEvidence2(event, entry.path)],
|
|
3512
4342
|
"metadata-only trace fields",
|
|
3513
4343
|
{ path: entry.path, valueType: valueType(entry.value) },
|
|
3514
4344
|
{
|
|
@@ -3546,7 +4376,7 @@ function createSafetySecretPatternRule(options = {}) {
|
|
|
3546
4376
|
failFinding(
|
|
3547
4377
|
"safety.secretPattern",
|
|
3548
4378
|
`Secret-like pattern ${pattern.id} matched at ${entry.path}.`,
|
|
3549
|
-
[
|
|
4379
|
+
[eventEvidence2(event, entry.path)],
|
|
3550
4380
|
"no secret-like strings",
|
|
3551
4381
|
{ pattern: pattern.id, path: entry.path },
|
|
3552
4382
|
{
|
|
@@ -3579,7 +4409,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
3579
4409
|
failFinding(
|
|
3580
4410
|
"safety.oversizedAttribute",
|
|
3581
4411
|
`String at ${entry.path} exceeds ${options.maxStringLength} characters.`,
|
|
3582
|
-
[
|
|
4412
|
+
[eventEvidence2(event, entry.path)],
|
|
3583
4413
|
{ maxStringLength: options.maxStringLength },
|
|
3584
4414
|
{ path: entry.path, length: entry.value.length },
|
|
3585
4415
|
{
|
|
@@ -3596,7 +4426,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
3596
4426
|
failFinding(
|
|
3597
4427
|
"safety.oversizedAttribute",
|
|
3598
4428
|
`Array at ${entry.path} exceeds ${options.maxArrayLength} items.`,
|
|
3599
|
-
[
|
|
4429
|
+
[eventEvidence2(event, entry.path)],
|
|
3600
4430
|
{ maxArrayLength: options.maxArrayLength },
|
|
3601
4431
|
{ path: entry.path, length: entry.value.length },
|
|
3602
4432
|
{
|
|
@@ -3613,7 +4443,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
3613
4443
|
failFinding(
|
|
3614
4444
|
"safety.oversizedAttribute",
|
|
3615
4445
|
`Object at ${entry.path} exceeds ${options.maxObjectKeys} keys.`,
|
|
3616
|
-
[
|
|
4446
|
+
[eventEvidence2(event, entry.path)],
|
|
3617
4447
|
{ maxObjectKeys: options.maxObjectKeys },
|
|
3618
4448
|
{ path: entry.path, keys: Object.keys(entry.value).length },
|
|
3619
4449
|
{
|
|
@@ -3632,7 +4462,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
3632
4462
|
failFinding(
|
|
3633
4463
|
"safety.oversizedAttribute",
|
|
3634
4464
|
`Value at ${entry.path} exceeds ${options.maxSerializedBytes} serialized bytes.`,
|
|
3635
|
-
[
|
|
4465
|
+
[eventEvidence2(event, entry.path)],
|
|
3636
4466
|
{ maxSerializedBytes: options.maxSerializedBytes },
|
|
3637
4467
|
{ path: entry.path, bytes },
|
|
3638
4468
|
{
|
|
@@ -3759,7 +4589,7 @@ function createBaselineRegressionRule(options) {
|
|
|
3759
4589
|
path: "guardrail",
|
|
3760
4590
|
expected: guardrailShape(baselineContext),
|
|
3761
4591
|
actual: guardrailShape(context),
|
|
3762
|
-
evidence: guardrailEvents(context)[0] ? [
|
|
4592
|
+
evidence: guardrailEvents(context)[0] ? [eventEvidence2(guardrailEvents(context)[0], "guardrail")] : runEvidence(context.selectedRun)
|
|
3763
4593
|
}
|
|
3764
4594
|
];
|
|
3765
4595
|
for (const comparison of comparisons) {
|
|
@@ -3928,13 +4758,17 @@ exports.createToolOrderingRule = createToolOrderingRule;
|
|
|
3928
4758
|
exports.createToolUsageRule = createToolUsageRule;
|
|
3929
4759
|
exports.defineTraceContract = defineTraceContract;
|
|
3930
4760
|
exports.deriveFailureFacts = deriveFailureFacts;
|
|
4761
|
+
exports.deriveRelationshipFacts = deriveRelationshipFacts;
|
|
4762
|
+
exports.evaluateToolArgumentValue = evaluateToolArgumentValue;
|
|
3931
4763
|
exports.evaluateTraceContract = evaluateTraceContract;
|
|
3932
4764
|
exports.evaluateTraceContractRead = evaluateTraceContractRead;
|
|
3933
4765
|
exports.explainTraceContract = explainTraceContract;
|
|
4766
|
+
exports.extractToolArgumentPayload = extractToolArgumentPayload;
|
|
3934
4767
|
exports.formatProgrammaticDiagnostic = formatProgrammaticDiagnostic;
|
|
3935
4768
|
exports.lintTraceContract = lintTraceContract;
|
|
3936
4769
|
exports.projectLogicalEvents = projectLogicalEvents;
|
|
3937
4770
|
exports.resolveCanonicalToolName = resolveCanonicalToolName;
|
|
4771
|
+
exports.resolveJsonPointer = resolveJsonPointer;
|
|
3938
4772
|
exports.resolveTraceContractScope = resolveTraceContractScope;
|
|
3939
4773
|
exports.runTraceChecks = runTraceChecks;
|
|
3940
4774
|
exports.summarizeSemanticParity = summarizeSemanticParity;
|