agent-inspect 6.21.0 → 6.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +1 -1
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-CRUHIL5J.mjs → chunk-CHBYF4LE.mjs} +11 -2
- package/packages/cli/dist/chunk-CHBYF4LE.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +29 -1
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +4 -4
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-MNJFYW3W.mjs → src-MY5WIBPF.mjs} +3 -3
- package/packages/cli/dist/{src-MNJFYW3W.mjs.map → src-MY5WIBPF.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +38 -0
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +36 -4
- package/packages/core/dist/advanced.d.ts +36 -4
- package/packages/core/dist/advanced.mjs +32 -4
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +230 -72
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +54 -5
- package/packages/core/dist/checks.d.ts +54 -5
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-HT54OLPE.mjs → chunk-NDRMIL7Z.mjs} +103 -3
- package/packages/core/dist/chunk-NDRMIL7Z.mjs.map +1 -0
- package/packages/core/dist/{chunk-43SHIPZM.mjs → chunk-OG3GAWCX.mjs} +178 -21
- package/packages/core/dist/chunk-OG3GAWCX.mjs.map +1 -0
- package/packages/core/dist/{index-BG-ZBESI.d.cts → index-CR4CCRqP.d.cts} +7 -2
- package/packages/core/dist/{index-xevIlqdd.d.ts → index-CtK_p-ps.d.ts} +7 -2
- package/packages/core/dist/{index-AMAGuO_i.d.ts → index-D72DPqAF.d.ts} +2 -1
- package/packages/core/dist/{index-vqr-g6ev.d.cts → index-DLLz9sce.d.cts} +2 -1
- package/packages/core/dist/programmatic-gZTWLObB.d.cts +25 -0
- package/packages/core/dist/programmatic-gZTWLObB.d.ts +25 -0
- package/packages/core/dist/readers.cjs +105 -0
- package/packages/core/dist/readers.cjs.map +1 -1
- package/packages/core/dist/readers.d.cts +43 -18
- package/packages/core/dist/readers.d.ts +43 -18
- package/packages/core/dist/readers.mjs +1 -1
- package/packages/cli/dist/chunk-CRUHIL5J.mjs.map +0 -1
- package/packages/core/dist/chunk-43SHIPZM.mjs.map +0 -1
- package/packages/core/dist/chunk-HT54OLPE.mjs.map +0 -1
|
@@ -38,6 +38,10 @@ function extractSessionWorkflowMetadata(record) {
|
|
|
38
38
|
assignString("toolCallId", record.toolCallId);
|
|
39
39
|
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
40
40
|
assignString("linkedStepId", record.linkedStepId);
|
|
41
|
+
assignString("operationId", record.operationId);
|
|
42
|
+
assignString("attemptId", record.attemptId);
|
|
43
|
+
assignString("fallbackOf", record.fallbackOf);
|
|
44
|
+
assignString("idempotencyKey", record.idempotencyKey);
|
|
41
45
|
assignString("correlationId", record.correlationId);
|
|
42
46
|
assignString("requestId", record.requestId);
|
|
43
47
|
assignString("decisionId", record.decisionId);
|
|
@@ -46,6 +50,11 @@ function extractSessionWorkflowMetadata(record) {
|
|
|
46
50
|
out.attempt = attempt;
|
|
47
51
|
found = true;
|
|
48
52
|
}
|
|
53
|
+
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
54
|
+
if (attemptNumber !== void 0) {
|
|
55
|
+
out.attemptNumber = attemptNumber;
|
|
56
|
+
found = true;
|
|
57
|
+
}
|
|
49
58
|
return found ? out : void 0;
|
|
50
59
|
}
|
|
51
60
|
function sessionKeyForRun(meta, options) {
|
|
@@ -333,10 +342,10 @@ function projectLogicalEvents(events) {
|
|
|
333
342
|
};
|
|
334
343
|
}
|
|
335
344
|
function resolveCanonicalToolName(event) {
|
|
336
|
-
const
|
|
337
|
-
const direct = pickString(
|
|
345
|
+
const attrs2 = event.attributes;
|
|
346
|
+
const direct = pickString(attrs2, ["toolName", "tool"]);
|
|
338
347
|
if (direct) return direct;
|
|
339
|
-
const metadata =
|
|
348
|
+
const metadata = attrs2?.metadata;
|
|
340
349
|
if (isRecord(metadata)) {
|
|
341
350
|
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
342
351
|
if (nested) return nested;
|
|
@@ -378,10 +387,10 @@ function pickNumber(record, keys) {
|
|
|
378
387
|
return void 0;
|
|
379
388
|
}
|
|
380
389
|
function eventMetadata(event) {
|
|
381
|
-
const
|
|
382
|
-
const nested =
|
|
390
|
+
const attrs2 = isRecord2(event.attributes) ? event.attributes : void 0;
|
|
391
|
+
const nested = attrs2 !== void 0 && isRecord2(attrs2.metadata) ? attrs2.metadata : void 0;
|
|
383
392
|
return {
|
|
384
|
-
...
|
|
393
|
+
...attrs2 ?? {},
|
|
385
394
|
...nested ?? {}
|
|
386
395
|
};
|
|
387
396
|
}
|
|
@@ -674,6 +683,151 @@ function deriveFailureFacts(logicalEvents) {
|
|
|
674
683
|
};
|
|
675
684
|
}
|
|
676
685
|
|
|
686
|
+
// packages/core/src/checks/relationship-facts.ts
|
|
687
|
+
function attrs(event) {
|
|
688
|
+
return event.attributes && typeof event.attributes === "object" ? event.attributes : {};
|
|
689
|
+
}
|
|
690
|
+
function stringAttr(record, key) {
|
|
691
|
+
const value = record[key];
|
|
692
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
693
|
+
}
|
|
694
|
+
function deriveRelationshipFacts(events) {
|
|
695
|
+
const relationships = [];
|
|
696
|
+
const diagnostics = [];
|
|
697
|
+
const byId = new Map(events.map((event) => [event.eventId, event]));
|
|
698
|
+
const seen = /* @__PURE__ */ new Set();
|
|
699
|
+
const push = (edge) => {
|
|
700
|
+
const key = `${edge.type}:${edge.fromEventId}:${edge.toEventId ?? ""}:${edge.externalRef ?? ""}`;
|
|
701
|
+
if (seen.has(key)) return;
|
|
702
|
+
seen.add(key);
|
|
703
|
+
relationships.push(edge);
|
|
704
|
+
};
|
|
705
|
+
for (const event of events) {
|
|
706
|
+
if (event.parentId) {
|
|
707
|
+
push({
|
|
708
|
+
type: "parent-child",
|
|
709
|
+
fromEventId: event.parentId,
|
|
710
|
+
toEventId: event.eventId,
|
|
711
|
+
confidence: byId.has(event.parentId) ? "explicit" : "unknown",
|
|
712
|
+
basis: ["persisted.parentId"]
|
|
713
|
+
});
|
|
714
|
+
if (!byId.has(event.parentId)) {
|
|
715
|
+
diagnostics.push({
|
|
716
|
+
code: "AI_RELATIONSHIP_PARENT_MISSING",
|
|
717
|
+
message: `parentId ${event.parentId} is not present in the event set.`,
|
|
718
|
+
eventId: event.eventId
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
const bag = attrs(event);
|
|
723
|
+
const meta = extractSessionWorkflowMetadata(bag) ?? {};
|
|
724
|
+
const nested = bag.metadata && typeof bag.metadata === "object" ? extractSessionWorkflowMetadata(bag.metadata) : void 0;
|
|
725
|
+
const workflow = { ...meta, ...nested };
|
|
726
|
+
if (workflow.retryOf) {
|
|
727
|
+
const target = events.find((candidate) => candidate.runId === workflow.retryOf);
|
|
728
|
+
push({
|
|
729
|
+
type: "retry-of",
|
|
730
|
+
fromEventId: event.eventId,
|
|
731
|
+
...target ? { toEventId: target.eventId } : {},
|
|
732
|
+
externalRef: workflow.retryOf,
|
|
733
|
+
confidence: target ? "explicit" : "correlated",
|
|
734
|
+
basis: ["attributes.retryOf"]
|
|
735
|
+
});
|
|
736
|
+
}
|
|
737
|
+
const attemptOf = stringAttr(bag, "attemptOf") ?? stringAttr(bag, "operationId");
|
|
738
|
+
if (attemptOf && workflow.attempt !== void 0) {
|
|
739
|
+
push({
|
|
740
|
+
type: "attempt-of",
|
|
741
|
+
fromEventId: event.eventId,
|
|
742
|
+
externalRef: attemptOf,
|
|
743
|
+
confidence: "explicit",
|
|
744
|
+
basis: workflow.attempt !== void 0 ? ["attributes.attempt", "attributes.operationId"] : ["attributes.operationId"]
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
const fallbackOf = stringAttr(bag, "fallbackOf");
|
|
748
|
+
if (fallbackOf) {
|
|
749
|
+
push({
|
|
750
|
+
type: "fallback-of",
|
|
751
|
+
fromEventId: event.eventId,
|
|
752
|
+
externalRef: fallbackOf,
|
|
753
|
+
confidence: "explicit",
|
|
754
|
+
basis: ["attributes.fallbackOf"]
|
|
755
|
+
});
|
|
756
|
+
}
|
|
757
|
+
const remediationOf = stringAttr(bag, "remediationOf");
|
|
758
|
+
if (remediationOf) {
|
|
759
|
+
push({
|
|
760
|
+
type: "remediation-of",
|
|
761
|
+
fromEventId: event.eventId,
|
|
762
|
+
externalRef: remediationOf,
|
|
763
|
+
confidence: "explicit",
|
|
764
|
+
basis: ["attributes.remediationOf"]
|
|
765
|
+
});
|
|
766
|
+
}
|
|
767
|
+
const evidenceFor = stringAttr(bag, "evidenceFor");
|
|
768
|
+
if (evidenceFor) {
|
|
769
|
+
push({
|
|
770
|
+
type: "evidence-for",
|
|
771
|
+
fromEventId: event.eventId,
|
|
772
|
+
...byId.has(evidenceFor) ? { toEventId: evidenceFor } : { externalRef: evidenceFor },
|
|
773
|
+
confidence: byId.has(evidenceFor) ? "explicit" : "correlated",
|
|
774
|
+
basis: ["attributes.evidenceFor"]
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
const acceptedBy = stringAttr(bag, "acceptedBy");
|
|
778
|
+
if (acceptedBy) {
|
|
779
|
+
push({
|
|
780
|
+
type: "accepted-by",
|
|
781
|
+
fromEventId: event.eventId,
|
|
782
|
+
...byId.has(acceptedBy) ? { toEventId: acceptedBy } : { externalRef: acceptedBy },
|
|
783
|
+
confidence: byId.has(acceptedBy) ? "explicit" : "correlated",
|
|
784
|
+
basis: ["attributes.acceptedBy"]
|
|
785
|
+
});
|
|
786
|
+
}
|
|
787
|
+
const supersedes = stringAttr(bag, "supersedes");
|
|
788
|
+
if (supersedes) {
|
|
789
|
+
push({
|
|
790
|
+
type: "supersedes",
|
|
791
|
+
fromEventId: event.eventId,
|
|
792
|
+
...byId.has(supersedes) ? { toEventId: supersedes } : { externalRef: supersedes },
|
|
793
|
+
confidence: byId.has(supersedes) ? "explicit" : "correlated",
|
|
794
|
+
basis: ["attributes.supersedes"]
|
|
795
|
+
});
|
|
796
|
+
}
|
|
797
|
+
const sourceLineage = stringAttr(bag, "sourceLineage") ?? stringAttr(bag, "sourceEventId");
|
|
798
|
+
if (sourceLineage) {
|
|
799
|
+
push({
|
|
800
|
+
type: "source-lineage",
|
|
801
|
+
fromEventId: event.eventId,
|
|
802
|
+
externalRef: sourceLineage,
|
|
803
|
+
confidence: "explicit",
|
|
804
|
+
basis: ["attributes.sourceLineage"]
|
|
805
|
+
});
|
|
806
|
+
}
|
|
807
|
+
const unsupported = stringAttr(bag, "relationshipType");
|
|
808
|
+
if (unsupported && unsupported !== "parent-child" && ![
|
|
809
|
+
"source-lineage",
|
|
810
|
+
"attempt-of",
|
|
811
|
+
"retry-of",
|
|
812
|
+
"fallback-of",
|
|
813
|
+
"remediation-of",
|
|
814
|
+
"evidence-for",
|
|
815
|
+
"accepted-by",
|
|
816
|
+
"supersedes"
|
|
817
|
+
].includes(unsupported)) {
|
|
818
|
+
diagnostics.push({
|
|
819
|
+
code: "AI_RELATIONSHIP_UNSUPPORTED_TYPE",
|
|
820
|
+
message: `Unsupported relationshipType ${unsupported} was reported without flattening.`,
|
|
821
|
+
eventId: event.eventId
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
return {
|
|
826
|
+
relationships: Object.freeze(relationships),
|
|
827
|
+
diagnostics: Object.freeze(diagnostics)
|
|
828
|
+
};
|
|
829
|
+
}
|
|
830
|
+
|
|
677
831
|
// packages/core/src/checks/trace-facts.ts
|
|
678
832
|
function summarizeSemanticParity(events) {
|
|
679
833
|
const projection = projectLogicalEvents(events);
|
|
@@ -760,6 +914,7 @@ function buildTraceFacts(input) {
|
|
|
760
914
|
toolsByName.set(name, Object.freeze([...list]));
|
|
761
915
|
}
|
|
762
916
|
const derived = deriveFailureFacts(projection.logicalEvents);
|
|
917
|
+
const relationship = deriveRelationshipFacts(events);
|
|
763
918
|
const summary = summarizeSemanticParity(events);
|
|
764
919
|
return {
|
|
765
920
|
rawEvents: Object.freeze([...events]),
|
|
@@ -773,7 +928,9 @@ function buildTraceFacts(input) {
|
|
|
773
928
|
failureRoleCounts: derived.failureRoleCounts
|
|
774
929
|
},
|
|
775
930
|
failureFacts: derived.failureFacts,
|
|
776
|
-
failuresByRole: derived.failuresByRole
|
|
931
|
+
failuresByRole: derived.failuresByRole,
|
|
932
|
+
relationships: relationship.relationships,
|
|
933
|
+
relationshipDiagnostics: relationship.diagnostics
|
|
777
934
|
};
|
|
778
935
|
}
|
|
779
936
|
|
|
@@ -816,10 +973,10 @@ function workflowMetadataForRun(events, runId) {
|
|
|
816
973
|
const fallback = {};
|
|
817
974
|
for (const event of events) {
|
|
818
975
|
if (event.runId !== runId) continue;
|
|
819
|
-
const
|
|
820
|
-
if (!
|
|
821
|
-
const direct = extractSessionWorkflowMetadata(
|
|
822
|
-
const nested =
|
|
976
|
+
const attrs2 = event.attributes;
|
|
977
|
+
if (!attrs2 || typeof attrs2 !== "object") continue;
|
|
978
|
+
const direct = extractSessionWorkflowMetadata(attrs2);
|
|
979
|
+
const nested = attrs2.metadata && typeof attrs2.metadata === "object" ? extractSessionWorkflowMetadata(attrs2.metadata) : void 0;
|
|
823
980
|
const bag = {};
|
|
824
981
|
mergeWorkflowMeta(bag, direct);
|
|
825
982
|
mergeWorkflowMeta(bag, nested);
|
|
@@ -1337,7 +1494,7 @@ function classifyRuleExecution(findings, threw) {
|
|
|
1337
1494
|
}
|
|
1338
1495
|
return "pass";
|
|
1339
1496
|
}
|
|
1340
|
-
function
|
|
1497
|
+
function stringAttr2(event, keys) {
|
|
1341
1498
|
for (const key of keys) {
|
|
1342
1499
|
const value = event.attributes?.[key];
|
|
1343
1500
|
if (typeof value === "string" && value.trim() !== "") return value;
|
|
@@ -1415,15 +1572,15 @@ function semanticEvents(context) {
|
|
|
1415
1572
|
}
|
|
1416
1573
|
function llmModel(event) {
|
|
1417
1574
|
const keys = ["model", "modelId", "responseModelId", "modelName", "model_name"];
|
|
1418
|
-
return
|
|
1575
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
1419
1576
|
}
|
|
1420
1577
|
function llmProvider(event) {
|
|
1421
1578
|
const keys = ["provider", "providerName", "provider_name"];
|
|
1422
|
-
return
|
|
1579
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys);
|
|
1423
1580
|
}
|
|
1424
1581
|
function llmFinishReason(event) {
|
|
1425
1582
|
const keys = ["finishReason", "rawFinishReason", "finish_reason"];
|
|
1426
|
-
return
|
|
1583
|
+
return stringAttr2(event, keys) ?? metadataStringAttr(event, keys);
|
|
1427
1584
|
}
|
|
1428
1585
|
function retryCount(event) {
|
|
1429
1586
|
return numericAttr(event, ["retryCount", "retryAttempt", "retry_attempt", "attempt"]);
|
|
@@ -1565,7 +1722,7 @@ function parentMarkedUnresolved(event) {
|
|
|
1565
1722
|
]) === true) {
|
|
1566
1723
|
return true;
|
|
1567
1724
|
}
|
|
1568
|
-
const resolution =
|
|
1725
|
+
const resolution = stringAttr2(event, [
|
|
1569
1726
|
"parentResolution",
|
|
1570
1727
|
"relationshipResolution",
|
|
1571
1728
|
"relationshipStatus"
|
|
@@ -1573,13 +1730,13 @@ function parentMarkedUnresolved(event) {
|
|
|
1573
1730
|
return resolution === "unresolved" || resolution === "missing-parent";
|
|
1574
1731
|
}
|
|
1575
1732
|
function signalName(event, attributeKeys, prefixes) {
|
|
1576
|
-
return
|
|
1733
|
+
return stringAttr2(event, attributeKeys) ?? stripPrefix(event.name, prefixes);
|
|
1577
1734
|
}
|
|
1578
1735
|
function guardrailEvents(context) {
|
|
1579
1736
|
return finishedEvents(context).filter((event) => {
|
|
1580
1737
|
const name = event.name.toLowerCase();
|
|
1581
1738
|
if (name.startsWith("guardrail:") || name.includes(".guardrail.")) return true;
|
|
1582
|
-
return
|
|
1739
|
+
return stringAttr2(event, ["guardrailName", "guardrail", "guardrailId"]) !== void 0;
|
|
1583
1740
|
});
|
|
1584
1741
|
}
|
|
1585
1742
|
function retryValue(event) {
|
|
@@ -3779,6 +3936,6 @@ function explainTraceContract(contract) {
|
|
|
3779
3936
|
return lines;
|
|
3780
3937
|
}
|
|
3781
3938
|
|
|
3782
|
-
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractSessionWorkflowMetadata, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveTraceContractScope, runTraceChecks, sessionKeyForRun, summarizeSemanticParity, workflowMetadataForRun };
|
|
3783
|
-
//# sourceMappingURL=chunk-
|
|
3784
|
-
//# sourceMappingURL=chunk-
|
|
3939
|
+
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractSessionWorkflowMetadata, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveTraceContractScope, runTraceChecks, sessionKeyForRun, summarizeSemanticParity, workflowMetadataForRun };
|
|
3940
|
+
//# sourceMappingURL=chunk-OG3GAWCX.mjs.map
|
|
3941
|
+
//# sourceMappingURL=chunk-OG3GAWCX.mjs.map
|