agent-inspect 6.7.2 → 6.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +1 -1
- package/docs/EXPORTS.md +9 -0
- package/docs/PERFORMANCE.md +18 -0
- package/docs/STANDARDS.md +4 -0
- package/package.json +2 -2
- package/packages/cli/dist/{chunk-M6PC5KUT.mjs → chunk-FVEQ7JFF.mjs} +251 -209
- package/packages/cli/dist/chunk-FVEQ7JFF.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +292 -229
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +35 -15
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-27LC2BB3.mjs → src-I5NA7BLK.mjs} +3 -3
- package/packages/cli/dist/{src-27LC2BB3.mjs.map → src-I5NA7BLK.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +249 -207
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.mjs +51 -250
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +56 -4
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-X2FLDF7M.mjs → chunk-CTA6XNDP.mjs} +3 -3
- package/packages/core/dist/{chunk-X2FLDF7M.mjs.map → chunk-CTA6XNDP.mjs.map} +1 -1
- package/packages/core/dist/{chunk-EWHN6NA4.mjs → chunk-EQVXDGGU.mjs} +58 -6
- package/packages/core/dist/chunk-EQVXDGGU.mjs.map +1 -0
- package/packages/core/dist/{chunk-3USJVBLA.mjs → chunk-KDDU2R5P.mjs} +236 -5
- package/packages/core/dist/chunk-KDDU2R5P.mjs.map +1 -0
- package/packages/core/dist/{chunk-E5F2LQCX.mjs → chunk-T5MASTIW.mjs} +15 -4
- package/packages/core/dist/chunk-T5MASTIW.mjs.map +1 -0
- package/packages/core/dist/{chunk-KNYL56KZ.mjs → chunk-UOPVGCOB.mjs} +3 -3
- package/packages/core/dist/{chunk-KNYL56KZ.mjs.map → chunk-UOPVGCOB.mjs.map} +1 -1
- package/packages/core/dist/diff.cjs +4 -0
- package/packages/core/dist/diff.cjs.map +1 -1
- package/packages/core/dist/diff.mjs +4 -0
- package/packages/core/dist/diff.mjs.map +1 -1
- package/packages/core/dist/exporters.cjs +145 -133
- package/packages/core/dist/exporters.cjs.map +1 -1
- package/packages/core/dist/exporters.d.cts +3 -2
- package/packages/core/dist/exporters.d.ts +3 -2
- package/packages/core/dist/exporters.mjs +10 -5
- package/packages/core/dist/exporters.mjs.map +1 -1
- package/packages/core/dist/logs.cjs +16 -4
- package/packages/core/dist/logs.cjs.map +1 -1
- package/packages/core/dist/logs.mjs +5 -4
- package/packages/core/dist/logs.mjs.map +1 -1
- package/packages/core/dist/persisted.cjs +13 -2
- package/packages/core/dist/persisted.cjs.map +1 -1
- package/packages/core/dist/persisted.mjs +3 -3
- package/packages/core/dist/readers.cjs +13 -2
- package/packages/core/dist/readers.cjs.map +1 -1
- package/packages/core/dist/readers.mjs +3 -3
- package/packages/cli/dist/chunk-M6PC5KUT.mjs.map +0 -1
- package/packages/core/dist/chunk-3USJVBLA.mjs.map +0 -1
- package/packages/core/dist/chunk-E5F2LQCX.mjs.map +0 -1
- package/packages/core/dist/chunk-EWHN6NA4.mjs.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createSafetyOversizedAttributeRule, resolveTraceDir, parseDuration, buildSessionIndex, enrichSessionRunRecord, TraceDirectory, loadTraceMetadataList, loadSessionRunRecords, extractMetadata, filterTraces, truncateName, getTraceFilePath, buildRunSummary, formatDuration, formatTimestamp, isAgentInspectTrace, openTrace, TraceReadError, filterMetasBySessionScope, runTraceChecks, aggregateSessionCheckResults, createBaselineRegressionRule, resolveBundleRunIds, aggregateBundleSafeStatus, bundleFailsOnSafety, bundleRunAssetRelativePath, buildPlaceholderArtifact, buildBundleMetadata, buildBundleSummaryMarkdown, buildRunTimeline, renderTimeline, buildTraceStats, renderTraceStats, parseDurationFilter, searchTraces, buildActivitySummary, renderActivitySummaryHuman, buildRunWhatSummary, renderRunWhat, buildLocalExplanation, resolveSuiteTemplate, defaultSuiteConfigTemplate, loadSuiteConfig, validateSuiteConfig, runSuite, renderSuiteReport, analyzeCohort, renderCohortReport, gateHasThresholds, runGate, renderGateReport, persistedInspectEventsToTraceEvents, renderStepLine, renderErrorLine, getIndent, Redactor, TreeBuilder, zeroKinds, validateEvent, traceEventToPersistedInspectEvent, isPersistedInspectEvent, createRunStatusRule, createStructureOrphanRule, createStructureCycleRule, createRunDurationRule, createMaxStepDurationRule, createRequireCompletedRule, createStallDetectionRule, createObservedOutcomeRule, createRunDepthRule, createToolUsageRule, createLlmUsageRule, createStructureRelationshipRule, createStructureParallelWidthRule, normalizeBundleOutputPath, sanitizeBundleRunId, defaultBundleOutputPath, assertBundlePathContained, summarizeObservedOutcomes, extractOutcomesFromTraceEvents, renderObservedOutcomesMarkdown, renderObservedOutcomesHtml, escapeMarkdown, escapeHtml, DEFAULT_SUITE_ARTIFACTS_DIR, parseCohortMetricList, COHORT_METRIC_IDS, parseGateList, nanoid, resolveRedactionProfile, applyProfileMetadataCaps, flattenTree, safeString, compactAttributes, stableJson, truncateStringForProfile } from './chunk-
|
|
2
|
+
import { createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createSafetyOversizedAttributeRule, resolveTraceDir, parseDuration, buildSessionIndex, enrichSessionRunRecord, TraceDirectory, loadTraceMetadataList, loadSessionRunRecords, extractMetadata, filterTraces, truncateName, getTraceFilePath, buildRunSummary, formatDuration, formatTimestamp, isAgentInspectTrace, openTrace, TraceReadError, filterMetasBySessionScope, runTraceChecks, aggregateSessionCheckResults, createBaselineRegressionRule, resolveBundleRunIds, aggregateBundleSafeStatus, bundleFailsOnSafety, bundleRunAssetRelativePath, buildPlaceholderArtifact, buildBundleMetadata, buildBundleSummaryMarkdown, buildRunTimeline, renderTimeline, buildTraceStats, renderTraceStats, parseDurationFilter, searchTraces, buildActivitySummary, renderActivitySummaryHuman, buildRunWhatSummary, renderRunWhat, buildLocalExplanation, resolveSuiteTemplate, defaultSuiteConfigTemplate, loadSuiteConfig, validateSuiteConfig, runSuite, renderSuiteReport, analyzeCohort, renderCohortReport, gateHasThresholds, runGate, renderGateReport, persistedInspectEventsToTraceEvents, renderStepLine, renderErrorLine, getIndent, Redactor, TreeBuilder, zeroKinds, validateEvent, traceEventToPersistedInspectEvent, isPersistedInspectEvent, createRunStatusRule, createStructureOrphanRule, createStructureCycleRule, createRunDurationRule, createMaxStepDurationRule, createRequireCompletedRule, createStallDetectionRule, createObservedOutcomeRule, createRunDepthRule, createToolUsageRule, createLlmUsageRule, createStructureRelationshipRule, createStructureParallelWidthRule, normalizeBundleOutputPath, sanitizeBundleRunId, defaultBundleOutputPath, assertBundlePathContained, summarizeObservedOutcomes, extractOutcomesFromTraceEvents, renderObservedOutcomesMarkdown, renderObservedOutcomesHtml, escapeMarkdown, escapeHtml, DEFAULT_SUITE_ARTIFACTS_DIR, parseCohortMetricList, COHORT_METRIC_IDS, parseGateList, nanoid, resolveRedactionProfile, applyProfileMetadataCaps, flattenTree, safeString, compactAttributes, stableJson, truncateStringForProfile } from './chunk-FVEQ7JFF.mjs';
|
|
3
3
|
import { realpathSync, constants as constants$1 } from 'fs';
|
|
4
4
|
import path14 from 'path';
|
|
5
5
|
import { fileURLToPath, pathToFileURL } from 'url';
|
|
@@ -11,7 +11,7 @@ import { createServer } from 'http';
|
|
|
11
11
|
import { createRequire } from 'module';
|
|
12
12
|
|
|
13
13
|
// package.json
|
|
14
|
-
var version = "6.7.
|
|
14
|
+
var version = "6.7.4";
|
|
15
15
|
|
|
16
16
|
// packages/cli/src/trace-dir-scale.ts
|
|
17
17
|
var TRACE_COUNT_WARN = 1e3;
|
|
@@ -939,13 +939,14 @@ function parseTimestamp(v) {
|
|
|
939
939
|
function hasToken(hay, token) {
|
|
940
940
|
return hay.includes(token);
|
|
941
941
|
}
|
|
942
|
+
var ERROR_TOKEN_RE = /\.(error|failed)(\.|$)/;
|
|
942
943
|
function inferKind(eventName) {
|
|
943
944
|
if (hasToken(eventName, ".llm.")) return "LLM";
|
|
944
945
|
if (hasToken(eventName, ".tool.")) return "TOOL";
|
|
945
946
|
if (hasToken(eventName, ".agent.")) return "AGENT";
|
|
946
947
|
if (hasToken(eventName, ".retriever.")) return "RETRIEVER";
|
|
947
948
|
if (hasToken(eventName, ".result.")) return "RESULT";
|
|
948
|
-
if (
|
|
949
|
+
if (ERROR_TOKEN_RE.test(eventName)) {
|
|
949
950
|
return "ERROR";
|
|
950
951
|
}
|
|
951
952
|
return "LOG";
|
|
@@ -1025,7 +1026,7 @@ var EventNormalizer = class {
|
|
|
1025
1026
|
status = mapping.status;
|
|
1026
1027
|
} else if (mapping?.kind === "ERROR") {
|
|
1027
1028
|
status = "error";
|
|
1028
|
-
} else if (
|
|
1029
|
+
} else if (ERROR_TOKEN_RE.test(eventName)) {
|
|
1029
1030
|
status = "error";
|
|
1030
1031
|
} else if (mapping?.startsRun || mapping?.startsStep) {
|
|
1031
1032
|
status = "running";
|
|
@@ -2213,6 +2214,11 @@ function exportMarkdown(tree, options) {
|
|
|
2213
2214
|
function hexFrom(seed, byteLen) {
|
|
2214
2215
|
return crypto.createHash("sha256").update(seed, "utf8").digest("hex").slice(0, byteLen * 2);
|
|
2215
2216
|
}
|
|
2217
|
+
function unixNano(ms) {
|
|
2218
|
+
const whole = Math.trunc(ms);
|
|
2219
|
+
const fractionNs = Math.round((ms - whole) * 1e6);
|
|
2220
|
+
return BigInt(whole) * 1000000n + BigInt(fractionNs);
|
|
2221
|
+
}
|
|
2216
2222
|
function mapInspectKindToOI(kind, warnings) {
|
|
2217
2223
|
switch (kind) {
|
|
2218
2224
|
case "LLM":
|
|
@@ -2262,10 +2268,10 @@ function exportOpenInference(tree, options) {
|
|
|
2262
2268
|
const ev = n.event;
|
|
2263
2269
|
const spanId = hexFrom(`${tree.runId}:${ev.eventId}`, 8);
|
|
2264
2270
|
const parentSpanHex = ev.parentId ? hexFrom(`${tree.runId}:${ev.parentId}`, 8) : void 0;
|
|
2265
|
-
const startNs =
|
|
2271
|
+
const startNs = unixNano(ev.timestamp);
|
|
2266
2272
|
let endNs;
|
|
2267
2273
|
if (ev.durationMs !== void 0 && Number.isFinite(ev.durationMs)) {
|
|
2268
|
-
endNs = startNs +
|
|
2274
|
+
endNs = startNs + unixNano(ev.durationMs);
|
|
2269
2275
|
}
|
|
2270
2276
|
const { openInferenceKind } = mapInspectKindToOI(ev.kind, warnings);
|
|
2271
2277
|
const attrs = {
|
|
@@ -2313,8 +2319,8 @@ function exportOpenInference(tree, options) {
|
|
|
2313
2319
|
span_id: spanId,
|
|
2314
2320
|
parent_span_id: parentSpanHex,
|
|
2315
2321
|
name: ev.name,
|
|
2316
|
-
start_time_unix_nano: startNs,
|
|
2317
|
-
end_time_unix_nano: endNs,
|
|
2322
|
+
start_time_unix_nano: startNs.toString(),
|
|
2323
|
+
end_time_unix_nano: endNs?.toString(),
|
|
2318
2324
|
attributes: attrs,
|
|
2319
2325
|
status
|
|
2320
2326
|
});
|
|
@@ -3049,6 +3055,10 @@ function manualTraceEventsToComparableRun(events) {
|
|
|
3049
3055
|
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
3050
3056
|
}
|
|
3051
3057
|
const outputPreview = extractOutputPreview(meta);
|
|
3058
|
+
if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
|
|
3059
|
+
delete meta.outputPreview;
|
|
3060
|
+
delete meta.resultPreview;
|
|
3061
|
+
}
|
|
3052
3062
|
const sc = {
|
|
3053
3063
|
id: acc.id,
|
|
3054
3064
|
name: acc.name,
|
|
@@ -3693,6 +3703,7 @@ async function searchCommand(options = {}) {
|
|
|
3693
3703
|
console.log(`Session not found: ${sessionId}`);
|
|
3694
3704
|
console.log(`Trace directory: ${traceDir}`);
|
|
3695
3705
|
}
|
|
3706
|
+
process.exitCode = 1;
|
|
3696
3707
|
return;
|
|
3697
3708
|
}
|
|
3698
3709
|
metas = scoped.metas;
|
|
@@ -3782,7 +3793,7 @@ async function newestTraceMtimeMs(traceDir) {
|
|
|
3782
3793
|
}
|
|
3783
3794
|
async function loadSessionRuns(traceDir) {
|
|
3784
3795
|
try {
|
|
3785
|
-
const mod = await import('./src-
|
|
3796
|
+
const mod = await import('./src-I5NA7BLK.mjs');
|
|
3786
3797
|
const dbPath = mod.resolveIndexDbPath(traceDir);
|
|
3787
3798
|
const status = mod.indexStatus(dbPath);
|
|
3788
3799
|
if (!status.healthy) {
|
|
@@ -6001,9 +6012,20 @@ function buildRules(config, options) {
|
|
|
6001
6012
|
...asStringArray(checks2.select) ?? [],
|
|
6002
6013
|
...options.rule ?? []
|
|
6003
6014
|
];
|
|
6015
|
+
if (select.length === 0) {
|
|
6016
|
+
const auto = new Set(DEFAULT_SELECT);
|
|
6017
|
+
for (const rule of rules) {
|
|
6018
|
+
if (rule.id !== "run.status") auto.add(rule.id);
|
|
6019
|
+
}
|
|
6020
|
+
return {
|
|
6021
|
+
rules,
|
|
6022
|
+
select: [...auto],
|
|
6023
|
+
diagnostics
|
|
6024
|
+
};
|
|
6025
|
+
}
|
|
6004
6026
|
return {
|
|
6005
6027
|
rules,
|
|
6006
|
-
select
|
|
6028
|
+
select,
|
|
6007
6029
|
diagnostics
|
|
6008
6030
|
};
|
|
6009
6031
|
}
|
|
@@ -7244,13 +7266,11 @@ async function studioCommand(options = {}) {
|
|
|
7244
7266
|
await new Promise(() => {
|
|
7245
7267
|
});
|
|
7246
7268
|
}
|
|
7247
|
-
|
|
7248
|
-
// packages/eval/src/index.ts
|
|
7249
7269
|
function isTraceReadResult(value) {
|
|
7250
7270
|
return value !== null && typeof value === "object" && "runs" in value && "events" in value && "format" in value;
|
|
7251
7271
|
}
|
|
7252
7272
|
function traceInputFrom(value) {
|
|
7253
|
-
return { type: "file", path: value instanceof URL ? value
|
|
7273
|
+
return { type: "file", path: value instanceof URL ? fileURLToPath(value) : value };
|
|
7254
7274
|
}
|
|
7255
7275
|
async function resolveRead(input3, options) {
|
|
7256
7276
|
try {
|
|
@@ -9571,7 +9591,7 @@ jobs:
|
|
|
9571
9591
|
async function planInit(options = {}) {
|
|
9572
9592
|
const framework = normalizeFramework(options.framework);
|
|
9573
9593
|
const cwd = path14.resolve(options.cwd ?? process.cwd());
|
|
9574
|
-
const demoPath = framework === "custom" ?
|
|
9594
|
+
const demoPath = framework === "custom" ? "examples/agent-inspect-demo.mjs" : `examples/agent-inspect-${framework}-demo.mjs`;
|
|
9575
9595
|
const candidates = [
|
|
9576
9596
|
{ rel: CONFIG_FILE, content: configTemplate(framework) },
|
|
9577
9597
|
{ rel: GITKEEP, content: "" },
|
|
@@ -10360,7 +10380,7 @@ function isModuleNotFound4(e) {
|
|
|
10360
10380
|
}
|
|
10361
10381
|
async function loadIndexSqlite() {
|
|
10362
10382
|
try {
|
|
10363
|
-
return await import('./src-
|
|
10383
|
+
return await import('./src-I5NA7BLK.mjs');
|
|
10364
10384
|
} catch (e) {
|
|
10365
10385
|
if (isModuleNotFound4(e)) {
|
|
10366
10386
|
console.error(
|