agent-inspect 6.31.31 → 6.31.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/docs/CLI.md +3 -1
- package/docs/TRACE-CONTRACTS.md +4 -0
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-4GTOJ3G2.mjs → chunk-AILSHDEF.mjs} +191 -31
- package/packages/cli/dist/chunk-AILSHDEF.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +209 -33
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +23 -7
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-BF74ZVDF.mjs → src-SP36M4VB.mjs} +3 -3
- package/packages/cli/dist/{src-BF74ZVDF.mjs.map → src-SP36M4VB.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +80 -18
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +2 -2
- package/packages/core/dist/advanced.d.ts +2 -2
- package/packages/core/dist/advanced.mjs +9 -9
- package/packages/core/dist/checks.cjs +185 -29
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +2 -2
- package/packages/core/dist/checks.d.ts +2 -2
- package/packages/core/dist/checks.mjs +3 -3
- package/packages/core/dist/{chunk-76QIWWJ5.mjs → chunk-6S666BF4.mjs} +3 -3
- package/packages/core/dist/{chunk-76QIWWJ5.mjs.map → chunk-6S666BF4.mjs.map} +1 -1
- package/packages/core/dist/{chunk-BYNAJPNH.mjs → chunk-EMDJLFRK.mjs} +188 -32
- package/packages/core/dist/chunk-EMDJLFRK.mjs.map +1 -0
- package/packages/core/dist/{chunk-QMO3ITYF.mjs → chunk-FBPDF7HO.mjs} +3 -3
- package/packages/core/dist/{chunk-QMO3ITYF.mjs.map → chunk-FBPDF7HO.mjs.map} +1 -1
- package/packages/core/dist/{chunk-XM27TZLD.mjs → chunk-RB6WANSW.mjs} +6 -2
- package/packages/core/dist/chunk-RB6WANSW.mjs.map +1 -0
- package/packages/core/dist/{chunk-JTJN4KBJ.mjs → chunk-VIEJJ5JB.mjs} +4 -4
- package/packages/core/dist/{chunk-JTJN4KBJ.mjs.map → chunk-VIEJJ5JB.mjs.map} +1 -1
- package/packages/core/dist/exporters.cjs +4 -0
- package/packages/core/dist/exporters.cjs.map +1 -1
- package/packages/core/dist/exporters.mjs +3 -3
- package/packages/core/dist/{index-DACU4Dy1.d.cts → index-Ceyw5x7b.d.cts} +4 -2
- package/packages/core/dist/{index-b4tIGrzq.d.ts → index-K5Cinf1r.d.ts} +4 -2
- package/packages/core/dist/index.cjs +4 -0
- package/packages/core/dist/index.cjs.map +1 -1
- package/packages/core/dist/index.mjs +4 -4
- package/packages/core/dist/writers.cjs +4 -0
- package/packages/core/dist/writers.cjs.map +1 -1
- package/packages/core/dist/writers.mjs +1 -1
- package/packages/cli/dist/chunk-4GTOJ3G2.mjs.map +0 -1
- package/packages/core/dist/chunk-BYNAJPNH.mjs.map +0 -1
- package/packages/core/dist/chunk-XM27TZLD.mjs.map +0 -1
|
@@ -6,8 +6,8 @@ import { a as InspectRunTree } from './inspect-event-ClM5Z8om.cjs';
|
|
|
6
6
|
export { b as ATTRIBUTION_CONFIDENCES, c as ATTRIBUTION_CONFIDENCE_RANK, A as AttributionConfidence, E as EventSource, d as InspectEvent, I as InspectKind, e as InspectNode, i as isAttributionConfidence } from './inspect-event-ClM5Z8om.cjs';
|
|
7
7
|
import { P as PersistedInspectEvent } from './persisted-inspect-event-q2ViiBW6.cjs';
|
|
8
8
|
import { Stats } from 'node:fs';
|
|
9
|
-
import { aZ as SessionWorkflowMetadata, a_ as SessionRunRecord, a$ as EnrichSessionSummaryOptions, b0 as SessionStatus, b1 as SessionSummary, b2 as SessionIndex, b3 as BuildActivitySummaryOptions, b4 as ActivitySummary, b5 as SessionWarning, b6 as BuildSessionIndexOptions, U as TraceCheckResult, _ as TraceCheckStatus, a5 as TraceContractInput, a0 as TraceContract } from './index-
|
|
10
|
-
export { b7 as ActivityEntry, b8 as CriticalPathStep, b9 as HandoffEdge, ba as RetryLink, bb as SESSION_WORKFLOW_KEYS, bc as SessionCheckSummary, bd as SessionConfidence, be as SessionEdgeSource, bf as SessionGroup, bg as SessionLastError, bh as SessionWorkflowKey } from './index-
|
|
9
|
+
import { aZ as SessionWorkflowMetadata, a_ as SessionRunRecord, a$ as EnrichSessionSummaryOptions, b0 as SessionStatus, b1 as SessionSummary, b2 as SessionIndex, b3 as BuildActivitySummaryOptions, b4 as ActivitySummary, b5 as SessionWarning, b6 as BuildSessionIndexOptions, U as TraceCheckResult, _ as TraceCheckStatus, a5 as TraceContractInput, a0 as TraceContract } from './index-Ceyw5x7b.cjs';
|
|
10
|
+
export { b7 as ActivityEntry, b8 as CriticalPathStep, b9 as HandoffEdge, ba as RetryLink, bb as SESSION_WORKFLOW_KEYS, bc as SessionCheckSummary, bd as SessionConfidence, be as SessionEdgeSource, bf as SessionGroup, bg as SessionLastError, bh as SessionWorkflowKey } from './index-Ceyw5x7b.cjs';
|
|
11
11
|
import { b as ObservedOutcome } from './types-BA_IOdg4.cjs';
|
|
12
12
|
export { a as ObservedOutcomeStatus } from './types-BA_IOdg4.cjs';
|
|
13
13
|
import './writers.cjs';
|
|
@@ -6,8 +6,8 @@ import { a as InspectRunTree } from './inspect-event-ClM5Z8om.js';
|
|
|
6
6
|
export { b as ATTRIBUTION_CONFIDENCES, c as ATTRIBUTION_CONFIDENCE_RANK, A as AttributionConfidence, E as EventSource, d as InspectEvent, I as InspectKind, e as InspectNode, i as isAttributionConfidence } from './inspect-event-ClM5Z8om.js';
|
|
7
7
|
import { P as PersistedInspectEvent } from './persisted-inspect-event-DG02LhEP.js';
|
|
8
8
|
import { Stats } from 'node:fs';
|
|
9
|
-
import { aZ as SessionWorkflowMetadata, a_ as SessionRunRecord, a$ as EnrichSessionSummaryOptions, b0 as SessionStatus, b1 as SessionSummary, b2 as SessionIndex, b3 as BuildActivitySummaryOptions, b4 as ActivitySummary, b5 as SessionWarning, b6 as BuildSessionIndexOptions, U as TraceCheckResult, _ as TraceCheckStatus, a5 as TraceContractInput, a0 as TraceContract } from './index-
|
|
10
|
-
export { b7 as ActivityEntry, b8 as CriticalPathStep, b9 as HandoffEdge, ba as RetryLink, bb as SESSION_WORKFLOW_KEYS, bc as SessionCheckSummary, bd as SessionConfidence, be as SessionEdgeSource, bf as SessionGroup, bg as SessionLastError, bh as SessionWorkflowKey } from './index-
|
|
9
|
+
import { aZ as SessionWorkflowMetadata, a_ as SessionRunRecord, a$ as EnrichSessionSummaryOptions, b0 as SessionStatus, b1 as SessionSummary, b2 as SessionIndex, b3 as BuildActivitySummaryOptions, b4 as ActivitySummary, b5 as SessionWarning, b6 as BuildSessionIndexOptions, U as TraceCheckResult, _ as TraceCheckStatus, a5 as TraceContractInput, a0 as TraceContract } from './index-K5Cinf1r.js';
|
|
10
|
+
export { b7 as ActivityEntry, b8 as CriticalPathStep, b9 as HandoffEdge, ba as RetryLink, bb as SESSION_WORKFLOW_KEYS, bc as SessionCheckSummary, bd as SessionConfidence, be as SessionEdgeSource, bf as SessionGroup, bg as SessionLastError, bh as SessionWorkflowKey } from './index-K5Cinf1r.js';
|
|
11
11
|
import { b as ObservedOutcome } from './types-BA_IOdg4.js';
|
|
12
12
|
export { a as ObservedOutcomeStatus } from './types-BA_IOdg4.js';
|
|
13
13
|
import './writers.js';
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { extractSessionWorkflowMetadata, sessionKeyForRun, defineTraceContract, runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createRunDepthRule, createToolFailureRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-
|
|
2
|
-
export { extractSessionWorkflowMetadata, sessionKeyForRun } from './chunk-
|
|
3
|
-
export { MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, TERMINAL_INDENT, TraceDirectory, createInspector, createInspectorRuntime, formatTerminalName, getCurrentContext, getCurrentCorrelationMetadata, getCurrentDepth, getCurrentRunId, getCurrentRunName, getCurrentStepId, getIndent, getParentStepId, getTraceDirFromContext, getTraceSafetyFromContext, hasActiveContext, isAgentInspectEnabled, isSilentContext, maybeInspectRun, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, renderErrorLine, renderRunSummary, renderStepLine, resolveTraceDir, runWithContext, runWithStepContext } from './chunk-
|
|
4
|
-
import { buildRunTimeline, filterTraces, extractMetadata, buildRunSummary } from './chunk-
|
|
5
|
-
export { buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildTraceStats, extractMetadata, filterTraces, formatStepLabel, renderRunWhat, renderTimeline, renderTraceStats } from './chunk-
|
|
6
|
-
import { extractOutcomesFromTraceEvents, parseObservationFilter, summarizeObservedOutcomes, extractOutcomesFromPersistedEvents, renderObservedOutcomesHtml } from './chunk-
|
|
7
|
-
export { extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents } from './chunk-
|
|
1
|
+
import { extractSessionWorkflowMetadata, sessionKeyForRun, defineTraceContract, runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createRunDepthRule, createToolFailureRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-EMDJLFRK.mjs';
|
|
2
|
+
export { extractSessionWorkflowMetadata, sessionKeyForRun } from './chunk-EMDJLFRK.mjs';
|
|
3
|
+
export { MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, TERMINAL_INDENT, TraceDirectory, createInspector, createInspectorRuntime, formatTerminalName, getCurrentContext, getCurrentCorrelationMetadata, getCurrentDepth, getCurrentRunId, getCurrentRunName, getCurrentStepId, getIndent, getParentStepId, getTraceDirFromContext, getTraceSafetyFromContext, hasActiveContext, isAgentInspectEnabled, isSilentContext, maybeInspectRun, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, renderErrorLine, renderRunSummary, renderStepLine, resolveTraceDir, runWithContext, runWithStepContext } from './chunk-VIEJJ5JB.mjs';
|
|
4
|
+
import { buildRunTimeline, filterTraces, extractMetadata, buildRunSummary } from './chunk-6S666BF4.mjs';
|
|
5
|
+
export { buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildTraceStats, extractMetadata, filterTraces, formatStepLabel, renderRunWhat, renderTimeline, renderTraceStats } from './chunk-6S666BF4.mjs';
|
|
6
|
+
import { extractOutcomesFromTraceEvents, parseObservationFilter, summarizeObservedOutcomes, extractOutcomesFromPersistedEvents, renderObservedOutcomesHtml } from './chunk-FBPDF7HO.mjs';
|
|
7
|
+
export { extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents } from './chunk-FBPDF7HO.mjs';
|
|
8
8
|
import { diffTraceEvents, renderRunDiff } from './chunk-F7STQ5JF.mjs';
|
|
9
9
|
import './chunk-BT7CATSD.mjs';
|
|
10
10
|
import { escapeHtml, flattenTree } from './chunk-CAWPF22J.mjs';
|
|
11
|
-
import { resolveRedactionProfile, readTraceEventsFromFile, stringContainsHighConfidenceCredential } from './chunk-
|
|
12
|
-
export { DEFAULT_MAX_EVENT_BYTES, DEFAULT_MAX_METADATA_VALUE_LENGTH, DEFAULT_MAX_PREVIEW_LENGTH, getRunIdFromTraceFileName, initializeTraceFile, listTraceFiles, prepareMetadataForDisk, prepareTraceEventForDisk, readTraceEvents, readTraceFile, resolveRedactionProfile, resolveTraceSafetyOptions, serializeEvent, validateEvent, writeTraceEvent } from './chunk-
|
|
11
|
+
import { resolveRedactionProfile, readTraceEventsFromFile, stringContainsHighConfidenceCredential } from './chunk-RB6WANSW.mjs';
|
|
12
|
+
export { DEFAULT_MAX_EVENT_BYTES, DEFAULT_MAX_METADATA_VALUE_LENGTH, DEFAULT_MAX_PREVIEW_LENGTH, getRunIdFromTraceFileName, initializeTraceFile, listTraceFiles, prepareMetadataForDisk, prepareTraceEventForDisk, readTraceEvents, readTraceFile, resolveRedactionProfile, resolveTraceSafetyOptions, serializeEvent, validateEvent, writeTraceEvent } from './chunk-RB6WANSW.mjs';
|
|
13
13
|
import { Redactor } from './chunk-VLVCLAPB.mjs';
|
|
14
14
|
import { openTrace } from './chunk-JJSL346L.mjs';
|
|
15
15
|
import './chunk-77PUPLPG.mjs';
|
|
@@ -3010,10 +3010,17 @@ function contractToRules(contract) {
|
|
|
3010
3010
|
(event) => event.status === "running"
|
|
3011
3011
|
);
|
|
3012
3012
|
if (running.length > 0) {
|
|
3013
|
+
const labels = running.map((event) => {
|
|
3014
|
+
if (event.kind === "TOOL") {
|
|
3015
|
+
return `tool ${resolveCanonicalToolName(event)}`;
|
|
3016
|
+
}
|
|
3017
|
+
return `${event.kind} ${event.name}`;
|
|
3018
|
+
});
|
|
3019
|
+
const message = labels.length === 1 ? `Run contains incomplete running events: ${labels[0]} never completed.` : `Run contains incomplete running events: ${labels.join("; ")}.`;
|
|
3013
3020
|
findings.push(
|
|
3014
3021
|
contractFailFinding(
|
|
3015
3022
|
"contract.run.allowedStatuses",
|
|
3016
|
-
|
|
3023
|
+
message,
|
|
3017
3024
|
running.map((event) => ({
|
|
3018
3025
|
runId: event.runId,
|
|
3019
3026
|
eventId: event.eventId,
|
|
@@ -3022,7 +3029,10 @@ function contractToRules(contract) {
|
|
|
3022
3029
|
status: event.status
|
|
3023
3030
|
})),
|
|
3024
3031
|
"no running events",
|
|
3025
|
-
|
|
3032
|
+
{
|
|
3033
|
+
count: running.length,
|
|
3034
|
+
unfinished: labels
|
|
3035
|
+
}
|
|
3026
3036
|
)
|
|
3027
3037
|
);
|
|
3028
3038
|
}
|
|
@@ -3972,6 +3982,12 @@ var SAFE_USAGE_LEAF_KEYS = /* @__PURE__ */ new Set([
|
|
|
3972
3982
|
var DEFAULT_SECRET_PATTERNS = [
|
|
3973
3983
|
{ id: "bearer-token", pattern: /Bearer\s+[A-Za-z0-9._~+/-]{12,}=*/ },
|
|
3974
3984
|
{ id: "openai-key", pattern: /sk-[A-Za-z0-9_-]{16,}/ },
|
|
3985
|
+
// Stripe + Slack: keep in sync with packages/redact + credential-value-patterns (#520).
|
|
3986
|
+
{
|
|
3987
|
+
id: "stripe-key",
|
|
3988
|
+
pattern: /\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}\b/
|
|
3989
|
+
},
|
|
3990
|
+
{ id: "slack-token", pattern: /\bxox[abposr]-[A-Za-z0-9-]{10,}\b/ },
|
|
3975
3991
|
{ id: "aws-access-key", pattern: /AKIA[0-9A-Z]{16}/ },
|
|
3976
3992
|
{ id: "github-token", pattern: /gh[opsu]_[A-Za-z0-9_]{20,}/ },
|
|
3977
3993
|
// Keep in sync with packages/redact/src/key-value-secret.ts (KEY_VALUE_SECRET_PATTERN_SOURCE).
|
|
@@ -4153,6 +4169,11 @@ function compareFindings(eventById) {
|
|
|
4153
4169
|
if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
|
|
4154
4170
|
return STATUS_RANK[a.status] - STATUS_RANK[b.status];
|
|
4155
4171
|
}
|
|
4172
|
+
if (a.ruleId === "tool.usage") {
|
|
4173
|
+
const aHasEvent = a.evidence[0]?.eventId !== void 0;
|
|
4174
|
+
const bHasEvent = b.evidence[0]?.eventId !== void 0;
|
|
4175
|
+
if (aHasEvent !== bHasEvent) return aHasEvent ? -1 : 1;
|
|
4176
|
+
}
|
|
4156
4177
|
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
4157
4178
|
if (byRun !== 0) return byRun;
|
|
4158
4179
|
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
@@ -4278,6 +4299,37 @@ function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
|
4278
4299
|
function toolName(event) {
|
|
4279
4300
|
return resolveCanonicalToolName(event);
|
|
4280
4301
|
}
|
|
4302
|
+
function formatToolNameForMessage(name) {
|
|
4303
|
+
const details = [];
|
|
4304
|
+
for (let index = 0; index < name.length; ) {
|
|
4305
|
+
const codePoint = name.codePointAt(index);
|
|
4306
|
+
const width = codePoint > 65535 ? 2 : 1;
|
|
4307
|
+
if (codePoint > 127) {
|
|
4308
|
+
details.push(
|
|
4309
|
+
`U+${codePoint.toString(16).toUpperCase().padStart(4, "0")} at index ${index}`
|
|
4310
|
+
);
|
|
4311
|
+
}
|
|
4312
|
+
index += width;
|
|
4313
|
+
}
|
|
4314
|
+
if (details.length === 0) return name;
|
|
4315
|
+
return `"${name}" (contains ${details.join("; ")})`;
|
|
4316
|
+
}
|
|
4317
|
+
function describeRunningEvent(event) {
|
|
4318
|
+
if (event.kind === "TOOL") {
|
|
4319
|
+
return `tool ${formatToolNameForMessage(resolveCanonicalToolName(event))}`;
|
|
4320
|
+
}
|
|
4321
|
+
if (event.kind === "LLM") {
|
|
4322
|
+
return `llm ${stepEndpointName(event, "LLM")}`;
|
|
4323
|
+
}
|
|
4324
|
+
return `${event.kind} ${event.name}`;
|
|
4325
|
+
}
|
|
4326
|
+
function incompleteRunningMessage(prefix, running) {
|
|
4327
|
+
const labels = running.map(describeRunningEvent);
|
|
4328
|
+
if (labels.length === 1) {
|
|
4329
|
+
return `${prefix}: ${labels[0]} never completed.`;
|
|
4330
|
+
}
|
|
4331
|
+
return `${prefix}: ${labels.join("; ")}.`;
|
|
4332
|
+
}
|
|
4281
4333
|
function stepEndpointName(event, kind) {
|
|
4282
4334
|
if (kind === "TOOL") return resolveCanonicalToolName(event);
|
|
4283
4335
|
for (const prefix of ["llm:", "generation:", "transcription:", "speech:"]) {
|
|
@@ -4352,6 +4404,56 @@ function eventEndMs(event) {
|
|
|
4352
4404
|
}
|
|
4353
4405
|
return void 0;
|
|
4354
4406
|
}
|
|
4407
|
+
function rowIndexByEventId(events) {
|
|
4408
|
+
const index = /* @__PURE__ */ new Map();
|
|
4409
|
+
for (const [position, event] of events.entries()) {
|
|
4410
|
+
if (!index.has(event.eventId)) index.set(event.eventId, position);
|
|
4411
|
+
}
|
|
4412
|
+
return index;
|
|
4413
|
+
}
|
|
4414
|
+
function recordedRowSpan(rowIndex, event) {
|
|
4415
|
+
const sourceIds = "sourceEventIds" in event && Array.isArray(event.sourceEventIds) ? event.sourceEventIds : [event.eventId];
|
|
4416
|
+
const rows = [];
|
|
4417
|
+
for (const id of sourceIds) {
|
|
4418
|
+
const row = typeof id === "string" ? rowIndex.get(id) : void 0;
|
|
4419
|
+
if (row === void 0) return void 0;
|
|
4420
|
+
rows.push(row);
|
|
4421
|
+
}
|
|
4422
|
+
if (rows.length === 0) return void 0;
|
|
4423
|
+
return { first: Math.min(...rows), last: Math.max(...rows) };
|
|
4424
|
+
}
|
|
4425
|
+
function recordedTieOrder(context, tiedBefore, tiedAfter) {
|
|
4426
|
+
const rowIndex = rowIndexByEventId(context.events);
|
|
4427
|
+
let lastFinished;
|
|
4428
|
+
let firstStarted;
|
|
4429
|
+
for (const event of tiedBefore) {
|
|
4430
|
+
const span = recordedRowSpan(rowIndex, event);
|
|
4431
|
+
if (span === void 0) return { inOrder: void 0 };
|
|
4432
|
+
if (lastFinished === void 0 || span.last > lastFinished.row) {
|
|
4433
|
+
lastFinished = { event, row: span.last };
|
|
4434
|
+
}
|
|
4435
|
+
}
|
|
4436
|
+
for (const event of tiedAfter) {
|
|
4437
|
+
const span = recordedRowSpan(rowIndex, event);
|
|
4438
|
+
if (span === void 0) return { inOrder: void 0 };
|
|
4439
|
+
if (firstStarted === void 0 || span.first < firstStarted.row) {
|
|
4440
|
+
firstStarted = { event, row: span.first };
|
|
4441
|
+
}
|
|
4442
|
+
}
|
|
4443
|
+
if (lastFinished === void 0 || firstStarted === void 0) return { inOrder: void 0 };
|
|
4444
|
+
return {
|
|
4445
|
+
inOrder: lastFinished.row < firstStarted.row,
|
|
4446
|
+
lastFinishedBefore: lastFinished.event,
|
|
4447
|
+
firstStartedAfter: firstStarted.event
|
|
4448
|
+
};
|
|
4449
|
+
}
|
|
4450
|
+
function tieResolution(order) {
|
|
4451
|
+
return order.inOrder === void 0 ? "recorded-order-unavailable" : "recorded-order";
|
|
4452
|
+
}
|
|
4453
|
+
function tieMessage(order, beforeLabel, afterLabel) {
|
|
4454
|
+
if (order === void 0 || order.inOrder === true) return "";
|
|
4455
|
+
return order.inOrder === void 0 ? " Their timestamps tie at the same millisecond and the recorded event order could not be resolved." : ` Their timestamps tie at the same millisecond; the recorded event order shows ${afterLabel} started before ${beforeLabel} finished.`;
|
|
4456
|
+
}
|
|
4355
4457
|
function sortedStrings(values) {
|
|
4356
4458
|
return [...values ?? []].sort((a, b) => a.localeCompare(b));
|
|
4357
4459
|
}
|
|
@@ -4572,10 +4674,13 @@ function createRunStatusRule(options = {}) {
|
|
|
4572
4674
|
findings.push(
|
|
4573
4675
|
failFinding(
|
|
4574
4676
|
"run.status",
|
|
4575
|
-
"Run contains incomplete running events
|
|
4677
|
+
incompleteRunningMessage("Run contains incomplete running events", running),
|
|
4576
4678
|
running.map((event) => eventEvidence3(event)),
|
|
4577
4679
|
"no running events",
|
|
4578
|
-
|
|
4680
|
+
{
|
|
4681
|
+
count: running.length,
|
|
4682
|
+
unfinished: running.map(describeRunningEvent)
|
|
4683
|
+
}
|
|
4579
4684
|
)
|
|
4580
4685
|
);
|
|
4581
4686
|
}
|
|
@@ -4781,6 +4886,34 @@ function createToolUsageRule(options) {
|
|
|
4781
4886
|
const names = tools.map(toolName);
|
|
4782
4887
|
const nameSet = new Set(names);
|
|
4783
4888
|
const findings = [];
|
|
4889
|
+
const forbidden = new Set(options.forbidden ?? []);
|
|
4890
|
+
const allowed = options.allowed?.length ? new Set(options.allowed) : void 0;
|
|
4891
|
+
for (const event of tools) {
|
|
4892
|
+
const name = toolName(event);
|
|
4893
|
+
const display = formatToolNameForMessage(name);
|
|
4894
|
+
if (forbidden.has(name)) {
|
|
4895
|
+
findings.push(
|
|
4896
|
+
failFinding(
|
|
4897
|
+
"tool.usage",
|
|
4898
|
+
`Forbidden tool ${display} appeared.`,
|
|
4899
|
+
[eventEvidence3(event)],
|
|
4900
|
+
"tool absent",
|
|
4901
|
+
name
|
|
4902
|
+
)
|
|
4903
|
+
);
|
|
4904
|
+
}
|
|
4905
|
+
if (allowed && !allowed.has(name)) {
|
|
4906
|
+
findings.push(
|
|
4907
|
+
failFinding(
|
|
4908
|
+
"tool.usage",
|
|
4909
|
+
`Tool ${display} is not in the allowed tool set.`,
|
|
4910
|
+
[eventEvidence3(event)],
|
|
4911
|
+
[...allowed].sort(),
|
|
4912
|
+
name
|
|
4913
|
+
)
|
|
4914
|
+
);
|
|
4915
|
+
}
|
|
4916
|
+
}
|
|
4784
4917
|
for (const required of options.required ?? []) {
|
|
4785
4918
|
if (!nameSet.has(required)) {
|
|
4786
4919
|
const otherKinds = [
|
|
@@ -4798,7 +4931,7 @@ function createToolUsageRule(options) {
|
|
|
4798
4931
|
findings.push(
|
|
4799
4932
|
failFinding(
|
|
4800
4933
|
"tool.usage",
|
|
4801
|
-
`Required tool ${required} did not appear.${hint}`,
|
|
4934
|
+
`Required tool ${formatToolNameForMessage(required)} did not appear.${hint}`,
|
|
4802
4935
|
runEvidence(context.selectedRun),
|
|
4803
4936
|
required,
|
|
4804
4937
|
names
|
|
@@ -4806,21 +4939,6 @@ function createToolUsageRule(options) {
|
|
|
4806
4939
|
);
|
|
4807
4940
|
}
|
|
4808
4941
|
}
|
|
4809
|
-
const forbidden = new Set(options.forbidden ?? []);
|
|
4810
|
-
const allowed = options.allowed?.length ? new Set(options.allowed) : void 0;
|
|
4811
|
-
for (const event of tools) {
|
|
4812
|
-
const name = toolName(event);
|
|
4813
|
-
if (forbidden.has(name)) {
|
|
4814
|
-
findings.push(
|
|
4815
|
-
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence3(event)], "tool absent", name)
|
|
4816
|
-
);
|
|
4817
|
-
}
|
|
4818
|
-
if (allowed && !allowed.has(name)) {
|
|
4819
|
-
findings.push(
|
|
4820
|
-
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence3(event)], [...allowed].sort(), name)
|
|
4821
|
-
);
|
|
4822
|
-
}
|
|
4823
|
-
}
|
|
4824
4942
|
if (options.minCount !== void 0 && tools.length < options.minCount) {
|
|
4825
4943
|
findings.push(
|
|
4826
4944
|
failFinding("tool.usage", `Tool count ${tools.length} was below minimum ${options.minCount}.`, runEvidence(context.selectedRun), { minCount: options.minCount }, tools.length)
|
|
@@ -4854,6 +4972,28 @@ function createStepOrderingRule(options) {
|
|
|
4854
4972
|
const afterEvents = finishedEvents(context, options.after.kind).filter(
|
|
4855
4973
|
(event) => stepEndpointName(event, options.after.kind) === options.after.name
|
|
4856
4974
|
);
|
|
4975
|
+
const beforeRunning = semanticEvents(context).filter(
|
|
4976
|
+
(event) => event.kind === options.before.kind && event.status === "running" && stepEndpointName(event, options.before.kind) === options.before.name
|
|
4977
|
+
);
|
|
4978
|
+
if (beforeEvents.length === 0 && afterEvents.length > 0 && beforeRunning.length > 0) {
|
|
4979
|
+
const endpointExpected2 = bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after };
|
|
4980
|
+
return [
|
|
4981
|
+
failFinding(
|
|
4982
|
+
ruleId,
|
|
4983
|
+
`${subject} ${beforeLabel} started but never finished before ${afterLabel} ran.`,
|
|
4984
|
+
[
|
|
4985
|
+
...beforeRunning.map((event) => eventEvidence3(event)),
|
|
4986
|
+
eventEvidence3(afterEvents[0])
|
|
4987
|
+
],
|
|
4988
|
+
{ ...endpointExpected2, mode },
|
|
4989
|
+
{
|
|
4990
|
+
code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
|
|
4991
|
+
unfinishedBefore: true,
|
|
4992
|
+
beforeStatus: "running"
|
|
4993
|
+
}
|
|
4994
|
+
)
|
|
4995
|
+
];
|
|
4996
|
+
}
|
|
4857
4997
|
if (beforeEvents.length === 0 || afterEvents.length === 0) {
|
|
4858
4998
|
if (!requireEndpoints) return [];
|
|
4859
4999
|
const findings = [];
|
|
@@ -4970,16 +5110,18 @@ function createStepOrderingRule(options) {
|
|
|
4970
5110
|
)
|
|
4971
5111
|
];
|
|
4972
5112
|
}
|
|
4973
|
-
|
|
5113
|
+
const tie = beforeEnd === afterStart ? recordedTieOrder(context, [beforeEvent], [afterEvent]) : void 0;
|
|
5114
|
+
if (beforeEnd > afterStart || tie !== void 0 && tie.inOrder !== true) {
|
|
4974
5115
|
return [
|
|
4975
5116
|
failFinding(
|
|
4976
5117
|
ruleId,
|
|
4977
|
-
`${subject} ${beforeLabel} must finish before ${afterLabel} starts
|
|
5118
|
+
`${subject} ${beforeLabel} must finish before ${afterLabel} starts.${tieMessage(tie, beforeLabel, afterLabel)}`,
|
|
4978
5119
|
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
4979
5120
|
expected,
|
|
4980
5121
|
{
|
|
4981
5122
|
beforeEndedAt: new Date(beforeEnd).toISOString(),
|
|
4982
|
-
afterStartedAt: new Date(afterStart).toISOString()
|
|
5123
|
+
afterStartedAt: new Date(afterStart).toISOString(),
|
|
5124
|
+
...tie !== void 0 ? { resolvedBy: tieResolution(tie) } : {}
|
|
4983
5125
|
}
|
|
4984
5126
|
)
|
|
4985
5127
|
];
|
|
@@ -5042,16 +5184,27 @@ function createStepOrderingRule(options) {
|
|
|
5042
5184
|
)
|
|
5043
5185
|
];
|
|
5044
5186
|
}
|
|
5045
|
-
|
|
5187
|
+
const tiedEnd = latestBefore.end;
|
|
5188
|
+
const tiedStart = earliestAfter.start;
|
|
5189
|
+
const tie = tiedEnd === tiedStart ? recordedTieOrder(
|
|
5190
|
+
context,
|
|
5191
|
+
beforeEvents.filter((event) => eventEndMs(event) === tiedEnd),
|
|
5192
|
+
afterEvents.filter((event) => eventStartMs(event) === tiedStart)
|
|
5193
|
+
) : void 0;
|
|
5194
|
+
if (latestBefore.end > earliestAfter.start || tie !== void 0 && tie.inOrder !== true) {
|
|
5046
5195
|
return [
|
|
5047
5196
|
failFinding(
|
|
5048
5197
|
ruleId,
|
|
5049
|
-
bothTools ? `Every ${options.before.name} tool call must finish before any ${options.after.name} tool call starts.` : `Every ${beforeLabel} must finish before any ${afterLabel} starts
|
|
5050
|
-
[
|
|
5198
|
+
(bothTools ? `Every ${options.before.name} tool call must finish before any ${options.after.name} tool call starts.` : `Every ${beforeLabel} must finish before any ${afterLabel} starts.`) + tieMessage(tie, beforeLabel, afterLabel),
|
|
5199
|
+
[
|
|
5200
|
+
eventEvidence3(tie?.lastFinishedBefore ?? latestBefore.event),
|
|
5201
|
+
eventEvidence3(tie?.firstStartedAfter ?? earliestAfter.event)
|
|
5202
|
+
],
|
|
5051
5203
|
expected,
|
|
5052
5204
|
{
|
|
5053
5205
|
latestBeforeEndedAt: new Date(latestBefore.end).toISOString(),
|
|
5054
|
-
earliestAfterStartedAt: new Date(earliestAfter.start).toISOString()
|
|
5206
|
+
earliestAfterStartedAt: new Date(earliestAfter.start).toISOString(),
|
|
5207
|
+
...tie !== void 0 ? { resolvedBy: tieResolution(tie) } : {}
|
|
5055
5208
|
}
|
|
5056
5209
|
)
|
|
5057
5210
|
];
|
|
@@ -5217,10 +5370,13 @@ function createStructureIncompleteRule(options = {}) {
|
|
|
5217
5370
|
findings.push(
|
|
5218
5371
|
failFinding(
|
|
5219
5372
|
"structure.incomplete",
|
|
5220
|
-
"Trace contains incomplete running events
|
|
5373
|
+
incompleteRunningMessage("Trace contains incomplete running events", running),
|
|
5221
5374
|
running.map((event) => eventEvidence3(event, "status")),
|
|
5222
5375
|
"no running events",
|
|
5223
|
-
|
|
5376
|
+
{
|
|
5377
|
+
count: running.length,
|
|
5378
|
+
unfinished: running.map(describeRunningEvent)
|
|
5379
|
+
}
|
|
5224
5380
|
)
|
|
5225
5381
|
);
|
|
5226
5382
|
}
|