agent-inspect 6.31.9 → 6.31.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/docs/NESTJS.md +56 -18
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-SWUZRRDT.mjs → chunk-MQYHJL3W.mjs} +2216 -1938
- package/packages/cli/dist/chunk-MQYHJL3W.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +975 -671
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +32 -9
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-XFIPYODW.mjs → src-76R6E2LQ.mjs} +3 -3
- package/packages/cli/dist/{src-XFIPYODW.mjs.map → src-76R6E2LQ.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +2079 -1879
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +56 -4
- package/packages/core/dist/advanced.d.ts +56 -4
- package/packages/core/dist/advanced.mjs +188 -23
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +126 -9
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +3 -3
- package/packages/core/dist/checks.d.ts +3 -3
- package/packages/core/dist/checks.mjs +3 -2
- package/packages/core/dist/{chunk-L5WLVBW4.mjs → chunk-AHUZNHFI.mjs} +4 -55
- package/packages/core/dist/chunk-AHUZNHFI.mjs.map +1 -0
- package/packages/core/dist/{chunk-2CCRUI42.mjs → chunk-CCYKNAOD.mjs} +119 -12
- package/packages/core/dist/chunk-CCYKNAOD.mjs.map +1 -0
- package/packages/core/dist/{chunk-7YYS742B.mjs → chunk-M3RJD3BP.mjs} +5 -5
- package/packages/core/dist/{chunk-7YYS742B.mjs.map → chunk-M3RJD3BP.mjs.map} +1 -1
- package/packages/core/dist/{chunk-2IRRUNIN.mjs → chunk-OAUNS5CS.mjs} +3 -3
- package/packages/core/dist/{chunk-2IRRUNIN.mjs.map → chunk-OAUNS5CS.mjs.map} +1 -1
- package/packages/core/dist/{chunk-IJJSPJXV.mjs → chunk-QOC3PJO3.mjs} +4 -4
- package/packages/core/dist/{chunk-IJJSPJXV.mjs.map → chunk-QOC3PJO3.mjs.map} +1 -1
- package/packages/core/dist/chunk-S6LQBLX5.mjs +67 -0
- package/packages/core/dist/chunk-S6LQBLX5.mjs.map +1 -0
- package/packages/core/dist/{chunk-3SPBELJB.mjs → chunk-SSG3ICI4.mjs} +16 -2
- package/packages/core/dist/chunk-SSG3ICI4.mjs.map +1 -0
- package/packages/core/dist/exporters.cjs +3 -3
- package/packages/core/dist/exporters.cjs.map +1 -1
- package/packages/core/dist/exporters.mjs +1 -1
- package/packages/core/dist/{index-EQihlEei.d.cts → index-DACU4Dy1.d.cts} +12 -0
- package/packages/core/dist/{index-CgOAGuM5.d.ts → index-b4tIGrzq.d.ts} +12 -0
- package/packages/core/dist/index.cjs +77 -0
- package/packages/core/dist/index.cjs.map +1 -1
- package/packages/core/dist/index.mjs +3 -2
- package/packages/core/dist/index.mjs.map +1 -1
- package/packages/core/dist/persisted.cjs +11 -0
- package/packages/core/dist/persisted.cjs.map +1 -1
- package/packages/core/dist/persisted.mjs +4 -3
- package/packages/core/dist/persisted.mjs.map +1 -1
- package/packages/core/dist/{programmatic-gZTWLObB.d.cts → programmatic-BLlC7Ytq.d.cts} +1 -1
- package/packages/core/dist/{programmatic-gZTWLObB.d.ts → programmatic-BLlC7Ytq.d.ts} +1 -1
- package/packages/core/dist/readers.cjs +11 -0
- package/packages/core/dist/readers.cjs.map +1 -1
- package/packages/core/dist/readers.d.cts +1 -1
- package/packages/core/dist/readers.d.ts +1 -1
- package/packages/core/dist/readers.mjs +4 -3
- package/packages/cli/dist/chunk-SWUZRRDT.mjs.map +0 -1
- package/packages/core/dist/chunk-2CCRUI42.mjs.map +0 -1
- package/packages/core/dist/chunk-3SPBELJB.mjs.map +0 -1
- package/packages/core/dist/chunk-L5WLVBW4.mjs.map +0 -1
|
@@ -6,8 +6,8 @@ import { a as InspectRunTree } from './inspect-event-ClM5Z8om.cjs';
|
|
|
6
6
|
export { b as ATTRIBUTION_CONFIDENCES, c as ATTRIBUTION_CONFIDENCE_RANK, A as AttributionConfidence, E as EventSource, d as InspectEvent, I as InspectKind, e as InspectNode, i as isAttributionConfidence } from './inspect-event-ClM5Z8om.cjs';
|
|
7
7
|
import { P as PersistedInspectEvent } from './persisted-inspect-event-q2ViiBW6.cjs';
|
|
8
8
|
import { Stats } from 'node:fs';
|
|
9
|
-
import { aZ as SessionWorkflowMetadata, a_ as SessionRunRecord, a$ as EnrichSessionSummaryOptions, b0 as SessionStatus, b1 as SessionSummary, b2 as SessionIndex, b3 as BuildActivitySummaryOptions, b4 as ActivitySummary, b5 as SessionWarning, b6 as BuildSessionIndexOptions, U as TraceCheckResult, _ as TraceCheckStatus, a5 as TraceContractInput, a0 as TraceContract } from './index-
|
|
10
|
-
export { b7 as ActivityEntry, b8 as CriticalPathStep, b9 as HandoffEdge, ba as RetryLink, bb as SESSION_WORKFLOW_KEYS, bc as SessionCheckSummary, bd as SessionConfidence, be as SessionEdgeSource, bf as SessionGroup, bg as SessionLastError, bh as SessionWorkflowKey } from './index-
|
|
9
|
+
import { aZ as SessionWorkflowMetadata, a_ as SessionRunRecord, a$ as EnrichSessionSummaryOptions, b0 as SessionStatus, b1 as SessionSummary, b2 as SessionIndex, b3 as BuildActivitySummaryOptions, b4 as ActivitySummary, b5 as SessionWarning, b6 as BuildSessionIndexOptions, U as TraceCheckResult, _ as TraceCheckStatus, a5 as TraceContractInput, a0 as TraceContract } from './index-DACU4Dy1.cjs';
|
|
10
|
+
export { b7 as ActivityEntry, b8 as CriticalPathStep, b9 as HandoffEdge, ba as RetryLink, bb as SESSION_WORKFLOW_KEYS, bc as SessionCheckSummary, bd as SessionConfidence, be as SessionEdgeSource, bf as SessionGroup, bg as SessionLastError, bh as SessionWorkflowKey } from './index-DACU4Dy1.cjs';
|
|
11
11
|
import { b as ObservedOutcome } from './types-BA_IOdg4.cjs';
|
|
12
12
|
export { a as ObservedOutcomeStatus } from './types-BA_IOdg4.cjs';
|
|
13
13
|
import './writers.cjs';
|
|
@@ -424,6 +424,11 @@ interface ExplainResult {
|
|
|
424
424
|
interface ExplainOptions {
|
|
425
425
|
mode?: ExplainMode;
|
|
426
426
|
redactionProfile?: RedactionProfile;
|
|
427
|
+
/**
|
|
428
|
+
* Optional raw persisted events for the selected run (or full read).
|
|
429
|
+
* When provided, explain adds lifecycle-projected counts and slowest logical step.
|
|
430
|
+
*/
|
|
431
|
+
events?: readonly PersistedInspectEvent[];
|
|
427
432
|
}
|
|
428
433
|
/**
|
|
429
434
|
* Build a local, deterministic explanation payload from a reader-selected run.
|
|
@@ -828,8 +833,31 @@ interface EvidenceManifest {
|
|
|
828
833
|
* Additive; older readers ignore unknown fields.
|
|
829
834
|
*/
|
|
830
835
|
contract?: EvidenceContractBinding;
|
|
836
|
+
/**
|
|
837
|
+
* Optional input provenance bindings (additive).
|
|
838
|
+
* Hashes bind integrity relative to this manifest — not who executed the run
|
|
839
|
+
* or a SLSA attestation level.
|
|
840
|
+
*/
|
|
841
|
+
inputs?: EvidenceInputBinding;
|
|
831
842
|
files: EvidenceFileEntry[];
|
|
832
843
|
}
|
|
844
|
+
/**
|
|
845
|
+
* Additive input bindings for Evidence v2 (app/scenario/config provenance).
|
|
846
|
+
* Older readers ignore unknown fields. Does not claim trusted-build attestation.
|
|
847
|
+
*/
|
|
848
|
+
interface EvidenceInputBinding {
|
|
849
|
+
scenarioId?: string;
|
|
850
|
+
profileId?: string;
|
|
851
|
+
/** SHA-256 of the lockfile bytes that materially affected the run. */
|
|
852
|
+
lockfileSha256?: string;
|
|
853
|
+
/** SHA-256 of an app snapshot / tree digest (dirty boolean alone is insufficient). */
|
|
854
|
+
appSnapshotSha256?: string;
|
|
855
|
+
/** SHA-256 of the exact dirty patch bytes when the tree was dirty. */
|
|
856
|
+
dirtyPatchSha256?: string;
|
|
857
|
+
/** SHA-256 of oracle implementation/configuration bytes when disclosed. */
|
|
858
|
+
oracleConfigSha256?: string;
|
|
859
|
+
note?: string;
|
|
860
|
+
}
|
|
833
861
|
/**
|
|
834
862
|
* Bounded semantic summary embedded in Evidence v2 (mirrors check parity summary).
|
|
835
863
|
* Does not embed raw events or prompts.
|
|
@@ -910,6 +938,7 @@ declare function buildEvidenceManifest(parts: {
|
|
|
910
938
|
note?: string;
|
|
911
939
|
semantics?: EvidenceManifest["semantics"];
|
|
912
940
|
contract?: EvidenceManifest["contract"];
|
|
941
|
+
inputs?: EvidenceManifest["inputs"];
|
|
913
942
|
}): EvidenceManifest;
|
|
914
943
|
/**
|
|
915
944
|
* Minimal structural validation for a parsed evidence manifest object.
|
|
@@ -1269,7 +1298,7 @@ interface EvidenceCiPackageFiles {
|
|
|
1269
1298
|
declare function buildEvidenceCiPackage(input: EvidenceCiPackageInput): EvidenceCiPackageFiles;
|
|
1270
1299
|
|
|
1271
1300
|
type SuiteCaseStatus = "pass" | "fail" | "error" | "skipped";
|
|
1272
|
-
type SuiteDiagnosticCode = "AI_SUITE_CONFIG_INVALID" | "AI_SUITE_CONFIG_LOAD_FAILED" | "AI_SUITE_CASE_TRACE_MISSING" | "AI_SUITE_CASE_CHECK_FAILED" | "AI_SUITE_CASE_EVAL_FAILED" | "AI_SUITE_CASE_OBSERVATION_FAILED" | "AI_SUITE_NO_ASSERTIONS" | "AI_SUITE_UNKNOWN_SELECTOR" | "AI_SUITE_TRACE_UNREADABLE";
|
|
1301
|
+
type SuiteDiagnosticCode = "AI_SUITE_CONFIG_INVALID" | "AI_SUITE_CONFIG_LOAD_FAILED" | "AI_SUITE_CASE_TRACE_MISSING" | "AI_SUITE_CASE_CHECK_FAILED" | "AI_SUITE_CASE_EVAL_FAILED" | "AI_SUITE_CASE_OBSERVATION_FAILED" | "AI_SUITE_NO_ASSERTIONS" | "AI_SUITE_UNKNOWN_SELECTOR" | "AI_SUITE_EXPECT_MISMATCH" | "AI_SUITE_TRACE_UNREADABLE";
|
|
1273
1302
|
interface SuiteDiagnostic {
|
|
1274
1303
|
code: SuiteDiagnosticCode;
|
|
1275
1304
|
severity: "error" | "warning" | "info";
|
|
@@ -1285,6 +1314,29 @@ interface SuiteCaseConfig {
|
|
|
1285
1314
|
forbidTools?: string[];
|
|
1286
1315
|
maxDurationMs?: number;
|
|
1287
1316
|
expectedObservations?: string[];
|
|
1317
|
+
/**
|
|
1318
|
+
* Additive expected semantic-failure assertion over the raw check result.
|
|
1319
|
+
* When set, the suite case can pass because the contract correctly failed.
|
|
1320
|
+
* Raw `check.ok` remains false; the suite assertion matches status + findings.
|
|
1321
|
+
*/
|
|
1322
|
+
expect?: SuiteCaseExpect;
|
|
1323
|
+
}
|
|
1324
|
+
/**
|
|
1325
|
+
* Typed expectation that a case fails for specific semantic reasons.
|
|
1326
|
+
* Does not invent captureOnly / nonGating modes.
|
|
1327
|
+
*/
|
|
1328
|
+
interface SuiteCaseExpect {
|
|
1329
|
+
/** Required check status (typically `"fail"` for semantic negatives). */
|
|
1330
|
+
checkStatus: "pass" | "fail" | "error";
|
|
1331
|
+
/** Finding rule ids that must appear among failed findings (order-insensitive). */
|
|
1332
|
+
findingRuleIds?: string[];
|
|
1333
|
+
/**
|
|
1334
|
+
* When true (default), any failed finding whose ruleId is not listed in
|
|
1335
|
+
* `findingRuleIds` rejects the expectation.
|
|
1336
|
+
*/
|
|
1337
|
+
rejectUnexpectedFindings?: boolean;
|
|
1338
|
+
/** Minimum effective assertion count (rules + observations). */
|
|
1339
|
+
minAssertions?: number;
|
|
1288
1340
|
}
|
|
1289
1341
|
interface SuiteChecksConfig {
|
|
1290
1342
|
select?: string[];
|
|
@@ -1596,4 +1648,4 @@ declare function createOmittedPayloadCommitment(payload: string | Uint8Array, op
|
|
|
1596
1648
|
capturePolicy?: OmittedPayloadCommitment["capturePolicy"];
|
|
1597
1649
|
}): OmittedPayloadCommitment;
|
|
1598
1650
|
|
|
1599
|
-
export { ADAPTER_CAPTURE_DIAGNOSTIC_CODES, ActivitySummary, type AdapterCaptureDiagnostic, type AdapterCaptureDiagnosticCode, type AdapterCaptureDiagnostics, type AdapterCaptureMode, type AdapterDiagnosticListener, type AdapterPreviewCapture, type AdapterPreviewCaptureOptions, type AnalyzeCohortOptions, BuildActivitySummaryOptions, type BuildEvidenceContractPackageInput, BuildSessionIndexOptions, type BundleCheckResults, type BundleCheckRunResult, type BundleMetadata, type BundlePlaceholderArtifact, type BundleRedactionProfile, type BundleRedactionReport, type BundleRedactionReportRun, type BundleResolveOptions, type BundleResolveResult, type BundleSafeStatus, type BundleSafeStatusMetadata, COHORT_METRIC_IDS, type CausalContractFindingInput, type CausalFailureKind, type CohortAggregateMetrics, type CohortAnalysisResult, type CohortMetricComparison, type CohortMetricId, type CohortRunMetrics, DEFAULT_ADAPTER_MAX_PREVIEW_CHARS, DEFAULT_SUITE_ARTIFACTS_DIR, DEFAULT_SUITE_CONFIG_NAMES, DEFAULT_TRACE_DIR_NAME, type DurationStats, EVIDENCE_ASSESSMENT_NOTE, EVIDENCE_CONTRACT_ASSURANCE_NOTE, EVIDENCE_CONTRACT_CANONICALIZATION_VERSION, EVIDENCE_CONTRACT_PARTIAL_NOTE, EVIDENCE_CONTRACT_UNAVAILABLE_NOTE, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, EVIDENCE_RESOLVED_CONTRACT_FILENAME, EVIDENCE_VIEW_CSS, EVIDENCE_VIEW_IDS, EnrichSessionSummaryOptions, ErrorInfo, type EvidenceCheckContractBinding, type EvidenceCheckFindingSummary, type EvidenceCiPackageFiles, type EvidenceCiPackageInput, type EvidenceContractBinding, type EvidenceContractBindingSource, type EvidenceContractBindingStatus, type EvidenceContractPackage, type EvidenceContractVerifyIssue, type EvidenceContractsViewInput, type EvidenceFileEntry, type EvidenceFileRole, type EvidenceFormatVersion, type EvidenceHtmlShellInput, type EvidenceManifest, type EvidencePackagedFile, type EvidenceProvenanceViewInput, type EvidenceRedactionProfile, type EvidenceSafeStatus, type EvidenceSafetyViewInput, type EvidenceSourceHash, type EvidenceVerificationPolicy, type EvidenceVerifyIssue, type EvidenceVerifyOptions, type EvidenceVerifyResult, type EvidenceVerifyStatus, type EvidenceViewId, type ExplainFact, type ExplainInference, type ExplainMode, type ExplainOptions, type ExplainResult, FALLBACK_TRACE_DIR, type FindFirstCausalFailureOptions, type FirstCausalFailure, type GateCheckId, type GateCheckResult, type GateExitCode, type GateResult, type GroupSessionCohortsOptions, InspectRunTree, type LoadSuiteConfigOptions, MAX_NAME_LENGTH, MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, OMITTED_PAYLOAD_MAX_DIGEST_INPUT_BYTES, ObservedOutcome, type OmittedPayloadAlgorithm, type OmittedPayloadCommitment, type ParseTraceJsonlOptions, type ParseTraceJsonlResult, type ParsedDurationFilter, RUNS_DIR_NAME, RedactionProfile, type RenderCohortReportOptions, type RenderGateReportOptions, type RenderSuiteReportOptions, type RenderTimelineOptions, type RenderWhatOptions, type ResolvedContractDocument, type ResolvedRedactionProfile, type ResolvedSuiteCase, type RunGateOptions, RunStatus, type RunSuiteOptions, RunSummary, type RunTimeline, type RunWhatSummary, SUITE_TEMPLATE_IDS, type SessionCohort, type SessionCohortKind, SessionIndex, SessionRunRecord, type SessionScopeOptions, type SessionScopeResult, SessionStatus, SessionSummary, SessionWarning, SessionWorkflowMetadata, StepStatus, StepType, type SuiteArtifactsConfig, type SuiteCaseConfig, type SuiteCaseResult, type SuiteCaseStatus, type SuiteChecksConfig, type SuiteConfig, type SuiteDiagnostic, type SuiteDiagnosticCode, type SuiteEvalConfig, type SuiteRunResult, type SuiteRunSummary, type SuiteTemplateId, TERMINAL_INDENT, type TimelineEntry, type TimelineFocus, type TimelineOptions, TraceCorrelationMetadata, TraceDirectory, type TraceDirectoryOptions, TraceEvent, type TraceFilterOptions, type TraceJsonlFormat, TraceMetadata, TraceMetadataStatus, type TraceSearchOptions, type TraceSearchResult, type TraceSessionCheckResult, type TraceStats, type TraceStatsOptions, type TraceStatsRankedRun, type TraceStatsRankedStep, type ValidateSuiteConfigResult, type ZipEntry, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, assertBundlePathContained, assertEvidenceRelativePath, bindCheckResultsToContract, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractPackage, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceFileEntries, buildEvidenceHtmlShell, buildEvidenceHtmlShellFromManifest, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectResolvedContractRuleIds, collectTraceSchemaVersions, compareCohortAggregates, createAdapterPreviewCapture, createOmittedPayloadCommitment, createRunId, createStepId, defaultBundleOutputPath, defaultSuiteConfigTemplate, deriveSessionStatus, encodeEmbeddedEvidenceJson, enrichSessionRunRecord, enrichSessionSummary, ensureTraceDir, extractMetadata, extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, findFirstCausalFailure, formatDuration, formatError, formatStepLabel, formatTerminalName, formatTimestamp, gateHasThresholds, getDefaultTraceDir, getIndent, getRunIdFromTraceFileName, getSuiteTemplate, getTraceFilePath, groupSessionCohorts, inferEvidenceFileRole, initializeTraceFile, isAgentInspectTrace, isSha256Hex, listSuiteTemplates, listTraceFiles, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, normalizeBundleOutputPath, normalizeSuiteConfig, omittedPayloadByteLength, parseCohortMetricList, parseDuration, parseDurationFilter, parseEvidenceManifestJson, parseGateList, parseGateNumber, parseGroupBySpec, parseTraceJsonl, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, readCheckResultsContractBinding, readTraceEvents, readTraceFile, renderActivitySummaryHuman, renderCohortReport, renderCohortSummaryMarkdown, renderErrorLine, renderGateGithubAnnotations, renderGateGithubStepSummary, renderGateJUnit, renderGateReport, renderGateSummaryMarkdown, renderRunSummary, renderRunWhat, renderStepLine, renderSuiteReport, renderSuiteReportMarkdown, renderTimeline, renderTraceStats, resolveAdapterMaxPreviewChars, resolveBundleRunIds, resolveRedactionProfile, resolveSerializableTraceContract, resolveSuiteCaseTrace, resolveSuiteConfigPath, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, sanitizeBundleRunId, searchTraces, serializeAdapterPreview, serializeCanonicalJson, serializeCheckResultsJson, serializeEvent, serializeEvidenceManifest, sessionKeyForRun, sha256Equals, sha256Hex, toMetadataSafeStatus, traceMetasToSessionRunRecords, truncateName, unknownTraceFormatMessage, validateEvent, validateEvidenceManifest, validateSuiteConfig, verifyEvidenceContractBinding, verifyEvidenceDirectory, warn, writeTraceEvent };
|
|
1651
|
+
export { ADAPTER_CAPTURE_DIAGNOSTIC_CODES, ActivitySummary, type AdapterCaptureDiagnostic, type AdapterCaptureDiagnosticCode, type AdapterCaptureDiagnostics, type AdapterCaptureMode, type AdapterDiagnosticListener, type AdapterPreviewCapture, type AdapterPreviewCaptureOptions, type AnalyzeCohortOptions, BuildActivitySummaryOptions, type BuildEvidenceContractPackageInput, BuildSessionIndexOptions, type BundleCheckResults, type BundleCheckRunResult, type BundleMetadata, type BundlePlaceholderArtifact, type BundleRedactionProfile, type BundleRedactionReport, type BundleRedactionReportRun, type BundleResolveOptions, type BundleResolveResult, type BundleSafeStatus, type BundleSafeStatusMetadata, COHORT_METRIC_IDS, type CausalContractFindingInput, type CausalFailureKind, type CohortAggregateMetrics, type CohortAnalysisResult, type CohortMetricComparison, type CohortMetricId, type CohortRunMetrics, DEFAULT_ADAPTER_MAX_PREVIEW_CHARS, DEFAULT_SUITE_ARTIFACTS_DIR, DEFAULT_SUITE_CONFIG_NAMES, DEFAULT_TRACE_DIR_NAME, type DurationStats, EVIDENCE_ASSESSMENT_NOTE, EVIDENCE_CONTRACT_ASSURANCE_NOTE, EVIDENCE_CONTRACT_CANONICALIZATION_VERSION, EVIDENCE_CONTRACT_PARTIAL_NOTE, EVIDENCE_CONTRACT_UNAVAILABLE_NOTE, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, EVIDENCE_RESOLVED_CONTRACT_FILENAME, EVIDENCE_VIEW_CSS, EVIDENCE_VIEW_IDS, EnrichSessionSummaryOptions, ErrorInfo, type EvidenceCheckContractBinding, type EvidenceCheckFindingSummary, type EvidenceCiPackageFiles, type EvidenceCiPackageInput, type EvidenceContractBinding, type EvidenceContractBindingSource, type EvidenceContractBindingStatus, type EvidenceContractPackage, type EvidenceContractVerifyIssue, type EvidenceContractsViewInput, type EvidenceFileEntry, type EvidenceFileRole, type EvidenceFormatVersion, type EvidenceHtmlShellInput, type EvidenceInputBinding, type EvidenceManifest, type EvidencePackagedFile, type EvidenceProvenanceViewInput, type EvidenceRedactionProfile, type EvidenceSafeStatus, type EvidenceSafetyViewInput, type EvidenceSourceHash, type EvidenceVerificationPolicy, type EvidenceVerifyIssue, type EvidenceVerifyOptions, type EvidenceVerifyResult, type EvidenceVerifyStatus, type EvidenceViewId, type ExplainFact, type ExplainInference, type ExplainMode, type ExplainOptions, type ExplainResult, FALLBACK_TRACE_DIR, type FindFirstCausalFailureOptions, type FirstCausalFailure, type GateCheckId, type GateCheckResult, type GateExitCode, type GateResult, type GroupSessionCohortsOptions, InspectRunTree, type LoadSuiteConfigOptions, MAX_NAME_LENGTH, MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, OMITTED_PAYLOAD_MAX_DIGEST_INPUT_BYTES, ObservedOutcome, type OmittedPayloadAlgorithm, type OmittedPayloadCommitment, type ParseTraceJsonlOptions, type ParseTraceJsonlResult, type ParsedDurationFilter, RUNS_DIR_NAME, RedactionProfile, type RenderCohortReportOptions, type RenderGateReportOptions, type RenderSuiteReportOptions, type RenderTimelineOptions, type RenderWhatOptions, type ResolvedContractDocument, type ResolvedRedactionProfile, type ResolvedSuiteCase, type RunGateOptions, RunStatus, type RunSuiteOptions, RunSummary, type RunTimeline, type RunWhatSummary, SUITE_TEMPLATE_IDS, type SessionCohort, type SessionCohortKind, SessionIndex, SessionRunRecord, type SessionScopeOptions, type SessionScopeResult, SessionStatus, SessionSummary, SessionWarning, SessionWorkflowMetadata, StepStatus, StepType, type SuiteArtifactsConfig, type SuiteCaseConfig, type SuiteCaseResult, type SuiteCaseStatus, type SuiteChecksConfig, type SuiteConfig, type SuiteDiagnostic, type SuiteDiagnosticCode, type SuiteEvalConfig, type SuiteRunResult, type SuiteRunSummary, type SuiteTemplateId, TERMINAL_INDENT, type TimelineEntry, type TimelineFocus, type TimelineOptions, TraceCorrelationMetadata, TraceDirectory, type TraceDirectoryOptions, TraceEvent, type TraceFilterOptions, type TraceJsonlFormat, TraceMetadata, TraceMetadataStatus, type TraceSearchOptions, type TraceSearchResult, type TraceSessionCheckResult, type TraceStats, type TraceStatsOptions, type TraceStatsRankedRun, type TraceStatsRankedStep, type ValidateSuiteConfigResult, type ZipEntry, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, assertBundlePathContained, assertEvidenceRelativePath, bindCheckResultsToContract, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractPackage, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceFileEntries, buildEvidenceHtmlShell, buildEvidenceHtmlShellFromManifest, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectResolvedContractRuleIds, collectTraceSchemaVersions, compareCohortAggregates, createAdapterPreviewCapture, createOmittedPayloadCommitment, createRunId, createStepId, defaultBundleOutputPath, defaultSuiteConfigTemplate, deriveSessionStatus, encodeEmbeddedEvidenceJson, enrichSessionRunRecord, enrichSessionSummary, ensureTraceDir, extractMetadata, extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, findFirstCausalFailure, formatDuration, formatError, formatStepLabel, formatTerminalName, formatTimestamp, gateHasThresholds, getDefaultTraceDir, getIndent, getRunIdFromTraceFileName, getSuiteTemplate, getTraceFilePath, groupSessionCohorts, inferEvidenceFileRole, initializeTraceFile, isAgentInspectTrace, isSha256Hex, listSuiteTemplates, listTraceFiles, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, normalizeBundleOutputPath, normalizeSuiteConfig, omittedPayloadByteLength, parseCohortMetricList, parseDuration, parseDurationFilter, parseEvidenceManifestJson, parseGateList, parseGateNumber, parseGroupBySpec, parseTraceJsonl, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, readCheckResultsContractBinding, readTraceEvents, readTraceFile, renderActivitySummaryHuman, renderCohortReport, renderCohortSummaryMarkdown, renderErrorLine, renderGateGithubAnnotations, renderGateGithubStepSummary, renderGateJUnit, renderGateReport, renderGateSummaryMarkdown, renderRunSummary, renderRunWhat, renderStepLine, renderSuiteReport, renderSuiteReportMarkdown, renderTimeline, renderTraceStats, resolveAdapterMaxPreviewChars, resolveBundleRunIds, resolveRedactionProfile, resolveSerializableTraceContract, resolveSuiteCaseTrace, resolveSuiteConfigPath, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, sanitizeBundleRunId, searchTraces, serializeAdapterPreview, serializeCanonicalJson, serializeCheckResultsJson, serializeEvent, serializeEvidenceManifest, sessionKeyForRun, sha256Equals, sha256Hex, toMetadataSafeStatus, traceMetasToSessionRunRecords, truncateName, unknownTraceFormatMessage, validateEvent, validateEvidenceManifest, validateSuiteConfig, verifyEvidenceContractBinding, verifyEvidenceDirectory, warn, writeTraceEvent };
|
|
@@ -6,8 +6,8 @@ import { a as InspectRunTree } from './inspect-event-ClM5Z8om.js';
|
|
|
6
6
|
export { b as ATTRIBUTION_CONFIDENCES, c as ATTRIBUTION_CONFIDENCE_RANK, A as AttributionConfidence, E as EventSource, d as InspectEvent, I as InspectKind, e as InspectNode, i as isAttributionConfidence } from './inspect-event-ClM5Z8om.js';
|
|
7
7
|
import { P as PersistedInspectEvent } from './persisted-inspect-event-DG02LhEP.js';
|
|
8
8
|
import { Stats } from 'node:fs';
|
|
9
|
-
import { aZ as SessionWorkflowMetadata, a_ as SessionRunRecord, a$ as EnrichSessionSummaryOptions, b0 as SessionStatus, b1 as SessionSummary, b2 as SessionIndex, b3 as BuildActivitySummaryOptions, b4 as ActivitySummary, b5 as SessionWarning, b6 as BuildSessionIndexOptions, U as TraceCheckResult, _ as TraceCheckStatus, a5 as TraceContractInput, a0 as TraceContract } from './index-
|
|
10
|
-
export { b7 as ActivityEntry, b8 as CriticalPathStep, b9 as HandoffEdge, ba as RetryLink, bb as SESSION_WORKFLOW_KEYS, bc as SessionCheckSummary, bd as SessionConfidence, be as SessionEdgeSource, bf as SessionGroup, bg as SessionLastError, bh as SessionWorkflowKey } from './index-
|
|
9
|
+
import { aZ as SessionWorkflowMetadata, a_ as SessionRunRecord, a$ as EnrichSessionSummaryOptions, b0 as SessionStatus, b1 as SessionSummary, b2 as SessionIndex, b3 as BuildActivitySummaryOptions, b4 as ActivitySummary, b5 as SessionWarning, b6 as BuildSessionIndexOptions, U as TraceCheckResult, _ as TraceCheckStatus, a5 as TraceContractInput, a0 as TraceContract } from './index-b4tIGrzq.js';
|
|
10
|
+
export { b7 as ActivityEntry, b8 as CriticalPathStep, b9 as HandoffEdge, ba as RetryLink, bb as SESSION_WORKFLOW_KEYS, bc as SessionCheckSummary, bd as SessionConfidence, be as SessionEdgeSource, bf as SessionGroup, bg as SessionLastError, bh as SessionWorkflowKey } from './index-b4tIGrzq.js';
|
|
11
11
|
import { b as ObservedOutcome } from './types-BA_IOdg4.js';
|
|
12
12
|
export { a as ObservedOutcomeStatus } from './types-BA_IOdg4.js';
|
|
13
13
|
import './writers.js';
|
|
@@ -424,6 +424,11 @@ interface ExplainResult {
|
|
|
424
424
|
interface ExplainOptions {
|
|
425
425
|
mode?: ExplainMode;
|
|
426
426
|
redactionProfile?: RedactionProfile;
|
|
427
|
+
/**
|
|
428
|
+
* Optional raw persisted events for the selected run (or full read).
|
|
429
|
+
* When provided, explain adds lifecycle-projected counts and slowest logical step.
|
|
430
|
+
*/
|
|
431
|
+
events?: readonly PersistedInspectEvent[];
|
|
427
432
|
}
|
|
428
433
|
/**
|
|
429
434
|
* Build a local, deterministic explanation payload from a reader-selected run.
|
|
@@ -828,8 +833,31 @@ interface EvidenceManifest {
|
|
|
828
833
|
* Additive; older readers ignore unknown fields.
|
|
829
834
|
*/
|
|
830
835
|
contract?: EvidenceContractBinding;
|
|
836
|
+
/**
|
|
837
|
+
* Optional input provenance bindings (additive).
|
|
838
|
+
* Hashes bind integrity relative to this manifest — not who executed the run
|
|
839
|
+
* or a SLSA attestation level.
|
|
840
|
+
*/
|
|
841
|
+
inputs?: EvidenceInputBinding;
|
|
831
842
|
files: EvidenceFileEntry[];
|
|
832
843
|
}
|
|
844
|
+
/**
|
|
845
|
+
* Additive input bindings for Evidence v2 (app/scenario/config provenance).
|
|
846
|
+
* Older readers ignore unknown fields. Does not claim trusted-build attestation.
|
|
847
|
+
*/
|
|
848
|
+
interface EvidenceInputBinding {
|
|
849
|
+
scenarioId?: string;
|
|
850
|
+
profileId?: string;
|
|
851
|
+
/** SHA-256 of the lockfile bytes that materially affected the run. */
|
|
852
|
+
lockfileSha256?: string;
|
|
853
|
+
/** SHA-256 of an app snapshot / tree digest (dirty boolean alone is insufficient). */
|
|
854
|
+
appSnapshotSha256?: string;
|
|
855
|
+
/** SHA-256 of the exact dirty patch bytes when the tree was dirty. */
|
|
856
|
+
dirtyPatchSha256?: string;
|
|
857
|
+
/** SHA-256 of oracle implementation/configuration bytes when disclosed. */
|
|
858
|
+
oracleConfigSha256?: string;
|
|
859
|
+
note?: string;
|
|
860
|
+
}
|
|
833
861
|
/**
|
|
834
862
|
* Bounded semantic summary embedded in Evidence v2 (mirrors check parity summary).
|
|
835
863
|
* Does not embed raw events or prompts.
|
|
@@ -910,6 +938,7 @@ declare function buildEvidenceManifest(parts: {
|
|
|
910
938
|
note?: string;
|
|
911
939
|
semantics?: EvidenceManifest["semantics"];
|
|
912
940
|
contract?: EvidenceManifest["contract"];
|
|
941
|
+
inputs?: EvidenceManifest["inputs"];
|
|
913
942
|
}): EvidenceManifest;
|
|
914
943
|
/**
|
|
915
944
|
* Minimal structural validation for a parsed evidence manifest object.
|
|
@@ -1269,7 +1298,7 @@ interface EvidenceCiPackageFiles {
|
|
|
1269
1298
|
declare function buildEvidenceCiPackage(input: EvidenceCiPackageInput): EvidenceCiPackageFiles;
|
|
1270
1299
|
|
|
1271
1300
|
type SuiteCaseStatus = "pass" | "fail" | "error" | "skipped";
|
|
1272
|
-
type SuiteDiagnosticCode = "AI_SUITE_CONFIG_INVALID" | "AI_SUITE_CONFIG_LOAD_FAILED" | "AI_SUITE_CASE_TRACE_MISSING" | "AI_SUITE_CASE_CHECK_FAILED" | "AI_SUITE_CASE_EVAL_FAILED" | "AI_SUITE_CASE_OBSERVATION_FAILED" | "AI_SUITE_NO_ASSERTIONS" | "AI_SUITE_UNKNOWN_SELECTOR" | "AI_SUITE_TRACE_UNREADABLE";
|
|
1301
|
+
type SuiteDiagnosticCode = "AI_SUITE_CONFIG_INVALID" | "AI_SUITE_CONFIG_LOAD_FAILED" | "AI_SUITE_CASE_TRACE_MISSING" | "AI_SUITE_CASE_CHECK_FAILED" | "AI_SUITE_CASE_EVAL_FAILED" | "AI_SUITE_CASE_OBSERVATION_FAILED" | "AI_SUITE_NO_ASSERTIONS" | "AI_SUITE_UNKNOWN_SELECTOR" | "AI_SUITE_EXPECT_MISMATCH" | "AI_SUITE_TRACE_UNREADABLE";
|
|
1273
1302
|
interface SuiteDiagnostic {
|
|
1274
1303
|
code: SuiteDiagnosticCode;
|
|
1275
1304
|
severity: "error" | "warning" | "info";
|
|
@@ -1285,6 +1314,29 @@ interface SuiteCaseConfig {
|
|
|
1285
1314
|
forbidTools?: string[];
|
|
1286
1315
|
maxDurationMs?: number;
|
|
1287
1316
|
expectedObservations?: string[];
|
|
1317
|
+
/**
|
|
1318
|
+
* Additive expected semantic-failure assertion over the raw check result.
|
|
1319
|
+
* When set, the suite case can pass because the contract correctly failed.
|
|
1320
|
+
* Raw `check.ok` remains false; the suite assertion matches status + findings.
|
|
1321
|
+
*/
|
|
1322
|
+
expect?: SuiteCaseExpect;
|
|
1323
|
+
}
|
|
1324
|
+
/**
|
|
1325
|
+
* Typed expectation that a case fails for specific semantic reasons.
|
|
1326
|
+
* Does not invent captureOnly / nonGating modes.
|
|
1327
|
+
*/
|
|
1328
|
+
interface SuiteCaseExpect {
|
|
1329
|
+
/** Required check status (typically `"fail"` for semantic negatives). */
|
|
1330
|
+
checkStatus: "pass" | "fail" | "error";
|
|
1331
|
+
/** Finding rule ids that must appear among failed findings (order-insensitive). */
|
|
1332
|
+
findingRuleIds?: string[];
|
|
1333
|
+
/**
|
|
1334
|
+
* When true (default), any failed finding whose ruleId is not listed in
|
|
1335
|
+
* `findingRuleIds` rejects the expectation.
|
|
1336
|
+
*/
|
|
1337
|
+
rejectUnexpectedFindings?: boolean;
|
|
1338
|
+
/** Minimum effective assertion count (rules + observations). */
|
|
1339
|
+
minAssertions?: number;
|
|
1288
1340
|
}
|
|
1289
1341
|
interface SuiteChecksConfig {
|
|
1290
1342
|
select?: string[];
|
|
@@ -1596,4 +1648,4 @@ declare function createOmittedPayloadCommitment(payload: string | Uint8Array, op
|
|
|
1596
1648
|
capturePolicy?: OmittedPayloadCommitment["capturePolicy"];
|
|
1597
1649
|
}): OmittedPayloadCommitment;
|
|
1598
1650
|
|
|
1599
|
-
export { ADAPTER_CAPTURE_DIAGNOSTIC_CODES, ActivitySummary, type AdapterCaptureDiagnostic, type AdapterCaptureDiagnosticCode, type AdapterCaptureDiagnostics, type AdapterCaptureMode, type AdapterDiagnosticListener, type AdapterPreviewCapture, type AdapterPreviewCaptureOptions, type AnalyzeCohortOptions, BuildActivitySummaryOptions, type BuildEvidenceContractPackageInput, BuildSessionIndexOptions, type BundleCheckResults, type BundleCheckRunResult, type BundleMetadata, type BundlePlaceholderArtifact, type BundleRedactionProfile, type BundleRedactionReport, type BundleRedactionReportRun, type BundleResolveOptions, type BundleResolveResult, type BundleSafeStatus, type BundleSafeStatusMetadata, COHORT_METRIC_IDS, type CausalContractFindingInput, type CausalFailureKind, type CohortAggregateMetrics, type CohortAnalysisResult, type CohortMetricComparison, type CohortMetricId, type CohortRunMetrics, DEFAULT_ADAPTER_MAX_PREVIEW_CHARS, DEFAULT_SUITE_ARTIFACTS_DIR, DEFAULT_SUITE_CONFIG_NAMES, DEFAULT_TRACE_DIR_NAME, type DurationStats, EVIDENCE_ASSESSMENT_NOTE, EVIDENCE_CONTRACT_ASSURANCE_NOTE, EVIDENCE_CONTRACT_CANONICALIZATION_VERSION, EVIDENCE_CONTRACT_PARTIAL_NOTE, EVIDENCE_CONTRACT_UNAVAILABLE_NOTE, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, EVIDENCE_RESOLVED_CONTRACT_FILENAME, EVIDENCE_VIEW_CSS, EVIDENCE_VIEW_IDS, EnrichSessionSummaryOptions, ErrorInfo, type EvidenceCheckContractBinding, type EvidenceCheckFindingSummary, type EvidenceCiPackageFiles, type EvidenceCiPackageInput, type EvidenceContractBinding, type EvidenceContractBindingSource, type EvidenceContractBindingStatus, type EvidenceContractPackage, type EvidenceContractVerifyIssue, type EvidenceContractsViewInput, type EvidenceFileEntry, type EvidenceFileRole, type EvidenceFormatVersion, type EvidenceHtmlShellInput, type EvidenceManifest, type EvidencePackagedFile, type EvidenceProvenanceViewInput, type EvidenceRedactionProfile, type EvidenceSafeStatus, type EvidenceSafetyViewInput, type EvidenceSourceHash, type EvidenceVerificationPolicy, type EvidenceVerifyIssue, type EvidenceVerifyOptions, type EvidenceVerifyResult, type EvidenceVerifyStatus, type EvidenceViewId, type ExplainFact, type ExplainInference, type ExplainMode, type ExplainOptions, type ExplainResult, FALLBACK_TRACE_DIR, type FindFirstCausalFailureOptions, type FirstCausalFailure, type GateCheckId, type GateCheckResult, type GateExitCode, type GateResult, type GroupSessionCohortsOptions, InspectRunTree, type LoadSuiteConfigOptions, MAX_NAME_LENGTH, MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, OMITTED_PAYLOAD_MAX_DIGEST_INPUT_BYTES, ObservedOutcome, type OmittedPayloadAlgorithm, type OmittedPayloadCommitment, type ParseTraceJsonlOptions, type ParseTraceJsonlResult, type ParsedDurationFilter, RUNS_DIR_NAME, RedactionProfile, type RenderCohortReportOptions, type RenderGateReportOptions, type RenderSuiteReportOptions, type RenderTimelineOptions, type RenderWhatOptions, type ResolvedContractDocument, type ResolvedRedactionProfile, type ResolvedSuiteCase, type RunGateOptions, RunStatus, type RunSuiteOptions, RunSummary, type RunTimeline, type RunWhatSummary, SUITE_TEMPLATE_IDS, type SessionCohort, type SessionCohortKind, SessionIndex, SessionRunRecord, type SessionScopeOptions, type SessionScopeResult, SessionStatus, SessionSummary, SessionWarning, SessionWorkflowMetadata, StepStatus, StepType, type SuiteArtifactsConfig, type SuiteCaseConfig, type SuiteCaseResult, type SuiteCaseStatus, type SuiteChecksConfig, type SuiteConfig, type SuiteDiagnostic, type SuiteDiagnosticCode, type SuiteEvalConfig, type SuiteRunResult, type SuiteRunSummary, type SuiteTemplateId, TERMINAL_INDENT, type TimelineEntry, type TimelineFocus, type TimelineOptions, TraceCorrelationMetadata, TraceDirectory, type TraceDirectoryOptions, TraceEvent, type TraceFilterOptions, type TraceJsonlFormat, TraceMetadata, TraceMetadataStatus, type TraceSearchOptions, type TraceSearchResult, type TraceSessionCheckResult, type TraceStats, type TraceStatsOptions, type TraceStatsRankedRun, type TraceStatsRankedStep, type ValidateSuiteConfigResult, type ZipEntry, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, assertBundlePathContained, assertEvidenceRelativePath, bindCheckResultsToContract, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractPackage, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceFileEntries, buildEvidenceHtmlShell, buildEvidenceHtmlShellFromManifest, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectResolvedContractRuleIds, collectTraceSchemaVersions, compareCohortAggregates, createAdapterPreviewCapture, createOmittedPayloadCommitment, createRunId, createStepId, defaultBundleOutputPath, defaultSuiteConfigTemplate, deriveSessionStatus, encodeEmbeddedEvidenceJson, enrichSessionRunRecord, enrichSessionSummary, ensureTraceDir, extractMetadata, extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, findFirstCausalFailure, formatDuration, formatError, formatStepLabel, formatTerminalName, formatTimestamp, gateHasThresholds, getDefaultTraceDir, getIndent, getRunIdFromTraceFileName, getSuiteTemplate, getTraceFilePath, groupSessionCohorts, inferEvidenceFileRole, initializeTraceFile, isAgentInspectTrace, isSha256Hex, listSuiteTemplates, listTraceFiles, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, normalizeBundleOutputPath, normalizeSuiteConfig, omittedPayloadByteLength, parseCohortMetricList, parseDuration, parseDurationFilter, parseEvidenceManifestJson, parseGateList, parseGateNumber, parseGroupBySpec, parseTraceJsonl, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, readCheckResultsContractBinding, readTraceEvents, readTraceFile, renderActivitySummaryHuman, renderCohortReport, renderCohortSummaryMarkdown, renderErrorLine, renderGateGithubAnnotations, renderGateGithubStepSummary, renderGateJUnit, renderGateReport, renderGateSummaryMarkdown, renderRunSummary, renderRunWhat, renderStepLine, renderSuiteReport, renderSuiteReportMarkdown, renderTimeline, renderTraceStats, resolveAdapterMaxPreviewChars, resolveBundleRunIds, resolveRedactionProfile, resolveSerializableTraceContract, resolveSuiteCaseTrace, resolveSuiteConfigPath, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, sanitizeBundleRunId, searchTraces, serializeAdapterPreview, serializeCanonicalJson, serializeCheckResultsJson, serializeEvent, serializeEvidenceManifest, sessionKeyForRun, sha256Equals, sha256Hex, toMetadataSafeStatus, traceMetasToSessionRunRecords, truncateName, unknownTraceFormatMessage, validateEvent, validateEvidenceManifest, validateSuiteConfig, verifyEvidenceContractBinding, verifyEvidenceDirectory, warn, writeTraceEvent };
|
|
1651
|
+
export { ADAPTER_CAPTURE_DIAGNOSTIC_CODES, ActivitySummary, type AdapterCaptureDiagnostic, type AdapterCaptureDiagnosticCode, type AdapterCaptureDiagnostics, type AdapterCaptureMode, type AdapterDiagnosticListener, type AdapterPreviewCapture, type AdapterPreviewCaptureOptions, type AnalyzeCohortOptions, BuildActivitySummaryOptions, type BuildEvidenceContractPackageInput, BuildSessionIndexOptions, type BundleCheckResults, type BundleCheckRunResult, type BundleMetadata, type BundlePlaceholderArtifact, type BundleRedactionProfile, type BundleRedactionReport, type BundleRedactionReportRun, type BundleResolveOptions, type BundleResolveResult, type BundleSafeStatus, type BundleSafeStatusMetadata, COHORT_METRIC_IDS, type CausalContractFindingInput, type CausalFailureKind, type CohortAggregateMetrics, type CohortAnalysisResult, type CohortMetricComparison, type CohortMetricId, type CohortRunMetrics, DEFAULT_ADAPTER_MAX_PREVIEW_CHARS, DEFAULT_SUITE_ARTIFACTS_DIR, DEFAULT_SUITE_CONFIG_NAMES, DEFAULT_TRACE_DIR_NAME, type DurationStats, EVIDENCE_ASSESSMENT_NOTE, EVIDENCE_CONTRACT_ASSURANCE_NOTE, EVIDENCE_CONTRACT_CANONICALIZATION_VERSION, EVIDENCE_CONTRACT_PARTIAL_NOTE, EVIDENCE_CONTRACT_UNAVAILABLE_NOTE, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, EVIDENCE_RESOLVED_CONTRACT_FILENAME, EVIDENCE_VIEW_CSS, EVIDENCE_VIEW_IDS, EnrichSessionSummaryOptions, ErrorInfo, type EvidenceCheckContractBinding, type EvidenceCheckFindingSummary, type EvidenceCiPackageFiles, type EvidenceCiPackageInput, type EvidenceContractBinding, type EvidenceContractBindingSource, type EvidenceContractBindingStatus, type EvidenceContractPackage, type EvidenceContractVerifyIssue, type EvidenceContractsViewInput, type EvidenceFileEntry, type EvidenceFileRole, type EvidenceFormatVersion, type EvidenceHtmlShellInput, type EvidenceInputBinding, type EvidenceManifest, type EvidencePackagedFile, type EvidenceProvenanceViewInput, type EvidenceRedactionProfile, type EvidenceSafeStatus, type EvidenceSafetyViewInput, type EvidenceSourceHash, type EvidenceVerificationPolicy, type EvidenceVerifyIssue, type EvidenceVerifyOptions, type EvidenceVerifyResult, type EvidenceVerifyStatus, type EvidenceViewId, type ExplainFact, type ExplainInference, type ExplainMode, type ExplainOptions, type ExplainResult, FALLBACK_TRACE_DIR, type FindFirstCausalFailureOptions, type FirstCausalFailure, type GateCheckId, type GateCheckResult, type GateExitCode, type GateResult, type GroupSessionCohortsOptions, InspectRunTree, type LoadSuiteConfigOptions, MAX_NAME_LENGTH, MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, OMITTED_PAYLOAD_MAX_DIGEST_INPUT_BYTES, ObservedOutcome, type OmittedPayloadAlgorithm, type OmittedPayloadCommitment, type ParseTraceJsonlOptions, type ParseTraceJsonlResult, type ParsedDurationFilter, RUNS_DIR_NAME, RedactionProfile, type RenderCohortReportOptions, type RenderGateReportOptions, type RenderSuiteReportOptions, type RenderTimelineOptions, type RenderWhatOptions, type ResolvedContractDocument, type ResolvedRedactionProfile, type ResolvedSuiteCase, type RunGateOptions, RunStatus, type RunSuiteOptions, RunSummary, type RunTimeline, type RunWhatSummary, SUITE_TEMPLATE_IDS, type SessionCohort, type SessionCohortKind, SessionIndex, SessionRunRecord, type SessionScopeOptions, type SessionScopeResult, SessionStatus, SessionSummary, SessionWarning, SessionWorkflowMetadata, StepStatus, StepType, type SuiteArtifactsConfig, type SuiteCaseConfig, type SuiteCaseResult, type SuiteCaseStatus, type SuiteChecksConfig, type SuiteConfig, type SuiteDiagnostic, type SuiteDiagnosticCode, type SuiteEvalConfig, type SuiteRunResult, type SuiteRunSummary, type SuiteTemplateId, TERMINAL_INDENT, type TimelineEntry, type TimelineFocus, type TimelineOptions, TraceCorrelationMetadata, TraceDirectory, type TraceDirectoryOptions, TraceEvent, type TraceFilterOptions, type TraceJsonlFormat, TraceMetadata, TraceMetadataStatus, type TraceSearchOptions, type TraceSearchResult, type TraceSessionCheckResult, type TraceStats, type TraceStatsOptions, type TraceStatsRankedRun, type TraceStatsRankedStep, type ValidateSuiteConfigResult, type ZipEntry, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, assertBundlePathContained, assertEvidenceRelativePath, bindCheckResultsToContract, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractPackage, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceFileEntries, buildEvidenceHtmlShell, buildEvidenceHtmlShellFromManifest, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectResolvedContractRuleIds, collectTraceSchemaVersions, compareCohortAggregates, createAdapterPreviewCapture, createOmittedPayloadCommitment, createRunId, createStepId, defaultBundleOutputPath, defaultSuiteConfigTemplate, deriveSessionStatus, encodeEmbeddedEvidenceJson, enrichSessionRunRecord, enrichSessionSummary, ensureTraceDir, extractMetadata, extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, findFirstCausalFailure, formatDuration, formatError, formatStepLabel, formatTerminalName, formatTimestamp, gateHasThresholds, getDefaultTraceDir, getIndent, getRunIdFromTraceFileName, getSuiteTemplate, getTraceFilePath, groupSessionCohorts, inferEvidenceFileRole, initializeTraceFile, isAgentInspectTrace, isSha256Hex, listSuiteTemplates, listTraceFiles, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, normalizeBundleOutputPath, normalizeSuiteConfig, omittedPayloadByteLength, parseCohortMetricList, parseDuration, parseDurationFilter, parseEvidenceManifestJson, parseGateList, parseGateNumber, parseGroupBySpec, parseTraceJsonl, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, readCheckResultsContractBinding, readTraceEvents, readTraceFile, renderActivitySummaryHuman, renderCohortReport, renderCohortSummaryMarkdown, renderErrorLine, renderGateGithubAnnotations, renderGateGithubStepSummary, renderGateJUnit, renderGateReport, renderGateSummaryMarkdown, renderRunSummary, renderRunWhat, renderStepLine, renderSuiteReport, renderSuiteReportMarkdown, renderTimeline, renderTraceStats, resolveAdapterMaxPreviewChars, resolveBundleRunIds, resolveRedactionProfile, resolveSerializableTraceContract, resolveSuiteCaseTrace, resolveSuiteConfigPath, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, sanitizeBundleRunId, searchTraces, serializeAdapterPreview, serializeCanonicalJson, serializeCheckResultsJson, serializeEvent, serializeEvidenceManifest, sessionKeyForRun, sha256Equals, sha256Hex, toMetadataSafeStatus, traceMetasToSessionRunRecords, truncateName, unknownTraceFormatMessage, validateEvent, validateEvidenceManifest, validateSuiteConfig, verifyEvidenceContractBinding, verifyEvidenceDirectory, warn, writeTraceEvent };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { extractSessionWorkflowMetadata, sessionKeyForRun, defineTraceContract, runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createRunDepthRule, createToolFailureRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-
|
|
2
|
-
export { extractSessionWorkflowMetadata, sessionKeyForRun } from './chunk-
|
|
3
|
-
export { MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, TERMINAL_INDENT, TraceDirectory, createInspector, createInspectorRuntime, formatTerminalName, getCurrentContext, getCurrentCorrelationMetadata, getCurrentDepth, getCurrentRunId, getCurrentRunName, getCurrentStepId, getIndent, getParentStepId, getTraceDirFromContext, getTraceSafetyFromContext, hasActiveContext, isAgentInspectEnabled, isSilentContext, maybeInspectRun, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, renderErrorLine, renderRunSummary, renderStepLine, resolveTraceDir, runWithContext, runWithStepContext } from './chunk-
|
|
4
|
-
import { buildRunTimeline, filterTraces, extractMetadata, buildRunSummary } from './chunk-
|
|
5
|
-
export { buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildTraceStats, extractMetadata, filterTraces, formatStepLabel, renderRunWhat, renderTimeline, renderTraceStats } from './chunk-
|
|
1
|
+
import { extractSessionWorkflowMetadata, sessionKeyForRun, defineTraceContract, runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createRunDepthRule, createToolFailureRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-CCYKNAOD.mjs';
|
|
2
|
+
export { extractSessionWorkflowMetadata, sessionKeyForRun } from './chunk-CCYKNAOD.mjs';
|
|
3
|
+
export { MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, TERMINAL_INDENT, TraceDirectory, createInspector, createInspectorRuntime, formatTerminalName, getCurrentContext, getCurrentCorrelationMetadata, getCurrentDepth, getCurrentRunId, getCurrentRunName, getCurrentStepId, getIndent, getParentStepId, getTraceDirFromContext, getTraceSafetyFromContext, hasActiveContext, isAgentInspectEnabled, isSilentContext, maybeInspectRun, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, renderErrorLine, renderRunSummary, renderStepLine, resolveTraceDir, runWithContext, runWithStepContext } from './chunk-SSG3ICI4.mjs';
|
|
4
|
+
import { buildRunTimeline, filterTraces, extractMetadata, buildRunSummary } from './chunk-M3RJD3BP.mjs';
|
|
5
|
+
export { buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildTraceStats, extractMetadata, filterTraces, formatStepLabel, renderRunWhat, renderTimeline, renderTraceStats } from './chunk-M3RJD3BP.mjs';
|
|
6
6
|
import { extractOutcomesFromTraceEvents, parseObservationFilter, summarizeObservedOutcomes, extractOutcomesFromPersistedEvents, renderObservedOutcomesHtml } from './chunk-N5PZAI6Q.mjs';
|
|
7
7
|
export { extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents } from './chunk-N5PZAI6Q.mjs';
|
|
8
8
|
import { diffTraceEvents, renderRunDiff } from './chunk-F7STQ5JF.mjs';
|
|
@@ -11,10 +11,11 @@ import { escapeHtml, flattenTree } from './chunk-CAWPF22J.mjs';
|
|
|
11
11
|
import { resolveRedactionProfile, readTraceEventsFromFile, stringContainsHighConfidenceCredential } from './chunk-DK3OW6UI.mjs';
|
|
12
12
|
export { DEFAULT_MAX_EVENT_BYTES, DEFAULT_MAX_METADATA_VALUE_LENGTH, DEFAULT_MAX_PREVIEW_LENGTH, getRunIdFromTraceFileName, initializeTraceFile, listTraceFiles, prepareMetadataForDisk, prepareTraceEventForDisk, readTraceEvents, readTraceFile, resolveRedactionProfile, resolveTraceSafetyOptions, serializeEvent, validateEvent, writeTraceEvent } from './chunk-DK3OW6UI.mjs';
|
|
13
13
|
import { Redactor } from './chunk-VLVCLAPB.mjs';
|
|
14
|
-
import { openTrace } from './chunk-
|
|
15
|
-
import './chunk-
|
|
16
|
-
import './chunk-
|
|
14
|
+
import { openTrace } from './chunk-QOC3PJO3.mjs';
|
|
15
|
+
import './chunk-OAUNS5CS.mjs';
|
|
16
|
+
import { projectLogicalEvents } from './chunk-AHUZNHFI.mjs';
|
|
17
17
|
import './chunk-S4YWKV4G.mjs';
|
|
18
|
+
import './chunk-S6LQBLX5.mjs';
|
|
18
19
|
import './chunk-AK5EN4U2.mjs';
|
|
19
20
|
import { parseDuration, isTraceEvent, getTraceFilePath } from './chunk-7DHSWDKB.mjs';
|
|
20
21
|
export { DEFAULT_TRACE_DIR_NAME, FALLBACK_TRACE_DIR, MAX_NAME_LENGTH, RUNS_DIR_NAME, createRunId, createStepId, ensureTraceDir, formatDuration, formatError, formatTimestamp, getDefaultTraceDir, getTraceFilePath, isStepStatus, isStepType, isTraceEvent, parseDuration, parseTraceJsonl, truncateName, unknownTraceFormatMessage, warn } from './chunk-7DHSWDKB.mjs';
|
|
@@ -409,7 +410,7 @@ function attributeFacts(nodes, redactor) {
|
|
|
409
410
|
}
|
|
410
411
|
return facts;
|
|
411
412
|
}
|
|
412
|
-
function buildFacts(run, redactor) {
|
|
413
|
+
function buildFacts(run, redactor, events) {
|
|
413
414
|
const nodes = flatten(run.children);
|
|
414
415
|
const facts = [
|
|
415
416
|
fact("run.id", "Run id", run.runId, redactor),
|
|
@@ -434,6 +435,39 @@ function buildFacts(run, redactor) {
|
|
|
434
435
|
}, redactor)
|
|
435
436
|
);
|
|
436
437
|
}
|
|
438
|
+
if (events !== void 0) {
|
|
439
|
+
const scoped = events.filter((event) => event.runId === run.runId);
|
|
440
|
+
const { logicalEvents: logical } = projectLogicalEvents(scoped);
|
|
441
|
+
const logicalSteps = logical.filter((event) => event.kind !== "RUN");
|
|
442
|
+
facts.push(
|
|
443
|
+
fact("run.rawEventCount", "Raw persisted event count", scoped.length, redactor),
|
|
444
|
+
fact(
|
|
445
|
+
"run.logicalStepCount",
|
|
446
|
+
"Logical lifecycle step count",
|
|
447
|
+
logicalSteps.length,
|
|
448
|
+
redactor
|
|
449
|
+
),
|
|
450
|
+
fact(
|
|
451
|
+
"run.logicalEventCount",
|
|
452
|
+
"Logical lifecycle event count (including RUN)",
|
|
453
|
+
logical.length,
|
|
454
|
+
redactor
|
|
455
|
+
)
|
|
456
|
+
);
|
|
457
|
+
const slowestLogical = [...logicalSteps].filter((event) => typeof event.durationMs === "number").sort((a, b) => {
|
|
458
|
+
const delta = (b.durationMs ?? 0) - (a.durationMs ?? 0);
|
|
459
|
+
return delta !== 0 ? delta : a.eventId.localeCompare(b.eventId);
|
|
460
|
+
})[0];
|
|
461
|
+
if (slowestLogical !== void 0) {
|
|
462
|
+
facts.push(
|
|
463
|
+
fact("run.slowestLogicalStep", "Slowest logical lifecycle step", {
|
|
464
|
+
name: slowestLogical.name,
|
|
465
|
+
kind: slowestLogical.kind,
|
|
466
|
+
durationMs: slowestLogical.durationMs
|
|
467
|
+
}, redactor)
|
|
468
|
+
);
|
|
469
|
+
}
|
|
470
|
+
}
|
|
437
471
|
facts.push(...attributeFacts(nodes, redactor));
|
|
438
472
|
return facts;
|
|
439
473
|
}
|
|
@@ -485,7 +519,7 @@ function buildLocalExplanation(run, options = {}) {
|
|
|
485
519
|
const resolved = resolveRedactionProfile(redactionProfile);
|
|
486
520
|
const redactor = new Redactor({ extraKeys: resolved.extraKeys });
|
|
487
521
|
const mode = options.mode ?? "local";
|
|
488
|
-
const facts = buildFacts(run, redactor);
|
|
522
|
+
const facts = buildFacts(run, redactor, options.events);
|
|
489
523
|
return {
|
|
490
524
|
mode,
|
|
491
525
|
runId: String(redactValue(redactor, "runId", run.runId)),
|
|
@@ -1171,7 +1205,17 @@ function groupSessionCohorts(runs, options = {}) {
|
|
|
1171
1205
|
|
|
1172
1206
|
// packages/core/src/sessions/checks.ts
|
|
1173
1207
|
function emptySummary() {
|
|
1174
|
-
return {
|
|
1208
|
+
return {
|
|
1209
|
+
passed: 0,
|
|
1210
|
+
failed: 0,
|
|
1211
|
+
warnings: 0,
|
|
1212
|
+
errors: 0,
|
|
1213
|
+
rulesEvaluated: 0,
|
|
1214
|
+
rulesPassed: 0,
|
|
1215
|
+
rulesWarning: 0,
|
|
1216
|
+
rulesFailed: 0,
|
|
1217
|
+
rulesError: 0
|
|
1218
|
+
};
|
|
1175
1219
|
}
|
|
1176
1220
|
function mergeSummary(target, source) {
|
|
1177
1221
|
return {
|
|
@@ -1179,7 +1223,11 @@ function mergeSummary(target, source) {
|
|
|
1179
1223
|
failed: target.failed + source.failed,
|
|
1180
1224
|
warnings: target.warnings + source.warnings,
|
|
1181
1225
|
errors: target.errors + source.errors,
|
|
1182
|
-
rulesEvaluated: target.rulesEvaluated + source.rulesEvaluated
|
|
1226
|
+
rulesEvaluated: target.rulesEvaluated + source.rulesEvaluated,
|
|
1227
|
+
rulesPassed: target.rulesPassed + source.rulesPassed,
|
|
1228
|
+
rulesWarning: target.rulesWarning + source.rulesWarning,
|
|
1229
|
+
rulesFailed: target.rulesFailed + source.rulesFailed,
|
|
1230
|
+
rulesError: target.rulesError + source.rulesError
|
|
1183
1231
|
};
|
|
1184
1232
|
}
|
|
1185
1233
|
function sessionDiagnostic(code, message) {
|
|
@@ -1903,6 +1951,7 @@ function buildEvidenceManifest(parts) {
|
|
|
1903
1951
|
assessment,
|
|
1904
1952
|
...parts.semantics !== void 0 ? { semantics: { ...parts.semantics } } : {},
|
|
1905
1953
|
...parts.contract !== void 0 ? { contract: { ...parts.contract } } : {},
|
|
1954
|
+
...parts.inputs !== void 0 ? { inputs: { ...parts.inputs } } : {},
|
|
1906
1955
|
files: buildEvidenceFileEntries(parts.files)
|
|
1907
1956
|
};
|
|
1908
1957
|
}
|
|
@@ -3349,6 +3398,28 @@ function asPositiveNumber(value, label) {
|
|
|
3349
3398
|
}
|
|
3350
3399
|
return value;
|
|
3351
3400
|
}
|
|
3401
|
+
function asExpectConfig(value, label) {
|
|
3402
|
+
if (value === void 0) return void 0;
|
|
3403
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
3404
|
+
throw new Error(`${label} must be an object.`);
|
|
3405
|
+
}
|
|
3406
|
+
const raw = value;
|
|
3407
|
+
const checkStatus = raw.checkStatus;
|
|
3408
|
+
if (checkStatus !== "pass" && checkStatus !== "fail" && checkStatus !== "error") {
|
|
3409
|
+
throw new Error(`${label}.checkStatus must be "pass", "fail", or "error".`);
|
|
3410
|
+
}
|
|
3411
|
+
const findingRuleIds = asStringArray(raw.findingRuleIds, `${label}.findingRuleIds`);
|
|
3412
|
+
const minAssertions = asPositiveNumber(raw.minAssertions, `${label}.minAssertions`);
|
|
3413
|
+
const rejectUnexpectedFindings = raw.rejectUnexpectedFindings === void 0 ? void 0 : typeof raw.rejectUnexpectedFindings === "boolean" ? raw.rejectUnexpectedFindings : (() => {
|
|
3414
|
+
throw new Error(`${label}.rejectUnexpectedFindings must be a boolean.`);
|
|
3415
|
+
})();
|
|
3416
|
+
return {
|
|
3417
|
+
checkStatus,
|
|
3418
|
+
...findingRuleIds !== void 0 ? { findingRuleIds } : {},
|
|
3419
|
+
...rejectUnexpectedFindings !== void 0 ? { rejectUnexpectedFindings } : {},
|
|
3420
|
+
...minAssertions !== void 0 ? { minAssertions } : {}
|
|
3421
|
+
};
|
|
3422
|
+
}
|
|
3352
3423
|
function validateCaseConfig(value, index) {
|
|
3353
3424
|
const diagnostics = [];
|
|
3354
3425
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
@@ -3391,7 +3462,8 @@ function validateCaseConfig(value, index) {
|
|
|
3391
3462
|
raw.expectedObservations,
|
|
3392
3463
|
`cases[${index}].expectedObservations`
|
|
3393
3464
|
)
|
|
3394
|
-
} : {}
|
|
3465
|
+
} : {},
|
|
3466
|
+
...asExpectConfig(raw.expect, `cases[${index}].expect`) !== void 0 ? { expect: asExpectConfig(raw.expect, `cases[${index}].expect`) } : {}
|
|
3395
3467
|
},
|
|
3396
3468
|
diagnostics
|
|
3397
3469
|
};
|
|
@@ -3770,6 +3842,78 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
3770
3842
|
}
|
|
3771
3843
|
return { ok: diagnostics.length === 0, diagnostics };
|
|
3772
3844
|
}
|
|
3845
|
+
function evaluateCaseExpect(suiteCase, checkResult, observationCount) {
|
|
3846
|
+
const expectation = suiteCase.expect;
|
|
3847
|
+
if (expectation === void 0) {
|
|
3848
|
+
return { ok: true, diagnostics: [] };
|
|
3849
|
+
}
|
|
3850
|
+
const diagnostics = [];
|
|
3851
|
+
if (checkResult.status !== expectation.checkStatus) {
|
|
3852
|
+
diagnostics.push(
|
|
3853
|
+
diagnostic3(
|
|
3854
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
3855
|
+
`Expected check status "${expectation.checkStatus}", got "${checkResult.status}".`,
|
|
3856
|
+
"error",
|
|
3857
|
+
suiteCase.id
|
|
3858
|
+
)
|
|
3859
|
+
);
|
|
3860
|
+
}
|
|
3861
|
+
const failedFindings = checkResult.findings.filter(
|
|
3862
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
3863
|
+
);
|
|
3864
|
+
const requiredIds = expectation.findingRuleIds ?? [];
|
|
3865
|
+
for (const ruleId of requiredIds) {
|
|
3866
|
+
if (!failedFindings.some((finding) => finding.ruleId === ruleId)) {
|
|
3867
|
+
diagnostics.push(
|
|
3868
|
+
diagnostic3(
|
|
3869
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
3870
|
+
`Expected failing finding for rule "${ruleId}" was not present.`,
|
|
3871
|
+
"error",
|
|
3872
|
+
suiteCase.id
|
|
3873
|
+
)
|
|
3874
|
+
);
|
|
3875
|
+
}
|
|
3876
|
+
}
|
|
3877
|
+
const rejectUnexpected = expectation.rejectUnexpectedFindings !== false;
|
|
3878
|
+
if (rejectUnexpected && requiredIds.length > 0) {
|
|
3879
|
+
for (const finding of failedFindings) {
|
|
3880
|
+
if (!requiredIds.includes(finding.ruleId)) {
|
|
3881
|
+
diagnostics.push(
|
|
3882
|
+
diagnostic3(
|
|
3883
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
3884
|
+
`Unexpected failing finding for rule "${finding.ruleId}".`,
|
|
3885
|
+
"error",
|
|
3886
|
+
suiteCase.id
|
|
3887
|
+
)
|
|
3888
|
+
);
|
|
3889
|
+
}
|
|
3890
|
+
}
|
|
3891
|
+
}
|
|
3892
|
+
if (expectation.minAssertions !== void 0) {
|
|
3893
|
+
const assertionCount = checkResult.ruleExecutions.length + observationCount;
|
|
3894
|
+
if (assertionCount < expectation.minAssertions) {
|
|
3895
|
+
diagnostics.push(
|
|
3896
|
+
diagnostic3(
|
|
3897
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
3898
|
+
`Expected at least ${expectation.minAssertions} assertion(s), got ${assertionCount}.`,
|
|
3899
|
+
"error",
|
|
3900
|
+
suiteCase.id
|
|
3901
|
+
)
|
|
3902
|
+
);
|
|
3903
|
+
}
|
|
3904
|
+
}
|
|
3905
|
+
if (expectation.checkStatus === "fail" && checkResult.diagnostics.some((item) => item.severity === "error") && checkResult.status === "error") {
|
|
3906
|
+
diagnostics.push(
|
|
3907
|
+
diagnostic3(
|
|
3908
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
3909
|
+
'Check ended in evaluator/config error; semantic failure expectation requires status "fail".',
|
|
3910
|
+
"error",
|
|
3911
|
+
suiteCase.id
|
|
3912
|
+
)
|
|
3913
|
+
);
|
|
3914
|
+
}
|
|
3915
|
+
return { ok: diagnostics.length === 0, diagnostics };
|
|
3916
|
+
}
|
|
3773
3917
|
async function runSuiteCase(suiteCase, config, options) {
|
|
3774
3918
|
const resolved = await resolveSuiteCaseTrace(suiteCase, options);
|
|
3775
3919
|
if (resolved.missing || resolved.tracePath === void 0) {
|
|
@@ -3822,9 +3966,21 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
3822
3966
|
status: "pass",
|
|
3823
3967
|
format: read.format,
|
|
3824
3968
|
findings: [],
|
|
3825
|
-
diagnostics: []
|
|
3969
|
+
diagnostics: [],
|
|
3970
|
+
ruleExecutions: []
|
|
3971
|
+
};
|
|
3826
3972
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
3827
|
-
const
|
|
3973
|
+
const checkOk = checkResult.ok;
|
|
3974
|
+
const observationsOk = observationResult.ok;
|
|
3975
|
+
const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
|
|
3976
|
+
(finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
|
|
3977
|
+
) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
|
|
3978
|
+
const expectResult = evaluateCaseExpect(
|
|
3979
|
+
suiteCase,
|
|
3980
|
+
checkResult,
|
|
3981
|
+
compiled.observationCount
|
|
3982
|
+
);
|
|
3983
|
+
const checkFailureDiagnostics = [
|
|
3828
3984
|
...checkResult.diagnostics.map(
|
|
3829
3985
|
(item) => diagnostic3(
|
|
3830
3986
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
@@ -3840,15 +3996,24 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
3840
3996
|
finding.severity,
|
|
3841
3997
|
suiteCase.id
|
|
3842
3998
|
)
|
|
3843
|
-
)
|
|
3844
|
-
...observationResult.diagnostics
|
|
3999
|
+
)
|
|
3845
4000
|
];
|
|
3846
|
-
const
|
|
3847
|
-
const
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
4001
|
+
const expectMatched = suiteCase.expect !== void 0 && expectResult.ok && observationsOk;
|
|
4002
|
+
const diagnostics = expectMatched ? [
|
|
4003
|
+
...observationResult.diagnostics,
|
|
4004
|
+
...expectResult.diagnostics,
|
|
4005
|
+
diagnostic3(
|
|
4006
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
4007
|
+
`Expected semantic failure matched (check.status=${checkResult.status}).`,
|
|
4008
|
+
"info",
|
|
4009
|
+
suiteCase.id
|
|
4010
|
+
)
|
|
4011
|
+
] : [
|
|
4012
|
+
...checkFailureDiagnostics,
|
|
4013
|
+
...observationResult.diagnostics,
|
|
4014
|
+
...expectResult.diagnostics
|
|
4015
|
+
];
|
|
4016
|
+
const ok = suiteCase.expect !== void 0 ? expectResult.ok && observationsOk : checkOk && observationsOk;
|
|
3852
4017
|
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
3853
4018
|
return {
|
|
3854
4019
|
id: suiteCase.id,
|