agent-inspect 6.3.0 → 6.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +2 -2
- package/package.json +2 -2
- package/packages/cli/dist/{chunk-YS2I722C.mjs → chunk-M6PC5KUT.mjs} +78 -31
- package/packages/cli/dist/chunk-M6PC5KUT.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +153 -42
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +88 -36
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-FVNE44UA.mjs → src-27LC2BB3.mjs} +8 -5
- package/packages/cli/dist/src-27LC2BB3.mjs.map +1 -0
- package/packages/core/dist/advanced.cjs +91 -35
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +36 -12
- package/packages/core/dist/advanced.d.ts +36 -12
- package/packages/core/dist/advanced.mjs +77 -24
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +113 -0
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +40 -599
- package/packages/core/dist/checks.d.ts +40 -599
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-QLYOCCEF.mjs → chunk-EWHN6NA4.mjs} +114 -3
- package/packages/core/dist/chunk-EWHN6NA4.mjs.map +1 -0
- package/packages/core/dist/{context-DyKbkmd4.d.cts → context-BHAgRjOe.d.cts} +4 -4
- package/packages/core/dist/{context-BJ5UQLDi.d.ts → context-Blyjb5my.d.ts} +4 -4
- package/packages/core/dist/diff.d.cts +3 -3
- package/packages/core/dist/diff.d.ts +3 -3
- package/packages/core/dist/exporters.cjs +75 -0
- package/packages/core/dist/exporters.cjs.map +1 -1
- package/packages/core/dist/exporters.d.cts +29 -4
- package/packages/core/dist/exporters.d.ts +29 -4
- package/packages/core/dist/exporters.mjs +71 -1
- package/packages/core/dist/exporters.mjs.map +1 -1
- package/packages/core/dist/index-BblJEdYZ.d.ts +618 -0
- package/packages/core/dist/index-DyKk7iR9.d.cts +618 -0
- package/packages/core/dist/index.d.cts +7 -7
- package/packages/core/dist/index.d.ts +7 -7
- package/packages/core/dist/{inspect-event-IqwOHbGo.d.cts → inspect-event-DVg84S4v.d.cts} +1 -1
- package/packages/core/dist/{inspect-event-IqwOHbGo.d.ts → inspect-event-DVg84S4v.d.ts} +1 -1
- package/packages/core/dist/{log-config-CW8w5YQ9.d.cts → log-config-BG1WmWf2.d.cts} +1 -1
- package/packages/core/dist/{log-config-SRA4xTp8.d.ts → log-config-Ck0aT_Ou.d.ts} +1 -1
- package/packages/core/dist/logs.d.cts +3 -3
- package/packages/core/dist/logs.d.ts +3 -3
- package/packages/core/dist/{persisted-inspect-event-DruwqARZ.d.ts → persisted-inspect-event-ChE7pGb7.d.ts} +1 -1
- package/packages/core/dist/{persisted-inspect-event-Gd-W8vB7.d.cts → persisted-inspect-event-d-mSP8kd.d.cts} +1 -1
- package/packages/core/dist/persisted.d.cts +5 -5
- package/packages/core/dist/persisted.d.ts +5 -5
- package/packages/core/dist/readers.d.cts +2 -2
- package/packages/core/dist/readers.d.ts +2 -2
- package/packages/core/dist/{types-J3Xtuf9x.d.cts → types-B562HgN_.d.cts} +1 -1
- package/packages/core/dist/{types-J3Xtuf9x.d.ts → types-B562HgN_.d.ts} +1 -1
- package/packages/core/dist/{types-B1X87dUW.d.cts → types-BLX3tiOW.d.cts} +1 -1
- package/packages/core/dist/{types-BVkjSd0q.d.ts → types-MuwXSJ4b.d.ts} +1 -1
- package/packages/core/dist/writers.d.cts +2 -2
- package/packages/core/dist/writers.d.ts +2 -2
- package/packages/cli/dist/chunk-YS2I722C.mjs.map +0 -1
- package/packages/cli/dist/src-FVNE44UA.mjs.map +0 -1
- package/packages/core/dist/chunk-QLYOCCEF.mjs.map +0 -1
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
export { C as CreateInspectorOptions, D as DEFAULT_MAX_EVENT_BYTES, f as DEFAULT_MAX_METADATA_VALUE_LENGTH, h as DEFAULT_MAX_PREVIEW_LENGTH, I as Inspector, a as InspectorCaptureOptions, b as InspectorObserveOptions, c as InspectorRunOptions, i as InspectorRuntime, j as InspectorRuntimeContext, k as InspectorRuntimeDiagnostics, l as InspectorRuntimeOptions, d as InspectorStepOptions, T as TraceSafetyOptions, e as createInspector, n as createInspectorRuntime, o as getCurrentContext, g as getCurrentCorrelationMetadata, p as getCurrentDepth, q as getCurrentRunId, r as getCurrentRunName, s as getCurrentStepId, t as getParentStepId, u as getTraceDirFromContext, v as getTraceSafetyFromContext, w as hasActiveContext, x as isAgentInspectEnabled, y as isSilentContext, m as maybeInspectRun, z as prepareMetadataForDisk, A as prepareTraceEventForDisk, B as resolveTraceSafetyOptions, E as runWithContext, F as runWithStepContext } from './context-
|
|
2
|
-
import { R as RedactionProfile, h as TraceEvent, E as ErrorInfo, b as RunStatus, e as StepStatus, j as RunSummary, k as TraceMetadata, l as TraceMetadataStatus, f as StepType, g as TraceCorrelationMetadata } from './types-
|
|
3
|
-
export { A as ActiveStepContext, i as ExecutionContext, I as InspectRunOptions, O as ObserveOptions, a as Run, m as RunCompletedEvent, n as RunStartedEvent, c as Step, o as StepCompletedEvent, d as StepMetadata, S as StepOptions, p as StepStartedEvent, T as TokenMetadata, q as TraceEventBase, r as TraceSchemaVersion, s as isStepStatus, t as isStepType, u as isTraceEvent } from './types-
|
|
4
|
-
import {
|
|
5
|
-
export { A as AttributionConfidence, E as EventSource,
|
|
6
|
-
import { P as PersistedInspectEvent } from './persisted-inspect-event-
|
|
1
|
+
export { C as CreateInspectorOptions, D as DEFAULT_MAX_EVENT_BYTES, f as DEFAULT_MAX_METADATA_VALUE_LENGTH, h as DEFAULT_MAX_PREVIEW_LENGTH, I as Inspector, a as InspectorCaptureOptions, b as InspectorObserveOptions, c as InspectorRunOptions, i as InspectorRuntime, j as InspectorRuntimeContext, k as InspectorRuntimeDiagnostics, l as InspectorRuntimeOptions, d as InspectorStepOptions, T as TraceSafetyOptions, e as createInspector, n as createInspectorRuntime, o as getCurrentContext, g as getCurrentCorrelationMetadata, p as getCurrentDepth, q as getCurrentRunId, r as getCurrentRunName, s as getCurrentStepId, t as getParentStepId, u as getTraceDirFromContext, v as getTraceSafetyFromContext, w as hasActiveContext, x as isAgentInspectEnabled, y as isSilentContext, m as maybeInspectRun, z as prepareMetadataForDisk, A as prepareTraceEventForDisk, B as resolveTraceSafetyOptions, E as runWithContext, F as runWithStepContext } from './context-BHAgRjOe.cjs';
|
|
2
|
+
import { R as RedactionProfile, h as TraceEvent, E as ErrorInfo, b as RunStatus, e as StepStatus, j as RunSummary, k as TraceMetadata, l as TraceMetadataStatus, f as StepType, g as TraceCorrelationMetadata } from './types-BLX3tiOW.cjs';
|
|
3
|
+
export { A as ActiveStepContext, i as ExecutionContext, I as InspectRunOptions, O as ObserveOptions, a as Run, m as RunCompletedEvent, n as RunStartedEvent, c as Step, o as StepCompletedEvent, d as StepMetadata, S as StepOptions, p as StepStartedEvent, T as TokenMetadata, q as TraceEventBase, r as TraceSchemaVersion, s as isStepStatus, t as isStepType, u as isTraceEvent } from './types-BLX3tiOW.cjs';
|
|
4
|
+
import { a as InspectRunTree } from './inspect-event-DVg84S4v.cjs';
|
|
5
|
+
export { A as AttributionConfidence, E as EventSource, c as InspectEvent, I as InspectKind, b as InspectNode } from './inspect-event-DVg84S4v.cjs';
|
|
6
|
+
import { P as PersistedInspectEvent } from './persisted-inspect-event-d-mSP8kd.cjs';
|
|
7
7
|
import { Stats } from 'node:fs';
|
|
8
|
-
import {
|
|
9
|
-
export {
|
|
10
|
-
import { TraceCheckResult, TraceCheckStatus } from './
|
|
8
|
+
import { b as ObservedOutcome } from './types-B562HgN_.cjs';
|
|
9
|
+
export { O as ObservedOutcomeStatus } from './types-B562HgN_.cjs';
|
|
10
|
+
import { a as TraceCheckResult, E as TraceCheckStatus } from './index-DyKk7iR9.cjs';
|
|
11
11
|
import './writers.cjs';
|
|
12
|
-
import './log-config-
|
|
12
|
+
import './log-config-BG1WmWf2.cjs';
|
|
13
13
|
import './readers.cjs';
|
|
14
14
|
|
|
15
15
|
interface ResolvedRedactionProfile {
|
|
@@ -734,6 +734,19 @@ declare function aggregateBundleSafeStatus(statuses: readonly BundleSafeStatus[]
|
|
|
734
734
|
declare function toMetadataSafeStatus(status: BundleSafeStatus): BundleSafeStatusMetadata;
|
|
735
735
|
declare function bundleFailsOnSafety(status: BundleSafeStatus, allowUnsafe: boolean): boolean;
|
|
736
736
|
|
|
737
|
+
/**
|
|
738
|
+
* Sanitizes a run id for bundle directory and asset filenames.
|
|
739
|
+
* Strips path segments and replaces unsafe characters.
|
|
740
|
+
*/
|
|
741
|
+
declare function sanitizeBundleRunId(runId: string): string;
|
|
742
|
+
/**
|
|
743
|
+
* Relative POSIX path for a run asset inside a bundle directory.
|
|
744
|
+
*/
|
|
745
|
+
declare function bundleRunAssetRelativePath(runId: string, extension: string): string;
|
|
746
|
+
/**
|
|
747
|
+
* Ensures a relative bundle path resolves inside the output directory.
|
|
748
|
+
*/
|
|
749
|
+
declare function assertBundlePathContained(outputDir: string, relativePath: string): string;
|
|
737
750
|
/**
|
|
738
751
|
* Normalizes bundle output path. Strips a `.zip` suffix (folder-first MVP).
|
|
739
752
|
*/
|
|
@@ -940,6 +953,12 @@ interface CohortAnalysisResult {
|
|
|
940
953
|
runs: CohortRunMetrics[];
|
|
941
954
|
warnings: string[];
|
|
942
955
|
}
|
|
956
|
+
interface CohortToleranceOptions {
|
|
957
|
+
/** Minimum runs per cohort label before comparisons are considered valid. */
|
|
958
|
+
minSampleSize?: number;
|
|
959
|
+
/** Allowed relative delta (0–1) before a metric comparison is a regression. */
|
|
960
|
+
maxRelativeDelta?: number;
|
|
961
|
+
}
|
|
943
962
|
interface AnalyzeCohortOptions {
|
|
944
963
|
traceDir: string;
|
|
945
964
|
baseline?: string;
|
|
@@ -947,6 +966,7 @@ interface AnalyzeCohortOptions {
|
|
|
947
966
|
cohortKey?: string;
|
|
948
967
|
groupBy?: string;
|
|
949
968
|
metrics?: CohortMetricId[];
|
|
969
|
+
tolerance?: CohortToleranceOptions;
|
|
950
970
|
}
|
|
951
971
|
interface RenderCohortReportOptions {
|
|
952
972
|
format?: "markdown" | "json" | "html";
|
|
@@ -959,6 +979,7 @@ declare function compareCohortAggregates(groups: readonly CohortAggregateMetrics
|
|
|
959
979
|
candidate: string;
|
|
960
980
|
metrics: readonly CohortMetricId[];
|
|
961
981
|
groupKey?: string;
|
|
982
|
+
maxRelativeDelta?: number;
|
|
962
983
|
}): CohortMetricComparison[];
|
|
963
984
|
|
|
964
985
|
declare function parseCohortMetricList(value: string | undefined): string[];
|
|
@@ -1005,7 +1026,10 @@ interface RenderGateReportOptions {
|
|
|
1005
1026
|
}
|
|
1006
1027
|
|
|
1007
1028
|
declare function parseGateList(value: string | undefined): string[];
|
|
1008
|
-
declare function parseGateNumber(value: string | undefined, label: string
|
|
1029
|
+
declare function parseGateNumber(value: string | undefined, label: string, options?: {
|
|
1030
|
+
min?: number;
|
|
1031
|
+
max?: number;
|
|
1032
|
+
}): number | undefined;
|
|
1009
1033
|
|
|
1010
1034
|
declare function gateHasThresholds(options: RunGateOptions): boolean;
|
|
1011
1035
|
|
|
@@ -1016,4 +1040,4 @@ declare function renderGateGithubStepSummary(result: GateResult): string;
|
|
|
1016
1040
|
declare function renderGateJUnit(result: GateResult): string;
|
|
1017
1041
|
declare function renderGateReport(result: GateResult, options?: RenderGateReportOptions): string;
|
|
1018
1042
|
|
|
1019
|
-
export { type ActivityEntry, type ActivitySummary, type AnalyzeCohortOptions, type BuildActivitySummaryOptions, type BuildSessionIndexOptions, type BundleCheckResults, type BundleCheckRunResult, type BundleMetadata, type BundlePlaceholderArtifact, type BundleRedactionProfile, type BundleRedactionReport, type BundleRedactionReportRun, type BundleResolveOptions, type BundleResolveResult, type BundleSafeStatus, type BundleSafeStatusMetadata, COHORT_METRIC_IDS, type CohortAggregateMetrics, type CohortAnalysisResult, type CohortMetricComparison, type CohortMetricId, type CohortRunMetrics, type CriticalPathStep, DEFAULT_SUITE_ARTIFACTS_DIR, DEFAULT_SUITE_CONFIG_NAMES, DEFAULT_TRACE_DIR_NAME, type DurationStats, type EnrichSessionSummaryOptions, ErrorInfo, type ExplainFact, type ExplainInference, type ExplainMode, type ExplainOptions, type ExplainResult, FALLBACK_TRACE_DIR, type GateCheckId, type GateCheckResult, type GateExitCode, type GateResult, type GroupSessionCohortsOptions, type HandoffEdge, InspectRunTree, type LoadSuiteConfigOptions, MAX_NAME_LENGTH, MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, ObservedOutcome, type ParseTraceJsonlOptions, type ParseTraceJsonlResult, type ParsedDurationFilter, RUNS_DIR_NAME, RedactionProfile, type RenderCohortReportOptions, type RenderGateReportOptions, type RenderSuiteReportOptions, type RenderTimelineOptions, type RenderWhatOptions, type ResolvedRedactionProfile, type ResolvedSuiteCase, type RetryLink, type RunGateOptions, RunStatus, type RunSuiteOptions, RunSummary, type RunTimeline, type RunWhatSummary, SESSION_WORKFLOW_KEYS, SUITE_TEMPLATE_IDS, type SessionCheckSummary, type SessionCohort, type SessionCohortKind, type SessionConfidence, type SessionEdgeSource, type SessionGroup, type SessionIndex, type SessionLastError, type SessionRunRecord, type SessionScopeOptions, type SessionScopeResult, type SessionStatus, type SessionSummary, type SessionWarning, type SessionWorkflowKey, type SessionWorkflowMetadata, StepStatus, StepType, type SuiteArtifactsConfig, type SuiteCaseConfig, type SuiteCaseResult, type SuiteCaseStatus, type SuiteChecksConfig, type SuiteConfig, type SuiteDiagnostic, type SuiteDiagnosticCode, type SuiteEvalConfig, type SuiteRunResult, type SuiteRunSummary, type SuiteTemplateId, TERMINAL_INDENT, type TimelineEntry, type TimelineFocus, type TimelineOptions, TraceCorrelationMetadata, TraceDirectory, type TraceDirectoryOptions, TraceEvent, type TraceFilterOptions, type TraceJsonlFormat, TraceMetadata, TraceMetadataStatus, type TraceSearchOptions, type TraceSearchResult, type TraceSessionCheckResult, type TraceStats, type TraceStatsOptions, type TraceStatsRankedRun, type TraceStatsRankedStep, type ValidateSuiteConfigResult, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, compareCohortAggregates, createRunId, createStepId, defaultBundleOutputPath, defaultSuiteConfigTemplate, deriveSessionStatus, enrichSessionRunRecord, enrichSessionSummary, ensureTraceDir, extractMetadata, extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, formatDuration, formatError, formatTerminalName, formatTimestamp, gateHasThresholds, getDefaultTraceDir, getIndent, getRunIdFromTraceFileName, getSuiteTemplate, getTraceFilePath, groupSessionCohorts, initializeTraceFile, isAgentInspectTrace, listSuiteTemplates, listTraceFiles, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, normalizeBundleOutputPath, normalizeSuiteConfig, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseGateNumber, parseGroupBySpec, parseTraceJsonl, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, readTraceEvents, readTraceFile, renderActivitySummaryHuman, renderCohortReport, renderCohortSummaryMarkdown, renderErrorLine, renderGateGithubStepSummary, renderGateJUnit, renderGateReport, renderGateSummaryMarkdown, renderRunSummary, renderRunWhat, renderStepLine, renderSuiteReport, renderSuiteReportMarkdown, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteCaseTrace, resolveSuiteConfigPath, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, searchTraces, serializeEvent, sessionKeyForRun, toMetadataSafeStatus, traceMetasToSessionRunRecords, truncateName, unknownTraceFormatMessage, validateEvent, validateSuiteConfig, warn, writeTraceEvent };
|
|
1043
|
+
export { type ActivityEntry, type ActivitySummary, type AnalyzeCohortOptions, type BuildActivitySummaryOptions, type BuildSessionIndexOptions, type BundleCheckResults, type BundleCheckRunResult, type BundleMetadata, type BundlePlaceholderArtifact, type BundleRedactionProfile, type BundleRedactionReport, type BundleRedactionReportRun, type BundleResolveOptions, type BundleResolveResult, type BundleSafeStatus, type BundleSafeStatusMetadata, COHORT_METRIC_IDS, type CohortAggregateMetrics, type CohortAnalysisResult, type CohortMetricComparison, type CohortMetricId, type CohortRunMetrics, type CriticalPathStep, DEFAULT_SUITE_ARTIFACTS_DIR, DEFAULT_SUITE_CONFIG_NAMES, DEFAULT_TRACE_DIR_NAME, type DurationStats, type EnrichSessionSummaryOptions, ErrorInfo, type ExplainFact, type ExplainInference, type ExplainMode, type ExplainOptions, type ExplainResult, FALLBACK_TRACE_DIR, type GateCheckId, type GateCheckResult, type GateExitCode, type GateResult, type GroupSessionCohortsOptions, type HandoffEdge, InspectRunTree, type LoadSuiteConfigOptions, MAX_NAME_LENGTH, MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, ObservedOutcome, type ParseTraceJsonlOptions, type ParseTraceJsonlResult, type ParsedDurationFilter, RUNS_DIR_NAME, RedactionProfile, type RenderCohortReportOptions, type RenderGateReportOptions, type RenderSuiteReportOptions, type RenderTimelineOptions, type RenderWhatOptions, type ResolvedRedactionProfile, type ResolvedSuiteCase, type RetryLink, type RunGateOptions, RunStatus, type RunSuiteOptions, RunSummary, type RunTimeline, type RunWhatSummary, SESSION_WORKFLOW_KEYS, SUITE_TEMPLATE_IDS, type SessionCheckSummary, type SessionCohort, type SessionCohortKind, type SessionConfidence, type SessionEdgeSource, type SessionGroup, type SessionIndex, type SessionLastError, type SessionRunRecord, type SessionScopeOptions, type SessionScopeResult, type SessionStatus, type SessionSummary, type SessionWarning, type SessionWorkflowKey, type SessionWorkflowMetadata, StepStatus, StepType, type SuiteArtifactsConfig, type SuiteCaseConfig, type SuiteCaseResult, type SuiteCaseStatus, type SuiteChecksConfig, type SuiteConfig, type SuiteDiagnostic, type SuiteDiagnosticCode, type SuiteEvalConfig, type SuiteRunResult, type SuiteRunSummary, type SuiteTemplateId, TERMINAL_INDENT, type TimelineEntry, type TimelineFocus, type TimelineOptions, TraceCorrelationMetadata, TraceDirectory, type TraceDirectoryOptions, TraceEvent, type TraceFilterOptions, type TraceJsonlFormat, TraceMetadata, TraceMetadataStatus, type TraceSearchOptions, type TraceSearchResult, type TraceSessionCheckResult, type TraceStats, type TraceStatsOptions, type TraceStatsRankedRun, type TraceStatsRankedStep, type ValidateSuiteConfigResult, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, assertBundlePathContained, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, bundleRunAssetRelativePath, compareCohortAggregates, createRunId, createStepId, defaultBundleOutputPath, defaultSuiteConfigTemplate, deriveSessionStatus, enrichSessionRunRecord, enrichSessionSummary, ensureTraceDir, extractMetadata, extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, formatDuration, formatError, formatTerminalName, formatTimestamp, gateHasThresholds, getDefaultTraceDir, getIndent, getRunIdFromTraceFileName, getSuiteTemplate, getTraceFilePath, groupSessionCohorts, initializeTraceFile, isAgentInspectTrace, listSuiteTemplates, listTraceFiles, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, normalizeBundleOutputPath, normalizeSuiteConfig, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseGateNumber, parseGroupBySpec, parseTraceJsonl, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, readTraceEvents, readTraceFile, renderActivitySummaryHuman, renderCohortReport, renderCohortSummaryMarkdown, renderErrorLine, renderGateGithubStepSummary, renderGateJUnit, renderGateReport, renderGateSummaryMarkdown, renderRunSummary, renderRunWhat, renderStepLine, renderSuiteReport, renderSuiteReportMarkdown, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteCaseTrace, resolveSuiteConfigPath, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, sanitizeBundleRunId, searchTraces, serializeEvent, sessionKeyForRun, toMetadataSafeStatus, traceMetasToSessionRunRecords, truncateName, unknownTraceFormatMessage, validateEvent, validateSuiteConfig, warn, writeTraceEvent };
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
export { C as CreateInspectorOptions, D as DEFAULT_MAX_EVENT_BYTES, f as DEFAULT_MAX_METADATA_VALUE_LENGTH, h as DEFAULT_MAX_PREVIEW_LENGTH, I as Inspector, a as InspectorCaptureOptions, b as InspectorObserveOptions, c as InspectorRunOptions, i as InspectorRuntime, j as InspectorRuntimeContext, k as InspectorRuntimeDiagnostics, l as InspectorRuntimeOptions, d as InspectorStepOptions, T as TraceSafetyOptions, e as createInspector, n as createInspectorRuntime, o as getCurrentContext, g as getCurrentCorrelationMetadata, p as getCurrentDepth, q as getCurrentRunId, r as getCurrentRunName, s as getCurrentStepId, t as getParentStepId, u as getTraceDirFromContext, v as getTraceSafetyFromContext, w as hasActiveContext, x as isAgentInspectEnabled, y as isSilentContext, m as maybeInspectRun, z as prepareMetadataForDisk, A as prepareTraceEventForDisk, B as resolveTraceSafetyOptions, E as runWithContext, F as runWithStepContext } from './context-
|
|
2
|
-
import { R as RedactionProfile, h as TraceEvent, E as ErrorInfo, b as RunStatus, e as StepStatus, j as RunSummary, k as TraceMetadata, l as TraceMetadataStatus, f as StepType, g as TraceCorrelationMetadata } from './types-
|
|
3
|
-
export { A as ActiveStepContext, i as ExecutionContext, I as InspectRunOptions, O as ObserveOptions, a as Run, m as RunCompletedEvent, n as RunStartedEvent, c as Step, o as StepCompletedEvent, d as StepMetadata, S as StepOptions, p as StepStartedEvent, T as TokenMetadata, q as TraceEventBase, r as TraceSchemaVersion, s as isStepStatus, t as isStepType, u as isTraceEvent } from './types-
|
|
4
|
-
import {
|
|
5
|
-
export { A as AttributionConfidence, E as EventSource,
|
|
6
|
-
import { P as PersistedInspectEvent } from './persisted-inspect-event-
|
|
1
|
+
export { C as CreateInspectorOptions, D as DEFAULT_MAX_EVENT_BYTES, f as DEFAULT_MAX_METADATA_VALUE_LENGTH, h as DEFAULT_MAX_PREVIEW_LENGTH, I as Inspector, a as InspectorCaptureOptions, b as InspectorObserveOptions, c as InspectorRunOptions, i as InspectorRuntime, j as InspectorRuntimeContext, k as InspectorRuntimeDiagnostics, l as InspectorRuntimeOptions, d as InspectorStepOptions, T as TraceSafetyOptions, e as createInspector, n as createInspectorRuntime, o as getCurrentContext, g as getCurrentCorrelationMetadata, p as getCurrentDepth, q as getCurrentRunId, r as getCurrentRunName, s as getCurrentStepId, t as getParentStepId, u as getTraceDirFromContext, v as getTraceSafetyFromContext, w as hasActiveContext, x as isAgentInspectEnabled, y as isSilentContext, m as maybeInspectRun, z as prepareMetadataForDisk, A as prepareTraceEventForDisk, B as resolveTraceSafetyOptions, E as runWithContext, F as runWithStepContext } from './context-Blyjb5my.js';
|
|
2
|
+
import { R as RedactionProfile, h as TraceEvent, E as ErrorInfo, b as RunStatus, e as StepStatus, j as RunSummary, k as TraceMetadata, l as TraceMetadataStatus, f as StepType, g as TraceCorrelationMetadata } from './types-MuwXSJ4b.js';
|
|
3
|
+
export { A as ActiveStepContext, i as ExecutionContext, I as InspectRunOptions, O as ObserveOptions, a as Run, m as RunCompletedEvent, n as RunStartedEvent, c as Step, o as StepCompletedEvent, d as StepMetadata, S as StepOptions, p as StepStartedEvent, T as TokenMetadata, q as TraceEventBase, r as TraceSchemaVersion, s as isStepStatus, t as isStepType, u as isTraceEvent } from './types-MuwXSJ4b.js';
|
|
4
|
+
import { a as InspectRunTree } from './inspect-event-DVg84S4v.js';
|
|
5
|
+
export { A as AttributionConfidence, E as EventSource, c as InspectEvent, I as InspectKind, b as InspectNode } from './inspect-event-DVg84S4v.js';
|
|
6
|
+
import { P as PersistedInspectEvent } from './persisted-inspect-event-ChE7pGb7.js';
|
|
7
7
|
import { Stats } from 'node:fs';
|
|
8
|
-
import {
|
|
9
|
-
export {
|
|
10
|
-
import { TraceCheckResult, TraceCheckStatus } from './
|
|
8
|
+
import { b as ObservedOutcome } from './types-B562HgN_.js';
|
|
9
|
+
export { O as ObservedOutcomeStatus } from './types-B562HgN_.js';
|
|
10
|
+
import { a as TraceCheckResult, E as TraceCheckStatus } from './index-BblJEdYZ.js';
|
|
11
11
|
import './writers.js';
|
|
12
|
-
import './log-config-
|
|
12
|
+
import './log-config-Ck0aT_Ou.js';
|
|
13
13
|
import './readers.js';
|
|
14
14
|
|
|
15
15
|
interface ResolvedRedactionProfile {
|
|
@@ -734,6 +734,19 @@ declare function aggregateBundleSafeStatus(statuses: readonly BundleSafeStatus[]
|
|
|
734
734
|
declare function toMetadataSafeStatus(status: BundleSafeStatus): BundleSafeStatusMetadata;
|
|
735
735
|
declare function bundleFailsOnSafety(status: BundleSafeStatus, allowUnsafe: boolean): boolean;
|
|
736
736
|
|
|
737
|
+
/**
|
|
738
|
+
* Sanitizes a run id for bundle directory and asset filenames.
|
|
739
|
+
* Strips path segments and replaces unsafe characters.
|
|
740
|
+
*/
|
|
741
|
+
declare function sanitizeBundleRunId(runId: string): string;
|
|
742
|
+
/**
|
|
743
|
+
* Relative POSIX path for a run asset inside a bundle directory.
|
|
744
|
+
*/
|
|
745
|
+
declare function bundleRunAssetRelativePath(runId: string, extension: string): string;
|
|
746
|
+
/**
|
|
747
|
+
* Ensures a relative bundle path resolves inside the output directory.
|
|
748
|
+
*/
|
|
749
|
+
declare function assertBundlePathContained(outputDir: string, relativePath: string): string;
|
|
737
750
|
/**
|
|
738
751
|
* Normalizes bundle output path. Strips a `.zip` suffix (folder-first MVP).
|
|
739
752
|
*/
|
|
@@ -940,6 +953,12 @@ interface CohortAnalysisResult {
|
|
|
940
953
|
runs: CohortRunMetrics[];
|
|
941
954
|
warnings: string[];
|
|
942
955
|
}
|
|
956
|
+
interface CohortToleranceOptions {
|
|
957
|
+
/** Minimum runs per cohort label before comparisons are considered valid. */
|
|
958
|
+
minSampleSize?: number;
|
|
959
|
+
/** Allowed relative delta (0–1) before a metric comparison is a regression. */
|
|
960
|
+
maxRelativeDelta?: number;
|
|
961
|
+
}
|
|
943
962
|
interface AnalyzeCohortOptions {
|
|
944
963
|
traceDir: string;
|
|
945
964
|
baseline?: string;
|
|
@@ -947,6 +966,7 @@ interface AnalyzeCohortOptions {
|
|
|
947
966
|
cohortKey?: string;
|
|
948
967
|
groupBy?: string;
|
|
949
968
|
metrics?: CohortMetricId[];
|
|
969
|
+
tolerance?: CohortToleranceOptions;
|
|
950
970
|
}
|
|
951
971
|
interface RenderCohortReportOptions {
|
|
952
972
|
format?: "markdown" | "json" | "html";
|
|
@@ -959,6 +979,7 @@ declare function compareCohortAggregates(groups: readonly CohortAggregateMetrics
|
|
|
959
979
|
candidate: string;
|
|
960
980
|
metrics: readonly CohortMetricId[];
|
|
961
981
|
groupKey?: string;
|
|
982
|
+
maxRelativeDelta?: number;
|
|
962
983
|
}): CohortMetricComparison[];
|
|
963
984
|
|
|
964
985
|
declare function parseCohortMetricList(value: string | undefined): string[];
|
|
@@ -1005,7 +1026,10 @@ interface RenderGateReportOptions {
|
|
|
1005
1026
|
}
|
|
1006
1027
|
|
|
1007
1028
|
declare function parseGateList(value: string | undefined): string[];
|
|
1008
|
-
declare function parseGateNumber(value: string | undefined, label: string
|
|
1029
|
+
declare function parseGateNumber(value: string | undefined, label: string, options?: {
|
|
1030
|
+
min?: number;
|
|
1031
|
+
max?: number;
|
|
1032
|
+
}): number | undefined;
|
|
1009
1033
|
|
|
1010
1034
|
declare function gateHasThresholds(options: RunGateOptions): boolean;
|
|
1011
1035
|
|
|
@@ -1016,4 +1040,4 @@ declare function renderGateGithubStepSummary(result: GateResult): string;
|
|
|
1016
1040
|
declare function renderGateJUnit(result: GateResult): string;
|
|
1017
1041
|
declare function renderGateReport(result: GateResult, options?: RenderGateReportOptions): string;
|
|
1018
1042
|
|
|
1019
|
-
export { type ActivityEntry, type ActivitySummary, type AnalyzeCohortOptions, type BuildActivitySummaryOptions, type BuildSessionIndexOptions, type BundleCheckResults, type BundleCheckRunResult, type BundleMetadata, type BundlePlaceholderArtifact, type BundleRedactionProfile, type BundleRedactionReport, type BundleRedactionReportRun, type BundleResolveOptions, type BundleResolveResult, type BundleSafeStatus, type BundleSafeStatusMetadata, COHORT_METRIC_IDS, type CohortAggregateMetrics, type CohortAnalysisResult, type CohortMetricComparison, type CohortMetricId, type CohortRunMetrics, type CriticalPathStep, DEFAULT_SUITE_ARTIFACTS_DIR, DEFAULT_SUITE_CONFIG_NAMES, DEFAULT_TRACE_DIR_NAME, type DurationStats, type EnrichSessionSummaryOptions, ErrorInfo, type ExplainFact, type ExplainInference, type ExplainMode, type ExplainOptions, type ExplainResult, FALLBACK_TRACE_DIR, type GateCheckId, type GateCheckResult, type GateExitCode, type GateResult, type GroupSessionCohortsOptions, type HandoffEdge, InspectRunTree, type LoadSuiteConfigOptions, MAX_NAME_LENGTH, MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, ObservedOutcome, type ParseTraceJsonlOptions, type ParseTraceJsonlResult, type ParsedDurationFilter, RUNS_DIR_NAME, RedactionProfile, type RenderCohortReportOptions, type RenderGateReportOptions, type RenderSuiteReportOptions, type RenderTimelineOptions, type RenderWhatOptions, type ResolvedRedactionProfile, type ResolvedSuiteCase, type RetryLink, type RunGateOptions, RunStatus, type RunSuiteOptions, RunSummary, type RunTimeline, type RunWhatSummary, SESSION_WORKFLOW_KEYS, SUITE_TEMPLATE_IDS, type SessionCheckSummary, type SessionCohort, type SessionCohortKind, type SessionConfidence, type SessionEdgeSource, type SessionGroup, type SessionIndex, type SessionLastError, type SessionRunRecord, type SessionScopeOptions, type SessionScopeResult, type SessionStatus, type SessionSummary, type SessionWarning, type SessionWorkflowKey, type SessionWorkflowMetadata, StepStatus, StepType, type SuiteArtifactsConfig, type SuiteCaseConfig, type SuiteCaseResult, type SuiteCaseStatus, type SuiteChecksConfig, type SuiteConfig, type SuiteDiagnostic, type SuiteDiagnosticCode, type SuiteEvalConfig, type SuiteRunResult, type SuiteRunSummary, type SuiteTemplateId, TERMINAL_INDENT, type TimelineEntry, type TimelineFocus, type TimelineOptions, TraceCorrelationMetadata, TraceDirectory, type TraceDirectoryOptions, TraceEvent, type TraceFilterOptions, type TraceJsonlFormat, TraceMetadata, TraceMetadataStatus, type TraceSearchOptions, type TraceSearchResult, type TraceSessionCheckResult, type TraceStats, type TraceStatsOptions, type TraceStatsRankedRun, type TraceStatsRankedStep, type ValidateSuiteConfigResult, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, compareCohortAggregates, createRunId, createStepId, defaultBundleOutputPath, defaultSuiteConfigTemplate, deriveSessionStatus, enrichSessionRunRecord, enrichSessionSummary, ensureTraceDir, extractMetadata, extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, formatDuration, formatError, formatTerminalName, formatTimestamp, gateHasThresholds, getDefaultTraceDir, getIndent, getRunIdFromTraceFileName, getSuiteTemplate, getTraceFilePath, groupSessionCohorts, initializeTraceFile, isAgentInspectTrace, listSuiteTemplates, listTraceFiles, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, normalizeBundleOutputPath, normalizeSuiteConfig, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseGateNumber, parseGroupBySpec, parseTraceJsonl, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, readTraceEvents, readTraceFile, renderActivitySummaryHuman, renderCohortReport, renderCohortSummaryMarkdown, renderErrorLine, renderGateGithubStepSummary, renderGateJUnit, renderGateReport, renderGateSummaryMarkdown, renderRunSummary, renderRunWhat, renderStepLine, renderSuiteReport, renderSuiteReportMarkdown, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteCaseTrace, resolveSuiteConfigPath, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, searchTraces, serializeEvent, sessionKeyForRun, toMetadataSafeStatus, traceMetasToSessionRunRecords, truncateName, unknownTraceFormatMessage, validateEvent, validateSuiteConfig, warn, writeTraceEvent };
|
|
1043
|
+
export { type ActivityEntry, type ActivitySummary, type AnalyzeCohortOptions, type BuildActivitySummaryOptions, type BuildSessionIndexOptions, type BundleCheckResults, type BundleCheckRunResult, type BundleMetadata, type BundlePlaceholderArtifact, type BundleRedactionProfile, type BundleRedactionReport, type BundleRedactionReportRun, type BundleResolveOptions, type BundleResolveResult, type BundleSafeStatus, type BundleSafeStatusMetadata, COHORT_METRIC_IDS, type CohortAggregateMetrics, type CohortAnalysisResult, type CohortMetricComparison, type CohortMetricId, type CohortRunMetrics, type CriticalPathStep, DEFAULT_SUITE_ARTIFACTS_DIR, DEFAULT_SUITE_CONFIG_NAMES, DEFAULT_TRACE_DIR_NAME, type DurationStats, type EnrichSessionSummaryOptions, ErrorInfo, type ExplainFact, type ExplainInference, type ExplainMode, type ExplainOptions, type ExplainResult, FALLBACK_TRACE_DIR, type GateCheckId, type GateCheckResult, type GateExitCode, type GateResult, type GroupSessionCohortsOptions, type HandoffEdge, InspectRunTree, type LoadSuiteConfigOptions, MAX_NAME_LENGTH, MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, ObservedOutcome, type ParseTraceJsonlOptions, type ParseTraceJsonlResult, type ParsedDurationFilter, RUNS_DIR_NAME, RedactionProfile, type RenderCohortReportOptions, type RenderGateReportOptions, type RenderSuiteReportOptions, type RenderTimelineOptions, type RenderWhatOptions, type ResolvedRedactionProfile, type ResolvedSuiteCase, type RetryLink, type RunGateOptions, RunStatus, type RunSuiteOptions, RunSummary, type RunTimeline, type RunWhatSummary, SESSION_WORKFLOW_KEYS, SUITE_TEMPLATE_IDS, type SessionCheckSummary, type SessionCohort, type SessionCohortKind, type SessionConfidence, type SessionEdgeSource, type SessionGroup, type SessionIndex, type SessionLastError, type SessionRunRecord, type SessionScopeOptions, type SessionScopeResult, type SessionStatus, type SessionSummary, type SessionWarning, type SessionWorkflowKey, type SessionWorkflowMetadata, StepStatus, StepType, type SuiteArtifactsConfig, type SuiteCaseConfig, type SuiteCaseResult, type SuiteCaseStatus, type SuiteChecksConfig, type SuiteConfig, type SuiteDiagnostic, type SuiteDiagnosticCode, type SuiteEvalConfig, type SuiteRunResult, type SuiteRunSummary, type SuiteTemplateId, TERMINAL_INDENT, type TimelineEntry, type TimelineFocus, type TimelineOptions, TraceCorrelationMetadata, TraceDirectory, type TraceDirectoryOptions, TraceEvent, type TraceFilterOptions, type TraceJsonlFormat, TraceMetadata, TraceMetadataStatus, type TraceSearchOptions, type TraceSearchResult, type TraceSessionCheckResult, type TraceStats, type TraceStatsOptions, type TraceStatsRankedRun, type TraceStatsRankedStep, type ValidateSuiteConfigResult, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, assertBundlePathContained, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, bundleRunAssetRelativePath, compareCohortAggregates, createRunId, createStepId, defaultBundleOutputPath, defaultSuiteConfigTemplate, deriveSessionStatus, enrichSessionRunRecord, enrichSessionSummary, ensureTraceDir, extractMetadata, extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, formatDuration, formatError, formatTerminalName, formatTimestamp, gateHasThresholds, getDefaultTraceDir, getIndent, getRunIdFromTraceFileName, getSuiteTemplate, getTraceFilePath, groupSessionCohorts, initializeTraceFile, isAgentInspectTrace, listSuiteTemplates, listTraceFiles, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, normalizeBundleOutputPath, normalizeSuiteConfig, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseGateNumber, parseGroupBySpec, parseTraceJsonl, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, readTraceEvents, readTraceFile, renderActivitySummaryHuman, renderCohortReport, renderCohortSummaryMarkdown, renderErrorLine, renderGateGithubStepSummary, renderGateJUnit, renderGateReport, renderGateSummaryMarkdown, renderRunSummary, renderRunWhat, renderStepLine, renderSuiteReport, renderSuiteReportMarkdown, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteCaseTrace, resolveSuiteConfigPath, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, sanitizeBundleRunId, searchTraces, serializeEvent, sessionKeyForRun, toMetadataSafeStatus, traceMetasToSessionRunRecords, truncateName, unknownTraceFormatMessage, validateEvent, validateSuiteConfig, warn, writeTraceEvent };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-
|
|
1
|
+
import { runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-EWHN6NA4.mjs';
|
|
2
2
|
export { MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, TERMINAL_INDENT, TraceDirectory, createInspector, createInspectorRuntime, formatTerminalName, getCurrentContext, getCurrentCorrelationMetadata, getCurrentDepth, getCurrentRunId, getCurrentRunName, getCurrentStepId, getIndent, getParentStepId, getTraceDirFromContext, getTraceSafetyFromContext, hasActiveContext, isAgentInspectEnabled, isSilentContext, maybeInspectRun, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, renderErrorLine, renderRunSummary, renderStepLine, resolveTraceDir, runWithContext, runWithStepContext } from './chunk-ZWA5RWMV.mjs';
|
|
3
3
|
import { buildRunSummary, extractMetadata } from './chunk-3USJVBLA.mjs';
|
|
4
4
|
export { buildRunSummary, buildRunTimeline, buildRunWhatSummary, extractMetadata, renderRunWhat, renderTimeline } from './chunk-3USJVBLA.mjs';
|
|
@@ -19,7 +19,7 @@ import './chunk-P366CCL2.mjs';
|
|
|
19
19
|
import './chunk-7TGZLWEE.mjs';
|
|
20
20
|
import { createReadStream } from 'fs';
|
|
21
21
|
import { createInterface } from 'readline';
|
|
22
|
-
import
|
|
22
|
+
import path from 'path';
|
|
23
23
|
import { access, readFile } from 'fs/promises';
|
|
24
24
|
import { pathToFileURL } from 'url';
|
|
25
25
|
|
|
@@ -1643,21 +1643,43 @@ function buildBundleSummaryMarkdown(parts) {
|
|
|
1643
1643
|
);
|
|
1644
1644
|
return lines.join("\n");
|
|
1645
1645
|
}
|
|
1646
|
+
function sanitizeBundleRunId(runId) {
|
|
1647
|
+
let safe = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
|
|
1648
|
+
safe = path.basename(safe);
|
|
1649
|
+
safe = safe.replace(/[^a-zA-Z0-9._-]+/g, "_");
|
|
1650
|
+
if (safe === "" || safe === "." || safe === "..") {
|
|
1651
|
+
safe = "run_unknown";
|
|
1652
|
+
}
|
|
1653
|
+
return safe;
|
|
1654
|
+
}
|
|
1655
|
+
function bundleRunAssetRelativePath(runId, extension) {
|
|
1656
|
+
const safe = sanitizeBundleRunId(runId);
|
|
1657
|
+
const ext = extension.startsWith(".") ? extension : `.${extension}`;
|
|
1658
|
+
return path.posix.join("assets", "runs", `${safe}${ext}`);
|
|
1659
|
+
}
|
|
1660
|
+
function assertBundlePathContained(outputDir, relativePath) {
|
|
1661
|
+
const base = path.resolve(outputDir);
|
|
1662
|
+
const resolved = path.resolve(base, relativePath);
|
|
1663
|
+
if (resolved !== base && !resolved.startsWith(base + path.sep)) {
|
|
1664
|
+
throw new Error(`Bundle path escapes output directory: ${relativePath}`);
|
|
1665
|
+
}
|
|
1666
|
+
return resolved;
|
|
1667
|
+
}
|
|
1646
1668
|
function normalizeBundleOutputPath(out) {
|
|
1647
1669
|
const trimmed = out.trim();
|
|
1648
1670
|
if (trimmed === "") {
|
|
1649
1671
|
throw new Error("--out requires a non-empty path.");
|
|
1650
1672
|
}
|
|
1651
|
-
const resolved =
|
|
1673
|
+
const resolved = path.resolve(trimmed);
|
|
1652
1674
|
if (resolved.toLowerCase().endsWith(".zip")) {
|
|
1653
1675
|
return resolved.slice(0, -4);
|
|
1654
1676
|
}
|
|
1655
1677
|
return resolved;
|
|
1656
1678
|
}
|
|
1657
1679
|
function defaultBundleOutputPath(runIds) {
|
|
1658
|
-
const label = runIds.length === 1 ? runIds[0] : `multi-${runIds.length}`;
|
|
1680
|
+
const label = runIds.length === 1 ? sanitizeBundleRunId(runIds[0]) : `multi-${runIds.length}`;
|
|
1659
1681
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
1660
|
-
return
|
|
1682
|
+
return path.resolve(`agent-inspect-bundle-${label}-${stamp}`);
|
|
1661
1683
|
}
|
|
1662
1684
|
|
|
1663
1685
|
// packages/core/src/suite/types.ts
|
|
@@ -1811,7 +1833,7 @@ function normalizeSuiteConfig(value) {
|
|
|
1811
1833
|
}
|
|
1812
1834
|
async function validateSuiteConfig(config, options) {
|
|
1813
1835
|
const diagnostics = [];
|
|
1814
|
-
const tracesDir =
|
|
1836
|
+
const tracesDir = path.resolve(options.configDir, config.traces);
|
|
1815
1837
|
try {
|
|
1816
1838
|
await access(tracesDir);
|
|
1817
1839
|
} catch {
|
|
@@ -1821,7 +1843,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
1821
1843
|
}
|
|
1822
1844
|
for (const suiteCase of config.cases) {
|
|
1823
1845
|
if (suiteCase.input !== void 0) {
|
|
1824
|
-
const inputPath =
|
|
1846
|
+
const inputPath = path.resolve(options.configDir, suiteCase.input);
|
|
1825
1847
|
try {
|
|
1826
1848
|
await access(inputPath);
|
|
1827
1849
|
} catch {
|
|
@@ -1854,12 +1876,12 @@ async function fileExists(filePath) {
|
|
|
1854
1876
|
}
|
|
1855
1877
|
}
|
|
1856
1878
|
async function resolveSuiteConfigPath(options = {}) {
|
|
1857
|
-
const cwd =
|
|
1879
|
+
const cwd = path.resolve(options.cwd ?? process.cwd());
|
|
1858
1880
|
if (options.configPath !== void 0 && options.configPath.trim() !== "") {
|
|
1859
|
-
return
|
|
1881
|
+
return path.resolve(cwd, options.configPath.trim());
|
|
1860
1882
|
}
|
|
1861
1883
|
for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
|
|
1862
|
-
const candidate =
|
|
1884
|
+
const candidate = path.join(cwd, name);
|
|
1863
1885
|
if (await fileExists(candidate)) return candidate;
|
|
1864
1886
|
}
|
|
1865
1887
|
throw new Error(
|
|
@@ -1876,7 +1898,7 @@ async function loadSuiteConfig(options = {}) {
|
|
|
1876
1898
|
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
1877
1899
|
});
|
|
1878
1900
|
}
|
|
1879
|
-
const extension =
|
|
1901
|
+
const extension = path.extname(configPath);
|
|
1880
1902
|
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
1881
1903
|
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
1882
1904
|
throw Object.assign(new Error(message), {
|
|
@@ -1901,7 +1923,7 @@ async function loadSuiteConfig(options = {}) {
|
|
|
1901
1923
|
return {
|
|
1902
1924
|
config,
|
|
1903
1925
|
configPath,
|
|
1904
|
-
configDir:
|
|
1926
|
+
configDir: path.dirname(configPath)
|
|
1905
1927
|
};
|
|
1906
1928
|
} catch (error) {
|
|
1907
1929
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -1942,7 +1964,7 @@ async function exists(filePath) {
|
|
|
1942
1964
|
}
|
|
1943
1965
|
async function resolveSuiteCaseTrace(suiteCase, options) {
|
|
1944
1966
|
if (suiteCase.trace !== void 0) {
|
|
1945
|
-
const tracePath =
|
|
1967
|
+
const tracePath = path.resolve(options.configDir, suiteCase.trace);
|
|
1946
1968
|
if (await exists(tracePath)) {
|
|
1947
1969
|
return { caseId: suiteCase.id, tracePath, missing: false };
|
|
1948
1970
|
}
|
|
@@ -1958,7 +1980,7 @@ async function resolveSuiteCaseTrace(suiteCase, options) {
|
|
|
1958
1980
|
if (await exists(directPath)) {
|
|
1959
1981
|
return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
|
|
1960
1982
|
}
|
|
1961
|
-
const nestedPath =
|
|
1983
|
+
const nestedPath = path.join(options.tracesDir, `${path.basename(runKey)}.jsonl`);
|
|
1962
1984
|
if (await exists(nestedPath)) {
|
|
1963
1985
|
return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
|
|
1964
1986
|
}
|
|
@@ -2137,7 +2159,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
2137
2159
|
async function runSuite(options = {}) {
|
|
2138
2160
|
const startedAt = new Date(options.nowMs ?? Date.now()).toISOString();
|
|
2139
2161
|
const { config, configPath, configDir } = await loadSuiteConfig(options);
|
|
2140
|
-
const tracesDir =
|
|
2162
|
+
const tracesDir = path.resolve(configDir, config.traces);
|
|
2141
2163
|
const cases = [];
|
|
2142
2164
|
const diagnostics = [];
|
|
2143
2165
|
for (const suiteCase of config.cases) {
|
|
@@ -2155,8 +2177,9 @@ async function runSuite(options = {}) {
|
|
|
2155
2177
|
skipped: cases.filter((item) => item.status === "skipped").length
|
|
2156
2178
|
};
|
|
2157
2179
|
const finishedAt = new Date(options.nowMs ?? Date.now()).toISOString();
|
|
2158
|
-
const
|
|
2159
|
-
const
|
|
2180
|
+
const allSkipped = cases.length > 0 && summary.skipped === cases.length;
|
|
2181
|
+
const ok = summary.failed === 0 && summary.errors === 0 && !allSkipped;
|
|
2182
|
+
const status = summary.errors > 0 ? "error" : allSkipped ? "fail" : summary.failed > 0 || !ok ? "fail" : "pass";
|
|
2160
2183
|
return {
|
|
2161
2184
|
ok,
|
|
2162
2185
|
status,
|
|
@@ -2348,10 +2371,14 @@ var COHORT_METRIC_IDS = [
|
|
|
2348
2371
|
];
|
|
2349
2372
|
|
|
2350
2373
|
// packages/core/src/cohort/compare.ts
|
|
2351
|
-
function compareNumber(metric, label, baseline, candidate, higherIsWorse = true) {
|
|
2374
|
+
function compareNumber(metric, label, baseline, candidate, higherIsWorse = true, maxRelativeDelta) {
|
|
2352
2375
|
if (baseline === void 0 && candidate === void 0) return void 0;
|
|
2353
2376
|
const delta = baseline !== void 0 && candidate !== void 0 ? candidate - baseline : void 0;
|
|
2354
|
-
|
|
2377
|
+
let regression = delta !== void 0 && (higherIsWorse && delta > 0 || !higherIsWorse && delta < 0);
|
|
2378
|
+
if (regression && maxRelativeDelta !== void 0 && baseline !== void 0 && baseline !== 0 && delta !== void 0) {
|
|
2379
|
+
const relative = Math.abs(delta / baseline);
|
|
2380
|
+
if (relative <= maxRelativeDelta) regression = false;
|
|
2381
|
+
}
|
|
2355
2382
|
return {
|
|
2356
2383
|
metric,
|
|
2357
2384
|
baseline,
|
|
@@ -2370,6 +2397,7 @@ function compareCohortAggregates(groups, options) {
|
|
|
2370
2397
|
const baselineAgg = pickAggregate(groups, options.baseline, options.groupKey);
|
|
2371
2398
|
const candidateAgg = pickAggregate(groups, options.candidate, options.groupKey);
|
|
2372
2399
|
const comparisons = [];
|
|
2400
|
+
const tolerance = options.maxRelativeDelta;
|
|
2373
2401
|
for (const metric of options.metrics) {
|
|
2374
2402
|
switch (metric) {
|
|
2375
2403
|
case "errorRate": {
|
|
@@ -2377,7 +2405,9 @@ function compareCohortAggregates(groups, options) {
|
|
|
2377
2405
|
metric,
|
|
2378
2406
|
"Error rate",
|
|
2379
2407
|
baselineAgg?.errorRate,
|
|
2380
|
-
candidateAgg?.errorRate
|
|
2408
|
+
candidateAgg?.errorRate,
|
|
2409
|
+
true,
|
|
2410
|
+
tolerance
|
|
2381
2411
|
);
|
|
2382
2412
|
if (item) comparisons.push(item);
|
|
2383
2413
|
break;
|
|
@@ -2731,13 +2761,27 @@ async function analyzeCohort(runsInput, options) {
|
|
|
2731
2761
|
...new Set(groups.map((group) => group.groupKey))
|
|
2732
2762
|
].sort((a, b) => a.localeCompare(b));
|
|
2733
2763
|
const items = [];
|
|
2764
|
+
const minSample = options.tolerance?.minSampleSize ?? 1;
|
|
2734
2765
|
for (const groupKey of groupKeys) {
|
|
2766
|
+
const baselineGroup = groups.find(
|
|
2767
|
+
(group) => group.groupKey === groupKey && group.cohortLabel === options.baseline
|
|
2768
|
+
);
|
|
2769
|
+
const candidateGroup = groups.find(
|
|
2770
|
+
(group) => group.groupKey === groupKey && group.cohortLabel === options.candidate
|
|
2771
|
+
);
|
|
2772
|
+
if (baselineGroup && baselineGroup.runCount < minSample || candidateGroup && candidateGroup.runCount < minSample) {
|
|
2773
|
+
warnings.push(
|
|
2774
|
+
`Insufficient sample for group ${groupKey}: need at least ${minSample} run(s) per cohort.`
|
|
2775
|
+
);
|
|
2776
|
+
continue;
|
|
2777
|
+
}
|
|
2735
2778
|
items.push(
|
|
2736
2779
|
...compareCohortAggregates(groups, {
|
|
2737
2780
|
baseline: options.baseline,
|
|
2738
2781
|
candidate: options.candidate,
|
|
2739
2782
|
metrics,
|
|
2740
|
-
groupKey
|
|
2783
|
+
groupKey,
|
|
2784
|
+
maxRelativeDelta: options.tolerance?.maxRelativeDelta
|
|
2741
2785
|
})
|
|
2742
2786
|
);
|
|
2743
2787
|
}
|
|
@@ -2853,12 +2897,18 @@ function parseGateList(value) {
|
|
|
2853
2897
|
if (value === void 0 || value.trim() === "") return [];
|
|
2854
2898
|
return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
|
|
2855
2899
|
}
|
|
2856
|
-
function parseGateNumber(value, label) {
|
|
2900
|
+
function parseGateNumber(value, label, options = {}) {
|
|
2857
2901
|
if (value === void 0 || value.trim() === "") return void 0;
|
|
2858
2902
|
const parsed = Number(value);
|
|
2859
2903
|
if (!Number.isFinite(parsed)) {
|
|
2860
2904
|
throw new Error(`Invalid ${label}: ${value}`);
|
|
2861
2905
|
}
|
|
2906
|
+
if (options.min !== void 0 && parsed < options.min) {
|
|
2907
|
+
throw new Error(`${label} must be at least ${options.min}.`);
|
|
2908
|
+
}
|
|
2909
|
+
if (options.max !== void 0 && parsed > options.max) {
|
|
2910
|
+
throw new Error(`${label} must be at most ${options.max}.`);
|
|
2911
|
+
}
|
|
2862
2912
|
return parsed;
|
|
2863
2913
|
}
|
|
2864
2914
|
|
|
@@ -3038,6 +3088,9 @@ function validateOptions(options) {
|
|
|
3038
3088
|
if (options.maxErrorRate !== void 0 && options.maxErrorRate < 0) {
|
|
3039
3089
|
errors.push("--max-error-rate must be a non-negative percentage.");
|
|
3040
3090
|
}
|
|
3091
|
+
if (options.maxErrorRate !== void 0 && options.maxErrorRate > 100) {
|
|
3092
|
+
errors.push("--max-error-rate must be at most 100.");
|
|
3093
|
+
}
|
|
3041
3094
|
if (options.maxP95DurationMs !== void 0 && options.maxP95DurationMs < 0) {
|
|
3042
3095
|
errors.push("--max-p95-duration must be a non-negative millisecond value.");
|
|
3043
3096
|
}
|
|
@@ -3045,7 +3098,7 @@ function validateOptions(options) {
|
|
|
3045
3098
|
}
|
|
3046
3099
|
function isConfigLoadError(error) {
|
|
3047
3100
|
if (!(error instanceof Error)) return false;
|
|
3048
|
-
const ext =
|
|
3101
|
+
const ext = path.extname(error.message);
|
|
3049
3102
|
if (error.message.includes("Unsupported suite config extension")) return true;
|
|
3050
3103
|
if (error.message.includes("TypeScript suite configs require")) return true;
|
|
3051
3104
|
if (error.message.includes("No suite config found")) return true;
|
|
@@ -3244,6 +3297,6 @@ function renderGateReport(result, options = {}) {
|
|
|
3244
3297
|
return renderGateSummaryMarkdown(result);
|
|
3245
3298
|
}
|
|
3246
3299
|
|
|
3247
|
-
export { COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, DEFAULT_SUITE_CONFIG_NAMES, SESSION_WORKFLOW_KEYS, SUITE_TEMPLATE_IDS, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, compareCohortAggregates, defaultBundleOutputPath, defaultSuiteConfigTemplate, deriveSessionStatus, enrichSessionRunRecord, enrichSessionSummary, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, gateHasThresholds, getSuiteTemplate, groupSessionCohorts, isAgentInspectTrace, listSuiteTemplates, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, normalizeBundleOutputPath, normalizeSuiteConfig, parseCohortMetricList, parseDurationFilter, parseGateList, parseGateNumber, parseGroupBySpec, renderActivitySummaryHuman, renderCohortReport, renderCohortSummaryMarkdown, renderGateGithubStepSummary, renderGateJUnit, renderGateReport, renderGateSummaryMarkdown, renderSuiteReport, renderSuiteReportMarkdown, renderTraceStats, resolveBundleRunIds, resolveSuiteCaseTrace, resolveSuiteConfigPath, resolveSuiteTemplate, runGate, runSuite, searchTraces, sessionKeyForRun, toMetadataSafeStatus, traceMetasToSessionRunRecords, validateSuiteConfig };
|
|
3300
|
+
export { COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, DEFAULT_SUITE_CONFIG_NAMES, SESSION_WORKFLOW_KEYS, SUITE_TEMPLATE_IDS, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, assertBundlePathContained, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, bundleRunAssetRelativePath, compareCohortAggregates, defaultBundleOutputPath, defaultSuiteConfigTemplate, deriveSessionStatus, enrichSessionRunRecord, enrichSessionSummary, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, gateHasThresholds, getSuiteTemplate, groupSessionCohorts, isAgentInspectTrace, listSuiteTemplates, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, normalizeBundleOutputPath, normalizeSuiteConfig, parseCohortMetricList, parseDurationFilter, parseGateList, parseGateNumber, parseGroupBySpec, renderActivitySummaryHuman, renderCohortReport, renderCohortSummaryMarkdown, renderGateGithubStepSummary, renderGateJUnit, renderGateReport, renderGateSummaryMarkdown, renderSuiteReport, renderSuiteReportMarkdown, renderTraceStats, resolveBundleRunIds, resolveSuiteCaseTrace, resolveSuiteConfigPath, resolveSuiteTemplate, runGate, runSuite, sanitizeBundleRunId, searchTraces, sessionKeyForRun, toMetadataSafeStatus, traceMetasToSessionRunRecords, validateSuiteConfig };
|
|
3248
3301
|
//# sourceMappingURL=advanced.mjs.map
|
|
3249
3302
|
//# sourceMappingURL=advanced.mjs.map
|