@sanity/ailf 2.9.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_vendor/ailf-core/artifact-registry.d.ts +1 -1
- package/dist/_vendor/ailf-core/artifact-registry.js +1 -18
- package/dist/_vendor/ailf-core/batch-signing.d.ts +64 -0
- package/dist/_vendor/ailf-core/batch-signing.js +23 -0
- package/dist/_vendor/ailf-core/index.d.ts +1 -1
- package/dist/_vendor/ailf-core/index.js +1 -1
- package/dist/_vendor/ailf-core/ports/context.d.ts +4 -20
- package/dist/_vendor/ailf-core/ports/index.d.ts +0 -2
- package/dist/adapters/config-sources/file-config-adapter.js +0 -4
- package/dist/artifact-capture/redact-artifact.d.ts +3 -5
- package/dist/artifact-capture/redact-artifact.js +3 -5
- package/dist/cli.js +56 -2
- package/dist/commands/explain-handler.js +1 -5
- package/dist/commands/pipeline-action.d.ts +0 -4
- package/dist/commands/pipeline-action.js +11 -45
- package/dist/commands/pipeline.d.ts +1 -5
- package/dist/commands/pipeline.js +1 -5
- package/dist/commands/runs.d.ts +18 -0
- package/dist/commands/runs.js +71 -0
- package/dist/composition-root.js +2 -28
- package/dist/orchestration/build-app-context.js +4 -7
- package/dist/orchestration/pipeline-orchestrator.js +3 -23
- package/dist/pipeline/map-request-to-config.js +0 -4
- package/package.json +1 -1
- package/dist/_vendor/ailf-core/artifact-capture/noop-collector.d.ts +0 -14
- package/dist/_vendor/ailf-core/artifact-capture/noop-collector.js +0 -25
- package/dist/_vendor/ailf-core/ports/artifact-collector.d.ts +0 -94
- package/dist/_vendor/ailf-core/ports/artifact-collector.js +0 -13
- package/dist/_vendor/ailf-core/ports/capture-comparator.d.ts +0 -138
- package/dist/_vendor/ailf-core/ports/capture-comparator.js +0 -10
- package/dist/artifact-capture/comparator.d.ts +0 -22
- package/dist/artifact-capture/comparator.js +0 -493
- package/dist/artifact-capture/filesystem-collector.d.ts +0 -60
- package/dist/artifact-capture/filesystem-collector.js +0 -262
- package/dist/artifact-capture/gcs-collector.d.ts +0 -55
- package/dist/artifact-capture/gcs-collector.js +0 -117
- package/dist/commands/capture-compare.d.ts +0 -15
- package/dist/commands/capture-compare.js +0 -253
- package/dist/commands/capture-list.d.ts +0 -12
- package/dist/commands/capture-list.js +0 -150
- package/dist/commands/capture.d.ts +0 -9
- package/dist/commands/capture.js +0 -16
|
@@ -41,7 +41,7 @@ export type ArtifactMime = "application/json" | "application/x-ndjson" | "text/m
|
|
|
41
41
|
*/
|
|
42
42
|
export type ArtifactTruncationPolicy = "reject" | "trailing-truncate" | "fielded-truncate" | "trial-oversize";
|
|
43
43
|
/** The union of every artifact type known to AILF. */
|
|
44
|
-
export type ArtifactType = "runManifest" | "scoreSummary" | "pipelineResult" | "pipelineContext" | "documentManifest" | "prComment" | "readinessReport" | "reportSnapshot" | "autoComparison" | "gapReport" | "sinkResults" | "callbackRequest" | "callbackResponse" | "configSnapshot" | "evalConfigGenerated" | "comparisonReport" | "discoveryReport" | "failureModes" | "taskDefinitions" | "renderedPrompts" | "rawResults" | "testOutputs" | "graderPrompts" | "graderJudgments" | "traces"
|
|
44
|
+
export type ArtifactType = "runManifest" | "scoreSummary" | "pipelineResult" | "pipelineContext" | "documentManifest" | "prComment" | "readinessReport" | "reportSnapshot" | "autoComparison" | "gapReport" | "sinkResults" | "callbackRequest" | "callbackResponse" | "configSnapshot" | "evalConfigGenerated" | "comparisonReport" | "discoveryReport" | "failureModes" | "taskDefinitions" | "renderedPrompts" | "rawResults" | "testOutputs" | "graderPrompts" | "graderJudgments" | "traces";
|
|
45
45
|
/**
|
|
46
46
|
* Result of parsing a per-entry key into a sanitized filename component.
|
|
47
47
|
* Success carries the sanitized value; failure carries a reason for 4xx responses.
|
|
@@ -333,7 +333,7 @@ function buildDescriptor(input) {
|
|
|
333
333
|
};
|
|
334
334
|
}
|
|
335
335
|
// ---------------------------------------------------------------------------
|
|
336
|
-
// The registry — 21 live descriptors
|
|
336
|
+
// The registry — 21 live descriptors
|
|
337
337
|
// ---------------------------------------------------------------------------
|
|
338
338
|
/**
|
|
339
339
|
* The canonical artifact descriptor for every artifact type. Iterate with
|
|
@@ -630,23 +630,6 @@ export const ARTIFACT_REGISTRY = {
|
|
|
630
630
|
capBytes: 10_000_000,
|
|
631
631
|
truncation: "trial-oversize",
|
|
632
632
|
}),
|
|
633
|
-
/**
|
|
634
|
-
* @deprecated Emit removed in W0050 (no producer calls `emit("evalResults")`
|
|
635
|
-
* any more — `emit-eval-results.ts` decomposes the promptfoo aggregate into
|
|
636
|
-
* per-entry rawResults / renderedPrompts / graderPrompts / graderJudgments
|
|
637
|
-
* instead). Descriptor retained for read-compat on pre-W0050 reports until
|
|
638
|
-
* W0052 removes it entirely. No code path should re-introduce emission.
|
|
639
|
-
*/
|
|
640
|
-
evalResults: buildDescriptor({
|
|
641
|
-
type: "evalResults",
|
|
642
|
-
slug: "eval-results",
|
|
643
|
-
layout: "bulk",
|
|
644
|
-
axes: ["run"],
|
|
645
|
-
entrySchema: unknownEntry,
|
|
646
|
-
mime: "application/json",
|
|
647
|
-
capBytes: 10_000_000,
|
|
648
|
-
optional: true,
|
|
649
|
-
}),
|
|
650
633
|
};
|
|
651
634
|
/** All artifact types in declaration order. */
|
|
652
635
|
export const ARTIFACT_TYPES = Object.keys(ARTIFACT_REGISTRY);
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Batch signed-URL types (D0033 / W0052 M3).
|
|
3
|
+
*
|
|
4
|
+
* Contract shared between the API Gateway and Studio for the two batch
|
|
5
|
+
* endpoints that amortise signing over many artifacts in a single round-trip:
|
|
6
|
+
*
|
|
7
|
+
* POST /v1/runs/:runId/artifacts/batch/read-urls
|
|
8
|
+
* POST /v1/runs/:runId/artifacts/batch/upload-urls
|
|
9
|
+
*
|
|
10
|
+
* The batch shape is intentionally symmetric: request carries the set of
|
|
11
|
+
* artifact types and (for per-entry layouts) the entry keys; response carries
|
|
12
|
+
* signed URLs keyed by type and entry key. For bulk layouts the response key
|
|
13
|
+
* is the empty string, since bulk artifacts have no entry dimension.
|
|
14
|
+
*
|
|
15
|
+
* Validation is all-or-nothing per AC 4 — a single malformed entry key, an
|
|
16
|
+
* unknown artifact type, or a missing keys list for a per-entry type causes
|
|
17
|
+
* the whole request to 400 with zero signed URLs emitted.
|
|
18
|
+
*
|
|
19
|
+
* @see docs/design-docs/unified-run-artifacts.md § M3
|
|
20
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
|
|
21
|
+
*/
|
|
22
|
+
import type { ArtifactType } from "./artifact-registry.js";
|
|
23
|
+
/** The empty entry-key sentinel used for bulk responses. */
|
|
24
|
+
export declare const BULK_ENTRY_KEY: "";
|
|
25
|
+
/**
|
|
26
|
+
* Batch request body.
|
|
27
|
+
*
|
|
28
|
+
* - `types` — the artifact types to sign. Must be non-empty. Unknown types
|
|
29
|
+
* cause a 400.
|
|
30
|
+
* - `keys` — per-type arrays of entry keys. Required for per-entry layouts;
|
|
31
|
+
* omitted (or an empty array) for bulk layouts. Extra keys for a bulk type
|
|
32
|
+
* are rejected; missing keys for a per-entry type are rejected.
|
|
33
|
+
*/
|
|
34
|
+
export interface BatchSignRequest {
|
|
35
|
+
readonly types: readonly ArtifactType[];
|
|
36
|
+
readonly keys?: Partial<Record<ArtifactType, readonly string[]>>;
|
|
37
|
+
}
|
|
38
|
+
/** One signed read URL in a batch read response. */
|
|
39
|
+
export interface BatchSignedReadUrl {
|
|
40
|
+
readonly url: string;
|
|
41
|
+
readonly path: string;
|
|
42
|
+
readonly expiresIn: number;
|
|
43
|
+
}
|
|
44
|
+
/** One signed upload URL in a batch upload response. */
|
|
45
|
+
export interface BatchSignedUploadUrl {
|
|
46
|
+
readonly url: string;
|
|
47
|
+
readonly method: "PUT";
|
|
48
|
+
readonly path: string;
|
|
49
|
+
readonly expiresIn: number;
|
|
50
|
+
readonly requiredHeaders: Readonly<Record<string, string>>;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Batch response. Outer map is keyed by artifact type; inner map is keyed by
|
|
54
|
+
* entry key (empty string for bulk layouts). All types requested in the
|
|
55
|
+
* request body are present in the response, even when `keys[type]` was empty.
|
|
56
|
+
*/
|
|
57
|
+
export interface BatchSignReadResponse {
|
|
58
|
+
readonly bucket: string;
|
|
59
|
+
readonly urls: Partial<Record<ArtifactType, Readonly<Record<string, BatchSignedReadUrl>>>>;
|
|
60
|
+
}
|
|
61
|
+
export interface BatchSignUploadResponse {
|
|
62
|
+
readonly bucket: string;
|
|
63
|
+
readonly urls: Partial<Record<ArtifactType, Readonly<Record<string, BatchSignedUploadUrl>>>>;
|
|
64
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Batch signed-URL types (D0033 / W0052 M3).
|
|
3
|
+
*
|
|
4
|
+
* Contract shared between the API Gateway and Studio for the two batch
|
|
5
|
+
* endpoints that amortise signing over many artifacts in a single round-trip:
|
|
6
|
+
*
|
|
7
|
+
* POST /v1/runs/:runId/artifacts/batch/read-urls
|
|
8
|
+
* POST /v1/runs/:runId/artifacts/batch/upload-urls
|
|
9
|
+
*
|
|
10
|
+
* The batch shape is intentionally symmetric: request carries the set of
|
|
11
|
+
* artifact types and (for per-entry layouts) the entry keys; response carries
|
|
12
|
+
* signed URLs keyed by type and entry key. For bulk layouts the response key
|
|
13
|
+
* is the empty string, since bulk artifacts have no entry dimension.
|
|
14
|
+
*
|
|
15
|
+
* Validation is all-or-nothing per AC 4 — a single malformed entry key, an
|
|
16
|
+
* unknown artifact type, or a missing keys list for a per-entry type causes
|
|
17
|
+
* the whole request to 400 with zero signed URLs emitted.
|
|
18
|
+
*
|
|
19
|
+
* @see docs/design-docs/unified-run-artifacts.md § M3
|
|
20
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
|
|
21
|
+
*/
|
|
22
|
+
/** The empty entry-key sentinel used for bulk responses. */
|
|
23
|
+
export const BULK_ENTRY_KEY = "";
|
|
@@ -16,9 +16,9 @@ export * from "./ports/index.js";
|
|
|
16
16
|
export * from "./services/index.js";
|
|
17
17
|
export * from "./examples/index.js";
|
|
18
18
|
export * from "./artifact-registry.js";
|
|
19
|
+
export * from "./batch-signing.js";
|
|
19
20
|
export { defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
|
|
20
21
|
export type { PricingEntry, PromptEntry, SourceEntry, } from "./config-helpers.js";
|
|
21
22
|
export { env } from "./env-helper.js";
|
|
22
|
-
export { NoOpArtifactCollector } from "./artifact-capture/noop-collector.js";
|
|
23
23
|
export { NoOpArtifactWriter, NotImplementedError, } from "./ports/artifact-writer.js";
|
|
24
24
|
export { assoc, type AssocContext } from "./artifact-capture/association.js";
|
|
@@ -16,11 +16,11 @@ export * from "./ports/index.js";
|
|
|
16
16
|
export * from "./services/index.js";
|
|
17
17
|
export * from "./examples/index.js";
|
|
18
18
|
export * from "./artifact-registry.js";
|
|
19
|
+
export * from "./batch-signing.js";
|
|
19
20
|
// ---------------------------------------------------------------------------
|
|
20
21
|
// Architecture overhaul — Phase 0 helpers
|
|
21
22
|
// ---------------------------------------------------------------------------
|
|
22
23
|
export { defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
|
|
23
24
|
export { env } from "./env-helper.js";
|
|
24
|
-
export { NoOpArtifactCollector } from "./artifact-capture/noop-collector.js";
|
|
25
25
|
export { NoOpArtifactWriter, NotImplementedError, } from "./ports/artifact-writer.js";
|
|
26
26
|
export { assoc } from "./artifact-capture/association.js";
|
|
@@ -13,7 +13,6 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import type { RunId } from "../types/branded-ids.js";
|
|
15
15
|
import type { DebugOptions, EvalMode, PluginRegistry } from "../types/index.js";
|
|
16
|
-
import type { ArtifactCollector } from "./artifact-collector.js";
|
|
17
16
|
import type { ArtifactWriter } from "./artifact-writer.js";
|
|
18
17
|
import type { CacheStore } from "./cache-store.js";
|
|
19
18
|
import type { DocFetcher } from "./doc-fetcher.js";
|
|
@@ -153,19 +152,10 @@ export interface ResolvedConfig {
|
|
|
153
152
|
apiKey?: string;
|
|
154
153
|
/** External preset file paths or npm package names to load */
|
|
155
154
|
presets?: string[];
|
|
156
|
-
/** Whether artifact capture is enabled for this run (default: false) */
|
|
157
|
-
captureEnabled?: boolean;
|
|
158
|
-
/** Base directory for capture output (default: results/captures/) */
|
|
159
|
-
captureDir?: string;
|
|
160
|
-
/** Whether to compress capture output to tar.gz (default: true) */
|
|
161
|
-
captureCompress?: boolean;
|
|
162
|
-
/** Whether to include mode-specific extra artifacts (default: true) */
|
|
163
|
-
captureExtras?: boolean;
|
|
164
155
|
/**
|
|
165
|
-
* D0033
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
* additive and do not replace the legacy `capture*` fields until W0052.
|
|
156
|
+
* D0033 unified artifact surface. Consumed by the writer factory to
|
|
157
|
+
* decide whether to attach a writer at all, where it writes to, and
|
|
158
|
+
* what to skip. Legacy `capture*` fields were retired in W0052.
|
|
169
159
|
*/
|
|
170
160
|
/** Disables all artifact writers — `--no-artifacts`. */
|
|
171
161
|
artifactsDisabled?: boolean;
|
|
@@ -173,12 +163,8 @@ export interface ResolvedConfig {
|
|
|
173
163
|
artifactsDir?: string;
|
|
174
164
|
/** Run writers in dry-run mode — `--artifacts-dry-run`. */
|
|
175
165
|
artifactsDryRun?: boolean;
|
|
176
|
-
/** Comma-separated artifact types to skip — `--
|
|
166
|
+
/** Comma-separated artifact types to skip — `--artifacts-exclude`. */
|
|
177
167
|
artifactsExclude?: readonly string[];
|
|
178
|
-
/** GCS bucket for capture upload (enables GCS decorator when set) */
|
|
179
|
-
captureGcsBucket?: string;
|
|
180
|
-
/** GCS object prefix for capture uploads (default: "captures/") */
|
|
181
|
-
captureGcsPrefix?: string;
|
|
182
168
|
/**
|
|
183
169
|
* GCS bucket for report artifact uploads. Defaults to "ailf-artifacts"
|
|
184
170
|
* at the composition root — only set this to override (e.g., self-hosted
|
|
@@ -221,8 +207,6 @@ export interface AppContext {
|
|
|
221
207
|
readonly artifactWriter: ArtifactWriter;
|
|
222
208
|
/** Evaluation caching (filesystem + optional Content Lake fallback) */
|
|
223
209
|
readonly cache?: CacheStore;
|
|
224
|
-
/** Artifact capture collector (no-op when --capture is not set) */
|
|
225
|
-
readonly collector: ArtifactCollector;
|
|
226
210
|
/** Resolved pipeline configuration */
|
|
227
211
|
readonly config: ResolvedConfig;
|
|
228
212
|
/** Documentation context fetcher */
|
|
@@ -4,10 +4,8 @@
|
|
|
4
4
|
* Ports define the contracts between the domain kernel and the outside world.
|
|
5
5
|
* Adapters (in packages/eval) implement these interfaces.
|
|
6
6
|
*/
|
|
7
|
-
export type { ArtifactCollector, CaptureFlushResult, CaptureManifest, CaptureManifestEntry, } from "./artifact-collector.js";
|
|
8
7
|
export type { ArtifactEntry, ArtifactWriter } from "./artifact-writer.js";
|
|
9
8
|
export { NoOpArtifactWriter } from "./artifact-writer.js";
|
|
10
|
-
export type { ArtifactContentDiff, CaptureDiffReport, ComparisonMode, ComparisonOptions, InventoryDiff, JsonDiffEntry, MetadataComparison, ScoreComparison, SecurityScan, TimingComparison, } from "./capture-comparator.js";
|
|
11
9
|
export type { CacheEntryMetadata, CacheKey, CacheLookupResult, CacheRecordInput, CacheStore, } from "./cache-store.js";
|
|
12
10
|
export type { ConfigSource } from "./config-source.js";
|
|
13
11
|
export type { AppContext, ReportSinkPort, ReportStorePort, ResolvedConfig, } from "./context.js";
|
|
@@ -120,10 +120,6 @@ function mapEvalConfigToResolvedConfig(config, rootDir) {
|
|
|
120
120
|
allowedOrigins: config.allowedOrigins,
|
|
121
121
|
searchMode: config.searchMode ?? "open",
|
|
122
122
|
concurrency: config.concurrency,
|
|
123
|
-
captureEnabled: false,
|
|
124
|
-
captureDir: undefined,
|
|
125
|
-
captureCompress: true,
|
|
126
|
-
captureExtras: true,
|
|
127
123
|
remote: false,
|
|
128
124
|
apiUrl: "https://ailf-api.sanity.build",
|
|
129
125
|
presets: config.presets,
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Artifact redaction — strips sensitive data from
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* Applied during FilesystemArtifactCollector.flush() so that secrets
|
|
6
|
-
* (resolved env vars, auth headers, session cookies) never reach storage.
|
|
2
|
+
* Artifact redaction — strips sensitive data from written artifacts so that
|
|
3
|
+
* secrets (resolved env vars, auth headers, session cookies) never reach
|
|
4
|
+
* local or remote storage.
|
|
7
5
|
*
|
|
8
6
|
* Two-layer approach:
|
|
9
7
|
* 1. **Header stripping** — known-sensitive HTTP header keys are replaced
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Artifact redaction — strips sensitive data from
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* Applied during FilesystemArtifactCollector.flush() so that secrets
|
|
6
|
-
* (resolved env vars, auth headers, session cookies) never reach storage.
|
|
2
|
+
* Artifact redaction — strips sensitive data from written artifacts so that
|
|
3
|
+
* secrets (resolved env vars, auth headers, session cookies) never reach
|
|
4
|
+
* local or remote storage.
|
|
7
5
|
*
|
|
8
6
|
* Two-layer approach:
|
|
9
7
|
* 1. **Header stripping** — known-sensitive HTTP header keys are replaced
|
package/dist/cli.js
CHANGED
|
@@ -76,6 +76,60 @@ else if (process.argv.includes("--quiet") || process.argv.includes("-q")) {
|
|
|
76
76
|
process.env.AILF_LOG_LEVEL = "quiet";
|
|
77
77
|
}
|
|
78
78
|
// ---------------------------------------------------------------------------
|
|
79
|
+
// W0052 — hard-error on retired capture flags and env vars.
|
|
80
|
+
// --------------------------------------------------------------------------
|
|
81
|
+
// The legacy collector has been removed. Callers still using
|
|
82
|
+
// --capture / --capture-dir / --no-capture-compress / --no-capture-extras
|
|
83
|
+
// or AILF_CAPTURE* / AILF_LEGACY_COLLECTOR / AILF_UNIFIED_ARTIFACTS must
|
|
84
|
+
// migrate to --artifacts-dir / --no-artifacts / --artifacts-exclude. We
|
|
85
|
+
// print a clear pointer so failures don't bubble up as opaque "unknown
|
|
86
|
+
// option" errors from Commander.
|
|
87
|
+
// ---------------------------------------------------------------------------
|
|
88
|
+
const RETIRED_FLAGS = [
|
|
89
|
+
"--capture",
|
|
90
|
+
"--capture-dir",
|
|
91
|
+
"--no-capture-compress",
|
|
92
|
+
"--no-capture-extras",
|
|
93
|
+
"--capture-exclude",
|
|
94
|
+
];
|
|
95
|
+
const RETIRED_ENV_VARS = [
|
|
96
|
+
"AILF_CAPTURE",
|
|
97
|
+
"AILF_CAPTURE_DIR",
|
|
98
|
+
"AILF_CAPTURE_COMPRESS",
|
|
99
|
+
"AILF_CAPTURE_EXTRAS",
|
|
100
|
+
"AILF_CAPTURE_GCS_BUCKET",
|
|
101
|
+
"AILF_CAPTURE_GCS_PREFIX",
|
|
102
|
+
"AILF_LEGACY_COLLECTOR",
|
|
103
|
+
"AILF_UNIFIED_ARTIFACTS",
|
|
104
|
+
];
|
|
105
|
+
function findRetiredFlag() {
|
|
106
|
+
for (const arg of process.argv) {
|
|
107
|
+
const bare = arg.split("=")[0];
|
|
108
|
+
if (RETIRED_FLAGS.includes(bare)) {
|
|
109
|
+
return bare;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return undefined;
|
|
113
|
+
}
|
|
114
|
+
function findRetiredEnv() {
|
|
115
|
+
for (const name of RETIRED_ENV_VARS) {
|
|
116
|
+
if (process.env[name] !== undefined)
|
|
117
|
+
return name;
|
|
118
|
+
}
|
|
119
|
+
return undefined;
|
|
120
|
+
}
|
|
121
|
+
const retiredFlag = findRetiredFlag();
|
|
122
|
+
const retiredEnv = findRetiredEnv();
|
|
123
|
+
if (retiredFlag || retiredEnv) {
|
|
124
|
+
const source = retiredFlag
|
|
125
|
+
? `flag "${retiredFlag}"`
|
|
126
|
+
: `environment variable "${retiredEnv}"`;
|
|
127
|
+
console.error(`❌ ${source} was retired in W0052 along with the legacy artifact collector.`);
|
|
128
|
+
console.error(" Use --artifacts-dir / --no-artifacts / --artifacts-exclude instead.");
|
|
129
|
+
console.error(" See docs/cli.md and docs/decisions/D0033-unified-run-anchored-artifact-capture.md.");
|
|
130
|
+
process.exit(2);
|
|
131
|
+
}
|
|
132
|
+
// ---------------------------------------------------------------------------
|
|
79
133
|
// Build CLI program
|
|
80
134
|
// ---------------------------------------------------------------------------
|
|
81
135
|
import { Command } from "commander";
|
|
@@ -134,6 +188,8 @@ import { createBaselineCommand } from "./commands/baseline.js";
|
|
|
134
188
|
program.addCommand(createBaselineCommand().helpGroup(CommandGroup.CoreWorkflow));
|
|
135
189
|
import { createPublishCommand } from "./commands/publish.js";
|
|
136
190
|
program.addCommand(createPublishCommand().helpGroup(CommandGroup.CoreWorkflow));
|
|
191
|
+
import { createRunsCommand } from "./commands/runs.js";
|
|
192
|
+
program.addCommand(createRunsCommand().helpGroup(CommandGroup.CoreWorkflow));
|
|
137
193
|
// ── Analysis & Reports ────────────────────────────────────────────────
|
|
138
194
|
import { createReadinessReportCommand } from "./commands/readiness-report.js";
|
|
139
195
|
program.addCommand(createReadinessReportCommand().helpGroup(CommandGroup.AnalysisReports));
|
|
@@ -179,8 +235,6 @@ program.addCommand(createLookupDocCommand().helpGroup(CommandGroup.PipelineInter
|
|
|
179
235
|
import { createWebhookServerCommand } from "./commands/webhook-server.js";
|
|
180
236
|
program.addCommand(createWebhookServerCommand().helpGroup(CommandGroup.PipelineInternals));
|
|
181
237
|
// ── Developer Tools ───────────────────────────────────────────────────
|
|
182
|
-
import { createCaptureCommand } from "./commands/capture.js";
|
|
183
|
-
program.addCommand(createCaptureCommand().helpGroup(CommandGroup.DeveloperTools));
|
|
184
238
|
import { createInteractiveCommand } from "./commands/interactive.js";
|
|
185
239
|
program.addCommand(createInteractiveCommand().helpGroup(CommandGroup.DeveloperTools));
|
|
186
240
|
// Shell completion — must be registered last (needs full program tree)
|
|
@@ -723,14 +723,10 @@ async function buildPipelineExplainPlan(actionCommand, rootDir) {
|
|
|
723
723
|
taskSource: raw.taskSource,
|
|
724
724
|
remoteCache: raw.remoteCache,
|
|
725
725
|
config: raw.config,
|
|
726
|
-
capture: raw.capture ?? false,
|
|
727
|
-
captureCompress: raw.captureCompress ?? true,
|
|
728
|
-
captureExtras: raw.captureExtras ?? true,
|
|
729
|
-
captureDir: raw.captureDir,
|
|
730
726
|
artifacts: raw.artifacts ?? true,
|
|
731
727
|
artifactsDir: raw.artifactsDir,
|
|
732
728
|
artifactsDryRun: raw.artifactsDryRun ?? false,
|
|
733
|
-
|
|
729
|
+
artifactsExclude: raw.artifactsExclude,
|
|
734
730
|
};
|
|
735
731
|
const resolved = computeResolvedOptions(withDefaults);
|
|
736
732
|
const planOpts = {
|
|
@@ -63,10 +63,6 @@ export interface ResolvedOptions {
|
|
|
63
63
|
urlArgs: string[];
|
|
64
64
|
apiUrl: string;
|
|
65
65
|
apiKey?: string;
|
|
66
|
-
captureEnabled: boolean;
|
|
67
|
-
captureDir?: string;
|
|
68
|
-
captureCompress: boolean;
|
|
69
|
-
captureExtras: boolean;
|
|
70
66
|
/** D0033 / W0049 — unified artifact surface (W0050 wires it into writer). */
|
|
71
67
|
artifactsDisabled: boolean;
|
|
72
68
|
artifactsDir?: string;
|
|
@@ -263,32 +263,25 @@ export function computeResolvedOptions(opts) {
|
|
|
263
263
|
tagOption,
|
|
264
264
|
taskSourceType: resolvedTaskSourceType,
|
|
265
265
|
urlArgs,
|
|
266
|
-
captureEnabled: opts.capture,
|
|
267
|
-
captureDir: resolveArtifactsDir(opts),
|
|
268
|
-
captureCompress: opts.captureCompress !== false &&
|
|
269
|
-
process.env.AILF_CAPTURE_COMPRESS !== "0",
|
|
270
|
-
captureExtras: opts.captureExtras !== false && process.env.AILF_CAPTURE_EXTRAS !== "0",
|
|
271
266
|
artifactsDisabled: opts.artifacts === false,
|
|
272
267
|
artifactsDir: resolveArtifactsDir(opts),
|
|
273
268
|
artifactsDryRun: opts.artifactsDryRun,
|
|
274
|
-
artifactsExclude:
|
|
269
|
+
artifactsExclude: parseArtifactsExcludeList(opts.artifactsExclude),
|
|
275
270
|
};
|
|
276
271
|
}
|
|
277
272
|
/**
|
|
278
|
-
* Resolve the artifacts
|
|
279
|
-
*
|
|
273
|
+
* Resolve the artifacts output directory from CLI flags and env vars.
|
|
274
|
+
* Precedence (highest first):
|
|
280
275
|
* 1. `--artifacts-dir` flag
|
|
281
|
-
* 2.
|
|
282
|
-
*
|
|
283
|
-
*
|
|
276
|
+
* 2. `AILF_ARTIFACTS_DIR` env var
|
|
277
|
+
*
|
|
278
|
+
* The `--capture-dir` / `AILF_CAPTURE_DIR` aliases were retired in W0052;
|
|
279
|
+
* callers of those names are rejected at CLI entry (see cli.ts).
|
|
284
280
|
*/
|
|
285
281
|
function resolveArtifactsDir(opts) {
|
|
286
|
-
return
|
|
287
|
-
opts.captureDir ??
|
|
288
|
-
process.env.AILF_ARTIFACTS_DIR ??
|
|
289
|
-
process.env.AILF_CAPTURE_DIR);
|
|
282
|
+
return opts.artifactsDir ?? process.env.AILF_ARTIFACTS_DIR;
|
|
290
283
|
}
|
|
291
|
-
function
|
|
284
|
+
function parseArtifactsExcludeList(raw) {
|
|
292
285
|
if (!raw)
|
|
293
286
|
return undefined;
|
|
294
287
|
const list = raw
|
|
@@ -309,20 +302,6 @@ function resolveTaskSourceType(raw) {
|
|
|
309
302
|
// ---------------------------------------------------------------------------
|
|
310
303
|
// Pipeline entry point
|
|
311
304
|
// ---------------------------------------------------------------------------
|
|
312
|
-
/**
|
|
313
|
-
* Module-level flag so the `--capture` deprecation warning fires exactly
|
|
314
|
-
* once per process even when `executePipeline` is invoked multiple times
|
|
315
|
-
* (e.g. tests, long-lived dev loops).
|
|
316
|
-
*/
|
|
317
|
-
let warnedCaptureDeprecation = false;
|
|
318
|
-
function warnCaptureDeprecationIfNeeded(cliOpts) {
|
|
319
|
-
if (!cliOpts.capture)
|
|
320
|
-
return;
|
|
321
|
-
if (warnedCaptureDeprecation)
|
|
322
|
-
return;
|
|
323
|
-
warnedCaptureDeprecation = true;
|
|
324
|
-
console.warn("--capture is deprecated and will be removed in a future release; use --artifacts-dir or --no-artifacts instead");
|
|
325
|
-
}
|
|
326
305
|
/**
|
|
327
306
|
* Execute the evaluation pipeline.
|
|
328
307
|
*
|
|
@@ -332,7 +311,6 @@ function warnCaptureDeprecationIfNeeded(cliOpts) {
|
|
|
332
311
|
* 4. Delegate to the PipelineOrchestrator
|
|
333
312
|
*/
|
|
334
313
|
export async function executePipeline(cliOpts) {
|
|
335
|
-
warnCaptureDeprecationIfNeeded(cliOpts);
|
|
336
314
|
// When --config is provided, resolve config from file instead of CLI flags
|
|
337
315
|
if (cliOpts.config) {
|
|
338
316
|
const { existsSync } = await import("fs");
|
|
@@ -357,25 +335,13 @@ export async function executePipeline(cliOpts) {
|
|
|
357
335
|
}
|
|
358
336
|
// Output dir: explicit CLI flag → $CWD/.ailf/results/latest/
|
|
359
337
|
config.outputDir = resolveOutputDir(cliOpts.outputDir);
|
|
360
|
-
//
|
|
338
|
+
// Artifact options — CLI flags and env vars aren't in the config file,
|
|
361
339
|
// so merge them here (same logic as resolveOptions).
|
|
362
|
-
// AILF_CAPTURE is a no-op post-W0049; only the flag toggles captureEnabled.
|
|
363
|
-
config.captureEnabled = cliOpts.capture;
|
|
364
340
|
const resolvedArtifactsDir = resolveArtifactsDir(cliOpts);
|
|
365
|
-
if (resolvedArtifactsDir) {
|
|
366
|
-
config.captureDir = resolvedArtifactsDir;
|
|
367
|
-
}
|
|
368
|
-
config.captureCompress =
|
|
369
|
-
cliOpts.captureCompress !== false &&
|
|
370
|
-
process.env.AILF_CAPTURE_COMPRESS !== "0";
|
|
371
|
-
config.captureExtras =
|
|
372
|
-
cliOpts.captureExtras !== false && process.env.AILF_CAPTURE_EXTRAS !== "0";
|
|
373
|
-
config.captureGcsBucket ??= process.env.AILF_CAPTURE_GCS_BUCKET;
|
|
374
|
-
config.captureGcsPrefix ??= process.env.AILF_CAPTURE_GCS_PREFIX;
|
|
375
341
|
config.artifactsDisabled ??= cliOpts.artifacts === false;
|
|
376
342
|
config.artifactsDir ??= resolvedArtifactsDir;
|
|
377
343
|
config.artifactsDryRun ??= cliOpts.artifactsDryRun;
|
|
378
|
-
const excludeList =
|
|
344
|
+
const excludeList = parseArtifactsExcludeList(cliOpts.artifactsExclude);
|
|
379
345
|
if (excludeList) {
|
|
380
346
|
config.artifactsExclude = excludeList;
|
|
381
347
|
}
|
|
@@ -64,13 +64,9 @@ export interface PipelineCliOptions {
|
|
|
64
64
|
url: string[];
|
|
65
65
|
urls: string[];
|
|
66
66
|
apiUrl?: string;
|
|
67
|
-
capture: boolean;
|
|
68
|
-
captureDir?: string;
|
|
69
|
-
captureCompress: boolean;
|
|
70
|
-
captureExtras: boolean;
|
|
71
67
|
artifacts: boolean;
|
|
72
68
|
artifactsDir?: string;
|
|
73
69
|
artifactsDryRun: boolean;
|
|
74
|
-
|
|
70
|
+
artifactsExclude?: string;
|
|
75
71
|
}
|
|
76
72
|
export declare function createPipelineCommand(): Command;
|
|
@@ -54,14 +54,10 @@ export function createPipelineCommand() {
|
|
|
54
54
|
.option("--repo-tasks-path <path>", "Path to repo-based task definitions (.ailf/tasks/ directory)")
|
|
55
55
|
.option("--remote", "Submit evaluation to the AILF API instead of running locally", false)
|
|
56
56
|
.option("--api-url <url>", "AILF API base URL (default: https://ailf-api.sanity.build)")
|
|
57
|
-
.option("--capture", "[DEPRECATED] Enable legacy artifact capture. Use --artifacts-dir / --no-artifacts instead.", false)
|
|
58
|
-
.option("--capture-dir <path>", "[DEPRECATED] Alias for --artifacts-dir.")
|
|
59
|
-
.option("--no-capture-compress", "Disable tar.gz compression of captures")
|
|
60
|
-
.option("--no-capture-extras", "Exclude mode-specific artifacts from captures")
|
|
61
57
|
.option("--no-artifacts", "Disable all artifact writers (D0033). Overrides --artifacts-dir.")
|
|
62
58
|
.option("--artifacts-dir <path>", "Root directory for local artifact output (D0033; default: .ailf/results/captures/)")
|
|
63
59
|
.option("--artifacts-dry-run", "Run artifact writers in dry-run mode — log intended writes, touch no storage", false)
|
|
64
|
-
.option("--
|
|
60
|
+
.option("--artifacts-exclude <types>", "Comma-separated artifact types to skip (e.g. traces,graderPrompts)")
|
|
65
61
|
.action(async (opts) => {
|
|
66
62
|
const { executePipeline } = await import("./pipeline-action.js");
|
|
67
63
|
await executePipeline(opts);
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* runs command — utilities for working with run-anchored artifacts on disk.
|
|
3
|
+
*
|
|
4
|
+
* The unified artifact writer (D0033 M4) lays artifacts down under
|
|
5
|
+
* `.ailf/results/captures/runs/{runId}/…`. Downstream tooling (CI archivers,
|
|
6
|
+
* external consumers that used to parse the legacy collector tarball) still
|
|
7
|
+
* want a single-file bundle. `runs export --format tarball` reproduces that
|
|
8
|
+
* bundle from the existing tree — no re-materialisation, just a tar of the
|
|
9
|
+
* files already written.
|
|
10
|
+
*
|
|
11
|
+
* The on-disk layout is identical to what the legacy FilesystemArtifactCollector
|
|
12
|
+
* produced (same paths, same filenames), so the tarball shape is stable for
|
|
13
|
+
* existing consumers.
|
|
14
|
+
*
|
|
15
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
|
|
16
|
+
*/
|
|
17
|
+
import { Command } from "commander";
|
|
18
|
+
export declare function createRunsCommand(): Command;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* runs command — utilities for working with run-anchored artifacts on disk.
|
|
3
|
+
*
|
|
4
|
+
* The unified artifact writer (D0033 M4) lays artifacts down under
|
|
5
|
+
* `.ailf/results/captures/runs/{runId}/…`. Downstream tooling (CI archivers,
|
|
6
|
+
* external consumers that used to parse the legacy collector tarball) still
|
|
7
|
+
* want a single-file bundle. `runs export --format tarball` reproduces that
|
|
8
|
+
* bundle from the existing tree — no re-materialisation, just a tar of the
|
|
9
|
+
* files already written.
|
|
10
|
+
*
|
|
11
|
+
* The on-disk layout is identical to what the legacy FilesystemArtifactCollector
|
|
12
|
+
* produced (same paths, same filenames), so the tarball shape is stable for
|
|
13
|
+
* existing consumers.
|
|
14
|
+
*
|
|
15
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
|
|
16
|
+
*/
|
|
17
|
+
import { spawnSync } from "node:child_process";
|
|
18
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
19
|
+
import { dirname, resolve } from "node:path";
|
|
20
|
+
import { Command } from "commander";
|
|
21
|
+
const DEFAULT_ARTIFACTS_DIR = ".ailf/results/captures";
|
|
22
|
+
function resolveArtifactsDir(flag) {
|
|
23
|
+
return resolve(flag ?? process.env.AILF_ARTIFACTS_DIR ?? DEFAULT_ARTIFACTS_DIR);
|
|
24
|
+
}
|
|
25
|
+
function resolveOutputPath(explicit, artifactsDir, runId) {
|
|
26
|
+
if (explicit)
|
|
27
|
+
return resolve(explicit);
|
|
28
|
+
return resolve(artifactsDir, `runs-${runId}.tar.gz`);
|
|
29
|
+
}
|
|
30
|
+
function exportTarball(opts) {
|
|
31
|
+
if (opts.format !== "tarball") {
|
|
32
|
+
console.error(`❌ Unsupported --format "${opts.format}". Supported: tarball.`);
|
|
33
|
+
return 2;
|
|
34
|
+
}
|
|
35
|
+
const artifactsDir = resolveArtifactsDir(opts.artifactsDir);
|
|
36
|
+
const runRoot = resolve(artifactsDir, "runs", opts.runId);
|
|
37
|
+
if (!existsSync(runRoot)) {
|
|
38
|
+
console.error(`❌ No artifacts on disk for run "${opts.runId}" at ${runRoot}`);
|
|
39
|
+
console.error(" Set --artifacts-dir / AILF_ARTIFACTS_DIR or check the runId.");
|
|
40
|
+
return 1;
|
|
41
|
+
}
|
|
42
|
+
const outputPath = resolveOutputPath(opts.output, artifactsDir, opts.runId);
|
|
43
|
+
const outputDir = dirname(outputPath);
|
|
44
|
+
mkdirSync(outputDir, { recursive: true });
|
|
45
|
+
// Spawn system tar — the unified tree is byte-for-byte identical to the
|
|
46
|
+
// legacy collector's output, so `tar -czf` produces a tarball with the
|
|
47
|
+
// same file list consumers depended on.
|
|
48
|
+
const tar = spawnSync("tar", ["-czf", outputPath, "-C", resolve(artifactsDir, "runs"), opts.runId], { stdio: "inherit" });
|
|
49
|
+
if (tar.status !== 0) {
|
|
50
|
+
console.error(`❌ tar exited with status ${tar.status}`);
|
|
51
|
+
return tar.status ?? 1;
|
|
52
|
+
}
|
|
53
|
+
console.log(` ✅ Exported ${runRoot} → ${outputPath}`);
|
|
54
|
+
return 0;
|
|
55
|
+
}
|
|
56
|
+
export function createRunsCommand() {
|
|
57
|
+
const cmd = new Command("runs").description("Utilities for working with run-anchored artifact trees on disk");
|
|
58
|
+
cmd
|
|
59
|
+
.command("export")
|
|
60
|
+
.description("Export a run's artifact tree to a portable archive. Only `--format tarball` is supported.")
|
|
61
|
+
.requiredOption("--run-id <runId>", "Run id to export (required)")
|
|
62
|
+
.option("--format <fmt>", "Output format (currently only: tarball)", "tarball")
|
|
63
|
+
.option("--artifacts-dir <path>", "Override the root directory containing runs/{runId}/… (default: .ailf/results/captures)")
|
|
64
|
+
.option("-o, --output <path>", "Output archive path (default: {artifacts-dir}/runs-{runId}.tar.gz)")
|
|
65
|
+
.action((opts) => {
|
|
66
|
+
const code = exportTarball(opts);
|
|
67
|
+
if (code !== 0)
|
|
68
|
+
process.exit(code);
|
|
69
|
+
});
|
|
70
|
+
return cmd;
|
|
71
|
+
}
|