@sanity/ailf 2.9.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_vendor/ailf-core/artifact-capture/association.d.ts +37 -0
- package/dist/_vendor/ailf-core/artifact-capture/association.js +19 -0
- package/dist/_vendor/ailf-core/artifact-registry.d.ts +1 -1
- package/dist/_vendor/ailf-core/artifact-registry.js +1 -18
- package/dist/_vendor/ailf-core/batch-signing.d.ts +64 -0
- package/dist/_vendor/ailf-core/batch-signing.js +23 -0
- package/dist/_vendor/ailf-core/index.d.ts +2 -2
- package/dist/_vendor/ailf-core/index.js +2 -2
- package/dist/_vendor/ailf-core/ports/context.d.ts +12 -20
- package/dist/_vendor/ailf-core/ports/index.d.ts +2 -2
- package/dist/_vendor/ailf-core/ports/index.js +1 -0
- package/dist/_vendor/ailf-core/ports/progress-reporter.d.ts +74 -0
- package/dist/_vendor/ailf-core/ports/progress-reporter.js +26 -0
- package/dist/_vendor/ailf-core/services/slim-report-summary.js +1 -16
- package/dist/adapters/config-sources/file-config-adapter.js +0 -4
- package/dist/adapters/progress/console-progress-reporter.d.ts +35 -0
- package/dist/adapters/progress/console-progress-reporter.js +110 -0
- package/dist/artifact-capture/api-gateway-artifact-writer.d.ts +8 -1
- package/dist/artifact-capture/api-gateway-artifact-writer.js +79 -42
- package/dist/artifact-capture/batching-api-gateway-artifact-writer.d.ts +108 -0
- package/dist/artifact-capture/batching-api-gateway-artifact-writer.js +492 -0
- package/dist/artifact-capture/fanout-artifact-writer.d.ts +14 -2
- package/dist/artifact-capture/fanout-artifact-writer.js +25 -4
- package/dist/artifact-capture/gcs-artifact-writer.d.ts +27 -1
- package/dist/artifact-capture/gcs-artifact-writer.js +168 -38
- package/dist/artifact-capture/instrumented-artifact-writer.d.ts +32 -0
- package/dist/artifact-capture/instrumented-artifact-writer.js +151 -0
- package/dist/artifact-capture/local-fs-artifact-writer.d.ts +8 -1
- package/dist/artifact-capture/local-fs-artifact-writer.js +23 -4
- package/dist/artifact-capture/parallel-emit.d.ts +43 -0
- package/dist/artifact-capture/parallel-emit.js +84 -0
- package/dist/artifact-capture/redact-artifact.d.ts +3 -5
- package/dist/artifact-capture/redact-artifact.js +3 -5
- package/dist/artifact-capture/upload-metrics.d.ts +62 -0
- package/dist/artifact-capture/upload-metrics.js +125 -0
- package/dist/cli.js +56 -2
- package/dist/commands/explain-handler.js +1 -5
- package/dist/commands/pipeline-action.d.ts +0 -4
- package/dist/commands/pipeline-action.js +11 -45
- package/dist/commands/pipeline.d.ts +1 -5
- package/dist/commands/pipeline.js +1 -5
- package/dist/commands/runs.d.ts +18 -0
- package/dist/commands/runs.js +71 -0
- package/dist/composition-root.d.ts +2 -2
- package/dist/composition-root.js +98 -38
- package/dist/orchestration/build-app-context.js +4 -7
- package/dist/orchestration/pipeline-orchestrator.js +100 -24
- package/dist/orchestration/steps/calculate-scores-step.js +1 -1
- package/dist/orchestration/steps/finalize-run-step.js +33 -2
- package/dist/pipeline/emit-eval-results.js +29 -11
- package/dist/pipeline/map-request-to-config.js +0 -4
- package/dist/pipeline/upload-test-outputs.d.ts +12 -5
- package/dist/pipeline/upload-test-outputs.js +27 -10
- package/package.json +3 -3
- package/dist/_vendor/ailf-core/artifact-capture/noop-collector.d.ts +0 -14
- package/dist/_vendor/ailf-core/artifact-capture/noop-collector.js +0 -25
- package/dist/_vendor/ailf-core/ports/artifact-collector.d.ts +0 -94
- package/dist/_vendor/ailf-core/ports/artifact-collector.js +0 -13
- package/dist/_vendor/ailf-core/ports/capture-comparator.d.ts +0 -138
- package/dist/_vendor/ailf-core/ports/capture-comparator.js +0 -10
- package/dist/artifact-capture/comparator.d.ts +0 -22
- package/dist/artifact-capture/comparator.js +0 -493
- package/dist/artifact-capture/filesystem-collector.d.ts +0 -60
- package/dist/artifact-capture/filesystem-collector.js +0 -262
- package/dist/artifact-capture/gcs-collector.d.ts +0 -55
- package/dist/artifact-capture/gcs-collector.js +0 -117
- package/dist/commands/capture-compare.d.ts +0 -15
- package/dist/commands/capture-compare.js +0 -253
- package/dist/commands/capture-list.d.ts +0 -12
- package/dist/commands/capture-list.js +0 -150
- package/dist/commands/capture.d.ts +0 -9
- package/dist/commands/capture.js +0 -16
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* parallel-emit.ts — W0056 prototype A (client-side parallelism).
|
|
3
|
+
*
|
|
4
|
+
* Bounded-concurrency helper for fanning out artifact emits. The baseline
|
|
5
|
+
* measurement (see `docs/design-docs/artifact-upload-throughput.md`) shows
|
|
6
|
+
* producer loops call `await writer.emit(...)` serially, and per-artifact
|
|
7
|
+
* wall clock is dominated by GCS response latency. A simple `p-limit(N)`
|
|
8
|
+
* turns that into a batched-parallel flow against the existing writers.
|
|
9
|
+
*
|
|
10
|
+
* Gated on `AILF_PARALLEL_UPLOAD`:
|
|
11
|
+
* - unset → use the per-writer default set at composition time.
|
|
12
|
+
* - "0" → forced serial (override when default-on is undesirable).
|
|
13
|
+
* - "1" → parallel, default concurrency 8.
|
|
14
|
+
* - "<N>" (N > 1 integer) → parallel with concurrency N.
|
|
15
|
+
*
|
|
16
|
+
* The per-writer default is set by the composition root via
|
|
17
|
+
* `setDefaultUploadConcurrency`. Writers with measured safe parallelism
|
|
18
|
+
* (GCS direct) set 8; writers still on serial (API Gateway, until the
|
|
19
|
+
* batching rollout completes) leave it at the module default of 1.
|
|
20
|
+
*/
|
|
21
|
+
const DEFAULT_CONCURRENCY = 8;
|
|
22
|
+
let moduleDefault = 1;
|
|
23
|
+
/**
|
|
24
|
+
* Set the default concurrency used when `AILF_PARALLEL_UPLOAD` is unset.
|
|
25
|
+
* Composition root calls this once per run based on the selected remote
|
|
26
|
+
* writer. Tests reset by passing 1.
|
|
27
|
+
*/
|
|
28
|
+
export function setDefaultUploadConcurrency(n) {
|
|
29
|
+
moduleDefault = n >= 1 ? n : 1;
|
|
30
|
+
}
|
|
31
|
+
/** Exposed for tests — returns the current module default. */
|
|
32
|
+
export function getDefaultUploadConcurrency() {
|
|
33
|
+
return moduleDefault;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Resolve the configured concurrency. Returns 1 (serial) when parallelism is
|
|
37
|
+
* explicitly disabled or the env value is invalid; otherwise returns the
|
|
38
|
+
* per-writer module default when `AILF_PARALLEL_UPLOAD` is unset.
|
|
39
|
+
*/
|
|
40
|
+
export function resolveUploadConcurrency() {
|
|
41
|
+
const raw = process.env.AILF_PARALLEL_UPLOAD ?? "";
|
|
42
|
+
if (raw === "0")
|
|
43
|
+
return 1;
|
|
44
|
+
if (raw === "")
|
|
45
|
+
return moduleDefault;
|
|
46
|
+
if (raw === "1")
|
|
47
|
+
return DEFAULT_CONCURRENCY;
|
|
48
|
+
const parsed = Number.parseInt(raw, 10);
|
|
49
|
+
if (Number.isFinite(parsed) && parsed > 1)
|
|
50
|
+
return parsed;
|
|
51
|
+
return moduleDefault;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Run `fn` against every item with at most `concurrency` active at once.
|
|
55
|
+
* Preserves input order in the result array. Rejections propagate — callers
|
|
56
|
+
* with non-blocking semantics should catch inside `fn`.
|
|
57
|
+
*
|
|
58
|
+
* When `concurrency <= 1`, runs strictly serially (drop-in equivalent of a
|
|
59
|
+
* `for … await` loop).
|
|
60
|
+
*/
|
|
61
|
+
export async function parallelMap(items, concurrency, fn) {
|
|
62
|
+
if (items.length === 0)
|
|
63
|
+
return [];
|
|
64
|
+
if (concurrency <= 1) {
|
|
65
|
+
const out = [];
|
|
66
|
+
for (let i = 0; i < items.length; i++) {
|
|
67
|
+
out.push(await fn(items[i], i));
|
|
68
|
+
}
|
|
69
|
+
return out;
|
|
70
|
+
}
|
|
71
|
+
const results = new Array(items.length);
|
|
72
|
+
let cursor = 0;
|
|
73
|
+
async function worker() {
|
|
74
|
+
while (true) {
|
|
75
|
+
const i = cursor++;
|
|
76
|
+
if (i >= items.length)
|
|
77
|
+
return;
|
|
78
|
+
results[i] = await fn(items[i], i);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
const width = Math.min(concurrency, items.length);
|
|
82
|
+
await Promise.all(Array.from({ length: width }, () => worker()));
|
|
83
|
+
return results;
|
|
84
|
+
}
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Artifact redaction — strips sensitive data from
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* Applied during FilesystemArtifactCollector.flush() so that secrets
|
|
6
|
-
* (resolved env vars, auth headers, session cookies) never reach storage.
|
|
2
|
+
* Artifact redaction — strips sensitive data from written artifacts so that
|
|
3
|
+
* secrets (resolved env vars, auth headers, session cookies) never reach
|
|
4
|
+
* local or remote storage.
|
|
7
5
|
*
|
|
8
6
|
* Two-layer approach:
|
|
9
7
|
* 1. **Header stripping** — known-sensitive HTTP header keys are replaced
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Artifact redaction — strips sensitive data from
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* Applied during FilesystemArtifactCollector.flush() so that secrets
|
|
6
|
-
* (resolved env vars, auth headers, session cookies) never reach storage.
|
|
2
|
+
* Artifact redaction — strips sensitive data from written artifacts so that
|
|
3
|
+
* secrets (resolved env vars, auth headers, session cookies) never reach
|
|
4
|
+
* local or remote storage.
|
|
7
5
|
*
|
|
8
6
|
* Two-layer approach:
|
|
9
7
|
* 1. **Header stripping** — known-sensitive HTTP header keys are replaced
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* UploadMetrics — spike instrumentation for W0056 (faster artifact upload).
|
|
3
|
+
*
|
|
4
|
+
* Captures per-operation timing on the artifact-upload path so the spike has
|
|
5
|
+
* a measured baseline: artifact count, total bytes, wall-clock, and the
|
|
6
|
+
* sign-RTT vs. PUT split. Gated on `AILF_UPLOAD_METRICS=1` in the composition
|
|
7
|
+
* root — a no-op when off.
|
|
8
|
+
*
|
|
9
|
+
* Design:
|
|
10
|
+
* - `UploadMetricsSink` is the narrow interface that writers depend on.
|
|
11
|
+
* - `UploadMetrics` is the in-process implementation that buffers events
|
|
12
|
+
* and emits both a stderr summary table and an NDJSON detail file.
|
|
13
|
+
* - `summarize()` is called by `InstrumentedArtifactWriter` once, after
|
|
14
|
+
* `writeManifest` succeeds (the natural end-of-run signal).
|
|
15
|
+
*
|
|
16
|
+
* This file is a spike deliverable — the API is intentionally ad hoc and
|
|
17
|
+
* may be promoted to `packages/core/src/ports/` if we ship anything.
|
|
18
|
+
*/
|
|
19
|
+
import type { Logger } from "../_vendor/ailf-core/index.d.ts";
|
|
20
|
+
export type UploadPhase = "sign" | "put" | "compose" | "emit" | "ndjson-part" | "manifest";
|
|
21
|
+
export interface UploadMetricEvent {
|
|
22
|
+
/** ISO timestamp the event was recorded. */
|
|
23
|
+
ts: string;
|
|
24
|
+
/** Phase being measured — writers record `sign`/`put`/`compose` at the call site; the decorator records `emit`/`ndjson-part`/`manifest` end-to-end. */
|
|
25
|
+
phase: UploadPhase;
|
|
26
|
+
/** Writer class that produced the event (e.g. "ApiGatewayArtifactWriter"). */
|
|
27
|
+
writer: string;
|
|
28
|
+
/** Artifact type (or `"manifest"`). */
|
|
29
|
+
type: string;
|
|
30
|
+
/** Wall-clock for the phase, in milliseconds. */
|
|
31
|
+
ms: number;
|
|
32
|
+
/** Body size in bytes, when applicable. */
|
|
33
|
+
bytes?: number;
|
|
34
|
+
/** True when the underlying call resolved without throwing / without a non-2xx response. */
|
|
35
|
+
success: boolean;
|
|
36
|
+
}
|
|
37
|
+
export interface UploadMetricsSink {
|
|
38
|
+
record(event: Omit<UploadMetricEvent, "ts">): void;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* No-op sink — writers default to this when metrics are off, so the
|
|
42
|
+
* instrumentation call sites remain uniform whether or not the collector is
|
|
43
|
+
* active.
|
|
44
|
+
*/
|
|
45
|
+
export declare const NO_OP_UPLOAD_METRICS: UploadMetricsSink;
|
|
46
|
+
export interface UploadMetricsOptions {
|
|
47
|
+
/** Logger used for the summary table. */
|
|
48
|
+
logger: Logger;
|
|
49
|
+
/** Absolute path where the NDJSON detail file is written. Skipped when undefined. */
|
|
50
|
+
detailFile?: string;
|
|
51
|
+
}
|
|
52
|
+
export declare class UploadMetrics implements UploadMetricsSink {
|
|
53
|
+
private readonly options;
|
|
54
|
+
private readonly events;
|
|
55
|
+
private summarized;
|
|
56
|
+
constructor(options: UploadMetricsOptions);
|
|
57
|
+
record(event: Omit<UploadMetricEvent, "ts">): void;
|
|
58
|
+
summarize(): Promise<void>;
|
|
59
|
+
/** Exposed for tests — returns a copy. */
|
|
60
|
+
snapshot(): readonly UploadMetricEvent[];
|
|
61
|
+
}
|
|
62
|
+
export declare function buildSummaryTable(events: readonly UploadMetricEvent[]): string;
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* UploadMetrics — spike instrumentation for W0056 (faster artifact upload).
|
|
3
|
+
*
|
|
4
|
+
* Captures per-operation timing on the artifact-upload path so the spike has
|
|
5
|
+
* a measured baseline: artifact count, total bytes, wall-clock, and the
|
|
6
|
+
* sign-RTT vs. PUT split. Gated on `AILF_UPLOAD_METRICS=1` in the composition
|
|
7
|
+
* root — a no-op when off.
|
|
8
|
+
*
|
|
9
|
+
* Design:
|
|
10
|
+
* - `UploadMetricsSink` is the narrow interface that writers depend on.
|
|
11
|
+
* - `UploadMetrics` is the in-process implementation that buffers events
|
|
12
|
+
* and emits both a stderr summary table and an NDJSON detail file.
|
|
13
|
+
* - `summarize()` is called by `InstrumentedArtifactWriter` once, after
|
|
14
|
+
* `writeManifest` succeeds (the natural end-of-run signal).
|
|
15
|
+
*
|
|
16
|
+
* This file is a spike deliverable — the API is intentionally ad hoc and
|
|
17
|
+
* may be promoted to `packages/core/src/ports/` if we ship anything.
|
|
18
|
+
*/
|
|
19
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
20
|
+
import { dirname } from "node:path";
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
// Implementation
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
/**
|
|
25
|
+
* No-op sink — writers default to this when metrics are off, so the
|
|
26
|
+
* instrumentation call sites remain uniform whether or not the collector is
|
|
27
|
+
* active.
|
|
28
|
+
*/
|
|
29
|
+
export const NO_OP_UPLOAD_METRICS = {
|
|
30
|
+
record() { },
|
|
31
|
+
};
|
|
32
|
+
export class UploadMetrics {
|
|
33
|
+
options;
|
|
34
|
+
events = [];
|
|
35
|
+
summarized = false;
|
|
36
|
+
constructor(options) {
|
|
37
|
+
this.options = options;
|
|
38
|
+
}
|
|
39
|
+
record(event) {
|
|
40
|
+
this.events.push({ ...event, ts: new Date().toISOString() });
|
|
41
|
+
}
|
|
42
|
+
async summarize() {
|
|
43
|
+
if (this.summarized)
|
|
44
|
+
return;
|
|
45
|
+
this.summarized = true;
|
|
46
|
+
const { logger, detailFile } = this.options;
|
|
47
|
+
if (this.events.length === 0) {
|
|
48
|
+
logger.info("[upload-metrics] no events recorded");
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
const table = buildSummaryTable(this.events);
|
|
52
|
+
logger.info(`[upload-metrics] ${this.events.length} events recorded\n${table}`);
|
|
53
|
+
if (detailFile) {
|
|
54
|
+
try {
|
|
55
|
+
await mkdir(dirname(detailFile), { recursive: true });
|
|
56
|
+
const body = this.events.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
|
57
|
+
await writeFile(detailFile, body, "utf-8");
|
|
58
|
+
logger.info(`[upload-metrics] detail written to ${detailFile}`);
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
62
|
+
logger.warn(`[upload-metrics] failed to write detail file "${detailFile}": ${message}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/** Exposed for tests — returns a copy. */
|
|
67
|
+
snapshot() {
|
|
68
|
+
return [...this.events];
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
export function buildSummaryTable(events) {
|
|
72
|
+
const byKey = new Map();
|
|
73
|
+
for (const ev of events) {
|
|
74
|
+
const key = `${ev.phase}\t${ev.writer}`;
|
|
75
|
+
let bucket = byKey.get(key);
|
|
76
|
+
if (!bucket) {
|
|
77
|
+
bucket = [];
|
|
78
|
+
byKey.set(key, bucket);
|
|
79
|
+
}
|
|
80
|
+
bucket.push(ev);
|
|
81
|
+
}
|
|
82
|
+
const rows = [];
|
|
83
|
+
for (const [key, bucket] of byKey) {
|
|
84
|
+
const phase = key.split("\t").join(" · ");
|
|
85
|
+
const durations = bucket.map((e) => e.ms).sort((a, b) => a - b);
|
|
86
|
+
const totalMs = durations.reduce((sum, ms) => sum + ms, 0);
|
|
87
|
+
const totalBytes = bucket.reduce((sum, e) => sum + (e.bytes ?? 0), 0);
|
|
88
|
+
const failures = bucket.filter((e) => !e.success).length;
|
|
89
|
+
rows.push({
|
|
90
|
+
phase,
|
|
91
|
+
count: bucket.length,
|
|
92
|
+
failures,
|
|
93
|
+
totalMs,
|
|
94
|
+
totalBytes,
|
|
95
|
+
p50: percentile(durations, 0.5),
|
|
96
|
+
p95: percentile(durations, 0.95),
|
|
97
|
+
max: durations[durations.length - 1] ?? 0,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
rows.sort((a, b) => b.totalMs - a.totalMs);
|
|
101
|
+
const header = "phase | n | fail | bytes | total ms | p50 | p95 | max";
|
|
102
|
+
const sep = "-----------------------------------------------+-----+------+-------------+----------+-----+-----+-----";
|
|
103
|
+
const body = rows
|
|
104
|
+
.map((r) => `${pad(r.phase, 47)}| ${pad(String(r.count), 4)}| ${pad(String(r.failures), 5)}| ${pad(formatBytes(r.totalBytes), 12)}| ${pad(String(Math.round(r.totalMs)), 9)}| ${pad(String(Math.round(r.p50)), 4)}| ${pad(String(Math.round(r.p95)), 4)}| ${Math.round(r.max)}`)
|
|
105
|
+
.join("\n");
|
|
106
|
+
return `${header}\n${sep}\n${body}`;
|
|
107
|
+
}
|
|
108
|
+
function percentile(sorted, p) {
|
|
109
|
+
if (sorted.length === 0)
|
|
110
|
+
return 0;
|
|
111
|
+
const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * p) - 1));
|
|
112
|
+
return sorted[idx] ?? 0;
|
|
113
|
+
}
|
|
114
|
+
function pad(s, width) {
|
|
115
|
+
return s.length >= width ? `${s} ` : s + " ".repeat(width - s.length);
|
|
116
|
+
}
|
|
117
|
+
function formatBytes(n) {
|
|
118
|
+
if (n < 1024)
|
|
119
|
+
return `${n} B`;
|
|
120
|
+
if (n < 1024 * 1024)
|
|
121
|
+
return `${(n / 1024).toFixed(1)} KB`;
|
|
122
|
+
if (n < 1024 * 1024 * 1024)
|
|
123
|
+
return `${(n / 1024 / 1024).toFixed(1)} MB`;
|
|
124
|
+
return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`;
|
|
125
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -76,6 +76,60 @@ else if (process.argv.includes("--quiet") || process.argv.includes("-q")) {
|
|
|
76
76
|
process.env.AILF_LOG_LEVEL = "quiet";
|
|
77
77
|
}
|
|
78
78
|
// ---------------------------------------------------------------------------
|
|
79
|
+
// W0052 — hard-error on retired capture flags and env vars.
|
|
80
|
+
// --------------------------------------------------------------------------
|
|
81
|
+
// The legacy collector has been removed. Callers still using
|
|
82
|
+
// --capture / --capture-dir / --no-capture-compress / --no-capture-extras
|
|
83
|
+
// or AILF_CAPTURE* / AILF_LEGACY_COLLECTOR / AILF_UNIFIED_ARTIFACTS must
|
|
84
|
+
// migrate to --artifacts-dir / --no-artifacts / --artifacts-exclude. We
|
|
85
|
+
// print a clear pointer so failures don't bubble up as opaque "unknown
|
|
86
|
+
// option" errors from Commander.
|
|
87
|
+
// ---------------------------------------------------------------------------
|
|
88
|
+
const RETIRED_FLAGS = [
|
|
89
|
+
"--capture",
|
|
90
|
+
"--capture-dir",
|
|
91
|
+
"--no-capture-compress",
|
|
92
|
+
"--no-capture-extras",
|
|
93
|
+
"--capture-exclude",
|
|
94
|
+
];
|
|
95
|
+
const RETIRED_ENV_VARS = [
|
|
96
|
+
"AILF_CAPTURE",
|
|
97
|
+
"AILF_CAPTURE_DIR",
|
|
98
|
+
"AILF_CAPTURE_COMPRESS",
|
|
99
|
+
"AILF_CAPTURE_EXTRAS",
|
|
100
|
+
"AILF_CAPTURE_GCS_BUCKET",
|
|
101
|
+
"AILF_CAPTURE_GCS_PREFIX",
|
|
102
|
+
"AILF_LEGACY_COLLECTOR",
|
|
103
|
+
"AILF_UNIFIED_ARTIFACTS",
|
|
104
|
+
];
|
|
105
|
+
function findRetiredFlag() {
|
|
106
|
+
for (const arg of process.argv) {
|
|
107
|
+
const bare = arg.split("=")[0];
|
|
108
|
+
if (RETIRED_FLAGS.includes(bare)) {
|
|
109
|
+
return bare;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return undefined;
|
|
113
|
+
}
|
|
114
|
+
function findRetiredEnv() {
|
|
115
|
+
for (const name of RETIRED_ENV_VARS) {
|
|
116
|
+
if (process.env[name] !== undefined)
|
|
117
|
+
return name;
|
|
118
|
+
}
|
|
119
|
+
return undefined;
|
|
120
|
+
}
|
|
121
|
+
const retiredFlag = findRetiredFlag();
|
|
122
|
+
const retiredEnv = findRetiredEnv();
|
|
123
|
+
if (retiredFlag || retiredEnv) {
|
|
124
|
+
const source = retiredFlag
|
|
125
|
+
? `flag "${retiredFlag}"`
|
|
126
|
+
: `environment variable "${retiredEnv}"`;
|
|
127
|
+
console.error(`❌ ${source} was retired in W0052 along with the legacy artifact collector.`);
|
|
128
|
+
console.error(" Use --artifacts-dir / --no-artifacts / --artifacts-exclude instead.");
|
|
129
|
+
console.error(" See docs/cli.md and docs/decisions/D0033-unified-run-anchored-artifact-capture.md.");
|
|
130
|
+
process.exit(2);
|
|
131
|
+
}
|
|
132
|
+
// ---------------------------------------------------------------------------
|
|
79
133
|
// Build CLI program
|
|
80
134
|
// ---------------------------------------------------------------------------
|
|
81
135
|
import { Command } from "commander";
|
|
@@ -134,6 +188,8 @@ import { createBaselineCommand } from "./commands/baseline.js";
|
|
|
134
188
|
program.addCommand(createBaselineCommand().helpGroup(CommandGroup.CoreWorkflow));
|
|
135
189
|
import { createPublishCommand } from "./commands/publish.js";
|
|
136
190
|
program.addCommand(createPublishCommand().helpGroup(CommandGroup.CoreWorkflow));
|
|
191
|
+
import { createRunsCommand } from "./commands/runs.js";
|
|
192
|
+
program.addCommand(createRunsCommand().helpGroup(CommandGroup.CoreWorkflow));
|
|
137
193
|
// ── Analysis & Reports ────────────────────────────────────────────────
|
|
138
194
|
import { createReadinessReportCommand } from "./commands/readiness-report.js";
|
|
139
195
|
program.addCommand(createReadinessReportCommand().helpGroup(CommandGroup.AnalysisReports));
|
|
@@ -179,8 +235,6 @@ program.addCommand(createLookupDocCommand().helpGroup(CommandGroup.PipelineInter
|
|
|
179
235
|
import { createWebhookServerCommand } from "./commands/webhook-server.js";
|
|
180
236
|
program.addCommand(createWebhookServerCommand().helpGroup(CommandGroup.PipelineInternals));
|
|
181
237
|
// ── Developer Tools ───────────────────────────────────────────────────
|
|
182
|
-
import { createCaptureCommand } from "./commands/capture.js";
|
|
183
|
-
program.addCommand(createCaptureCommand().helpGroup(CommandGroup.DeveloperTools));
|
|
184
238
|
import { createInteractiveCommand } from "./commands/interactive.js";
|
|
185
239
|
program.addCommand(createInteractiveCommand().helpGroup(CommandGroup.DeveloperTools));
|
|
186
240
|
// Shell completion — must be registered last (needs full program tree)
|
|
@@ -723,14 +723,10 @@ async function buildPipelineExplainPlan(actionCommand, rootDir) {
|
|
|
723
723
|
taskSource: raw.taskSource,
|
|
724
724
|
remoteCache: raw.remoteCache,
|
|
725
725
|
config: raw.config,
|
|
726
|
-
capture: raw.capture ?? false,
|
|
727
|
-
captureCompress: raw.captureCompress ?? true,
|
|
728
|
-
captureExtras: raw.captureExtras ?? true,
|
|
729
|
-
captureDir: raw.captureDir,
|
|
730
726
|
artifacts: raw.artifacts ?? true,
|
|
731
727
|
artifactsDir: raw.artifactsDir,
|
|
732
728
|
artifactsDryRun: raw.artifactsDryRun ?? false,
|
|
733
|
-
|
|
729
|
+
artifactsExclude: raw.artifactsExclude,
|
|
734
730
|
};
|
|
735
731
|
const resolved = computeResolvedOptions(withDefaults);
|
|
736
732
|
const planOpts = {
|
|
@@ -63,10 +63,6 @@ export interface ResolvedOptions {
|
|
|
63
63
|
urlArgs: string[];
|
|
64
64
|
apiUrl: string;
|
|
65
65
|
apiKey?: string;
|
|
66
|
-
captureEnabled: boolean;
|
|
67
|
-
captureDir?: string;
|
|
68
|
-
captureCompress: boolean;
|
|
69
|
-
captureExtras: boolean;
|
|
70
66
|
/** D0033 / W0049 — unified artifact surface (W0050 wires it into writer). */
|
|
71
67
|
artifactsDisabled: boolean;
|
|
72
68
|
artifactsDir?: string;
|
|
@@ -263,32 +263,25 @@ export function computeResolvedOptions(opts) {
|
|
|
263
263
|
tagOption,
|
|
264
264
|
taskSourceType: resolvedTaskSourceType,
|
|
265
265
|
urlArgs,
|
|
266
|
-
captureEnabled: opts.capture,
|
|
267
|
-
captureDir: resolveArtifactsDir(opts),
|
|
268
|
-
captureCompress: opts.captureCompress !== false &&
|
|
269
|
-
process.env.AILF_CAPTURE_COMPRESS !== "0",
|
|
270
|
-
captureExtras: opts.captureExtras !== false && process.env.AILF_CAPTURE_EXTRAS !== "0",
|
|
271
266
|
artifactsDisabled: opts.artifacts === false,
|
|
272
267
|
artifactsDir: resolveArtifactsDir(opts),
|
|
273
268
|
artifactsDryRun: opts.artifactsDryRun,
|
|
274
|
-
artifactsExclude:
|
|
269
|
+
artifactsExclude: parseArtifactsExcludeList(opts.artifactsExclude),
|
|
275
270
|
};
|
|
276
271
|
}
|
|
277
272
|
/**
|
|
278
|
-
* Resolve the artifacts
|
|
279
|
-
*
|
|
273
|
+
* Resolve the artifacts output directory from CLI flags and env vars.
|
|
274
|
+
* Precedence (highest first):
|
|
280
275
|
* 1. `--artifacts-dir` flag
|
|
281
|
-
* 2.
|
|
282
|
-
*
|
|
283
|
-
*
|
|
276
|
+
* 2. `AILF_ARTIFACTS_DIR` env var
|
|
277
|
+
*
|
|
278
|
+
* The `--capture-dir` / `AILF_CAPTURE_DIR` aliases were retired in W0052;
|
|
279
|
+
* callers of those names are rejected at CLI entry (see cli.ts).
|
|
284
280
|
*/
|
|
285
281
|
function resolveArtifactsDir(opts) {
|
|
286
|
-
return
|
|
287
|
-
opts.captureDir ??
|
|
288
|
-
process.env.AILF_ARTIFACTS_DIR ??
|
|
289
|
-
process.env.AILF_CAPTURE_DIR);
|
|
282
|
+
return opts.artifactsDir ?? process.env.AILF_ARTIFACTS_DIR;
|
|
290
283
|
}
|
|
291
|
-
function
|
|
284
|
+
function parseArtifactsExcludeList(raw) {
|
|
292
285
|
if (!raw)
|
|
293
286
|
return undefined;
|
|
294
287
|
const list = raw
|
|
@@ -309,20 +302,6 @@ function resolveTaskSourceType(raw) {
|
|
|
309
302
|
// ---------------------------------------------------------------------------
|
|
310
303
|
// Pipeline entry point
|
|
311
304
|
// ---------------------------------------------------------------------------
|
|
312
|
-
/**
|
|
313
|
-
* Module-level flag so the `--capture` deprecation warning fires exactly
|
|
314
|
-
* once per process even when `executePipeline` is invoked multiple times
|
|
315
|
-
* (e.g. tests, long-lived dev loops).
|
|
316
|
-
*/
|
|
317
|
-
let warnedCaptureDeprecation = false;
|
|
318
|
-
function warnCaptureDeprecationIfNeeded(cliOpts) {
|
|
319
|
-
if (!cliOpts.capture)
|
|
320
|
-
return;
|
|
321
|
-
if (warnedCaptureDeprecation)
|
|
322
|
-
return;
|
|
323
|
-
warnedCaptureDeprecation = true;
|
|
324
|
-
console.warn("--capture is deprecated and will be removed in a future release; use --artifacts-dir or --no-artifacts instead");
|
|
325
|
-
}
|
|
326
305
|
/**
|
|
327
306
|
* Execute the evaluation pipeline.
|
|
328
307
|
*
|
|
@@ -332,7 +311,6 @@ function warnCaptureDeprecationIfNeeded(cliOpts) {
|
|
|
332
311
|
* 4. Delegate to the PipelineOrchestrator
|
|
333
312
|
*/
|
|
334
313
|
export async function executePipeline(cliOpts) {
|
|
335
|
-
warnCaptureDeprecationIfNeeded(cliOpts);
|
|
336
314
|
// When --config is provided, resolve config from file instead of CLI flags
|
|
337
315
|
if (cliOpts.config) {
|
|
338
316
|
const { existsSync } = await import("fs");
|
|
@@ -357,25 +335,13 @@ export async function executePipeline(cliOpts) {
|
|
|
357
335
|
}
|
|
358
336
|
// Output dir: explicit CLI flag → $CWD/.ailf/results/latest/
|
|
359
337
|
config.outputDir = resolveOutputDir(cliOpts.outputDir);
|
|
360
|
-
//
|
|
338
|
+
// Artifact options — CLI flags and env vars aren't in the config file,
|
|
361
339
|
// so merge them here (same logic as resolveOptions).
|
|
362
|
-
// AILF_CAPTURE is a no-op post-W0049; only the flag toggles captureEnabled.
|
|
363
|
-
config.captureEnabled = cliOpts.capture;
|
|
364
340
|
const resolvedArtifactsDir = resolveArtifactsDir(cliOpts);
|
|
365
|
-
if (resolvedArtifactsDir) {
|
|
366
|
-
config.captureDir = resolvedArtifactsDir;
|
|
367
|
-
}
|
|
368
|
-
config.captureCompress =
|
|
369
|
-
cliOpts.captureCompress !== false &&
|
|
370
|
-
process.env.AILF_CAPTURE_COMPRESS !== "0";
|
|
371
|
-
config.captureExtras =
|
|
372
|
-
cliOpts.captureExtras !== false && process.env.AILF_CAPTURE_EXTRAS !== "0";
|
|
373
|
-
config.captureGcsBucket ??= process.env.AILF_CAPTURE_GCS_BUCKET;
|
|
374
|
-
config.captureGcsPrefix ??= process.env.AILF_CAPTURE_GCS_PREFIX;
|
|
375
341
|
config.artifactsDisabled ??= cliOpts.artifacts === false;
|
|
376
342
|
config.artifactsDir ??= resolvedArtifactsDir;
|
|
377
343
|
config.artifactsDryRun ??= cliOpts.artifactsDryRun;
|
|
378
|
-
const excludeList =
|
|
344
|
+
const excludeList = parseArtifactsExcludeList(cliOpts.artifactsExclude);
|
|
379
345
|
if (excludeList) {
|
|
380
346
|
config.artifactsExclude = excludeList;
|
|
381
347
|
}
|
|
@@ -64,13 +64,9 @@ export interface PipelineCliOptions {
|
|
|
64
64
|
url: string[];
|
|
65
65
|
urls: string[];
|
|
66
66
|
apiUrl?: string;
|
|
67
|
-
capture: boolean;
|
|
68
|
-
captureDir?: string;
|
|
69
|
-
captureCompress: boolean;
|
|
70
|
-
captureExtras: boolean;
|
|
71
67
|
artifacts: boolean;
|
|
72
68
|
artifactsDir?: string;
|
|
73
69
|
artifactsDryRun: boolean;
|
|
74
|
-
|
|
70
|
+
artifactsExclude?: string;
|
|
75
71
|
}
|
|
76
72
|
export declare function createPipelineCommand(): Command;
|
|
@@ -54,14 +54,10 @@ export function createPipelineCommand() {
|
|
|
54
54
|
.option("--repo-tasks-path <path>", "Path to repo-based task definitions (.ailf/tasks/ directory)")
|
|
55
55
|
.option("--remote", "Submit evaluation to the AILF API instead of running locally", false)
|
|
56
56
|
.option("--api-url <url>", "AILF API base URL (default: https://ailf-api.sanity.build)")
|
|
57
|
-
.option("--capture", "[DEPRECATED] Enable legacy artifact capture. Use --artifacts-dir / --no-artifacts instead.", false)
|
|
58
|
-
.option("--capture-dir <path>", "[DEPRECATED] Alias for --artifacts-dir.")
|
|
59
|
-
.option("--no-capture-compress", "Disable tar.gz compression of captures")
|
|
60
|
-
.option("--no-capture-extras", "Exclude mode-specific artifacts from captures")
|
|
61
57
|
.option("--no-artifacts", "Disable all artifact writers (D0033). Overrides --artifacts-dir.")
|
|
62
58
|
.option("--artifacts-dir <path>", "Root directory for local artifact output (D0033; default: .ailf/results/captures/)")
|
|
63
59
|
.option("--artifacts-dry-run", "Run artifact writers in dry-run mode — log intended writes, touch no storage", false)
|
|
64
|
-
.option("--
|
|
60
|
+
.option("--artifacts-exclude <types>", "Comma-separated artifact types to skip (e.g. traces,graderPrompts)")
|
|
65
61
|
.action(async (opts) => {
|
|
66
62
|
const { executePipeline } = await import("./pipeline-action.js");
|
|
67
63
|
await executePipeline(opts);
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* runs command — utilities for working with run-anchored artifacts on disk.
|
|
3
|
+
*
|
|
4
|
+
* The unified artifact writer (D0033 M4) lays artifacts down under
|
|
5
|
+
* `.ailf/results/captures/runs/{runId}/…`. Downstream tooling (CI archivers,
|
|
6
|
+
* external consumers that used to parse the legacy collector tarball) still
|
|
7
|
+
* want a single-file bundle. `runs export --format tarball` reproduces that
|
|
8
|
+
* bundle from the existing tree — no re-materialisation, just a tar of the
|
|
9
|
+
* files already written.
|
|
10
|
+
*
|
|
11
|
+
* The on-disk layout is identical to what the legacy FilesystemArtifactCollector
|
|
12
|
+
* produced (same paths, same filenames), so the tarball shape is stable for
|
|
13
|
+
* existing consumers.
|
|
14
|
+
*
|
|
15
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
|
|
16
|
+
*/
|
|
17
|
+
import { Command } from "commander";
|
|
18
|
+
export declare function createRunsCommand(): Command;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* runs command — utilities for working with run-anchored artifacts on disk.
|
|
3
|
+
*
|
|
4
|
+
* The unified artifact writer (D0033 M4) lays artifacts down under
|
|
5
|
+
* `.ailf/results/captures/runs/{runId}/…`. Downstream tooling (CI archivers,
|
|
6
|
+
* external consumers that used to parse the legacy collector tarball) still
|
|
7
|
+
* want a single-file bundle. `runs export --format tarball` reproduces that
|
|
8
|
+
* bundle from the existing tree — no re-materialisation, just a tar of the
|
|
9
|
+
* files already written.
|
|
10
|
+
*
|
|
11
|
+
* The on-disk layout is identical to what the legacy FilesystemArtifactCollector
|
|
12
|
+
* produced (same paths, same filenames), so the tarball shape is stable for
|
|
13
|
+
* existing consumers.
|
|
14
|
+
*
|
|
15
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
|
|
16
|
+
*/
|
|
17
|
+
import { spawnSync } from "node:child_process";
|
|
18
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
19
|
+
import { dirname, resolve } from "node:path";
|
|
20
|
+
import { Command } from "commander";
|
|
21
|
+
const DEFAULT_ARTIFACTS_DIR = ".ailf/results/captures";
|
|
22
|
+
function resolveArtifactsDir(flag) {
|
|
23
|
+
return resolve(flag ?? process.env.AILF_ARTIFACTS_DIR ?? DEFAULT_ARTIFACTS_DIR);
|
|
24
|
+
}
|
|
25
|
+
function resolveOutputPath(explicit, artifactsDir, runId) {
|
|
26
|
+
if (explicit)
|
|
27
|
+
return resolve(explicit);
|
|
28
|
+
return resolve(artifactsDir, `runs-${runId}.tar.gz`);
|
|
29
|
+
}
|
|
30
|
+
function exportTarball(opts) {
|
|
31
|
+
if (opts.format !== "tarball") {
|
|
32
|
+
console.error(`❌ Unsupported --format "${opts.format}". Supported: tarball.`);
|
|
33
|
+
return 2;
|
|
34
|
+
}
|
|
35
|
+
const artifactsDir = resolveArtifactsDir(opts.artifactsDir);
|
|
36
|
+
const runRoot = resolve(artifactsDir, "runs", opts.runId);
|
|
37
|
+
if (!existsSync(runRoot)) {
|
|
38
|
+
console.error(`❌ No artifacts on disk for run "${opts.runId}" at ${runRoot}`);
|
|
39
|
+
console.error(" Set --artifacts-dir / AILF_ARTIFACTS_DIR or check the runId.");
|
|
40
|
+
return 1;
|
|
41
|
+
}
|
|
42
|
+
const outputPath = resolveOutputPath(opts.output, artifactsDir, opts.runId);
|
|
43
|
+
const outputDir = dirname(outputPath);
|
|
44
|
+
mkdirSync(outputDir, { recursive: true });
|
|
45
|
+
// Spawn system tar — the unified tree is byte-for-byte identical to the
|
|
46
|
+
// legacy collector's output, so `tar -czf` produces a tarball with the
|
|
47
|
+
// same file list consumers depended on.
|
|
48
|
+
const tar = spawnSync("tar", ["-czf", outputPath, "-C", resolve(artifactsDir, "runs"), opts.runId], { stdio: "inherit" });
|
|
49
|
+
if (tar.status !== 0) {
|
|
50
|
+
console.error(`❌ tar exited with status ${tar.status}`);
|
|
51
|
+
return tar.status ?? 1;
|
|
52
|
+
}
|
|
53
|
+
console.log(` ✅ Exported ${runRoot} → ${outputPath}`);
|
|
54
|
+
return 0;
|
|
55
|
+
}
|
|
56
|
+
export function createRunsCommand() {
|
|
57
|
+
const cmd = new Command("runs").description("Utilities for working with run-anchored artifact trees on disk");
|
|
58
|
+
cmd
|
|
59
|
+
.command("export")
|
|
60
|
+
.description("Export a run's artifact tree to a portable archive. Only `--format tarball` is supported.")
|
|
61
|
+
.requiredOption("--run-id <runId>", "Run id to export (required)")
|
|
62
|
+
.option("--format <fmt>", "Output format (currently only: tarball)", "tarball")
|
|
63
|
+
.option("--artifacts-dir <path>", "Override the root directory containing runs/{runId}/… (default: .ailf/results/captures)")
|
|
64
|
+
.option("-o, --output <path>", "Output archive path (default: {artifacts-dir}/runs-{runId}.tar.gz)")
|
|
65
|
+
.action((opts) => {
|
|
66
|
+
const code = exportTarball(opts);
|
|
67
|
+
if (code !== 0)
|
|
68
|
+
process.exit(code);
|
|
69
|
+
});
|
|
70
|
+
return cmd;
|
|
71
|
+
}
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* @see packages/core/src/ports/context.ts — AppContext interface
|
|
16
16
|
* @see docs/archive/exec-plans/ports-and-adapters/phase-7-composition-root.md
|
|
17
17
|
*/
|
|
18
|
-
import { type AppContext, type ArtifactWriter, type AssertionRegistration, type Logger, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
|
|
18
|
+
import { type AppContext, type ArtifactWriter, type ArtifactWriterProgressOptions, type AssertionRegistration, type Logger, type ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
|
|
19
19
|
/**
|
|
20
20
|
* Create a fully wired AppContext from resolved configuration.
|
|
21
21
|
*
|
|
@@ -41,7 +41,7 @@ export declare function createAppContext(config: ResolvedConfig): AppContext;
|
|
|
41
41
|
*
|
|
42
42
|
* Exported for unit-test access; not part of the public package API.
|
|
43
43
|
*/
|
|
44
|
-
export declare function createArtifactWriter(config: ResolvedConfig, logger: Logger): ArtifactWriter;
|
|
44
|
+
export declare function createArtifactWriter(config: ResolvedConfig, logger: Logger, progress?: ArtifactWriterProgressOptions): ArtifactWriter;
|
|
45
45
|
/**
|
|
46
46
|
* Generic Promptfoo assertion types available to all evaluation modes.
|
|
47
47
|
*
|