@sanity/ailf 2.8.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_vendor/ailf-core/artifact-capture/association.d.ts +35 -0
- package/dist/_vendor/ailf-core/artifact-capture/association.js +28 -0
- package/dist/_vendor/ailf-core/artifact-registry.d.ts +124 -23
- package/dist/_vendor/ailf-core/artifact-registry.js +708 -64
- package/dist/_vendor/ailf-core/batch-signing.d.ts +64 -0
- package/dist/_vendor/ailf-core/batch-signing.js +23 -0
- package/dist/_vendor/ailf-core/index.d.ts +3 -2
- package/dist/_vendor/ailf-core/index.js +3 -2
- package/dist/_vendor/ailf-core/ports/artifact-writer.d.ts +59 -20
- package/dist/_vendor/ailf-core/ports/artifact-writer.js +33 -10
- package/dist/_vendor/ailf-core/ports/context.d.ts +20 -17
- package/dist/_vendor/ailf-core/ports/index.d.ts +0 -2
- package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +6 -6
- package/dist/_vendor/ailf-core/services/index.d.ts +1 -0
- package/dist/_vendor/ailf-core/services/index.js +1 -0
- package/dist/_vendor/ailf-core/services/slim-report-summary.d.ts +31 -0
- package/dist/_vendor/ailf-core/services/slim-report-summary.js +217 -0
- package/dist/_vendor/ailf-core/types/branded-ids.d.ts +33 -0
- package/dist/_vendor/ailf-core/types/index.d.ts +202 -23
- package/dist/adapters/config-sources/file-config-adapter.js +0 -4
- package/dist/artifact-capture/accumulating-artifact-writer.d.ts +50 -0
- package/dist/artifact-capture/accumulating-artifact-writer.js +111 -0
- package/dist/artifact-capture/api-gateway-artifact-writer.d.ts +17 -4
- package/dist/artifact-capture/api-gateway-artifact-writer.js +58 -7
- package/dist/artifact-capture/emit-file.d.ts +28 -0
- package/dist/artifact-capture/emit-file.js +56 -0
- package/dist/artifact-capture/fanout-artifact-writer.d.ts +39 -0
- package/dist/artifact-capture/fanout-artifact-writer.js +76 -0
- package/dist/artifact-capture/gcs-artifact-writer.d.ts +40 -3
- package/dist/artifact-capture/gcs-artifact-writer.js +238 -14
- package/dist/artifact-capture/local-fs-artifact-writer.d.ts +71 -0
- package/dist/artifact-capture/local-fs-artifact-writer.js +273 -0
- package/dist/artifact-capture/redact-artifact.d.ts +3 -5
- package/dist/artifact-capture/redact-artifact.js +3 -5
- package/dist/cli.js +56 -2
- package/dist/commands/explain-handler.js +4 -4
- package/dist/commands/pipeline-action.d.ts +5 -4
- package/dist/commands/pipeline-action.js +33 -16
- package/dist/commands/pipeline.d.ts +4 -4
- package/dist/commands/pipeline.js +4 -4
- package/dist/commands/publish.js +4 -1
- package/dist/commands/runs.d.ts +18 -0
- package/dist/commands/runs.js +71 -0
- package/dist/composition-root.d.ts +13 -10
- package/dist/composition-root.js +74 -46
- package/dist/orchestration/build-app-context.js +4 -7
- package/dist/orchestration/pipeline-orchestrator.d.ts +1 -1
- package/dist/orchestration/pipeline-orchestrator.js +37 -46
- package/dist/orchestration/steps/calculate-scores-step.d.ts +1 -1
- package/dist/orchestration/steps/calculate-scores-step.js +19 -19
- package/dist/orchestration/steps/callback-step.d.ts +1 -1
- package/dist/orchestration/steps/callback-step.js +6 -4
- package/dist/orchestration/steps/compare-step.d.ts +1 -1
- package/dist/orchestration/steps/compare-step.js +4 -2
- package/dist/orchestration/steps/discovery-report-step.d.ts +1 -1
- package/dist/orchestration/steps/discovery-report-step.js +4 -1
- package/dist/orchestration/steps/fetch-docs-step.js +9 -15
- package/dist/orchestration/steps/finalize-run-step.js +21 -7
- package/dist/orchestration/steps/gap-analysis-step.js +34 -6
- package/dist/orchestration/steps/generate-configs-step.d.ts +1 -1
- package/dist/orchestration/steps/generate-configs-step.js +11 -11
- package/dist/orchestration/steps/publish-report-step.d.ts +1 -1
- package/dist/orchestration/steps/publish-report-step.js +24 -19
- package/dist/orchestration/steps/readiness-step.d.ts +1 -1
- package/dist/orchestration/steps/readiness-step.js +4 -1
- package/dist/orchestration/steps/report-step.d.ts +1 -1
- package/dist/orchestration/steps/report-step.js +6 -3
- package/dist/orchestration/steps/run-eval-step.js +14 -9
- package/dist/pipeline/compare.d.ts +2 -2
- package/dist/pipeline/emit-eval-results.d.ts +38 -0
- package/dist/pipeline/emit-eval-results.js +100 -0
- package/dist/pipeline/map-request-to-config.js +0 -4
- package/package.json +1 -1
- package/dist/_vendor/ailf-core/artifact-capture/noop-collector.d.ts +0 -14
- package/dist/_vendor/ailf-core/artifact-capture/noop-collector.js +0 -25
- package/dist/_vendor/ailf-core/ports/artifact-collector.d.ts +0 -94
- package/dist/_vendor/ailf-core/ports/artifact-collector.js +0 -13
- package/dist/_vendor/ailf-core/ports/capture-comparator.d.ts +0 -138
- package/dist/_vendor/ailf-core/ports/capture-comparator.js +0 -10
- package/dist/artifact-capture/comparator.d.ts +0 -22
- package/dist/artifact-capture/comparator.js +0 -493
- package/dist/artifact-capture/filesystem-collector.d.ts +0 -42
- package/dist/artifact-capture/filesystem-collector.js +0 -237
- package/dist/artifact-capture/gcs-collector.d.ts +0 -55
- package/dist/artifact-capture/gcs-collector.js +0 -117
- package/dist/commands/capture-compare.d.ts +0 -15
- package/dist/commands/capture-compare.js +0 -253
- package/dist/commands/capture-list.d.ts +0 -12
- package/dist/commands/capture-list.js +0 -150
- package/dist/commands/capture.d.ts +0 -9
- package/dist/commands/capture.js +0 -16
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Artifact redaction — strips sensitive data from
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* Applied during FilesystemArtifactCollector.flush() so that secrets
|
|
6
|
-
* (resolved env vars, auth headers, session cookies) never reach storage.
|
|
2
|
+
* Artifact redaction — strips sensitive data from written artifacts so that
|
|
3
|
+
* secrets (resolved env vars, auth headers, session cookies) never reach
|
|
4
|
+
* local or remote storage.
|
|
7
5
|
*
|
|
8
6
|
* Two-layer approach:
|
|
9
7
|
* 1. **Header stripping** — known-sensitive HTTP header keys are replaced
|
package/dist/cli.js
CHANGED
|
@@ -76,6 +76,60 @@ else if (process.argv.includes("--quiet") || process.argv.includes("-q")) {
|
|
|
76
76
|
process.env.AILF_LOG_LEVEL = "quiet";
|
|
77
77
|
}
|
|
78
78
|
// ---------------------------------------------------------------------------
|
|
79
|
+
// W0052 — hard-error on retired capture flags and env vars.
|
|
80
|
+
// --------------------------------------------------------------------------
|
|
81
|
+
// The legacy collector has been removed. Callers still using
|
|
82
|
+
// --capture / --capture-dir / --no-capture-compress / --no-capture-extras
|
|
83
|
+
// or AILF_CAPTURE* / AILF_LEGACY_COLLECTOR / AILF_UNIFIED_ARTIFACTS must
|
|
84
|
+
// migrate to --artifacts-dir / --no-artifacts / --artifacts-exclude. We
|
|
85
|
+
// print a clear pointer so failures don't bubble up as opaque "unknown
|
|
86
|
+
// option" errors from Commander.
|
|
87
|
+
// ---------------------------------------------------------------------------
|
|
88
|
+
const RETIRED_FLAGS = [
|
|
89
|
+
"--capture",
|
|
90
|
+
"--capture-dir",
|
|
91
|
+
"--no-capture-compress",
|
|
92
|
+
"--no-capture-extras",
|
|
93
|
+
"--capture-exclude",
|
|
94
|
+
];
|
|
95
|
+
const RETIRED_ENV_VARS = [
|
|
96
|
+
"AILF_CAPTURE",
|
|
97
|
+
"AILF_CAPTURE_DIR",
|
|
98
|
+
"AILF_CAPTURE_COMPRESS",
|
|
99
|
+
"AILF_CAPTURE_EXTRAS",
|
|
100
|
+
"AILF_CAPTURE_GCS_BUCKET",
|
|
101
|
+
"AILF_CAPTURE_GCS_PREFIX",
|
|
102
|
+
"AILF_LEGACY_COLLECTOR",
|
|
103
|
+
"AILF_UNIFIED_ARTIFACTS",
|
|
104
|
+
];
|
|
105
|
+
function findRetiredFlag() {
|
|
106
|
+
for (const arg of process.argv) {
|
|
107
|
+
const bare = arg.split("=")[0];
|
|
108
|
+
if (RETIRED_FLAGS.includes(bare)) {
|
|
109
|
+
return bare;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return undefined;
|
|
113
|
+
}
|
|
114
|
+
function findRetiredEnv() {
|
|
115
|
+
for (const name of RETIRED_ENV_VARS) {
|
|
116
|
+
if (process.env[name] !== undefined)
|
|
117
|
+
return name;
|
|
118
|
+
}
|
|
119
|
+
return undefined;
|
|
120
|
+
}
|
|
121
|
+
const retiredFlag = findRetiredFlag();
|
|
122
|
+
const retiredEnv = findRetiredEnv();
|
|
123
|
+
if (retiredFlag || retiredEnv) {
|
|
124
|
+
const source = retiredFlag
|
|
125
|
+
? `flag "${retiredFlag}"`
|
|
126
|
+
: `environment variable "${retiredEnv}"`;
|
|
127
|
+
console.error(`❌ ${source} was retired in W0052 along with the legacy artifact collector.`);
|
|
128
|
+
console.error(" Use --artifacts-dir / --no-artifacts / --artifacts-exclude instead.");
|
|
129
|
+
console.error(" See docs/cli.md and docs/decisions/D0033-unified-run-anchored-artifact-capture.md.");
|
|
130
|
+
process.exit(2);
|
|
131
|
+
}
|
|
132
|
+
// ---------------------------------------------------------------------------
|
|
79
133
|
// Build CLI program
|
|
80
134
|
// ---------------------------------------------------------------------------
|
|
81
135
|
import { Command } from "commander";
|
|
@@ -134,6 +188,8 @@ import { createBaselineCommand } from "./commands/baseline.js";
|
|
|
134
188
|
program.addCommand(createBaselineCommand().helpGroup(CommandGroup.CoreWorkflow));
|
|
135
189
|
import { createPublishCommand } from "./commands/publish.js";
|
|
136
190
|
program.addCommand(createPublishCommand().helpGroup(CommandGroup.CoreWorkflow));
|
|
191
|
+
import { createRunsCommand } from "./commands/runs.js";
|
|
192
|
+
program.addCommand(createRunsCommand().helpGroup(CommandGroup.CoreWorkflow));
|
|
137
193
|
// ── Analysis & Reports ────────────────────────────────────────────────
|
|
138
194
|
import { createReadinessReportCommand } from "./commands/readiness-report.js";
|
|
139
195
|
program.addCommand(createReadinessReportCommand().helpGroup(CommandGroup.AnalysisReports));
|
|
@@ -179,8 +235,6 @@ program.addCommand(createLookupDocCommand().helpGroup(CommandGroup.PipelineInter
|
|
|
179
235
|
import { createWebhookServerCommand } from "./commands/webhook-server.js";
|
|
180
236
|
program.addCommand(createWebhookServerCommand().helpGroup(CommandGroup.PipelineInternals));
|
|
181
237
|
// ── Developer Tools ───────────────────────────────────────────────────
|
|
182
|
-
import { createCaptureCommand } from "./commands/capture.js";
|
|
183
|
-
program.addCommand(createCaptureCommand().helpGroup(CommandGroup.DeveloperTools));
|
|
184
238
|
import { createInteractiveCommand } from "./commands/interactive.js";
|
|
185
239
|
program.addCommand(createInteractiveCommand().helpGroup(CommandGroup.DeveloperTools));
|
|
186
240
|
// Shell completion — must be registered last (needs full program tree)
|
|
@@ -723,10 +723,10 @@ async function buildPipelineExplainPlan(actionCommand, rootDir) {
|
|
|
723
723
|
taskSource: raw.taskSource,
|
|
724
724
|
remoteCache: raw.remoteCache,
|
|
725
725
|
config: raw.config,
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
726
|
+
artifacts: raw.artifacts ?? true,
|
|
727
|
+
artifactsDir: raw.artifactsDir,
|
|
728
|
+
artifactsDryRun: raw.artifactsDryRun ?? false,
|
|
729
|
+
artifactsExclude: raw.artifactsExclude,
|
|
730
730
|
};
|
|
731
731
|
const resolved = computeResolvedOptions(withDefaults);
|
|
732
732
|
const planOpts = {
|
|
@@ -63,10 +63,11 @@ export interface ResolvedOptions {
|
|
|
63
63
|
urlArgs: string[];
|
|
64
64
|
apiUrl: string;
|
|
65
65
|
apiKey?: string;
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
/** D0033 / W0049 — unified artifact surface (W0050 wires it into writer). */
|
|
67
|
+
artifactsDisabled: boolean;
|
|
68
|
+
artifactsDir?: string;
|
|
69
|
+
artifactsDryRun: boolean;
|
|
70
|
+
artifactsExclude?: readonly string[];
|
|
70
71
|
}
|
|
71
72
|
/**
|
|
72
73
|
* Pure option resolution — computes ResolvedOptions from CLI flags without
|
|
@@ -263,13 +263,33 @@ export function computeResolvedOptions(opts) {
|
|
|
263
263
|
tagOption,
|
|
264
264
|
taskSourceType: resolvedTaskSourceType,
|
|
265
265
|
urlArgs,
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
captureExtras: opts.captureExtras !== false && process.env.AILF_CAPTURE_EXTRAS !== "0",
|
|
266
|
+
artifactsDisabled: opts.artifacts === false,
|
|
267
|
+
artifactsDir: resolveArtifactsDir(opts),
|
|
268
|
+
artifactsDryRun: opts.artifactsDryRun,
|
|
269
|
+
artifactsExclude: parseArtifactsExcludeList(opts.artifactsExclude),
|
|
271
270
|
};
|
|
272
271
|
}
|
|
272
|
+
/**
|
|
273
|
+
* Resolve the artifacts output directory from CLI flags and env vars.
|
|
274
|
+
* Precedence (highest first):
|
|
275
|
+
* 1. `--artifacts-dir` flag
|
|
276
|
+
* 2. `AILF_ARTIFACTS_DIR` env var
|
|
277
|
+
*
|
|
278
|
+
* The `--capture-dir` / `AILF_CAPTURE_DIR` aliases were retired in W0052;
|
|
279
|
+
* callers of those names are rejected at CLI entry (see cli.ts).
|
|
280
|
+
*/
|
|
281
|
+
function resolveArtifactsDir(opts) {
|
|
282
|
+
return opts.artifactsDir ?? process.env.AILF_ARTIFACTS_DIR;
|
|
283
|
+
}
|
|
284
|
+
function parseArtifactsExcludeList(raw) {
|
|
285
|
+
if (!raw)
|
|
286
|
+
return undefined;
|
|
287
|
+
const list = raw
|
|
288
|
+
.split(",")
|
|
289
|
+
.map((s) => s.trim())
|
|
290
|
+
.filter(Boolean);
|
|
291
|
+
return list.length > 0 ? list : undefined;
|
|
292
|
+
}
|
|
273
293
|
/** Resolve and validate the --task-source flag value. */
|
|
274
294
|
function resolveTaskSourceType(raw) {
|
|
275
295
|
if (!raw || raw === "content-lake")
|
|
@@ -315,19 +335,16 @@ export async function executePipeline(cliOpts) {
|
|
|
315
335
|
}
|
|
316
336
|
// Output dir: explicit CLI flag → $CWD/.ailf/results/latest/
|
|
317
337
|
config.outputDir = resolveOutputDir(cliOpts.outputDir);
|
|
318
|
-
//
|
|
338
|
+
// Artifact options — CLI flags and env vars aren't in the config file,
|
|
319
339
|
// so merge them here (same logic as resolveOptions).
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
340
|
+
const resolvedArtifactsDir = resolveArtifactsDir(cliOpts);
|
|
341
|
+
config.artifactsDisabled ??= cliOpts.artifacts === false;
|
|
342
|
+
config.artifactsDir ??= resolvedArtifactsDir;
|
|
343
|
+
config.artifactsDryRun ??= cliOpts.artifactsDryRun;
|
|
344
|
+
const excludeList = parseArtifactsExcludeList(cliOpts.artifactsExclude);
|
|
345
|
+
if (excludeList) {
|
|
346
|
+
config.artifactsExclude = excludeList;
|
|
323
347
|
}
|
|
324
|
-
config.captureCompress =
|
|
325
|
-
cliOpts.captureCompress !== false &&
|
|
326
|
-
process.env.AILF_CAPTURE_COMPRESS !== "0";
|
|
327
|
-
config.captureExtras =
|
|
328
|
-
cliOpts.captureExtras !== false && process.env.AILF_CAPTURE_EXTRAS !== "0";
|
|
329
|
-
config.captureGcsBucket ??= process.env.AILF_CAPTURE_GCS_BUCKET;
|
|
330
|
-
config.captureGcsPrefix ??= process.env.AILF_CAPTURE_GCS_PREFIX;
|
|
331
348
|
config.artifactGcsBucket ??= process.env.AILF_GCS_ARTIFACT_BUCKET;
|
|
332
349
|
config.artifactUpload ??= parseArtifactUploadEnv(process.env.AILF_ARTIFACT_UPLOAD);
|
|
333
350
|
// Create AppContext directly from the merged config so adapters
|
|
@@ -64,9 +64,9 @@ export interface PipelineCliOptions {
|
|
|
64
64
|
url: string[];
|
|
65
65
|
urls: string[];
|
|
66
66
|
apiUrl?: string;
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
67
|
+
artifacts: boolean;
|
|
68
|
+
artifactsDir?: string;
|
|
69
|
+
artifactsDryRun: boolean;
|
|
70
|
+
artifactsExclude?: string;
|
|
71
71
|
}
|
|
72
72
|
export declare function createPipelineCommand(): Command;
|
|
@@ -54,10 +54,10 @@ export function createPipelineCommand() {
|
|
|
54
54
|
.option("--repo-tasks-path <path>", "Path to repo-based task definitions (.ailf/tasks/ directory)")
|
|
55
55
|
.option("--remote", "Submit evaluation to the AILF API instead of running locally", false)
|
|
56
56
|
.option("--api-url <url>", "AILF API base URL (default: https://ailf-api.sanity.build)")
|
|
57
|
-
.option("--
|
|
58
|
-
.option("--
|
|
59
|
-
.option("--
|
|
60
|
-
.option("--
|
|
57
|
+
.option("--no-artifacts", "Disable all artifact writers (D0033). Overrides --artifacts-dir.")
|
|
58
|
+
.option("--artifacts-dir <path>", "Root directory for local artifact output (D0033; default: .ailf/results/captures/)")
|
|
59
|
+
.option("--artifacts-dry-run", "Run artifact writers in dry-run mode — log intended writes, touch no storage", false)
|
|
60
|
+
.option("--artifacts-exclude <types>", "Comma-separated artifact types to skip (e.g. traces,graderPrompts)")
|
|
61
61
|
.action(async (opts) => {
|
|
62
62
|
const { executePipeline } = await import("./pipeline-action.js");
|
|
63
63
|
await executePipeline(opts);
|
package/dist/commands/publish.js
CHANGED
|
@@ -27,6 +27,7 @@ import { addOutputDirOption } from "./shared/options.js";
|
|
|
27
27
|
import { getCallerCwd, resolveOutputDir } from "./shared/resolve-output-dir.js";
|
|
28
28
|
import { buildProvenance, } from "../pipeline/provenance.js";
|
|
29
29
|
import { generateReportTitle } from "../pipeline/report-title.js";
|
|
30
|
+
import { buildSlimReportSummary } from "../_vendor/ailf-core/index.js";
|
|
30
31
|
import { generateReportId, } from "../report-store.js";
|
|
31
32
|
import { withRetry } from "../sinks/retry.js";
|
|
32
33
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
@@ -174,13 +175,15 @@ async function runPublishCommand(summaryPath, outputDir, opts) {
|
|
|
174
175
|
}
|
|
175
176
|
const reportId = generateReportId();
|
|
176
177
|
const title = generateReportTitle({ provenance });
|
|
178
|
+
// W0051 Slice 3: slim the summary at publish time.
|
|
179
|
+
const slimSummary = buildSlimReportSummary(summary, provenance.mode);
|
|
177
180
|
const report = {
|
|
178
181
|
comparison: comparison ?? undefined,
|
|
179
182
|
completedAt: now,
|
|
180
183
|
durationMs: 0, // manual publish — no pipeline duration
|
|
181
184
|
id: reportId,
|
|
182
185
|
provenance,
|
|
183
|
-
summary,
|
|
186
|
+
summary: slimSummary,
|
|
184
187
|
tag: opts.tag,
|
|
185
188
|
title,
|
|
186
189
|
};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* runs command — utilities for working with run-anchored artifacts on disk.
|
|
3
|
+
*
|
|
4
|
+
* The unified artifact writer (D0033 M4) lays artifacts down under
|
|
5
|
+
* `.ailf/results/captures/runs/{runId}/…`. Downstream tooling (CI archivers,
|
|
6
|
+
* external consumers that used to parse the legacy collector tarball) still
|
|
7
|
+
* want a single-file bundle. `runs export --format tarball` reproduces that
|
|
8
|
+
* bundle from the existing tree — no re-materialisation, just a tar of the
|
|
9
|
+
* files already written.
|
|
10
|
+
*
|
|
11
|
+
* The on-disk layout is identical to what the legacy FilesystemArtifactCollector
|
|
12
|
+
* produced (same paths, same filenames), so the tarball shape is stable for
|
|
13
|
+
* existing consumers.
|
|
14
|
+
*
|
|
15
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
|
|
16
|
+
*/
|
|
17
|
+
import { Command } from "commander";
|
|
18
|
+
export declare function createRunsCommand(): Command;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* runs command — utilities for working with run-anchored artifacts on disk.
|
|
3
|
+
*
|
|
4
|
+
* The unified artifact writer (D0033 M4) lays artifacts down under
|
|
5
|
+
* `.ailf/results/captures/runs/{runId}/…`. Downstream tooling (CI archivers,
|
|
6
|
+
* external consumers that used to parse the legacy collector tarball) still
|
|
7
|
+
* want a single-file bundle. `runs export --format tarball` reproduces that
|
|
8
|
+
* bundle from the existing tree — no re-materialisation, just a tar of the
|
|
9
|
+
* files already written.
|
|
10
|
+
*
|
|
11
|
+
* The on-disk layout is identical to what the legacy FilesystemArtifactCollector
|
|
12
|
+
* produced (same paths, same filenames), so the tarball shape is stable for
|
|
13
|
+
* existing consumers.
|
|
14
|
+
*
|
|
15
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
|
|
16
|
+
*/
|
|
17
|
+
import { spawnSync } from "node:child_process";
|
|
18
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
19
|
+
import { dirname, resolve } from "node:path";
|
|
20
|
+
import { Command } from "commander";
|
|
21
|
+
const DEFAULT_ARTIFACTS_DIR = ".ailf/results/captures";
|
|
22
|
+
function resolveArtifactsDir(flag) {
|
|
23
|
+
return resolve(flag ?? process.env.AILF_ARTIFACTS_DIR ?? DEFAULT_ARTIFACTS_DIR);
|
|
24
|
+
}
|
|
25
|
+
function resolveOutputPath(explicit, artifactsDir, runId) {
|
|
26
|
+
if (explicit)
|
|
27
|
+
return resolve(explicit);
|
|
28
|
+
return resolve(artifactsDir, `runs-${runId}.tar.gz`);
|
|
29
|
+
}
|
|
30
|
+
function exportTarball(opts) {
|
|
31
|
+
if (opts.format !== "tarball") {
|
|
32
|
+
console.error(`❌ Unsupported --format "${opts.format}". Supported: tarball.`);
|
|
33
|
+
return 2;
|
|
34
|
+
}
|
|
35
|
+
const artifactsDir = resolveArtifactsDir(opts.artifactsDir);
|
|
36
|
+
const runRoot = resolve(artifactsDir, "runs", opts.runId);
|
|
37
|
+
if (!existsSync(runRoot)) {
|
|
38
|
+
console.error(`❌ No artifacts on disk for run "${opts.runId}" at ${runRoot}`);
|
|
39
|
+
console.error(" Set --artifacts-dir / AILF_ARTIFACTS_DIR or check the runId.");
|
|
40
|
+
return 1;
|
|
41
|
+
}
|
|
42
|
+
const outputPath = resolveOutputPath(opts.output, artifactsDir, opts.runId);
|
|
43
|
+
const outputDir = dirname(outputPath);
|
|
44
|
+
mkdirSync(outputDir, { recursive: true });
|
|
45
|
+
// Spawn system tar — the unified tree is byte-for-byte identical to the
|
|
46
|
+
// legacy collector's output, so `tar -czf` produces a tarball with the
|
|
47
|
+
// same file list consumers depended on.
|
|
48
|
+
const tar = spawnSync("tar", ["-czf", outputPath, "-C", resolve(artifactsDir, "runs"), opts.runId], { stdio: "inherit" });
|
|
49
|
+
if (tar.status !== 0) {
|
|
50
|
+
console.error(`❌ tar exited with status ${tar.status}`);
|
|
51
|
+
return tar.status ?? 1;
|
|
52
|
+
}
|
|
53
|
+
console.log(` ✅ Exported ${runRoot} → ${outputPath}`);
|
|
54
|
+
return 0;
|
|
55
|
+
}
|
|
56
|
+
export function createRunsCommand() {
|
|
57
|
+
const cmd = new Command("runs").description("Utilities for working with run-anchored artifact trees on disk");
|
|
58
|
+
cmd
|
|
59
|
+
.command("export")
|
|
60
|
+
.description("Export a run's artifact tree to a portable archive. Only `--format tarball` is supported.")
|
|
61
|
+
.requiredOption("--run-id <runId>", "Run id to export (required)")
|
|
62
|
+
.option("--format <fmt>", "Output format (currently only: tarball)", "tarball")
|
|
63
|
+
.option("--artifacts-dir <path>", "Override the root directory containing runs/{runId}/… (default: .ailf/results/captures)")
|
|
64
|
+
.option("-o, --output <path>", "Output archive path (default: {artifacts-dir}/runs-{runId}.tar.gz)")
|
|
65
|
+
.action((opts) => {
|
|
66
|
+
const code = exportTarball(opts);
|
|
67
|
+
if (code !== 0)
|
|
68
|
+
process.exit(code);
|
|
69
|
+
});
|
|
70
|
+
return cmd;
|
|
71
|
+
}
|
|
@@ -24,21 +24,24 @@ import { type AppContext, type ArtifactWriter, type AssertionRegistration, type
|
|
|
24
24
|
*/
|
|
25
25
|
export declare function createAppContext(config: ResolvedConfig): AppContext;
|
|
26
26
|
/**
|
|
27
|
-
* Selects
|
|
27
|
+
* Selects the `ArtifactWriter` wiring per D0033 M4:
|
|
28
28
|
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
* 2.
|
|
32
|
-
*
|
|
33
|
-
*
|
|
29
|
+
* 1. `--no-artifacts` (`config.artifactsDisabled === true`, or legacy
|
|
30
|
+
* `config.artifactUpload === false`) → `NoOpArtifactWriter`.
|
|
31
|
+
* 2. Otherwise: always attach `LocalFilesystemArtifactWriter` under
|
|
32
|
+
* `--artifacts-dir` (default `.ailf/results/captures`).
|
|
33
|
+
* 3. When a remote backend is reachable (ADC, GCLOUD_PROJECT, or an
|
|
34
|
+
* AILF API key + URL), layer it via `FanoutArtifactWriter([local, gcs])`.
|
|
35
|
+
* Local is listed first so a local success + remote failure still
|
|
36
|
+
* produces a non-null ref.
|
|
34
37
|
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
+
* Always returns a writer — pipeline code can assume `ctx.artifactWriter`
|
|
39
|
+
* is present. Producers post-W0050 drop their `if (ctx.artifactWriter)`
|
|
40
|
+
* guards in Slice 6.
|
|
38
41
|
*
|
|
39
42
|
* Exported for unit-test access; not part of the public package API.
|
|
40
43
|
*/
|
|
41
|
-
export declare function createArtifactWriter(config: ResolvedConfig, logger: Logger): ArtifactWriter
|
|
44
|
+
export declare function createArtifactWriter(config: ResolvedConfig, logger: Logger): ArtifactWriter;
|
|
42
45
|
/**
|
|
43
46
|
* Generic Promptfoo assertion types available to all evaluation modes.
|
|
44
47
|
*
|
package/dist/composition-root.js
CHANGED
|
@@ -15,12 +15,12 @@
|
|
|
15
15
|
* @see packages/core/src/ports/context.ts — AppContext interface
|
|
16
16
|
* @see docs/archive/exec-plans/ports-and-adapters/phase-7-composition-root.md
|
|
17
17
|
*/
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
18
|
+
import { InMemoryPluginRegistry, NoOpArtifactWriter, generateRunId, isArtifactType, } from "./_vendor/ailf-core/index.js";
|
|
19
|
+
import { AccumulatingArtifactWriter } from "./artifact-capture/accumulating-artifact-writer.js";
|
|
20
20
|
import { ApiGatewayArtifactWriter } from "./artifact-capture/api-gateway-artifact-writer.js";
|
|
21
|
-
import {
|
|
22
|
-
import { GcsArtifactCollector } from "./artifact-capture/gcs-collector.js";
|
|
21
|
+
import { FanoutArtifactWriter } from "./artifact-capture/fanout-artifact-writer.js";
|
|
23
22
|
import { GcsArtifactWriter } from "./artifact-capture/gcs-artifact-writer.js";
|
|
23
|
+
import { LocalFilesystemArtifactWriter } from "./artifact-capture/local-fs-artifact-writer.js";
|
|
24
24
|
import { ContentLakeCacheAdapter } from "./adapters/cache/content-lake-cache.js";
|
|
25
25
|
import { loadExternalPresets } from "./pipeline/compiler/preset-loader.js";
|
|
26
26
|
import { FilesystemCache } from "./adapters/cache/filesystem-cache.js";
|
|
@@ -60,28 +60,6 @@ export function createAppContext(config) {
|
|
|
60
60
|
const reportStore = createReportStore(config);
|
|
61
61
|
// Sinks — loaded from config/sinks
|
|
62
62
|
const sinks = loadSinks();
|
|
63
|
-
// Artifact collector — no-op by default, filesystem when --capture is set,
|
|
64
|
-
// GCS decorator when --capture-gcs-bucket is also provided (D0030/W0035)
|
|
65
|
-
let collector = new NoOpArtifactCollector();
|
|
66
|
-
if (config.captureEnabled) {
|
|
67
|
-
const fsCollector = new FilesystemArtifactCollector({
|
|
68
|
-
captureDir: config.captureDir ?? join(config.outputDir, "..", "captures"),
|
|
69
|
-
mode: config.mode,
|
|
70
|
-
compress: config.captureCompress ?? true,
|
|
71
|
-
extras: config.captureExtras ?? true,
|
|
72
|
-
pipeline: {
|
|
73
|
-
variant: config.variant,
|
|
74
|
-
source: config.source,
|
|
75
|
-
areas: config.areas,
|
|
76
|
-
},
|
|
77
|
-
});
|
|
78
|
-
collector = config.captureGcsBucket
|
|
79
|
-
? new GcsArtifactCollector(fsCollector, {
|
|
80
|
-
bucket: config.captureGcsBucket,
|
|
81
|
-
prefix: config.captureGcsPrefix,
|
|
82
|
-
})
|
|
83
|
-
: fsCollector;
|
|
84
|
-
}
|
|
85
63
|
// Artifact writer — writes run artifacts + manifest to GCS at known
|
|
86
64
|
// `runs/{runId}/…` paths (D0032). Auto-detects the right adapter from
|
|
87
65
|
// available credentials; defaults bucket to "ailf-artifacts". Set
|
|
@@ -94,7 +72,6 @@ export function createAppContext(config) {
|
|
|
94
72
|
return {
|
|
95
73
|
artifactWriter,
|
|
96
74
|
cache,
|
|
97
|
-
collector,
|
|
98
75
|
config,
|
|
99
76
|
docFetcher,
|
|
100
77
|
evalRunner,
|
|
@@ -129,44 +106,95 @@ function createLogger() {
|
|
|
129
106
|
*/
|
|
130
107
|
const DEFAULT_ARTIFACT_BUCKET = "ailf-artifacts";
|
|
131
108
|
/**
|
|
132
|
-
*
|
|
109
|
+
* D0033 M4 default root for local artifacts when `--artifacts-dir` is unset.
|
|
110
|
+
* Mirrors the pre-W0050 capture root so existing dev tooling (Studio
|
|
111
|
+
* retrieval, CI archivers) keeps finding files at the same path prefix.
|
|
112
|
+
*/
|
|
113
|
+
const DEFAULT_LOCAL_ARTIFACTS_DIR = ".ailf/results/captures";
|
|
114
|
+
/**
|
|
115
|
+
* Selects the `ArtifactWriter` wiring per D0033 M4:
|
|
133
116
|
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
* 2.
|
|
137
|
-
*
|
|
138
|
-
*
|
|
117
|
+
* 1. `--no-artifacts` (`config.artifactsDisabled === true`, or legacy
|
|
118
|
+
* `config.artifactUpload === false`) → `NoOpArtifactWriter`.
|
|
119
|
+
* 2. Otherwise: always attach `LocalFilesystemArtifactWriter` under
|
|
120
|
+
* `--artifacts-dir` (default `.ailf/results/captures`).
|
|
121
|
+
* 3. When a remote backend is reachable (ADC, GCLOUD_PROJECT, or an
|
|
122
|
+
* AILF API key + URL), layer it via `FanoutArtifactWriter([local, gcs])`.
|
|
123
|
+
* Local is listed first so a local success + remote failure still
|
|
124
|
+
* produces a non-null ref.
|
|
139
125
|
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
126
|
+
* Always returns a writer — pipeline code can assume `ctx.artifactWriter`
|
|
127
|
+
* is present. Producers post-W0050 drop their `if (ctx.artifactWriter)`
|
|
128
|
+
* guards in Slice 6.
|
|
143
129
|
*
|
|
144
130
|
* Exported for unit-test access; not part of the public package API.
|
|
145
131
|
*/
|
|
146
132
|
export function createArtifactWriter(config, logger) {
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
133
|
+
// Legacy `artifactUpload: false` still disables — treat as an alias for
|
|
134
|
+
// the canonical `artifactsDisabled: true` until W0052 removes it.
|
|
135
|
+
if (config.artifactsDisabled === true || config.artifactUpload === false) {
|
|
136
|
+
logger.debug("Artifact writer: NoOpArtifactWriter (--no-artifacts / artifactsDisabled / artifactUpload=false)");
|
|
137
|
+
return new NoOpArtifactWriter();
|
|
150
138
|
}
|
|
139
|
+
const exclude = resolveExcludeList(config.artifactsExclude, logger);
|
|
140
|
+
const rootDir = config.artifactsDir ?? DEFAULT_LOCAL_ARTIFACTS_DIR;
|
|
141
|
+
const local = new LocalFilesystemArtifactWriter({ rootDir, exclude });
|
|
142
|
+
const remote = createRemoteArtifactWriter(config, logger);
|
|
143
|
+
const base = remote
|
|
144
|
+
? new FanoutArtifactWriter([local, remote])
|
|
145
|
+
: local;
|
|
146
|
+
if (!remote) {
|
|
147
|
+
logger.debug(`Artifact writer: LocalFilesystemArtifactWriter only (rootDir=${rootDir})`);
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
logger.debug(`Artifact writer: FanoutArtifactWriter([local=${rootDir}, ${remote.constructor.name}])`);
|
|
151
|
+
}
|
|
152
|
+
// Wrap in the accumulator so FinalizeRunStep can build a populated
|
|
153
|
+
// RunManifest without each producer bookkeeping its own ArtifactRefs
|
|
154
|
+
// (W0051 Slice 3 revisit — Option B of the "manifest empty on real runs"
|
|
155
|
+
// fix).
|
|
156
|
+
return new AccumulatingArtifactWriter(base);
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Validate the exclude list against the registry. Unknown types are dropped
|
|
160
|
+
* with a warning — a typo'd CLI flag shouldn't silently match nothing.
|
|
161
|
+
*/
|
|
162
|
+
function resolveExcludeList(raw, logger) {
|
|
163
|
+
if (!raw || raw.length === 0)
|
|
164
|
+
return [];
|
|
165
|
+
const valid = [];
|
|
166
|
+
for (const name of raw) {
|
|
167
|
+
if (isArtifactType(name)) {
|
|
168
|
+
valid.push(name);
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
logger.warn(`--artifacts-exclude: "${name}" is not a known artifact type — ignored`);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
return valid;
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* The optional remote-backend writer layered on top of the local writer.
|
|
178
|
+
* Returns null when no credentials are available — the local writer stays
|
|
179
|
+
* the sole backend for that run, which is the D0033 M4 default for laptops
|
|
180
|
+
* and CI without GCS creds.
|
|
181
|
+
*/
|
|
182
|
+
function createRemoteArtifactWriter(config, logger) {
|
|
151
183
|
const bucket = config.artifactGcsBucket ?? DEFAULT_ARTIFACT_BUCKET;
|
|
152
|
-
// CI / GCP runtime — direct GCS upload (fastest, no extra hop).
|
|
153
|
-
// We treat the presence of either env var as the user opting in to ADC.
|
|
154
184
|
const hasGcsCredentials = Boolean(process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.GCLOUD_PROJECT);
|
|
155
185
|
if (hasGcsCredentials) {
|
|
156
|
-
logger.debug(`Artifact
|
|
186
|
+
logger.debug(`Artifact remote backend: GcsArtifactWriter (ADC, bucket=${bucket})`);
|
|
157
187
|
return new GcsArtifactWriter({ bucket });
|
|
158
188
|
}
|
|
159
|
-
// Local dev — request signed PUT URLs from the API gateway, no GCS creds needed.
|
|
160
189
|
if (config.apiKey && config.apiUrl) {
|
|
161
|
-
logger.debug(`Artifact
|
|
190
|
+
logger.debug(`Artifact remote backend: ApiGatewayArtifactWriter (via ${config.apiUrl}, bucket=${bucket})`);
|
|
162
191
|
return new ApiGatewayArtifactWriter({
|
|
163
192
|
apiBaseUrl: config.apiUrl,
|
|
164
193
|
apiKey: config.apiKey,
|
|
165
194
|
bucket,
|
|
166
195
|
});
|
|
167
196
|
}
|
|
168
|
-
|
|
169
|
-
return undefined;
|
|
197
|
+
return null;
|
|
170
198
|
}
|
|
171
199
|
function createCache(config) {
|
|
172
200
|
const local = new FilesystemCache(config.rootDir);
|
|
@@ -8,7 +8,6 @@
|
|
|
8
8
|
* Once all commands construct ResolvedConfig directly (or use --config),
|
|
9
9
|
* this bridge can be deleted.
|
|
10
10
|
*/
|
|
11
|
-
import { join } from "node:path";
|
|
12
11
|
import { createAppContext } from "../composition-root.js";
|
|
13
12
|
import { tryLoadConfigFile } from "../pipeline/compiler/config-loader.js";
|
|
14
13
|
/**
|
|
@@ -78,12 +77,10 @@ export function mapToResolvedConfig(opts, rootDir) {
|
|
|
78
77
|
remote: opts.remote ?? false,
|
|
79
78
|
apiUrl: opts.apiUrl ?? "https://ailf-api.sanity.build",
|
|
80
79
|
apiKey: opts.apiKey,
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
captureGcsBucket: process.env.AILF_CAPTURE_GCS_BUCKET,
|
|
86
|
-
captureGcsPrefix: process.env.AILF_CAPTURE_GCS_PREFIX,
|
|
80
|
+
artifactsDisabled: opts.artifactsDisabled,
|
|
81
|
+
artifactsDir: opts.artifactsDir,
|
|
82
|
+
artifactsDryRun: opts.artifactsDryRun,
|
|
83
|
+
artifactsExclude: opts.artifactsExclude,
|
|
87
84
|
artifactGcsBucket: process.env.AILF_GCS_ARTIFACT_BUCKET,
|
|
88
85
|
artifactUpload: parseArtifactUploadEnv(process.env.AILF_ARTIFACT_UPLOAD),
|
|
89
86
|
};
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* each step completes. This enables the GET /v1/jobs/:jobId polling
|
|
12
12
|
* endpoint to show real-time progress.
|
|
13
13
|
*/
|
|
14
|
-
import type
|
|
14
|
+
import { type AppContext, type PipelineResult, type PipelineStep } from "../_vendor/ailf-core/index.d.ts";
|
|
15
15
|
/**
|
|
16
16
|
* Run a sequence of pipeline steps, short-circuiting on required step failure.
|
|
17
17
|
*
|