@sanity/ailf 2.7.1 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_vendor/ailf-core/artifact-capture/association.d.ts +35 -0
- package/dist/_vendor/ailf-core/artifact-capture/association.js +28 -0
- package/dist/_vendor/ailf-core/artifact-registry.d.ts +173 -0
- package/dist/_vendor/ailf-core/artifact-registry.js +811 -0
- package/dist/_vendor/ailf-core/index.d.ts +3 -1
- package/dist/_vendor/ailf-core/index.js +3 -1
- package/dist/_vendor/ailf-core/ports/artifact-collector.d.ts +3 -3
- package/dist/_vendor/ailf-core/ports/artifact-writer.d.ts +95 -0
- package/dist/_vendor/ailf-core/ports/artifact-writer.js +51 -0
- package/dist/_vendor/ailf-core/ports/context.d.ts +32 -3
- package/dist/_vendor/ailf-core/ports/index.d.ts +3 -3
- package/dist/_vendor/ailf-core/ports/index.js +1 -1
- package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +6 -6
- package/dist/_vendor/ailf-core/services/index.d.ts +1 -0
- package/dist/_vendor/ailf-core/services/index.js +1 -0
- package/dist/_vendor/ailf-core/services/slim-report-summary.d.ts +31 -0
- package/dist/_vendor/ailf-core/services/slim-report-summary.js +217 -0
- package/dist/_vendor/ailf-core/types/branded-ids.d.ts +42 -0
- package/dist/_vendor/ailf-core/types/branded-ids.js +21 -0
- package/dist/_vendor/ailf-core/types/index.d.ts +298 -77
- package/dist/_vendor/ailf-core/types/index.js +1 -1
- package/dist/_vendor/ailf-shared/index.d.ts +2 -0
- package/dist/_vendor/ailf-shared/index.js +2 -0
- package/dist/_vendor/ailf-shared/run-context.d.ts +55 -0
- package/dist/_vendor/ailf-shared/run-context.js +17 -0
- package/dist/_vendor/ailf-shared/run-trigger.d.ts +30 -0
- package/dist/_vendor/ailf-shared/run-trigger.js +13 -0
- package/dist/artifact-capture/accumulating-artifact-writer.d.ts +50 -0
- package/dist/artifact-capture/accumulating-artifact-writer.js +111 -0
- package/dist/artifact-capture/api-gateway-artifact-writer.d.ts +52 -0
- package/dist/artifact-capture/api-gateway-artifact-writer.js +199 -0
- package/dist/artifact-capture/emit-file.d.ts +28 -0
- package/dist/artifact-capture/emit-file.js +56 -0
- package/dist/artifact-capture/fanout-artifact-writer.d.ts +39 -0
- package/dist/artifact-capture/fanout-artifact-writer.js +76 -0
- package/dist/artifact-capture/filesystem-collector.d.ts +22 -4
- package/dist/artifact-capture/filesystem-collector.js +48 -23
- package/dist/artifact-capture/gcs-artifact-writer.d.ts +67 -0
- package/dist/artifact-capture/gcs-artifact-writer.js +343 -0
- package/dist/artifact-capture/local-fs-artifact-writer.d.ts +71 -0
- package/dist/artifact-capture/local-fs-artifact-writer.js +273 -0
- package/dist/commands/explain-handler.js +4 -0
- package/dist/commands/pipeline-action.d.ts +5 -0
- package/dist/commands/pipeline-action.js +56 -5
- package/dist/commands/pipeline.d.ts +4 -0
- package/dist/commands/pipeline.js +6 -2
- package/dist/commands/publish.js +7 -3
- package/dist/composition-root.d.ts +14 -11
- package/dist/composition-root.js +90 -31
- package/dist/orchestration/build-step-sequence.js +6 -1
- package/dist/orchestration/pipeline-orchestrator.d.ts +1 -1
- package/dist/orchestration/pipeline-orchestrator.js +41 -30
- package/dist/orchestration/steps/calculate-scores-step.d.ts +1 -1
- package/dist/orchestration/steps/calculate-scores-step.js +50 -10
- package/dist/orchestration/steps/callback-step.d.ts +1 -1
- package/dist/orchestration/steps/callback-step.js +6 -4
- package/dist/orchestration/steps/compare-step.d.ts +1 -1
- package/dist/orchestration/steps/compare-step.js +4 -2
- package/dist/orchestration/steps/discovery-report-step.d.ts +1 -1
- package/dist/orchestration/steps/discovery-report-step.js +4 -1
- package/dist/orchestration/steps/fetch-docs-step.js +9 -15
- package/dist/orchestration/steps/finalize-run-step.d.ts +29 -0
- package/dist/orchestration/steps/finalize-run-step.js +117 -0
- package/dist/orchestration/steps/gap-analysis-step.js +34 -6
- package/dist/orchestration/steps/generate-configs-step.d.ts +1 -1
- package/dist/orchestration/steps/generate-configs-step.js +11 -11
- package/dist/orchestration/steps/publish-report-step.d.ts +1 -1
- package/dist/orchestration/steps/publish-report-step.js +40 -55
- package/dist/orchestration/steps/readiness-step.d.ts +1 -1
- package/dist/orchestration/steps/readiness-step.js +4 -1
- package/dist/orchestration/steps/report-step.d.ts +1 -1
- package/dist/orchestration/steps/report-step.js +6 -3
- package/dist/orchestration/steps/run-eval-step.js +14 -9
- package/dist/pipeline/calculate-scores.js +13 -2
- package/dist/pipeline/compare.d.ts +2 -2
- package/dist/pipeline/emit-eval-results.d.ts +38 -0
- package/dist/pipeline/emit-eval-results.js +100 -0
- package/dist/pipeline/provenance.d.ts +24 -44
- package/dist/pipeline/provenance.js +17 -165
- package/dist/pipeline/report-title.d.ts +2 -2
- package/dist/pipeline/run-context.d.ts +57 -0
- package/dist/pipeline/run-context.js +156 -0
- package/dist/pipeline/upload-test-outputs.d.ts +26 -0
- package/dist/pipeline/upload-test-outputs.js +34 -0
- package/dist/report-store.js +4 -2
- package/package.json +3 -3
- package/dist/_vendor/ailf-core/ports/artifact-uploader.d.ts +0 -35
- package/dist/_vendor/ailf-core/ports/artifact-uploader.js +0 -18
- package/dist/artifact-capture/api-gateway-artifact-uploader.d.ts +0 -41
- package/dist/artifact-capture/api-gateway-artifact-uploader.js +0 -123
- package/dist/artifact-capture/gcs-report-artifact-uploader.d.ts +0 -31
- package/dist/artifact-capture/gcs-report-artifact-uploader.js +0 -66
|
@@ -1,17 +1,34 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* FilesystemArtifactCollector —
|
|
2
|
+
* FilesystemArtifactCollector — DEPRECATED legacy capture path.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* W0050 Slice 5: the unified artifact writer (`ctx.artifactWriter.emit()`)
|
|
5
|
+
* owns artifact production starting this release. This class is retained
|
|
6
|
+
* for one release cycle so producer steps that haven't migrated yet still
|
|
7
|
+
* have a surface to call against; W0052 deletes the class + port.
|
|
8
|
+
*
|
|
9
|
+
* Behavioral changes vs. pre-W0050:
|
|
10
|
+
*
|
|
11
|
+
* - **No tarball.** The `compress` option is ignored; flush() always writes
|
|
12
|
+
* a loose directory tree. The legacy `.tar.gz` artifact archive is no
|
|
13
|
+
* longer produced (D0033 AC8 — "no new capture tarballs").
|
|
14
|
+
* - **First-call deprecation warning.** The first invocation of capture()
|
|
15
|
+
* or captureFile() on any instance emits a one-time-per-process warning
|
|
16
|
+
* pointing callers at `ctx.artifactWriter.emit()`.
|
|
17
|
+
*
|
|
18
|
+
* Everything else (in-memory accumulation, redaction, loose-file output at
|
|
19
|
+
* flush time) is unchanged — existing tests continue to pass. This is the
|
|
20
|
+
* **minimum** pass-through posture. Full delegation to
|
|
21
|
+
* `LocalFilesystemArtifactWriter` is W0052 scope, at which point the port
|
|
22
|
+
* and this class are both deleted.
|
|
7
23
|
*
|
|
8
24
|
* Design principles:
|
|
9
25
|
* - capture() and captureFile() are synchronous (no I/O during step execution)
|
|
10
26
|
* - flush() does all I/O at pipeline end
|
|
11
27
|
* - Failures in capture/captureFile are swallowed (P5: non-blocking)
|
|
28
|
+
*
|
|
29
|
+
* @deprecated Use `ctx.artifactWriter.emit()` instead; removal scheduled for W0052.
|
|
12
30
|
*/
|
|
13
|
-
import {
|
|
14
|
-
import { copyFileSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync, } from "node:fs";
|
|
31
|
+
import { copyFileSync, mkdirSync, readFileSync, statSync, writeFileSync, } from "node:fs";
|
|
15
32
|
import path from "node:path";
|
|
16
33
|
import { redactArtifactData } from "./redact-artifact.js";
|
|
17
34
|
// ---------------------------------------------------------------------------
|
|
@@ -69,6 +86,17 @@ function fileExtension(format) {
|
|
|
69
86
|
// ---------------------------------------------------------------------------
|
|
70
87
|
// Collector
|
|
71
88
|
// ---------------------------------------------------------------------------
|
|
89
|
+
/**
|
|
90
|
+
* Module-level flag so the W0050 deprecation warning fires exactly once
|
|
91
|
+
* per process even when multiple collectors are constructed.
|
|
92
|
+
*/
|
|
93
|
+
let warnedLegacyCollector = false;
|
|
94
|
+
function warnLegacyCollectorOnce() {
|
|
95
|
+
if (warnedLegacyCollector)
|
|
96
|
+
return;
|
|
97
|
+
warnedLegacyCollector = true;
|
|
98
|
+
console.warn("FilesystemArtifactCollector is deprecated (W0050). Producers should migrate to ctx.artifactWriter.emit(); this class will be removed in W0052.");
|
|
99
|
+
}
|
|
72
100
|
export class FilesystemArtifactCollector {
|
|
73
101
|
enabled = true;
|
|
74
102
|
extrasEnabled;
|
|
@@ -89,6 +117,7 @@ export class FilesystemArtifactCollector {
|
|
|
89
117
|
this.outputDir = path.join(options.captureDir, `${options.mode}-${timestamp}-${shortId}`);
|
|
90
118
|
}
|
|
91
119
|
capture(step, type, data, meta) {
|
|
120
|
+
warnLegacyCollectorOnce();
|
|
92
121
|
try {
|
|
93
122
|
this.entries.push({
|
|
94
123
|
step,
|
|
@@ -103,6 +132,7 @@ export class FilesystemArtifactCollector {
|
|
|
103
132
|
}
|
|
104
133
|
}
|
|
105
134
|
captureFile(step, type, filePath, meta) {
|
|
135
|
+
warnLegacyCollectorOnce();
|
|
106
136
|
try {
|
|
107
137
|
this.entries.push({
|
|
108
138
|
step,
|
|
@@ -208,24 +238,12 @@ export class FilesystemArtifactCollector {
|
|
|
208
238
|
};
|
|
209
239
|
const manifestContent = JSON.stringify(manifest, null, 2);
|
|
210
240
|
writeFileSync(path.join(this.outputDir, "manifest.json"), manifestContent, "utf-8");
|
|
211
|
-
//
|
|
241
|
+
// W0050 Slice 5 — compression disabled unconditionally. The legacy
|
|
242
|
+
// .tar.gz archive is no longer produced (D0033 AC8 "no new capture
|
|
243
|
+
// tarballs"); callers that asked for it get the loose directory plus
|
|
244
|
+
// a one-time warning.
|
|
212
245
|
if (this.options.compress) {
|
|
213
|
-
|
|
214
|
-
const archivePath = `${this.outputDir}.tar.gz`;
|
|
215
|
-
const parentDir = path.dirname(this.outputDir);
|
|
216
|
-
const dirName = path.basename(this.outputDir);
|
|
217
|
-
execFileSync("tar", ["-czf", archivePath, "-C", parentDir, dirName]);
|
|
218
|
-
rmSync(this.outputDir, { recursive: true });
|
|
219
|
-
return {
|
|
220
|
-
artifactCount: manifestEntries.length,
|
|
221
|
-
destination: archivePath,
|
|
222
|
-
totalBytes,
|
|
223
|
-
compressed: true,
|
|
224
|
-
};
|
|
225
|
-
}
|
|
226
|
-
catch {
|
|
227
|
-
// Non-blocking: compression failed, keep the raw directory
|
|
228
|
-
}
|
|
246
|
+
warnTarballSuppressedOnce();
|
|
229
247
|
}
|
|
230
248
|
return {
|
|
231
249
|
artifactCount: manifestEntries.length,
|
|
@@ -235,3 +253,10 @@ export class FilesystemArtifactCollector {
|
|
|
235
253
|
};
|
|
236
254
|
}
|
|
237
255
|
}
|
|
256
|
+
let warnedTarballSuppressed = false;
|
|
257
|
+
function warnTarballSuppressedOnce() {
|
|
258
|
+
if (warnedTarballSuppressed)
|
|
259
|
+
return;
|
|
260
|
+
warnedTarballSuppressed = true;
|
|
261
|
+
console.warn("capture.compress is deprecated (W0050); tarball output is no longer produced. The loose directory at <captureDir>/<run>-<ts>-<id>/ contains the same files.");
|
|
262
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GcsArtifactWriter — writes AILF run artifacts + manifest directly to GCS.
|
|
3
|
+
*
|
|
4
|
+
* Uses Application Default Credentials (ADC). Used when the CLI runs in CI or
|
|
5
|
+
* anywhere ADC is configured — the client talks to GCS without the API Gateway
|
|
6
|
+
* acting as a middleman.
|
|
7
|
+
*
|
|
8
|
+
* Paths come from `ARTIFACT_REGISTRY` so writers, signers, and readers agree.
|
|
9
|
+
*
|
|
10
|
+
* ## W0049 API surface
|
|
11
|
+
*
|
|
12
|
+
* - `emit(type, association, payload)` — the canonical single-shot write.
|
|
13
|
+
* Dispatch on `descriptor.layout` is internal; callers pass axis values
|
|
14
|
+
* and the writer resolves the path.
|
|
15
|
+
* - `appendNdjson(type, association, rows)` — streaming-append for `traces`.
|
|
16
|
+
* Each call writes a numbered part object (`.ndjson.part-NNNN`); the
|
|
17
|
+
* parts are composed into the final object lazily at `writeManifest` time
|
|
18
|
+
* via GCS object compose. When a stream accumulates > 32 parts the writer
|
|
19
|
+
* rolls up parts into intermediate composites to stay under the GCS
|
|
20
|
+
* compose cap.
|
|
21
|
+
* - `writeBulk` / `writePerEntry` — @deprecated legacy surface. Removal in W0052.
|
|
22
|
+
*
|
|
23
|
+
* Design principles:
|
|
24
|
+
* - P5: Non-blocking — upload failure returns null, never throws.
|
|
25
|
+
* - Lazy client — Storage created on first write.
|
|
26
|
+
*
|
|
27
|
+
* @see docs/decisions/D0032-run-anchored-artifact-store.md
|
|
28
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
|
|
29
|
+
*/
|
|
30
|
+
import { Storage } from "@google-cloud/storage";
|
|
31
|
+
import { type ArtifactEntry, type ArtifactRef, type ArtifactType, type ArtifactWriter, type AssociationValues, type RunId, type RunManifest } from "../_vendor/ailf-core/index.d.ts";
|
|
32
|
+
export interface GcsArtifactWriterOptions {
|
|
33
|
+
/** GCS bucket name (e.g., "ailf-artifacts") */
|
|
34
|
+
bucket: string;
|
|
35
|
+
/**
|
|
36
|
+
* Optional pre-constructed Storage client. When omitted the writer
|
|
37
|
+
* constructs one lazily from Application Default Credentials. Test code
|
|
38
|
+
* supplies a fake here to avoid real network calls.
|
|
39
|
+
*/
|
|
40
|
+
storage?: Storage;
|
|
41
|
+
}
|
|
42
|
+
export declare class GcsArtifactWriter implements ArtifactWriter {
|
|
43
|
+
private client;
|
|
44
|
+
private readonly options;
|
|
45
|
+
private readonly ndjsonStreams;
|
|
46
|
+
constructor(options: GcsArtifactWriterOptions);
|
|
47
|
+
emit<T extends ArtifactType>(type: T, association: AssociationValues, payload: unknown): Promise<ArtifactRef | null>;
|
|
48
|
+
appendNdjson<T extends ArtifactType>(type: T, association: AssociationValues, rows: readonly unknown[]): Promise<ArtifactRef | null>;
|
|
49
|
+
writeManifest(runId: RunId, manifest: RunManifest): Promise<ArtifactRef | null>;
|
|
50
|
+
/** @deprecated Use `emit()` instead. Routes through the same GCS I/O. */
|
|
51
|
+
writeBulk(type: ArtifactType, runId: RunId, data: unknown): Promise<ArtifactRef | null>;
|
|
52
|
+
/** @deprecated Use `emit()` per entry instead. */
|
|
53
|
+
writePerEntry(type: ArtifactType, runId: RunId, entries: readonly ArtifactEntry[]): Promise<ArtifactRef | null>;
|
|
54
|
+
/**
|
|
55
|
+
* Compose all buffered NDJSON streams into their final objects. Called
|
|
56
|
+
* from `writeManifest` so the manifest is the single sync point for
|
|
57
|
+
* NDJSON finalization.
|
|
58
|
+
*
|
|
59
|
+
* When `partCount > GCS_COMPOSE_MAX`, parts are rolled up in groups of
|
|
60
|
+
* `GCS_COMPOSE_MAX` into intermediate composites, then composed. This
|
|
61
|
+
* stays under the per-call source cap at the cost of one extra round
|
|
62
|
+
* trip per 32 additional parts.
|
|
63
|
+
*/
|
|
64
|
+
private finalizeNdjsonStreams;
|
|
65
|
+
private putBody;
|
|
66
|
+
private getClient;
|
|
67
|
+
}
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GcsArtifactWriter — writes AILF run artifacts + manifest directly to GCS.
|
|
3
|
+
*
|
|
4
|
+
* Uses Application Default Credentials (ADC). Used when the CLI runs in CI or
|
|
5
|
+
* anywhere ADC is configured — the client talks to GCS without the API Gateway
|
|
6
|
+
* acting as a middleman.
|
|
7
|
+
*
|
|
8
|
+
* Paths come from `ARTIFACT_REGISTRY` so writers, signers, and readers agree.
|
|
9
|
+
*
|
|
10
|
+
* ## W0049 API surface
|
|
11
|
+
*
|
|
12
|
+
* - `emit(type, association, payload)` — the canonical single-shot write.
|
|
13
|
+
* Dispatch on `descriptor.layout` is internal; callers pass axis values
|
|
14
|
+
* and the writer resolves the path.
|
|
15
|
+
* - `appendNdjson(type, association, rows)` — streaming-append for `traces`.
|
|
16
|
+
* Each call writes a numbered part object (`.ndjson.part-NNNN`); the
|
|
17
|
+
* parts are composed into the final object lazily at `writeManifest` time
|
|
18
|
+
* via GCS object compose. When a stream accumulates > 32 parts the writer
|
|
19
|
+
* rolls up parts into intermediate composites to stay under the GCS
|
|
20
|
+
* compose cap.
|
|
21
|
+
* - `writeBulk` / `writePerEntry` — @deprecated legacy surface. Removal in W0052.
|
|
22
|
+
*
|
|
23
|
+
* Design principles:
|
|
24
|
+
* - P5: Non-blocking — upload failure returns null, never throws.
|
|
25
|
+
* - Lazy client — Storage created on first write.
|
|
26
|
+
*
|
|
27
|
+
* @see docs/decisions/D0032-run-anchored-artifact-store.md
|
|
28
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
|
|
29
|
+
*/
|
|
30
|
+
import { Storage } from "@google-cloud/storage";
|
|
31
|
+
import { ARTIFACT_REGISTRY, buildManifestPreview, } from "../_vendor/ailf-core/index.js";
|
|
32
|
+
import { redactArtifactData } from "./redact-artifact.js";
|
|
33
|
+
/**
|
|
34
|
+
* GCS's object compose operation accepts at most 32 source objects per call
|
|
35
|
+
* ([docs](https://cloud.google.com/storage/docs/composite-objects)). When an
|
|
36
|
+
* NDJSON stream accumulates more than 32 parts the writer rolls up groups of
|
|
37
|
+
* 32 into intermediate composites and composes those — one extra round trip
|
|
38
|
+
* per 32 additional parts.
|
|
39
|
+
*/
|
|
40
|
+
const GCS_COMPOSE_MAX = 32;
|
|
41
|
+
export class GcsArtifactWriter {
|
|
42
|
+
client = null;
|
|
43
|
+
options;
|
|
44
|
+
ndjsonStreams = new Map();
|
|
45
|
+
constructor(options) {
|
|
46
|
+
this.options = options;
|
|
47
|
+
if (options.storage) {
|
|
48
|
+
this.client = options.storage;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// ---- Canonical W0049 API ------------------------------------------------
|
|
52
|
+
async emit(type, association, payload) {
|
|
53
|
+
const descriptor = ARTIFACT_REGISTRY[type];
|
|
54
|
+
const runId = association.run;
|
|
55
|
+
if (!runId) {
|
|
56
|
+
console.warn(` ⚠️ emit("${type}"): association.run is required, skipping`);
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
// AC10 — redact at the writer boundary so secrets never reach GCS.
|
|
60
|
+
const redacted = redactArtifactData(payload);
|
|
61
|
+
// Preview reads the pre-redaction payload (same as local writer — the
|
|
62
|
+
// preview carries a descriptor-controlled summary bounded by capBytes,
|
|
63
|
+
// not the raw entry bytes).
|
|
64
|
+
const preview = buildManifestPreview(descriptor, payload);
|
|
65
|
+
if (descriptor.layout === "bulk") {
|
|
66
|
+
const path = descriptor.objectPath(runId);
|
|
67
|
+
const ref = await this.putBody(path, serializeForMime(redacted, descriptor.mime), {
|
|
68
|
+
layout: "bulk",
|
|
69
|
+
mime: descriptor.mime,
|
|
70
|
+
entryCount: entryCountOf(redacted),
|
|
71
|
+
});
|
|
72
|
+
if (!ref)
|
|
73
|
+
return null;
|
|
74
|
+
return preview === undefined ? ref : { ...ref, preview };
|
|
75
|
+
}
|
|
76
|
+
// per-entry
|
|
77
|
+
const entryKey = descriptor.formatEntryKey(association);
|
|
78
|
+
const path = descriptor.objectPath(runId, entryKey);
|
|
79
|
+
const ref = await this.putBody(path, serializeForMime(redacted, descriptor.mime), { layout: "per-entry", mime: descriptor.mime });
|
|
80
|
+
if (!ref)
|
|
81
|
+
return null;
|
|
82
|
+
return {
|
|
83
|
+
...ref,
|
|
84
|
+
path: `runs/${runId}/${descriptor.slug}`,
|
|
85
|
+
entryCount: 1,
|
|
86
|
+
entries: [
|
|
87
|
+
{
|
|
88
|
+
key: entryKey,
|
|
89
|
+
bytes: ref.bytes ?? 0,
|
|
90
|
+
association,
|
|
91
|
+
...(preview === undefined ? {} : { preview }),
|
|
92
|
+
},
|
|
93
|
+
],
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
async appendNdjson(type, association, rows) {
|
|
97
|
+
const descriptor = ARTIFACT_REGISTRY[type];
|
|
98
|
+
if (descriptor.mime !== "application/x-ndjson") {
|
|
99
|
+
console.warn(` ⚠️ appendNdjson("${type}"): descriptor mime is ${descriptor.mime}, not application/x-ndjson — skipping`);
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
const runId = association.run;
|
|
103
|
+
if (!runId) {
|
|
104
|
+
console.warn(` ⚠️ appendNdjson("${type}"): association.run is required, skipping`);
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
if (rows.length === 0)
|
|
108
|
+
return null;
|
|
109
|
+
const entryKey = descriptor.formatEntryKey(association);
|
|
110
|
+
const finalPath = descriptor.objectPath(runId, entryKey);
|
|
111
|
+
const streamKey = `${type}::${entryKey}`;
|
|
112
|
+
let state = this.ndjsonStreams.get(streamKey);
|
|
113
|
+
if (!state) {
|
|
114
|
+
state = { finalPath, partCount: 0, totalBytes: 0 };
|
|
115
|
+
this.ndjsonStreams.set(streamKey, state);
|
|
116
|
+
}
|
|
117
|
+
const partPath = `${finalPath}.part-${String(state.partCount).padStart(4, "0")}`;
|
|
118
|
+
// AC10 — redact per row before serializing the NDJSON batch.
|
|
119
|
+
const redactedRows = rows.map((r) => redactArtifactData(r));
|
|
120
|
+
const body = redactedRows.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
121
|
+
const bytes = Buffer.byteLength(body, "utf-8");
|
|
122
|
+
try {
|
|
123
|
+
const storage = this.getClient();
|
|
124
|
+
await storage
|
|
125
|
+
.bucket(this.options.bucket)
|
|
126
|
+
.file(partPath)
|
|
127
|
+
.save(body, { contentType: "application/x-ndjson" });
|
|
128
|
+
state.partCount++;
|
|
129
|
+
state.totalBytes += bytes;
|
|
130
|
+
}
|
|
131
|
+
catch (err) {
|
|
132
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
133
|
+
console.warn(` ⚠️ NDJSON part upload failed (non-blocking): ${partPath} — ${message}`);
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
return {
|
|
137
|
+
store: "gcs",
|
|
138
|
+
bucket: this.options.bucket,
|
|
139
|
+
path: `runs/${runId}/${descriptor.slug}`,
|
|
140
|
+
bytes: state.totalBytes,
|
|
141
|
+
entryCount: 1,
|
|
142
|
+
layout: "per-entry",
|
|
143
|
+
entries: [
|
|
144
|
+
{
|
|
145
|
+
key: entryKey,
|
|
146
|
+
bytes: state.totalBytes,
|
|
147
|
+
association,
|
|
148
|
+
},
|
|
149
|
+
],
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
async writeManifest(runId, manifest) {
|
|
153
|
+
await this.finalizeNdjsonStreams();
|
|
154
|
+
const path = `runs/${runId}/manifest.json`;
|
|
155
|
+
return this.putBody(path, JSON.stringify(manifest), {
|
|
156
|
+
layout: "bulk",
|
|
157
|
+
mime: "application/json",
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
// ---- Deprecated legacy surface (W0052) ----------------------------------
|
|
161
|
+
/** @deprecated Use `emit()` instead. Routes through the same GCS I/O. */
|
|
162
|
+
async writeBulk(type, runId, data) {
|
|
163
|
+
const descriptor = ARTIFACT_REGISTRY[type];
|
|
164
|
+
if (descriptor.layout !== "bulk") {
|
|
165
|
+
console.warn(` ⚠️ writeBulk("${type}"): descriptor layout is "${descriptor.layout}", not "bulk" — skipping`);
|
|
166
|
+
return null;
|
|
167
|
+
}
|
|
168
|
+
const path = descriptor.objectPath(runId);
|
|
169
|
+
const redacted = redactArtifactData(data);
|
|
170
|
+
return this.putBody(path, serializeForMime(redacted, descriptor.mime), {
|
|
171
|
+
layout: "bulk",
|
|
172
|
+
mime: descriptor.mime,
|
|
173
|
+
entryCount: entryCountOf(redacted),
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
/** @deprecated Use `emit()` per entry instead. */
|
|
177
|
+
async writePerEntry(type, runId, entries) {
|
|
178
|
+
const descriptor = ARTIFACT_REGISTRY[type];
|
|
179
|
+
if (descriptor.layout !== "per-entry") {
|
|
180
|
+
console.warn(` ⚠️ writePerEntry("${type}"): descriptor layout is "${descriptor.layout}", not "per-entry" — skipping`);
|
|
181
|
+
return null;
|
|
182
|
+
}
|
|
183
|
+
if (!descriptor.parseEntryKey) {
|
|
184
|
+
console.warn(` ⚠️ writePerEntry("${type}"): descriptor has no parseEntryKey`);
|
|
185
|
+
return null;
|
|
186
|
+
}
|
|
187
|
+
const storage = this.getClient();
|
|
188
|
+
const uploaded = [];
|
|
189
|
+
let totalBytes = 0;
|
|
190
|
+
for (const entry of entries) {
|
|
191
|
+
const parsed = descriptor.parseEntryKey(entry.key);
|
|
192
|
+
if (!parsed.ok) {
|
|
193
|
+
console.warn(` ⚠️ Skipping entry with invalid key "${entry.key}": ${parsed.reason}`);
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
const path = descriptor.objectPath(runId, entry.key);
|
|
197
|
+
const redacted = redactArtifactData(entry.data);
|
|
198
|
+
const body = serializeForMime(redacted, descriptor.mime);
|
|
199
|
+
const bytes = Buffer.byteLength(body, "utf-8");
|
|
200
|
+
try {
|
|
201
|
+
await storage
|
|
202
|
+
.bucket(this.options.bucket)
|
|
203
|
+
.file(path)
|
|
204
|
+
.save(body, { contentType: descriptor.mime });
|
|
205
|
+
uploaded.push({ key: entry.key, bytes });
|
|
206
|
+
totalBytes += bytes;
|
|
207
|
+
}
|
|
208
|
+
catch (err) {
|
|
209
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
210
|
+
console.warn(` ⚠️ Artifact entry upload failed (non-blocking): ${path} — ${message}`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
if (uploaded.length === 0)
|
|
214
|
+
return null;
|
|
215
|
+
return {
|
|
216
|
+
store: "gcs",
|
|
217
|
+
bucket: this.options.bucket,
|
|
218
|
+
path: `runs/${runId}/${descriptor.slug}`,
|
|
219
|
+
bytes: totalBytes,
|
|
220
|
+
entryCount: uploaded.length,
|
|
221
|
+
layout: "per-entry",
|
|
222
|
+
entries: uploaded,
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
// ---- Internals ----------------------------------------------------------
|
|
226
|
+
/**
|
|
227
|
+
* Compose all buffered NDJSON streams into their final objects. Called
|
|
228
|
+
* from `writeManifest` so the manifest is the single sync point for
|
|
229
|
+
* NDJSON finalization.
|
|
230
|
+
*
|
|
231
|
+
* When `partCount > GCS_COMPOSE_MAX`, parts are rolled up in groups of
|
|
232
|
+
* `GCS_COMPOSE_MAX` into intermediate composites, then composed. This
|
|
233
|
+
* stays under the per-call source cap at the cost of one extra round
|
|
234
|
+
* trip per 32 additional parts.
|
|
235
|
+
*/
|
|
236
|
+
async finalizeNdjsonStreams() {
|
|
237
|
+
for (const state of this.ndjsonStreams.values()) {
|
|
238
|
+
if (state.partCount === 0)
|
|
239
|
+
continue;
|
|
240
|
+
const storage = this.getClient();
|
|
241
|
+
const bucket = storage.bucket(this.options.bucket);
|
|
242
|
+
const partPaths = Array.from({ length: state.partCount }, (_, i) => `${state.finalPath}.part-${String(i).padStart(4, "0")}`);
|
|
243
|
+
try {
|
|
244
|
+
const finalPath = await composeInGroups(bucket, partPaths, state.finalPath);
|
|
245
|
+
if (finalPath !== state.finalPath) {
|
|
246
|
+
console.warn(` ⚠️ NDJSON compose produced "${finalPath}" (expected "${state.finalPath}")`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
catch (err) {
|
|
250
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
251
|
+
console.warn(` ⚠️ NDJSON compose failed (non-blocking): ${state.finalPath} — ${message}`);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
this.ndjsonStreams.clear();
|
|
255
|
+
}
|
|
256
|
+
async putBody(path, body, meta) {
|
|
257
|
+
const bytes = Buffer.byteLength(body, "utf-8");
|
|
258
|
+
try {
|
|
259
|
+
const storage = this.getClient();
|
|
260
|
+
await storage
|
|
261
|
+
.bucket(this.options.bucket)
|
|
262
|
+
.file(path)
|
|
263
|
+
.save(body, { contentType: meta.mime });
|
|
264
|
+
return {
|
|
265
|
+
store: "gcs",
|
|
266
|
+
bucket: this.options.bucket,
|
|
267
|
+
path,
|
|
268
|
+
bytes,
|
|
269
|
+
entryCount: meta.entryCount,
|
|
270
|
+
layout: meta.layout,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
catch (err) {
|
|
274
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
275
|
+
console.warn(` ⚠️ Artifact upload failed (non-blocking): ${path} — ${message}`);
|
|
276
|
+
return null;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
getClient() {
|
|
280
|
+
if (this.client)
|
|
281
|
+
return this.client;
|
|
282
|
+
this.client = new Storage();
|
|
283
|
+
return this.client;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
// ---------------------------------------------------------------------------
|
|
287
|
+
// Helpers
|
|
288
|
+
// ---------------------------------------------------------------------------
|
|
289
|
+
function serializeForMime(payload, mime) {
|
|
290
|
+
if (mime === "text/markdown" || mime === "application/yaml") {
|
|
291
|
+
if (typeof payload === "string")
|
|
292
|
+
return payload;
|
|
293
|
+
return String(payload ?? "");
|
|
294
|
+
}
|
|
295
|
+
return JSON.stringify(payload);
|
|
296
|
+
}
|
|
297
|
+
function entryCountOf(data) {
|
|
298
|
+
if (typeof data === "object" &&
|
|
299
|
+
data !== null &&
|
|
300
|
+
"entries" in data &&
|
|
301
|
+
typeof data.entries === "object") {
|
|
302
|
+
return Object.keys(data.entries)
|
|
303
|
+
.length;
|
|
304
|
+
}
|
|
305
|
+
return undefined;
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Compose a list of GCS parts into a destination object. When the part count
|
|
309
|
+
* exceeds `GCS_COMPOSE_MAX`, roll up groups into intermediate composites and
|
|
310
|
+
* recurse until a single compose call suffices.
|
|
311
|
+
*
|
|
312
|
+
* Returns the destination path on success. Intermediate composites are
|
|
313
|
+
* written to `{dest}.roll-{n}` and left in place — they are cheap, and
|
|
314
|
+
* keeping them simplifies failure recovery.
|
|
315
|
+
*/
|
|
316
|
+
async function composeInGroups(bucket, partPaths, destPath) {
|
|
317
|
+
if (partPaths.length === 0) {
|
|
318
|
+
throw new Error(`composeInGroups: no parts to compose for ${destPath}`);
|
|
319
|
+
}
|
|
320
|
+
if (partPaths.length === 1) {
|
|
321
|
+
// Single part — server-side copy into place. Avoids streaming the
|
|
322
|
+
// object through this process (unlike download + re-upload) and
|
|
323
|
+
// preserves the source object's Content-Type / metadata on the
|
|
324
|
+
// destination (W0049 review finding C2).
|
|
325
|
+
await bucket.file(partPaths[0]).copy(bucket.file(destPath));
|
|
326
|
+
return destPath;
|
|
327
|
+
}
|
|
328
|
+
if (partPaths.length <= GCS_COMPOSE_MAX) {
|
|
329
|
+
const sources = partPaths.map((p) => bucket.file(p));
|
|
330
|
+
await bucket.combine(sources, bucket.file(destPath));
|
|
331
|
+
return destPath;
|
|
332
|
+
}
|
|
333
|
+
// >32 parts: roll up in groups of GCS_COMPOSE_MAX and recurse.
|
|
334
|
+
const rollups = [];
|
|
335
|
+
for (let i = 0; i < partPaths.length; i += GCS_COMPOSE_MAX) {
|
|
336
|
+
const group = partPaths.slice(i, i + GCS_COMPOSE_MAX);
|
|
337
|
+
const rollupPath = `${destPath}.roll-${String(i / GCS_COMPOSE_MAX).padStart(4, "0")}`;
|
|
338
|
+
const sources = group.map((p) => bucket.file(p));
|
|
339
|
+
await bucket.combine(sources, bucket.file(rollupPath));
|
|
340
|
+
rollups.push(rollupPath);
|
|
341
|
+
}
|
|
342
|
+
return composeInGroups(bucket, rollups, destPath);
|
|
343
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LocalFilesystemArtifactWriter — writes AILF run artifacts to the local
|
|
3
|
+
* filesystem under `{rootDir}/runs/{runId}/…`.
|
|
4
|
+
*
|
|
5
|
+
* D0033 M4 inverts D0032's "GCS default, local fallback" stance: the local
|
|
6
|
+
* writer is **always** attached, and GCS layers on top via a fanout writer
|
|
7
|
+
* when credentials are present. The result — every run produces a usable
|
|
8
|
+
* artifact bundle on disk even on airplanes, laptops, and CI without GCS
|
|
9
|
+
* creds. Studio retrieval works uniformly; only `ArtifactRef.store`
|
|
10
|
+
* differentiates.
|
|
11
|
+
*
|
|
12
|
+
* ## Path layout
|
|
13
|
+
*
|
|
14
|
+
* Paths mirror the GCS tree exactly, so the same descriptor's `objectPath`
|
|
15
|
+
* is used verbatim:
|
|
16
|
+
* - bulk: `{rootDir}/runs/{runId}/{slug}.{ext}`
|
|
17
|
+
* - per-entry: `{rootDir}/runs/{runId}/{slug}/{sanitizedKey}.{ext}`
|
|
18
|
+
* - manifest: `{rootDir}/runs/{runId}/manifest.json`
|
|
19
|
+
*
|
|
20
|
+
* This keeps the L6 cross-reader contract test simple — a byte-compare of
|
|
21
|
+
* every object at every path, modulo timestamped manifest fields.
|
|
22
|
+
*
|
|
23
|
+
* ## Design choices
|
|
24
|
+
*
|
|
25
|
+
* - **Redaction at emit boundary (AC10).** `redactArtifactData` runs per
|
|
26
|
+
* write, not post-hoc on a tarball. Same rules as the legacy collector.
|
|
27
|
+
* - **Exclude list gating (Q3).** `--capture-exclude=LIST` passes through
|
|
28
|
+
* to the constructor; `emit()` returns null for excluded types before
|
|
29
|
+
* touching disk.
|
|
30
|
+
* - **NDJSON uses plain `fs.appendFile`.** No compose rollover needed —
|
|
31
|
+
* unlike GCS, local fs appends are atomic and unbounded. A crash mid-
|
|
32
|
+
* append leaves a partial row visible; acceptable for dev runs.
|
|
33
|
+
* - **Non-blocking per P5.** Any fs error returns null + warns, never
|
|
34
|
+
* throws. The pipeline must not fail because local disk is full.
|
|
35
|
+
*
|
|
36
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md (§ M4)
|
|
37
|
+
* @see packages/eval/src/artifact-capture/gcs-artifact-writer.ts (mirror)
|
|
38
|
+
*/
|
|
39
|
+
import { type ArtifactEntry, type ArtifactRef, type ArtifactType, type ArtifactWriter, type AssociationValues, type RunId, type RunManifest } from "../_vendor/ailf-core/index.d.ts";
|
|
40
|
+
export interface LocalFilesystemArtifactWriterOptions {
|
|
41
|
+
/**
|
|
42
|
+
* Absolute or cwd-relative root directory under which `runs/{runId}/…`
|
|
43
|
+
* is written. Defaults to `.ailf/results/captures/` at the composition
|
|
44
|
+
* root; tests inject their own temp dirs.
|
|
45
|
+
*/
|
|
46
|
+
rootDir: string;
|
|
47
|
+
/**
|
|
48
|
+
* Artifact types to skip. When `emit()` is called for an excluded
|
|
49
|
+
* type, it returns null without touching disk. Driven by
|
|
50
|
+
* `--capture-exclude=LIST` in W0050 Slice 4.
|
|
51
|
+
*/
|
|
52
|
+
exclude?: readonly ArtifactType[];
|
|
53
|
+
}
|
|
54
|
+
export declare class LocalFilesystemArtifactWriter implements ArtifactWriter {
|
|
55
|
+
private readonly options;
|
|
56
|
+
private readonly excludeSet;
|
|
57
|
+
constructor(options: LocalFilesystemArtifactWriterOptions);
|
|
58
|
+
emit<T extends ArtifactType>(type: T, association: AssociationValues, payload: unknown): Promise<ArtifactRef | null>;
|
|
59
|
+
appendNdjson<T extends ArtifactType>(type: T, association: AssociationValues, rows: readonly unknown[]): Promise<ArtifactRef | null>;
|
|
60
|
+
writeManifest(runId: RunId, manifest: RunManifest): Promise<ArtifactRef | null>;
|
|
61
|
+
/** @deprecated Use `emit()` instead. */
|
|
62
|
+
writeBulk(type: ArtifactType, runId: RunId, data: unknown): Promise<ArtifactRef | null>;
|
|
63
|
+
/** @deprecated Use `emit()` per entry instead. */
|
|
64
|
+
writePerEntry(type: ArtifactType, runId: RunId, entries: readonly ArtifactEntry[]): Promise<ArtifactRef | null>;
|
|
65
|
+
private resolve;
|
|
66
|
+
/**
|
|
67
|
+
* Write the body to `absPath`, creating parent dirs as needed. Returns
|
|
68
|
+
* false + warns on any fs error (P5 non-blocking).
|
|
69
|
+
*/
|
|
70
|
+
private writeAtomic;
|
|
71
|
+
}
|