@sanity/ailf 2.8.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_vendor/ailf-core/artifact-capture/association.d.ts +35 -0
- package/dist/_vendor/ailf-core/artifact-capture/association.js +28 -0
- package/dist/_vendor/ailf-core/artifact-registry.d.ts +124 -23
- package/dist/_vendor/ailf-core/artifact-registry.js +708 -64
- package/dist/_vendor/ailf-core/batch-signing.d.ts +64 -0
- package/dist/_vendor/ailf-core/batch-signing.js +23 -0
- package/dist/_vendor/ailf-core/index.d.ts +3 -2
- package/dist/_vendor/ailf-core/index.js +3 -2
- package/dist/_vendor/ailf-core/ports/artifact-writer.d.ts +59 -20
- package/dist/_vendor/ailf-core/ports/artifact-writer.js +33 -10
- package/dist/_vendor/ailf-core/ports/context.d.ts +20 -17
- package/dist/_vendor/ailf-core/ports/index.d.ts +0 -2
- package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +6 -6
- package/dist/_vendor/ailf-core/services/index.d.ts +1 -0
- package/dist/_vendor/ailf-core/services/index.js +1 -0
- package/dist/_vendor/ailf-core/services/slim-report-summary.d.ts +31 -0
- package/dist/_vendor/ailf-core/services/slim-report-summary.js +217 -0
- package/dist/_vendor/ailf-core/types/branded-ids.d.ts +33 -0
- package/dist/_vendor/ailf-core/types/index.d.ts +202 -23
- package/dist/adapters/config-sources/file-config-adapter.js +0 -4
- package/dist/artifact-capture/accumulating-artifact-writer.d.ts +50 -0
- package/dist/artifact-capture/accumulating-artifact-writer.js +111 -0
- package/dist/artifact-capture/api-gateway-artifact-writer.d.ts +17 -4
- package/dist/artifact-capture/api-gateway-artifact-writer.js +58 -7
- package/dist/artifact-capture/emit-file.d.ts +28 -0
- package/dist/artifact-capture/emit-file.js +56 -0
- package/dist/artifact-capture/fanout-artifact-writer.d.ts +39 -0
- package/dist/artifact-capture/fanout-artifact-writer.js +76 -0
- package/dist/artifact-capture/gcs-artifact-writer.d.ts +40 -3
- package/dist/artifact-capture/gcs-artifact-writer.js +238 -14
- package/dist/artifact-capture/local-fs-artifact-writer.d.ts +71 -0
- package/dist/artifact-capture/local-fs-artifact-writer.js +273 -0
- package/dist/artifact-capture/redact-artifact.d.ts +3 -5
- package/dist/artifact-capture/redact-artifact.js +3 -5
- package/dist/cli.js +56 -2
- package/dist/commands/explain-handler.js +4 -4
- package/dist/commands/pipeline-action.d.ts +5 -4
- package/dist/commands/pipeline-action.js +33 -16
- package/dist/commands/pipeline.d.ts +4 -4
- package/dist/commands/pipeline.js +4 -4
- package/dist/commands/publish.js +4 -1
- package/dist/commands/runs.d.ts +18 -0
- package/dist/commands/runs.js +71 -0
- package/dist/composition-root.d.ts +13 -10
- package/dist/composition-root.js +74 -46
- package/dist/orchestration/build-app-context.js +4 -7
- package/dist/orchestration/pipeline-orchestrator.d.ts +1 -1
- package/dist/orchestration/pipeline-orchestrator.js +37 -46
- package/dist/orchestration/steps/calculate-scores-step.d.ts +1 -1
- package/dist/orchestration/steps/calculate-scores-step.js +19 -19
- package/dist/orchestration/steps/callback-step.d.ts +1 -1
- package/dist/orchestration/steps/callback-step.js +6 -4
- package/dist/orchestration/steps/compare-step.d.ts +1 -1
- package/dist/orchestration/steps/compare-step.js +4 -2
- package/dist/orchestration/steps/discovery-report-step.d.ts +1 -1
- package/dist/orchestration/steps/discovery-report-step.js +4 -1
- package/dist/orchestration/steps/fetch-docs-step.js +9 -15
- package/dist/orchestration/steps/finalize-run-step.js +21 -7
- package/dist/orchestration/steps/gap-analysis-step.js +34 -6
- package/dist/orchestration/steps/generate-configs-step.d.ts +1 -1
- package/dist/orchestration/steps/generate-configs-step.js +11 -11
- package/dist/orchestration/steps/publish-report-step.d.ts +1 -1
- package/dist/orchestration/steps/publish-report-step.js +24 -19
- package/dist/orchestration/steps/readiness-step.d.ts +1 -1
- package/dist/orchestration/steps/readiness-step.js +4 -1
- package/dist/orchestration/steps/report-step.d.ts +1 -1
- package/dist/orchestration/steps/report-step.js +6 -3
- package/dist/orchestration/steps/run-eval-step.js +14 -9
- package/dist/pipeline/compare.d.ts +2 -2
- package/dist/pipeline/emit-eval-results.d.ts +38 -0
- package/dist/pipeline/emit-eval-results.js +100 -0
- package/dist/pipeline/map-request-to-config.js +0 -4
- package/package.json +1 -1
- package/dist/_vendor/ailf-core/artifact-capture/noop-collector.d.ts +0 -14
- package/dist/_vendor/ailf-core/artifact-capture/noop-collector.js +0 -25
- package/dist/_vendor/ailf-core/ports/artifact-collector.d.ts +0 -94
- package/dist/_vendor/ailf-core/ports/artifact-collector.js +0 -13
- package/dist/_vendor/ailf-core/ports/capture-comparator.d.ts +0 -138
- package/dist/_vendor/ailf-core/ports/capture-comparator.js +0 -10
- package/dist/artifact-capture/comparator.d.ts +0 -22
- package/dist/artifact-capture/comparator.js +0 -493
- package/dist/artifact-capture/filesystem-collector.d.ts +0 -42
- package/dist/artifact-capture/filesystem-collector.js +0 -237
- package/dist/artifact-capture/gcs-collector.d.ts +0 -55
- package/dist/artifact-capture/gcs-collector.js +0 -117
- package/dist/commands/capture-compare.d.ts +0 -15
- package/dist/commands/capture-compare.js +0 -253
- package/dist/commands/capture-list.d.ts +0 -12
- package/dist/commands/capture-list.js +0 -150
- package/dist/commands/capture.d.ts +0 -9
- package/dist/commands/capture.js +0 -16
|
@@ -7,32 +7,181 @@
|
|
|
7
7
|
*
|
|
8
8
|
* Paths come from `ARTIFACT_REGISTRY` so writers, signers, and readers agree.
|
|
9
9
|
*
|
|
10
|
+
* ## W0049 API surface
|
|
11
|
+
*
|
|
12
|
+
* - `emit(type, association, payload)` — the canonical single-shot write.
|
|
13
|
+
* Dispatch on `descriptor.layout` is internal; callers pass axis values
|
|
14
|
+
* and the writer resolves the path.
|
|
15
|
+
* - `appendNdjson(type, association, rows)` — streaming-append for `traces`.
|
|
16
|
+
* Each call writes a numbered part object (`.ndjson.part-NNNN`); the
|
|
17
|
+
* parts are composed into the final object lazily at `writeManifest` time
|
|
18
|
+
* via GCS object compose. When a stream accumulates > 32 parts the writer
|
|
19
|
+
* rolls up parts into intermediate composites to stay under the GCS
|
|
20
|
+
* compose cap.
|
|
21
|
+
* - `writeBulk` / `writePerEntry` — @deprecated legacy surface. Removal in W0052.
|
|
22
|
+
*
|
|
10
23
|
* Design principles:
|
|
11
24
|
* - P5: Non-blocking — upload failure returns null, never throws.
|
|
12
25
|
* - Lazy client — Storage created on first write.
|
|
13
26
|
*
|
|
14
27
|
* @see docs/decisions/D0032-run-anchored-artifact-store.md
|
|
28
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md
|
|
15
29
|
*/
|
|
16
30
|
import { Storage } from "@google-cloud/storage";
|
|
17
|
-
import { ARTIFACT_REGISTRY, } from "../_vendor/ailf-core/index.js";
|
|
31
|
+
import { ARTIFACT_REGISTRY, buildManifestPreview, } from "../_vendor/ailf-core/index.js";
|
|
32
|
+
import { redactArtifactData } from "./redact-artifact.js";
|
|
33
|
+
/**
|
|
34
|
+
* GCS's object compose operation accepts at most 32 source objects per call
|
|
35
|
+
* ([docs](https://cloud.google.com/storage/docs/composite-objects)). When an
|
|
36
|
+
* NDJSON stream accumulates more than 32 parts the writer rolls up groups of
|
|
37
|
+
* 32 into intermediate composites and composes those — one extra round trip
|
|
38
|
+
* per 32 additional parts.
|
|
39
|
+
*/
|
|
40
|
+
const GCS_COMPOSE_MAX = 32;
|
|
18
41
|
export class GcsArtifactWriter {
|
|
19
42
|
client = null;
|
|
20
43
|
options;
|
|
44
|
+
ndjsonStreams = new Map();
|
|
21
45
|
constructor(options) {
|
|
22
46
|
this.options = options;
|
|
47
|
+
if (options.storage) {
|
|
48
|
+
this.client = options.storage;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// ---- Canonical W0049 API ------------------------------------------------
|
|
52
|
+
async emit(type, association, payload) {
|
|
53
|
+
const descriptor = ARTIFACT_REGISTRY[type];
|
|
54
|
+
const runId = association.run;
|
|
55
|
+
if (!runId) {
|
|
56
|
+
console.warn(` ⚠️ emit("${type}"): association.run is required, skipping`);
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
// AC10 — redact at the writer boundary so secrets never reach GCS.
|
|
60
|
+
const redacted = redactArtifactData(payload);
|
|
61
|
+
// Preview reads the pre-redaction payload (same as local writer — the
|
|
62
|
+
// preview carries a descriptor-controlled summary bounded by capBytes,
|
|
63
|
+
// not the raw entry bytes).
|
|
64
|
+
const preview = buildManifestPreview(descriptor, payload);
|
|
65
|
+
if (descriptor.layout === "bulk") {
|
|
66
|
+
const path = descriptor.objectPath(runId);
|
|
67
|
+
const ref = await this.putBody(path, serializeForMime(redacted, descriptor.mime), {
|
|
68
|
+
layout: "bulk",
|
|
69
|
+
mime: descriptor.mime,
|
|
70
|
+
entryCount: entryCountOf(redacted),
|
|
71
|
+
});
|
|
72
|
+
if (!ref)
|
|
73
|
+
return null;
|
|
74
|
+
return preview === undefined ? ref : { ...ref, preview };
|
|
75
|
+
}
|
|
76
|
+
// per-entry
|
|
77
|
+
const entryKey = descriptor.formatEntryKey(association);
|
|
78
|
+
const path = descriptor.objectPath(runId, entryKey);
|
|
79
|
+
const ref = await this.putBody(path, serializeForMime(redacted, descriptor.mime), { layout: "per-entry", mime: descriptor.mime });
|
|
80
|
+
if (!ref)
|
|
81
|
+
return null;
|
|
82
|
+
return {
|
|
83
|
+
...ref,
|
|
84
|
+
path: `runs/${runId}/${descriptor.slug}`,
|
|
85
|
+
entryCount: 1,
|
|
86
|
+
entries: [
|
|
87
|
+
{
|
|
88
|
+
key: entryKey,
|
|
89
|
+
bytes: ref.bytes ?? 0,
|
|
90
|
+
association,
|
|
91
|
+
...(preview === undefined ? {} : { preview }),
|
|
92
|
+
},
|
|
93
|
+
],
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
async appendNdjson(type, association, rows) {
|
|
97
|
+
const descriptor = ARTIFACT_REGISTRY[type];
|
|
98
|
+
if (descriptor.mime !== "application/x-ndjson") {
|
|
99
|
+
console.warn(` ⚠️ appendNdjson("${type}"): descriptor mime is ${descriptor.mime}, not application/x-ndjson — skipping`);
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
const runId = association.run;
|
|
103
|
+
if (!runId) {
|
|
104
|
+
console.warn(` ⚠️ appendNdjson("${type}"): association.run is required, skipping`);
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
if (rows.length === 0)
|
|
108
|
+
return null;
|
|
109
|
+
const entryKey = descriptor.formatEntryKey(association);
|
|
110
|
+
const finalPath = descriptor.objectPath(runId, entryKey);
|
|
111
|
+
const streamKey = `${type}::${entryKey}`;
|
|
112
|
+
let state = this.ndjsonStreams.get(streamKey);
|
|
113
|
+
if (!state) {
|
|
114
|
+
state = { finalPath, partCount: 0, totalBytes: 0 };
|
|
115
|
+
this.ndjsonStreams.set(streamKey, state);
|
|
116
|
+
}
|
|
117
|
+
const partPath = `${finalPath}.part-${String(state.partCount).padStart(4, "0")}`;
|
|
118
|
+
// AC10 — redact per row before serializing the NDJSON batch.
|
|
119
|
+
const redactedRows = rows.map((r) => redactArtifactData(r));
|
|
120
|
+
const body = redactedRows.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
121
|
+
const bytes = Buffer.byteLength(body, "utf-8");
|
|
122
|
+
try {
|
|
123
|
+
const storage = this.getClient();
|
|
124
|
+
await storage
|
|
125
|
+
.bucket(this.options.bucket)
|
|
126
|
+
.file(partPath)
|
|
127
|
+
.save(body, { contentType: "application/x-ndjson" });
|
|
128
|
+
state.partCount++;
|
|
129
|
+
state.totalBytes += bytes;
|
|
130
|
+
}
|
|
131
|
+
catch (err) {
|
|
132
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
133
|
+
console.warn(` ⚠️ NDJSON part upload failed (non-blocking): ${partPath} — ${message}`);
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
return {
|
|
137
|
+
store: "gcs",
|
|
138
|
+
bucket: this.options.bucket,
|
|
139
|
+
path: `runs/${runId}/${descriptor.slug}`,
|
|
140
|
+
bytes: state.totalBytes,
|
|
141
|
+
entryCount: 1,
|
|
142
|
+
layout: "per-entry",
|
|
143
|
+
entries: [
|
|
144
|
+
{
|
|
145
|
+
key: entryKey,
|
|
146
|
+
bytes: state.totalBytes,
|
|
147
|
+
association,
|
|
148
|
+
},
|
|
149
|
+
],
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
async writeManifest(runId, manifest) {
|
|
153
|
+
await this.finalizeNdjsonStreams();
|
|
154
|
+
const path = `runs/${runId}/manifest.json`;
|
|
155
|
+
return this.putBody(path, JSON.stringify(manifest), {
|
|
156
|
+
layout: "bulk",
|
|
157
|
+
mime: "application/json",
|
|
158
|
+
});
|
|
23
159
|
}
|
|
160
|
+
// ---- Deprecated legacy surface (W0052) ----------------------------------
|
|
161
|
+
/** @deprecated Use `emit()` instead. Routes through the same GCS I/O. */
|
|
24
162
|
async writeBulk(type, runId, data) {
|
|
25
163
|
const descriptor = ARTIFACT_REGISTRY[type];
|
|
164
|
+
if (descriptor.layout !== "bulk") {
|
|
165
|
+
console.warn(` ⚠️ writeBulk("${type}"): descriptor layout is "${descriptor.layout}", not "bulk" — skipping`);
|
|
166
|
+
return null;
|
|
167
|
+
}
|
|
26
168
|
const path = descriptor.objectPath(runId);
|
|
27
|
-
|
|
169
|
+
const redacted = redactArtifactData(data);
|
|
170
|
+
return this.putBody(path, serializeForMime(redacted, descriptor.mime), {
|
|
28
171
|
layout: "bulk",
|
|
29
|
-
|
|
172
|
+
mime: descriptor.mime,
|
|
173
|
+
entryCount: entryCountOf(redacted),
|
|
30
174
|
});
|
|
31
175
|
}
|
|
176
|
+
/** @deprecated Use `emit()` per entry instead. */
|
|
32
177
|
async writePerEntry(type, runId, entries) {
|
|
33
178
|
const descriptor = ARTIFACT_REGISTRY[type];
|
|
179
|
+
if (descriptor.layout !== "per-entry") {
|
|
180
|
+
console.warn(` ⚠️ writePerEntry("${type}"): descriptor layout is "${descriptor.layout}", not "per-entry" — skipping`);
|
|
181
|
+
return null;
|
|
182
|
+
}
|
|
34
183
|
if (!descriptor.parseEntryKey) {
|
|
35
|
-
console.warn(` ⚠️ writePerEntry
|
|
184
|
+
console.warn(` ⚠️ writePerEntry("${type}"): descriptor has no parseEntryKey`);
|
|
36
185
|
return null;
|
|
37
186
|
}
|
|
38
187
|
const storage = this.getClient();
|
|
@@ -45,13 +194,14 @@ export class GcsArtifactWriter {
|
|
|
45
194
|
continue;
|
|
46
195
|
}
|
|
47
196
|
const path = descriptor.objectPath(runId, entry.key);
|
|
48
|
-
const
|
|
49
|
-
const
|
|
197
|
+
const redacted = redactArtifactData(entry.data);
|
|
198
|
+
const body = serializeForMime(redacted, descriptor.mime);
|
|
199
|
+
const bytes = Buffer.byteLength(body, "utf-8");
|
|
50
200
|
try {
|
|
51
201
|
await storage
|
|
52
202
|
.bucket(this.options.bucket)
|
|
53
203
|
.file(path)
|
|
54
|
-
.save(
|
|
204
|
+
.save(body, { contentType: descriptor.mime });
|
|
55
205
|
uploaded.push({ key: entry.key, bytes });
|
|
56
206
|
totalBytes += bytes;
|
|
57
207
|
}
|
|
@@ -72,19 +222,45 @@ export class GcsArtifactWriter {
|
|
|
72
222
|
entries: uploaded,
|
|
73
223
|
};
|
|
74
224
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
225
|
+
// ---- Internals ----------------------------------------------------------
|
|
226
|
+
/**
|
|
227
|
+
* Compose all buffered NDJSON streams into their final objects. Called
|
|
228
|
+
* from `writeManifest` so the manifest is the single sync point for
|
|
229
|
+
* NDJSON finalization.
|
|
230
|
+
*
|
|
231
|
+
* When `partCount > GCS_COMPOSE_MAX`, parts are rolled up in groups of
|
|
232
|
+
* `GCS_COMPOSE_MAX` into intermediate composites, then composed. This
|
|
233
|
+
* stays under the per-call source cap at the cost of one extra round
|
|
234
|
+
* trip per 32 additional parts.
|
|
235
|
+
*/
|
|
236
|
+
async finalizeNdjsonStreams() {
|
|
237
|
+
for (const state of this.ndjsonStreams.values()) {
|
|
238
|
+
if (state.partCount === 0)
|
|
239
|
+
continue;
|
|
240
|
+
const storage = this.getClient();
|
|
241
|
+
const bucket = storage.bucket(this.options.bucket);
|
|
242
|
+
const partPaths = Array.from({ length: state.partCount }, (_, i) => `${state.finalPath}.part-${String(i).padStart(4, "0")}`);
|
|
243
|
+
try {
|
|
244
|
+
const finalPath = await composeInGroups(bucket, partPaths, state.finalPath);
|
|
245
|
+
if (finalPath !== state.finalPath) {
|
|
246
|
+
console.warn(` ⚠️ NDJSON compose produced "${finalPath}" (expected "${state.finalPath}")`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
catch (err) {
|
|
250
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
251
|
+
console.warn(` ⚠️ NDJSON compose failed (non-blocking): ${state.finalPath} — ${message}`);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
this.ndjsonStreams.clear();
|
|
78
255
|
}
|
|
79
|
-
async
|
|
80
|
-
const
|
|
81
|
-
const bytes = Buffer.byteLength(json, "utf-8");
|
|
256
|
+
async putBody(path, body, meta) {
|
|
257
|
+
const bytes = Buffer.byteLength(body, "utf-8");
|
|
82
258
|
try {
|
|
83
259
|
const storage = this.getClient();
|
|
84
260
|
await storage
|
|
85
261
|
.bucket(this.options.bucket)
|
|
86
262
|
.file(path)
|
|
87
|
-
.save(
|
|
263
|
+
.save(body, { contentType: meta.mime });
|
|
88
264
|
return {
|
|
89
265
|
store: "gcs",
|
|
90
266
|
bucket: this.options.bucket,
|
|
@@ -107,6 +283,17 @@ export class GcsArtifactWriter {
|
|
|
107
283
|
return this.client;
|
|
108
284
|
}
|
|
109
285
|
}
|
|
286
|
+
// ---------------------------------------------------------------------------
|
|
287
|
+
// Helpers
|
|
288
|
+
// ---------------------------------------------------------------------------
|
|
289
|
+
function serializeForMime(payload, mime) {
|
|
290
|
+
if (mime === "text/markdown" || mime === "application/yaml") {
|
|
291
|
+
if (typeof payload === "string")
|
|
292
|
+
return payload;
|
|
293
|
+
return String(payload ?? "");
|
|
294
|
+
}
|
|
295
|
+
return JSON.stringify(payload);
|
|
296
|
+
}
|
|
110
297
|
function entryCountOf(data) {
|
|
111
298
|
if (typeof data === "object" &&
|
|
112
299
|
data !== null &&
|
|
@@ -117,3 +304,40 @@ function entryCountOf(data) {
|
|
|
117
304
|
}
|
|
118
305
|
return undefined;
|
|
119
306
|
}
|
|
307
|
+
/**
|
|
308
|
+
* Compose a list of GCS parts into a destination object. When the part count
|
|
309
|
+
* exceeds `GCS_COMPOSE_MAX`, roll up groups into intermediate composites and
|
|
310
|
+
* recurse until a single compose call suffices.
|
|
311
|
+
*
|
|
312
|
+
* Returns the destination path on success. Intermediate composites are
|
|
313
|
+
* written to `{dest}.roll-{n}` and left in place — they are cheap, and
|
|
314
|
+
* keeping them simplifies failure recovery.
|
|
315
|
+
*/
|
|
316
|
+
async function composeInGroups(bucket, partPaths, destPath) {
|
|
317
|
+
if (partPaths.length === 0) {
|
|
318
|
+
throw new Error(`composeInGroups: no parts to compose for ${destPath}`);
|
|
319
|
+
}
|
|
320
|
+
if (partPaths.length === 1) {
|
|
321
|
+
// Single part — server-side copy into place. Avoids streaming the
|
|
322
|
+
// object through this process (unlike download + re-upload) and
|
|
323
|
+
// preserves the source object's Content-Type / metadata on the
|
|
324
|
+
// destination (W0049 review finding C2).
|
|
325
|
+
await bucket.file(partPaths[0]).copy(bucket.file(destPath));
|
|
326
|
+
return destPath;
|
|
327
|
+
}
|
|
328
|
+
if (partPaths.length <= GCS_COMPOSE_MAX) {
|
|
329
|
+
const sources = partPaths.map((p) => bucket.file(p));
|
|
330
|
+
await bucket.combine(sources, bucket.file(destPath));
|
|
331
|
+
return destPath;
|
|
332
|
+
}
|
|
333
|
+
// >32 parts: roll up in groups of GCS_COMPOSE_MAX and recurse.
|
|
334
|
+
const rollups = [];
|
|
335
|
+
for (let i = 0; i < partPaths.length; i += GCS_COMPOSE_MAX) {
|
|
336
|
+
const group = partPaths.slice(i, i + GCS_COMPOSE_MAX);
|
|
337
|
+
const rollupPath = `${destPath}.roll-${String(i / GCS_COMPOSE_MAX).padStart(4, "0")}`;
|
|
338
|
+
const sources = group.map((p) => bucket.file(p));
|
|
339
|
+
await bucket.combine(sources, bucket.file(rollupPath));
|
|
340
|
+
rollups.push(rollupPath);
|
|
341
|
+
}
|
|
342
|
+
return composeInGroups(bucket, rollups, destPath);
|
|
343
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LocalFilesystemArtifactWriter — writes AILF run artifacts to the local
|
|
3
|
+
* filesystem under `{rootDir}/runs/{runId}/…`.
|
|
4
|
+
*
|
|
5
|
+
* D0033 M4 inverts D0032's "GCS default, local fallback" stance: the local
|
|
6
|
+
* writer is **always** attached, and GCS layers on top via a fanout writer
|
|
7
|
+
* when credentials are present. The result — every run produces a usable
|
|
8
|
+
* artifact bundle on disk even on airplanes, laptops, and CI without GCS
|
|
9
|
+
* creds. Studio retrieval works uniformly; only `ArtifactRef.store`
|
|
10
|
+
* differentiates.
|
|
11
|
+
*
|
|
12
|
+
* ## Path layout
|
|
13
|
+
*
|
|
14
|
+
* Paths mirror the GCS tree exactly, so the same descriptor's `objectPath`
|
|
15
|
+
* is used verbatim:
|
|
16
|
+
* - bulk: `{rootDir}/runs/{runId}/{slug}.{ext}`
|
|
17
|
+
* - per-entry: `{rootDir}/runs/{runId}/{slug}/{sanitizedKey}.{ext}`
|
|
18
|
+
* - manifest: `{rootDir}/runs/{runId}/manifest.json`
|
|
19
|
+
*
|
|
20
|
+
* This keeps the L6 cross-reader contract test simple — a byte-compare of
|
|
21
|
+
* every object at every path, modulo timestamped manifest fields.
|
|
22
|
+
*
|
|
23
|
+
* ## Design choices
|
|
24
|
+
*
|
|
25
|
+
* - **Redaction at emit boundary (AC10).** `redactArtifactData` runs per
|
|
26
|
+
* write, not post-hoc on a tarball. Same rules as the legacy collector.
|
|
27
|
+
* - **Exclude list gating (Q3).** `--capture-exclude=LIST` passes through
|
|
28
|
+
* to the constructor; `emit()` returns null for excluded types before
|
|
29
|
+
* touching disk.
|
|
30
|
+
* - **NDJSON uses plain `fs.appendFile`.** No compose rollover needed —
|
|
31
|
+
* unlike GCS, local fs appends are atomic and unbounded. A crash mid-
|
|
32
|
+
* append leaves a partial row visible; acceptable for dev runs.
|
|
33
|
+
* - **Non-blocking per P5.** Any fs error returns null + warns, never
|
|
34
|
+
* throws. The pipeline must not fail because local disk is full.
|
|
35
|
+
*
|
|
36
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md (§ M4)
|
|
37
|
+
* @see packages/eval/src/artifact-capture/gcs-artifact-writer.ts (mirror)
|
|
38
|
+
*/
|
|
39
|
+
import { type ArtifactEntry, type ArtifactRef, type ArtifactType, type ArtifactWriter, type AssociationValues, type RunId, type RunManifest } from "../_vendor/ailf-core/index.d.ts";
|
|
40
|
+
export interface LocalFilesystemArtifactWriterOptions {
|
|
41
|
+
/**
|
|
42
|
+
* Absolute or cwd-relative root directory under which `runs/{runId}/…`
|
|
43
|
+
* is written. Defaults to `.ailf/results/captures/` at the composition
|
|
44
|
+
* root; tests inject their own temp dirs.
|
|
45
|
+
*/
|
|
46
|
+
rootDir: string;
|
|
47
|
+
/**
|
|
48
|
+
* Artifact types to skip. When `emit()` is called for an excluded
|
|
49
|
+
* type, it returns null without touching disk. Driven by
|
|
50
|
+
* `--capture-exclude=LIST` in W0050 Slice 4.
|
|
51
|
+
*/
|
|
52
|
+
exclude?: readonly ArtifactType[];
|
|
53
|
+
}
|
|
54
|
+
export declare class LocalFilesystemArtifactWriter implements ArtifactWriter {
|
|
55
|
+
private readonly options;
|
|
56
|
+
private readonly excludeSet;
|
|
57
|
+
constructor(options: LocalFilesystemArtifactWriterOptions);
|
|
58
|
+
emit<T extends ArtifactType>(type: T, association: AssociationValues, payload: unknown): Promise<ArtifactRef | null>;
|
|
59
|
+
appendNdjson<T extends ArtifactType>(type: T, association: AssociationValues, rows: readonly unknown[]): Promise<ArtifactRef | null>;
|
|
60
|
+
writeManifest(runId: RunId, manifest: RunManifest): Promise<ArtifactRef | null>;
|
|
61
|
+
/** @deprecated Use `emit()` instead. */
|
|
62
|
+
writeBulk(type: ArtifactType, runId: RunId, data: unknown): Promise<ArtifactRef | null>;
|
|
63
|
+
/** @deprecated Use `emit()` per entry instead. */
|
|
64
|
+
writePerEntry(type: ArtifactType, runId: RunId, entries: readonly ArtifactEntry[]): Promise<ArtifactRef | null>;
|
|
65
|
+
private resolve;
|
|
66
|
+
/**
|
|
67
|
+
* Write the body to `absPath`, creating parent dirs as needed. Returns
|
|
68
|
+
* false + warns on any fs error (P5 non-blocking).
|
|
69
|
+
*/
|
|
70
|
+
private writeAtomic;
|
|
71
|
+
}
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LocalFilesystemArtifactWriter — writes AILF run artifacts to the local
|
|
3
|
+
* filesystem under `{rootDir}/runs/{runId}/…`.
|
|
4
|
+
*
|
|
5
|
+
* D0033 M4 inverts D0032's "GCS default, local fallback" stance: the local
|
|
6
|
+
* writer is **always** attached, and GCS layers on top via a fanout writer
|
|
7
|
+
* when credentials are present. The result — every run produces a usable
|
|
8
|
+
* artifact bundle on disk even on airplanes, laptops, and CI without GCS
|
|
9
|
+
* creds. Studio retrieval works uniformly; only `ArtifactRef.store`
|
|
10
|
+
* differentiates.
|
|
11
|
+
*
|
|
12
|
+
* ## Path layout
|
|
13
|
+
*
|
|
14
|
+
* Paths mirror the GCS tree exactly, so the same descriptor's `objectPath`
|
|
15
|
+
* is used verbatim:
|
|
16
|
+
* - bulk: `{rootDir}/runs/{runId}/{slug}.{ext}`
|
|
17
|
+
* - per-entry: `{rootDir}/runs/{runId}/{slug}/{sanitizedKey}.{ext}`
|
|
18
|
+
* - manifest: `{rootDir}/runs/{runId}/manifest.json`
|
|
19
|
+
*
|
|
20
|
+
* This keeps the L6 cross-reader contract test simple — a byte-compare of
|
|
21
|
+
* every object at every path, modulo timestamped manifest fields.
|
|
22
|
+
*
|
|
23
|
+
* ## Design choices
|
|
24
|
+
*
|
|
25
|
+
* - **Redaction at emit boundary (AC10).** `redactArtifactData` runs per
|
|
26
|
+
* write, not post-hoc on a tarball. Same rules as the legacy collector.
|
|
27
|
+
* - **Exclude list gating (Q3).** `--capture-exclude=LIST` passes through
|
|
28
|
+
* to the constructor; `emit()` returns null for excluded types before
|
|
29
|
+
* touching disk.
|
|
30
|
+
* - **NDJSON uses plain `fs.appendFile`.** No compose rollover needed —
|
|
31
|
+
* unlike GCS, local fs appends are atomic and unbounded. A crash mid-
|
|
32
|
+
* append leaves a partial row visible; acceptable for dev runs.
|
|
33
|
+
* - **Non-blocking per P5.** Any fs error returns null + warns, never
|
|
34
|
+
* throws. The pipeline must not fail because local disk is full.
|
|
35
|
+
*
|
|
36
|
+
* @see docs/decisions/D0033-unified-run-anchored-artifact-capture.md (§ M4)
|
|
37
|
+
* @see packages/eval/src/artifact-capture/gcs-artifact-writer.ts (mirror)
|
|
38
|
+
*/
|
|
39
|
+
import { promises as fs } from "node:fs";
|
|
40
|
+
import path from "node:path";
|
|
41
|
+
import { ARTIFACT_REGISTRY, buildManifestPreview, } from "../_vendor/ailf-core/index.js";
|
|
42
|
+
import { redactArtifactData } from "./redact-artifact.js";
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
// Implementation
|
|
45
|
+
// ---------------------------------------------------------------------------
|
|
46
|
+
export class LocalFilesystemArtifactWriter {
|
|
47
|
+
options;
|
|
48
|
+
excludeSet;
|
|
49
|
+
constructor(options) {
|
|
50
|
+
this.options = options;
|
|
51
|
+
this.excludeSet = new Set(options.exclude ?? []);
|
|
52
|
+
}
|
|
53
|
+
// ---- Canonical W0049 API ------------------------------------------------
|
|
54
|
+
async emit(type, association, payload) {
|
|
55
|
+
if (this.excludeSet.has(type))
|
|
56
|
+
return null;
|
|
57
|
+
const descriptor = ARTIFACT_REGISTRY[type];
|
|
58
|
+
const runId = association.run;
|
|
59
|
+
if (!runId) {
|
|
60
|
+
console.warn(` ⚠️ emit("${type}"): association.run is required, skipping`);
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
const redacted = redactArtifactData(payload);
|
|
64
|
+
const body = serializeForMime(redacted, descriptor.mime);
|
|
65
|
+
const bytes = Buffer.byteLength(body, "utf-8");
|
|
66
|
+
// Preview is built from the pre-redaction payload so the extract sees
|
|
67
|
+
// the same shape the producer handed us. The full entry is still redacted
|
|
68
|
+
// on disk; the preview lives only on the manifest and is bounded by the
|
|
69
|
+
// descriptor's capBytes.
|
|
70
|
+
const preview = buildManifestPreview(descriptor, payload);
|
|
71
|
+
if (descriptor.layout === "bulk") {
|
|
72
|
+
const relPath = descriptor.objectPath(runId);
|
|
73
|
+
const absPath = this.resolve(relPath);
|
|
74
|
+
const wrote = await this.writeAtomic(absPath, body);
|
|
75
|
+
if (!wrote)
|
|
76
|
+
return null;
|
|
77
|
+
return {
|
|
78
|
+
store: "local",
|
|
79
|
+
bucket: this.options.rootDir,
|
|
80
|
+
path: relPath,
|
|
81
|
+
bytes,
|
|
82
|
+
entryCount: entryCountOf(redacted),
|
|
83
|
+
layout: "bulk",
|
|
84
|
+
...(preview === undefined ? {} : { preview }),
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
// per-entry
|
|
88
|
+
const entryKey = descriptor.formatEntryKey(association);
|
|
89
|
+
const relPath = descriptor.objectPath(runId, entryKey);
|
|
90
|
+
const absPath = this.resolve(relPath);
|
|
91
|
+
const wrote = await this.writeAtomic(absPath, body);
|
|
92
|
+
if (!wrote)
|
|
93
|
+
return null;
|
|
94
|
+
return {
|
|
95
|
+
store: "local",
|
|
96
|
+
bucket: this.options.rootDir,
|
|
97
|
+
path: `runs/${runId}/${descriptor.slug}`,
|
|
98
|
+
bytes,
|
|
99
|
+
entryCount: 1,
|
|
100
|
+
layout: "per-entry",
|
|
101
|
+
entries: [
|
|
102
|
+
{
|
|
103
|
+
key: entryKey,
|
|
104
|
+
bytes,
|
|
105
|
+
association,
|
|
106
|
+
...(preview === undefined ? {} : { preview }),
|
|
107
|
+
},
|
|
108
|
+
],
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
async appendNdjson(type, association, rows) {
|
|
112
|
+
if (this.excludeSet.has(type))
|
|
113
|
+
return null;
|
|
114
|
+
const descriptor = ARTIFACT_REGISTRY[type];
|
|
115
|
+
if (descriptor.mime !== "application/x-ndjson") {
|
|
116
|
+
console.warn(` ⚠️ appendNdjson("${type}"): descriptor mime is ${descriptor.mime}, not application/x-ndjson — skipping`);
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
const runId = association.run;
|
|
120
|
+
if (!runId) {
|
|
121
|
+
console.warn(` ⚠️ appendNdjson("${type}"): association.run is required, skipping`);
|
|
122
|
+
return null;
|
|
123
|
+
}
|
|
124
|
+
if (rows.length === 0)
|
|
125
|
+
return null;
|
|
126
|
+
const entryKey = descriptor.formatEntryKey(association);
|
|
127
|
+
const relPath = descriptor.objectPath(runId, entryKey);
|
|
128
|
+
const absPath = this.resolve(relPath);
|
|
129
|
+
const redactedRows = rows.map((r) => redactArtifactData(r));
|
|
130
|
+
const body = redactedRows.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
131
|
+
const bytes = Buffer.byteLength(body, "utf-8");
|
|
132
|
+
try {
|
|
133
|
+
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
|
134
|
+
await fs.appendFile(absPath, body, "utf-8");
|
|
135
|
+
}
|
|
136
|
+
catch (err) {
|
|
137
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
138
|
+
console.warn(` ⚠️ NDJSON append failed (non-blocking): ${absPath} — ${message}`);
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
// Local fs appendFile keeps no part state, so the returned ref always
|
|
142
|
+
// reports the *cumulative* size on disk — same semantic as the GCS
|
|
143
|
+
// writer's `state.totalBytes`, just computed differently.
|
|
144
|
+
let cumulative = bytes;
|
|
145
|
+
try {
|
|
146
|
+
const stat = await fs.stat(absPath);
|
|
147
|
+
cumulative = stat.size;
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
// If stat fails we still have the current batch's bytes — acceptable.
|
|
151
|
+
}
|
|
152
|
+
return {
|
|
153
|
+
store: "local",
|
|
154
|
+
bucket: this.options.rootDir,
|
|
155
|
+
path: `runs/${runId}/${descriptor.slug}`,
|
|
156
|
+
bytes: cumulative,
|
|
157
|
+
entryCount: 1,
|
|
158
|
+
layout: "per-entry",
|
|
159
|
+
entries: [{ key: entryKey, bytes: cumulative, association }],
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
async writeManifest(runId, manifest) {
|
|
163
|
+
const relPath = `runs/${runId}/manifest.json`;
|
|
164
|
+
const absPath = this.resolve(relPath);
|
|
165
|
+
const body = JSON.stringify(manifest);
|
|
166
|
+
const wrote = await this.writeAtomic(absPath, body);
|
|
167
|
+
if (!wrote)
|
|
168
|
+
return null;
|
|
169
|
+
return {
|
|
170
|
+
store: "local",
|
|
171
|
+
bucket: this.options.rootDir,
|
|
172
|
+
path: relPath,
|
|
173
|
+
bytes: Buffer.byteLength(body, "utf-8"),
|
|
174
|
+
layout: "bulk",
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
// ---- Deprecated legacy surface (W0052) ----------------------------------
|
|
178
|
+
/** @deprecated Use `emit()` instead. */
|
|
179
|
+
async writeBulk(type, runId, data) {
|
|
180
|
+
const descriptor = ARTIFACT_REGISTRY[type];
|
|
181
|
+
if (descriptor.layout !== "bulk") {
|
|
182
|
+
console.warn(` ⚠️ writeBulk("${type}"): descriptor layout is "${descriptor.layout}", not "bulk" — skipping`);
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
return this.emit(type, { run: runId }, data);
|
|
186
|
+
}
|
|
187
|
+
/** @deprecated Use `emit()` per entry instead. */
|
|
188
|
+
async writePerEntry(type, runId, entries) {
|
|
189
|
+
const descriptor = ARTIFACT_REGISTRY[type];
|
|
190
|
+
if (descriptor.layout !== "per-entry") {
|
|
191
|
+
console.warn(` ⚠️ writePerEntry("${type}"): descriptor layout is "${descriptor.layout}", not "per-entry" — skipping`);
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
if (!descriptor.parseEntryKey) {
|
|
195
|
+
console.warn(` ⚠️ writePerEntry("${type}"): descriptor has no parseEntryKey`);
|
|
196
|
+
return null;
|
|
197
|
+
}
|
|
198
|
+
// Legacy shim: write each entry directly by its raw key, bypassing
|
|
199
|
+
// formatEntryKey. Preserves byte-equivalence with pre-W0050 paths
|
|
200
|
+
// for producers still on the legacy call surface.
|
|
201
|
+
const uploaded = [];
|
|
202
|
+
let totalBytes = 0;
|
|
203
|
+
for (const entry of entries) {
|
|
204
|
+
const parsed = descriptor.parseEntryKey(entry.key);
|
|
205
|
+
if (!parsed.ok) {
|
|
206
|
+
console.warn(` ⚠️ Skipping entry with invalid key "${entry.key}": ${parsed.reason}`);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
const redacted = redactArtifactData(entry.data);
|
|
210
|
+
const body = serializeForMime(redacted, descriptor.mime);
|
|
211
|
+
const bytes = Buffer.byteLength(body, "utf-8");
|
|
212
|
+
const relPath = descriptor.objectPath(runId, entry.key);
|
|
213
|
+
const absPath = this.resolve(relPath);
|
|
214
|
+
const wrote = await this.writeAtomic(absPath, body);
|
|
215
|
+
if (!wrote)
|
|
216
|
+
continue;
|
|
217
|
+
uploaded.push({ key: entry.key, bytes });
|
|
218
|
+
totalBytes += bytes;
|
|
219
|
+
}
|
|
220
|
+
if (uploaded.length === 0)
|
|
221
|
+
return null;
|
|
222
|
+
return {
|
|
223
|
+
store: "local",
|
|
224
|
+
bucket: this.options.rootDir,
|
|
225
|
+
path: `runs/${runId}/${descriptor.slug}`,
|
|
226
|
+
bytes: totalBytes,
|
|
227
|
+
entryCount: uploaded.length,
|
|
228
|
+
layout: "per-entry",
|
|
229
|
+
entries: uploaded,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
// ---- Internals ----------------------------------------------------------
|
|
233
|
+
resolve(relPath) {
|
|
234
|
+
return path.resolve(this.options.rootDir, relPath);
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Write the body to `absPath`, creating parent dirs as needed. Returns
|
|
238
|
+
* false + warns on any fs error (P5 non-blocking).
|
|
239
|
+
*/
|
|
240
|
+
async writeAtomic(absPath, body) {
|
|
241
|
+
try {
|
|
242
|
+
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
|
243
|
+
await fs.writeFile(absPath, body, "utf-8");
|
|
244
|
+
return true;
|
|
245
|
+
}
|
|
246
|
+
catch (err) {
|
|
247
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
248
|
+
console.warn(` ⚠️ Artifact write failed (non-blocking): ${absPath} — ${message}`);
|
|
249
|
+
return false;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
// ---------------------------------------------------------------------------
|
|
254
|
+
// Helpers (shared shape with GcsArtifactWriter)
|
|
255
|
+
// ---------------------------------------------------------------------------
|
|
256
|
+
function serializeForMime(payload, mime) {
|
|
257
|
+
if (mime === "text/markdown" || mime === "application/yaml") {
|
|
258
|
+
if (typeof payload === "string")
|
|
259
|
+
return payload;
|
|
260
|
+
return String(payload ?? "");
|
|
261
|
+
}
|
|
262
|
+
return JSON.stringify(payload);
|
|
263
|
+
}
|
|
264
|
+
function entryCountOf(data) {
|
|
265
|
+
if (typeof data === "object" &&
|
|
266
|
+
data !== null &&
|
|
267
|
+
"entries" in data &&
|
|
268
|
+
typeof data.entries === "object") {
|
|
269
|
+
return Object.keys(data.entries)
|
|
270
|
+
.length;
|
|
271
|
+
}
|
|
272
|
+
return undefined;
|
|
273
|
+
}
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Artifact redaction — strips sensitive data from
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* Applied during FilesystemArtifactCollector.flush() so that secrets
|
|
6
|
-
* (resolved env vars, auth headers, session cookies) never reach storage.
|
|
2
|
+
* Artifact redaction — strips sensitive data from written artifacts so that
|
|
3
|
+
* secrets (resolved env vars, auth headers, session cookies) never reach
|
|
4
|
+
* local or remote storage.
|
|
7
5
|
*
|
|
8
6
|
* Two-layer approach:
|
|
9
7
|
* 1. **Header stripping** — known-sensitive HTTP header keys are replaced
|