@sanity/ailf 2.7.1 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_vendor/ailf-core/artifact-capture/association.d.ts +35 -0
- package/dist/_vendor/ailf-core/artifact-capture/association.js +28 -0
- package/dist/_vendor/ailf-core/artifact-registry.d.ts +173 -0
- package/dist/_vendor/ailf-core/artifact-registry.js +811 -0
- package/dist/_vendor/ailf-core/index.d.ts +3 -1
- package/dist/_vendor/ailf-core/index.js +3 -1
- package/dist/_vendor/ailf-core/ports/artifact-collector.d.ts +3 -3
- package/dist/_vendor/ailf-core/ports/artifact-writer.d.ts +95 -0
- package/dist/_vendor/ailf-core/ports/artifact-writer.js +51 -0
- package/dist/_vendor/ailf-core/ports/context.d.ts +32 -3
- package/dist/_vendor/ailf-core/ports/index.d.ts +3 -3
- package/dist/_vendor/ailf-core/ports/index.js +1 -1
- package/dist/_vendor/ailf-core/schemas/pipeline.d.ts +6 -6
- package/dist/_vendor/ailf-core/services/index.d.ts +1 -0
- package/dist/_vendor/ailf-core/services/index.js +1 -0
- package/dist/_vendor/ailf-core/services/slim-report-summary.d.ts +31 -0
- package/dist/_vendor/ailf-core/services/slim-report-summary.js +217 -0
- package/dist/_vendor/ailf-core/types/branded-ids.d.ts +42 -0
- package/dist/_vendor/ailf-core/types/branded-ids.js +21 -0
- package/dist/_vendor/ailf-core/types/index.d.ts +298 -77
- package/dist/_vendor/ailf-core/types/index.js +1 -1
- package/dist/_vendor/ailf-shared/index.d.ts +2 -0
- package/dist/_vendor/ailf-shared/index.js +2 -0
- package/dist/_vendor/ailf-shared/run-context.d.ts +55 -0
- package/dist/_vendor/ailf-shared/run-context.js +17 -0
- package/dist/_vendor/ailf-shared/run-trigger.d.ts +30 -0
- package/dist/_vendor/ailf-shared/run-trigger.js +13 -0
- package/dist/artifact-capture/accumulating-artifact-writer.d.ts +50 -0
- package/dist/artifact-capture/accumulating-artifact-writer.js +111 -0
- package/dist/artifact-capture/api-gateway-artifact-writer.d.ts +52 -0
- package/dist/artifact-capture/api-gateway-artifact-writer.js +199 -0
- package/dist/artifact-capture/emit-file.d.ts +28 -0
- package/dist/artifact-capture/emit-file.js +56 -0
- package/dist/artifact-capture/fanout-artifact-writer.d.ts +39 -0
- package/dist/artifact-capture/fanout-artifact-writer.js +76 -0
- package/dist/artifact-capture/filesystem-collector.d.ts +22 -4
- package/dist/artifact-capture/filesystem-collector.js +48 -23
- package/dist/artifact-capture/gcs-artifact-writer.d.ts +67 -0
- package/dist/artifact-capture/gcs-artifact-writer.js +343 -0
- package/dist/artifact-capture/local-fs-artifact-writer.d.ts +71 -0
- package/dist/artifact-capture/local-fs-artifact-writer.js +273 -0
- package/dist/commands/explain-handler.js +4 -0
- package/dist/commands/pipeline-action.d.ts +5 -0
- package/dist/commands/pipeline-action.js +56 -5
- package/dist/commands/pipeline.d.ts +4 -0
- package/dist/commands/pipeline.js +6 -2
- package/dist/commands/publish.js +7 -3
- package/dist/composition-root.d.ts +14 -11
- package/dist/composition-root.js +90 -31
- package/dist/orchestration/build-step-sequence.js +6 -1
- package/dist/orchestration/pipeline-orchestrator.d.ts +1 -1
- package/dist/orchestration/pipeline-orchestrator.js +41 -30
- package/dist/orchestration/steps/calculate-scores-step.d.ts +1 -1
- package/dist/orchestration/steps/calculate-scores-step.js +50 -10
- package/dist/orchestration/steps/callback-step.d.ts +1 -1
- package/dist/orchestration/steps/callback-step.js +6 -4
- package/dist/orchestration/steps/compare-step.d.ts +1 -1
- package/dist/orchestration/steps/compare-step.js +4 -2
- package/dist/orchestration/steps/discovery-report-step.d.ts +1 -1
- package/dist/orchestration/steps/discovery-report-step.js +4 -1
- package/dist/orchestration/steps/fetch-docs-step.js +9 -15
- package/dist/orchestration/steps/finalize-run-step.d.ts +29 -0
- package/dist/orchestration/steps/finalize-run-step.js +117 -0
- package/dist/orchestration/steps/gap-analysis-step.js +34 -6
- package/dist/orchestration/steps/generate-configs-step.d.ts +1 -1
- package/dist/orchestration/steps/generate-configs-step.js +11 -11
- package/dist/orchestration/steps/publish-report-step.d.ts +1 -1
- package/dist/orchestration/steps/publish-report-step.js +40 -55
- package/dist/orchestration/steps/readiness-step.d.ts +1 -1
- package/dist/orchestration/steps/readiness-step.js +4 -1
- package/dist/orchestration/steps/report-step.d.ts +1 -1
- package/dist/orchestration/steps/report-step.js +6 -3
- package/dist/orchestration/steps/run-eval-step.js +14 -9
- package/dist/pipeline/calculate-scores.js +13 -2
- package/dist/pipeline/compare.d.ts +2 -2
- package/dist/pipeline/emit-eval-results.d.ts +38 -0
- package/dist/pipeline/emit-eval-results.js +100 -0
- package/dist/pipeline/provenance.d.ts +24 -44
- package/dist/pipeline/provenance.js +17 -165
- package/dist/pipeline/report-title.d.ts +2 -2
- package/dist/pipeline/run-context.d.ts +57 -0
- package/dist/pipeline/run-context.js +156 -0
- package/dist/pipeline/upload-test-outputs.d.ts +26 -0
- package/dist/pipeline/upload-test-outputs.js +34 -0
- package/dist/report-store.js +4 -2
- package/package.json +3 -3
- package/dist/_vendor/ailf-core/ports/artifact-uploader.d.ts +0 -35
- package/dist/_vendor/ailf-core/ports/artifact-uploader.js +0 -18
- package/dist/artifact-capture/api-gateway-artifact-uploader.d.ts +0 -41
- package/dist/artifact-capture/api-gateway-artifact-uploader.js +0 -123
- package/dist/artifact-capture/gcs-report-artifact-uploader.d.ts +0 -31
- package/dist/artifact-capture/gcs-report-artifact-uploader.js +0 -66
package/dist/composition-root.js
CHANGED
|
@@ -16,11 +16,14 @@
|
|
|
16
16
|
* @see docs/archive/exec-plans/ports-and-adapters/phase-7-composition-root.md
|
|
17
17
|
*/
|
|
18
18
|
import { join } from "node:path";
|
|
19
|
-
import { InMemoryPluginRegistry, NoOpArtifactCollector, } from "./_vendor/ailf-core/index.js";
|
|
20
|
-
import {
|
|
19
|
+
import { InMemoryPluginRegistry, NoOpArtifactCollector, NoOpArtifactWriter, generateRunId, isArtifactType, } from "./_vendor/ailf-core/index.js";
|
|
20
|
+
import { AccumulatingArtifactWriter } from "./artifact-capture/accumulating-artifact-writer.js";
|
|
21
|
+
import { ApiGatewayArtifactWriter } from "./artifact-capture/api-gateway-artifact-writer.js";
|
|
22
|
+
import { FanoutArtifactWriter } from "./artifact-capture/fanout-artifact-writer.js";
|
|
21
23
|
import { FilesystemArtifactCollector } from "./artifact-capture/filesystem-collector.js";
|
|
22
24
|
import { GcsArtifactCollector } from "./artifact-capture/gcs-collector.js";
|
|
23
|
-
import {
|
|
25
|
+
import { GcsArtifactWriter } from "./artifact-capture/gcs-artifact-writer.js";
|
|
26
|
+
import { LocalFilesystemArtifactWriter } from "./artifact-capture/local-fs-artifact-writer.js";
|
|
24
27
|
import { ContentLakeCacheAdapter } from "./adapters/cache/content-lake-cache.js";
|
|
25
28
|
import { loadExternalPresets } from "./pipeline/compiler/preset-loader.js";
|
|
26
29
|
import { FilesystemCache } from "./adapters/cache/filesystem-cache.js";
|
|
@@ -82,13 +85,17 @@ export function createAppContext(config) {
|
|
|
82
85
|
})
|
|
83
86
|
: fsCollector;
|
|
84
87
|
}
|
|
85
|
-
//
|
|
86
|
-
// paths
|
|
87
|
-
//
|
|
88
|
-
//
|
|
89
|
-
const
|
|
88
|
+
// Artifact writer — writes run artifacts + manifest to GCS at known
|
|
89
|
+
// `runs/{runId}/…` paths (D0032). Auto-detects the right adapter from
|
|
90
|
+
// available credentials; defaults bucket to "ailf-artifacts". Set
|
|
91
|
+
// artifactUpload: false to opt out entirely.
|
|
92
|
+
const artifactWriter = createArtifactWriter(config, logger);
|
|
93
|
+
// Generate the pipeline's RunId once; every downstream step reads it
|
|
94
|
+
// from the context (D0032).
|
|
95
|
+
const runId = generateRunId();
|
|
96
|
+
logger.debug(`Pipeline runId: ${runId}`);
|
|
90
97
|
return {
|
|
91
|
-
|
|
98
|
+
artifactWriter,
|
|
92
99
|
cache,
|
|
93
100
|
collector,
|
|
94
101
|
config,
|
|
@@ -97,6 +104,7 @@ export function createAppContext(config) {
|
|
|
97
104
|
logger,
|
|
98
105
|
registry,
|
|
99
106
|
reportStore,
|
|
107
|
+
runId,
|
|
100
108
|
sinks,
|
|
101
109
|
taskSource,
|
|
102
110
|
};
|
|
@@ -124,44 +132,95 @@ function createLogger() {
|
|
|
124
132
|
*/
|
|
125
133
|
const DEFAULT_ARTIFACT_BUCKET = "ailf-artifacts";
|
|
126
134
|
/**
|
|
127
|
-
*
|
|
135
|
+
* D0033 M4 default root for local artifacts when `--artifacts-dir` is unset.
|
|
136
|
+
* Mirrors the pre-W0050 capture root so existing dev tooling (Studio
|
|
137
|
+
* retrieval, CI archivers) keeps finding files at the same path prefix.
|
|
138
|
+
*/
|
|
139
|
+
const DEFAULT_LOCAL_ARTIFACTS_DIR = ".ailf/results/captures";
|
|
140
|
+
/**
|
|
141
|
+
* Selects the `ArtifactWriter` wiring per D0033 M4:
|
|
128
142
|
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
* 2.
|
|
132
|
-
*
|
|
133
|
-
*
|
|
143
|
+
* 1. `--no-artifacts` (`config.artifactsDisabled === true`, or legacy
|
|
144
|
+
* `config.artifactUpload === false`) → `NoOpArtifactWriter`.
|
|
145
|
+
* 2. Otherwise: always attach `LocalFilesystemArtifactWriter` under
|
|
146
|
+
* `--artifacts-dir` (default `.ailf/results/captures`).
|
|
147
|
+
* 3. When a remote backend is reachable (ADC, GCLOUD_PROJECT, or an
|
|
148
|
+
* AILF API key + URL), layer it via `FanoutArtifactWriter([local, gcs])`.
|
|
149
|
+
* Local is listed first so a local success + remote failure still
|
|
150
|
+
* produces a non-null ref.
|
|
134
151
|
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
152
|
+
* Always returns a writer — pipeline code can assume `ctx.artifactWriter`
|
|
153
|
+
* is present. Producers post-W0050 drop their `if (ctx.artifactWriter)`
|
|
154
|
+
* guards in Slice 6.
|
|
138
155
|
*
|
|
139
156
|
* Exported for unit-test access; not part of the public package API.
|
|
140
157
|
*/
|
|
141
|
-
export function
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
158
|
+
export function createArtifactWriter(config, logger) {
|
|
159
|
+
// Legacy `artifactUpload: false` still disables — treat as an alias for
|
|
160
|
+
// the canonical `artifactsDisabled: true` until W0052 removes it.
|
|
161
|
+
if (config.artifactsDisabled === true || config.artifactUpload === false) {
|
|
162
|
+
logger.debug("Artifact writer: NoOpArtifactWriter (--no-artifacts / artifactsDisabled / artifactUpload=false)");
|
|
163
|
+
return new NoOpArtifactWriter();
|
|
164
|
+
}
|
|
165
|
+
const exclude = resolveExcludeList(config.artifactsExclude, logger);
|
|
166
|
+
const rootDir = config.artifactsDir ?? DEFAULT_LOCAL_ARTIFACTS_DIR;
|
|
167
|
+
const local = new LocalFilesystemArtifactWriter({ rootDir, exclude });
|
|
168
|
+
const remote = createRemoteArtifactWriter(config, logger);
|
|
169
|
+
const base = remote
|
|
170
|
+
? new FanoutArtifactWriter([local, remote])
|
|
171
|
+
: local;
|
|
172
|
+
if (!remote) {
|
|
173
|
+
logger.debug(`Artifact writer: LocalFilesystemArtifactWriter only (rootDir=${rootDir})`);
|
|
174
|
+
}
|
|
175
|
+
else {
|
|
176
|
+
logger.debug(`Artifact writer: FanoutArtifactWriter([local=${rootDir}, ${remote.constructor.name}])`);
|
|
145
177
|
}
|
|
178
|
+
// Wrap in the accumulator so FinalizeRunStep can build a populated
|
|
179
|
+
// RunManifest without each producer bookkeeping its own ArtifactRefs
|
|
180
|
+
// (W0051 Slice 3 revisit — Option B of the "manifest empty on real runs"
|
|
181
|
+
// fix).
|
|
182
|
+
return new AccumulatingArtifactWriter(base);
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Validate the exclude list against the registry. Unknown types are dropped
|
|
186
|
+
* with a warning — a typo'd CLI flag shouldn't silently match nothing.
|
|
187
|
+
*/
|
|
188
|
+
function resolveExcludeList(raw, logger) {
|
|
189
|
+
if (!raw || raw.length === 0)
|
|
190
|
+
return [];
|
|
191
|
+
const valid = [];
|
|
192
|
+
for (const name of raw) {
|
|
193
|
+
if (isArtifactType(name)) {
|
|
194
|
+
valid.push(name);
|
|
195
|
+
}
|
|
196
|
+
else {
|
|
197
|
+
logger.warn(`--capture-exclude: "${name}" is not a known artifact type — ignored`);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
return valid;
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* The optional remote-backend writer layered on top of the local writer.
|
|
204
|
+
* Returns null when no credentials are available — the local writer stays
|
|
205
|
+
* the sole backend for that run, which is the D0033 M4 default for laptops
|
|
206
|
+
* and CI without GCS creds.
|
|
207
|
+
*/
|
|
208
|
+
function createRemoteArtifactWriter(config, logger) {
|
|
146
209
|
const bucket = config.artifactGcsBucket ?? DEFAULT_ARTIFACT_BUCKET;
|
|
147
|
-
// CI / GCP runtime — direct GCS upload (fastest, no extra hop).
|
|
148
|
-
// We treat the presence of either env var as the user opting in to ADC.
|
|
149
210
|
const hasGcsCredentials = Boolean(process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.GCLOUD_PROJECT);
|
|
150
211
|
if (hasGcsCredentials) {
|
|
151
|
-
logger.debug(`Artifact
|
|
152
|
-
return new
|
|
212
|
+
logger.debug(`Artifact remote backend: GcsArtifactWriter (ADC, bucket=${bucket})`);
|
|
213
|
+
return new GcsArtifactWriter({ bucket });
|
|
153
214
|
}
|
|
154
|
-
// Local dev — request signed PUT URLs from the API gateway, no GCS creds needed.
|
|
155
215
|
if (config.apiKey && config.apiUrl) {
|
|
156
|
-
logger.debug(`Artifact
|
|
157
|
-
return new
|
|
216
|
+
logger.debug(`Artifact remote backend: ApiGatewayArtifactWriter (via ${config.apiUrl}, bucket=${bucket})`);
|
|
217
|
+
return new ApiGatewayArtifactWriter({
|
|
158
218
|
apiBaseUrl: config.apiUrl,
|
|
159
219
|
apiKey: config.apiKey,
|
|
160
220
|
bucket,
|
|
161
221
|
});
|
|
162
222
|
}
|
|
163
|
-
|
|
164
|
-
return undefined;
|
|
223
|
+
return null;
|
|
165
224
|
}
|
|
166
225
|
function createCache(config) {
|
|
167
226
|
const local = new FilesystemCache(config.rootDir);
|
|
@@ -11,6 +11,7 @@ import { CalculateScoresStep } from "./steps/calculate-scores-step.js";
|
|
|
11
11
|
import { CompareStep } from "./steps/compare-step.js";
|
|
12
12
|
import { DiscoveryReportStep } from "./steps/discovery-report-step.js";
|
|
13
13
|
import { FetchDocsStep } from "./steps/fetch-docs-step.js";
|
|
14
|
+
import { FinalizeRunStep } from "./steps/finalize-run-step.js";
|
|
14
15
|
import { GapAnalysisStep } from "./steps/gap-analysis-step.js";
|
|
15
16
|
import { GenerateConfigsStep } from "./steps/generate-configs-step.js";
|
|
16
17
|
import { GraderConsistencyStep } from "./steps/grader-consistency-step.js";
|
|
@@ -76,7 +77,11 @@ export function buildStepSequence(ctx, pipelineStart = Date.now()) {
|
|
|
76
77
|
if (config.gapAnalysisEnabled) {
|
|
77
78
|
steps.push(new GapAnalysisStep());
|
|
78
79
|
}
|
|
79
|
-
// Step
|
|
80
|
+
// Step 4c: Finalize the run — write `runs/{runId}/manifest.json` with the
|
|
81
|
+
// catalog of artifacts produced so far. Skipped silently when no
|
|
82
|
+
// artifactWriter is wired (D0032).
|
|
83
|
+
steps.push(new FinalizeRunStep(pipelineStart));
|
|
84
|
+
// Step 4d: Publish report (optional, when token is configured)
|
|
80
85
|
if (config.publishEnabled) {
|
|
81
86
|
steps.push(new PublishReportStep(pipelineStart, {
|
|
82
87
|
publishTag: config.publishTag,
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* each step completes. This enables the GET /v1/jobs/:jobId polling
|
|
12
12
|
* endpoint to show real-time progress.
|
|
13
13
|
*/
|
|
14
|
-
import type
|
|
14
|
+
import { type AppContext, type PipelineResult, type PipelineStep } from "../_vendor/ailf-core/index.d.ts";
|
|
15
15
|
/**
|
|
16
16
|
* Run a sequence of pipeline steps, short-circuiting on required step failure.
|
|
17
17
|
*
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* each step completes. This enables the GET /v1/jobs/:jobId polling
|
|
12
12
|
* endpoint to show real-time progress.
|
|
13
13
|
*/
|
|
14
|
+
import { assoc, } from "../_vendor/ailf-core/index.js";
|
|
14
15
|
import { runStep } from "./step-runner.js";
|
|
15
16
|
// ---------------------------------------------------------------------------
|
|
16
17
|
// Job progress reporter
|
|
@@ -75,28 +76,40 @@ async function reportJobProgress(ctx, stepName, completedSteps, totalSteps, stat
|
|
|
75
76
|
* Capture a snapshot of the pipeline config, final state, and step results.
|
|
76
77
|
* Strips secrets (API keys, tokens) from the config.
|
|
77
78
|
*/
|
|
78
|
-
function capturePipelineContext(ctx, state, results) {
|
|
79
|
-
if (!ctx.collector.enabled)
|
|
80
|
-
return;
|
|
79
|
+
async function capturePipelineContext(ctx, state, results) {
|
|
81
80
|
const sanitized = Object.fromEntries(Object.entries(ctx.config).filter(([k]) => !/token|secret|key/i.test(k)));
|
|
82
|
-
ctx.collector.capture("pipeline", "pipeline-context",
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
81
|
+
// W0050 — migrated from ctx.collector.capture("pipeline", "pipeline-context", …)
|
|
82
|
+
// to the registry-driven emit() path. The writer handles redaction,
|
|
83
|
+
// --capture-exclude gating, and local+GCS fanout internally.
|
|
84
|
+
//
|
|
85
|
+
// Awaited (not fire-and-forget) so the write is observable by the
|
|
86
|
+
// orchestrator's caller — a fire-and-forget let the emit fall through
|
|
87
|
+
// to runtime teardown in tests with aggressive afterEach cleanup.
|
|
88
|
+
// `emit` is non-blocking internally (P5): failures return null + warn,
|
|
89
|
+
// never throw, so awaiting can't surface a rejected promise either.
|
|
90
|
+
try {
|
|
91
|
+
await ctx.artifactWriter.emit("pipelineContext", assoc(ctx), {
|
|
92
|
+
config: sanitized,
|
|
93
|
+
state: {
|
|
94
|
+
reportId: state.reportId,
|
|
95
|
+
evalFingerprint: state.evalFingerprint,
|
|
96
|
+
belowCritical: state.belowCritical,
|
|
97
|
+
remoteCacheHits: state.remoteCacheHits
|
|
98
|
+
? [...state.remoteCacheHits]
|
|
99
|
+
: undefined,
|
|
100
|
+
releaseAutoScope: state.releaseAutoScope,
|
|
101
|
+
testSummary: state.testSummary,
|
|
102
|
+
},
|
|
103
|
+
steps: Object.entries(results).map(([name, result]) => ({
|
|
104
|
+
name,
|
|
105
|
+
status: result.status,
|
|
106
|
+
durationMs: result.status !== "skipped" ? result.durationMs : undefined,
|
|
107
|
+
})),
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
ctx.logger.debug(`pipelineContext emit rejected: ${err instanceof Error ? err.message : String(err)}`);
|
|
112
|
+
}
|
|
100
113
|
}
|
|
101
114
|
/**
|
|
102
115
|
* Flush captured artifacts to disk. Non-blocking — failures are logged
|
|
@@ -170,10 +183,10 @@ export async function orchestratePipeline(ctx, steps) {
|
|
|
170
183
|
}, jobUpdates);
|
|
171
184
|
}
|
|
172
185
|
// Capture pipeline context and job updates before flushing
|
|
173
|
-
capturePipelineContext(ctx, state, results);
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
186
|
+
await capturePipelineContext(ctx, state, results);
|
|
187
|
+
// W0050 — `job-updates` was an observability-only capture not tied
|
|
188
|
+
// to a registered artifact type; dropped here. Use the JobStore
|
|
189
|
+
// path if job telemetry is needed.
|
|
177
190
|
// Flush captured artifacts even on failure (partial capture is useful)
|
|
178
191
|
await flushArtifacts(ctx);
|
|
179
192
|
return {
|
|
@@ -229,11 +242,9 @@ export async function orchestratePipeline(ctx, steps) {
|
|
|
229
242
|
ctx.logger.warn("Failed to report job completion — continuing");
|
|
230
243
|
}
|
|
231
244
|
}
|
|
232
|
-
// Capture pipeline context
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
ctx.collector.capture("job-store", "job-updates", jobUpdates);
|
|
236
|
-
}
|
|
245
|
+
// Capture pipeline context. `job-updates` observability captures were
|
|
246
|
+
// dropped in Slice 6.1 — JobStore is the supported telemetry path.
|
|
247
|
+
await capturePipelineContext(ctx, state, results);
|
|
237
248
|
// Flush captured artifacts (non-blocking — failures never affect pipeline result)
|
|
238
249
|
await flushArtifacts(ctx);
|
|
239
250
|
return {
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Calls calculateAndWriteScores() from pipeline/calculate-scores.ts with
|
|
5
5
|
* typed options derived from AppContext. No env bridge needed.
|
|
6
6
|
*/
|
|
7
|
-
import type
|
|
7
|
+
import { type AppContext, type PipelineState, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
|
|
8
8
|
export declare class CalculateScoresStep implements PipelineStep {
|
|
9
9
|
readonly name = "calculate-scores";
|
|
10
10
|
check(): ValidationIssue[];
|
|
@@ -4,8 +4,10 @@
|
|
|
4
4
|
* Calls calculateAndWriteScores() from pipeline/calculate-scores.ts with
|
|
5
5
|
* typed options derived from AppContext. No env bridge needed.
|
|
6
6
|
*/
|
|
7
|
-
import { existsSync } from "node:fs";
|
|
8
|
-
import { join } from "path";
|
|
7
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
8
|
+
import { join, resolve } from "path";
|
|
9
|
+
import { assoc, } from "../../_vendor/ailf-core/index.js";
|
|
10
|
+
import { emitFileContents } from "../../artifact-capture/emit-file.js";
|
|
9
11
|
import { LiteracyVariant } from "../../pipeline/normalize-mode.js";
|
|
10
12
|
import { getStepInputPaths } from "../../pipeline/cache.js";
|
|
11
13
|
import { buildCacheContext } from "../cache-context.js";
|
|
@@ -13,6 +15,7 @@ import { calculateAndWriteScores } from "../../pipeline/calculate-scores.js";
|
|
|
13
15
|
import { checkResultsExist, checkScoreSummaryValid, } from "../../pipeline/checks.js";
|
|
14
16
|
import { resultsFileForMode } from "../../pipeline/eval-constants.js";
|
|
15
17
|
import { loadSource } from "../../sources.js";
|
|
18
|
+
import { uploadTestOutputs } from "../../pipeline/upload-test-outputs.js";
|
|
16
19
|
import { configToSourceOverrides } from "../config-to-source-overrides.js";
|
|
17
20
|
export class CalculateScoresStep {
|
|
18
21
|
name = "calculate-scores";
|
|
@@ -121,15 +124,34 @@ export class CalculateScoresStep {
|
|
|
121
124
|
state.belowCritical = belowCritical;
|
|
122
125
|
}
|
|
123
126
|
// Capture score artifacts
|
|
127
|
+
// W0050 — score-summary → scoreSummary (run-scoped bulk).
|
|
128
|
+
// grader-judgments.json and test-results.json were aggregated captures
|
|
129
|
+
// without registered descriptors. graderJudgments is now per-entry
|
|
130
|
+
// ({run, mode, task, model, grader}) and lands via run-eval-step in
|
|
131
|
+
// Slice 6.6; the aggregated file is dropped.
|
|
124
132
|
const resultsDir = join(ctx.config.rootDir, "results", "latest");
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
+
const summaryPath = join(resultsDir, "score-summary.json");
|
|
134
|
+
if (existsSync(summaryPath)) {
|
|
135
|
+
await emitFileContents(ctx.artifactWriter, "scoreSummary", assoc(ctx), summaryPath);
|
|
136
|
+
}
|
|
137
|
+
// Upload testOutputs to GCS (D0032 — non-blocking, P5).
|
|
138
|
+
// Read from test-results.json rather than score-summary.json: the
|
|
139
|
+
// gap-analysis step (downstream) is the one that enriches score-summary
|
|
140
|
+
// with testResults, so at this point the summary still has an empty
|
|
141
|
+
// testResults[]. test-results.json is written by calculateAndWriteScores
|
|
142
|
+
// above and carries the full per-test shape we need for per-entry upload.
|
|
143
|
+
// The full responseOutput lives in the GCS artifact; PublishReportStep
|
|
144
|
+
// later strips it from the inline Content Lake document when this
|
|
145
|
+
// upload succeeds.
|
|
146
|
+
// W0050 — ctx.artifactWriter is always present; no guard needed.
|
|
147
|
+
const testResults = tryReadTestResults(ctx.config.rootDir);
|
|
148
|
+
if (testResults?.length) {
|
|
149
|
+
const artifactRef = await uploadTestOutputs(ctx.artifactWriter, ctx.runId, testResults);
|
|
150
|
+
if (artifactRef) {
|
|
151
|
+
state.artifactRefs = {
|
|
152
|
+
...state.artifactRefs,
|
|
153
|
+
testOutputs: artifactRef,
|
|
154
|
+
};
|
|
133
155
|
}
|
|
134
156
|
}
|
|
135
157
|
const criticalSuffix = belowCritical.length > 0
|
|
@@ -148,3 +170,21 @@ export class CalculateScoresStep {
|
|
|
148
170
|
return buildCacheContext(ctx.config);
|
|
149
171
|
}
|
|
150
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* Read the per-test result set written by `calculateAndWriteScores`.
|
|
175
|
+
*
|
|
176
|
+
* This is the authoritative source for `uploadTestOutputs` at the time
|
|
177
|
+
* CalculateScoresStep runs — `score-summary.json` doesn't carry
|
|
178
|
+
* `testResults[]` until `gap-analysis-step` enriches it downstream.
|
|
179
|
+
*/
|
|
180
|
+
function tryReadTestResults(rootDir) {
|
|
181
|
+
const path = resolve(rootDir, "results", "latest", "test-results.json");
|
|
182
|
+
if (!existsSync(path))
|
|
183
|
+
return undefined;
|
|
184
|
+
try {
|
|
185
|
+
return JSON.parse(readFileSync(path, "utf-8"));
|
|
186
|
+
}
|
|
187
|
+
catch {
|
|
188
|
+
return undefined;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* @see packages/eval/src/pipeline/callback-delivery.ts
|
|
12
12
|
* @see docs/design-docs/api-service-gateway.md
|
|
13
13
|
*/
|
|
14
|
-
import type
|
|
14
|
+
import { type AppContext, type PipelineState, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
|
|
15
15
|
import { type CallbackConfig } from "../../pipeline/callback-delivery.js";
|
|
16
16
|
export declare class CallbackStep implements PipelineStep {
|
|
17
17
|
private readonly callback;
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { readFileSync } from "fs";
|
|
15
15
|
import { resolve } from "path";
|
|
16
|
+
import { assoc, } from "../../_vendor/ailf-core/index.js";
|
|
16
17
|
import { deliverCallback, } from "../../pipeline/callback-delivery.js";
|
|
17
18
|
export class CallbackStep {
|
|
18
19
|
callback;
|
|
@@ -58,11 +59,12 @@ export class CallbackStep {
|
|
|
58
59
|
reportId: state.reportId,
|
|
59
60
|
summary,
|
|
60
61
|
};
|
|
61
|
-
//
|
|
62
|
-
|
|
62
|
+
// W0050 — callbackRequest/callbackResponse are per-entry artifacts
|
|
63
|
+
// keyed by the callback target URL (the `name` slot on the association).
|
|
64
|
+
const callbackName = this.callback.url;
|
|
65
|
+
await ctx.artifactWriter.emit("callbackRequest", assoc(ctx, { name: callbackName }), callbackPayload);
|
|
63
66
|
const result = await deliverCallback(this.callback, callbackPayload);
|
|
64
|
-
|
|
65
|
-
ctx.collector.capture("callback", "callback-response", {
|
|
67
|
+
await ctx.artifactWriter.emit("callbackResponse", assoc(ctx, { name: callbackName }), {
|
|
66
68
|
ok: result.ok,
|
|
67
69
|
attempts: result.attempts,
|
|
68
70
|
error: result.error,
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* inlined directly from the former pipeline/steps/compare-step.ts.
|
|
6
6
|
* This is an optional step — failure doesn't stop the pipeline.
|
|
7
7
|
*/
|
|
8
|
-
import type
|
|
8
|
+
import { type AppContext, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
|
|
9
9
|
export declare class CompareStep implements PipelineStep {
|
|
10
10
|
readonly name = "compare";
|
|
11
11
|
readonly optional = true;
|
|
@@ -7,6 +7,8 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, } from "fs";
|
|
9
9
|
import { join, resolve } from "path";
|
|
10
|
+
import { assoc, } from "../../_vendor/ailf-core/index.js";
|
|
11
|
+
import { emitFileContents } from "../../artifact-capture/emit-file.js";
|
|
10
12
|
import { compare } from "../../pipeline/compare.js";
|
|
11
13
|
export class CompareStep {
|
|
12
14
|
name = "compare";
|
|
@@ -69,8 +71,8 @@ export class CompareStep {
|
|
|
69
71
|
mkdirSync(ctx.config.outputDir, { recursive: true });
|
|
70
72
|
const reportPath = resolve(ctx.config.outputDir, "comparison-report.json");
|
|
71
73
|
writeFileSync(reportPath, JSON.stringify(report, null, 2));
|
|
72
|
-
//
|
|
73
|
-
ctx.
|
|
74
|
+
// W0050 — comparisonReport is per-entry keyed by mode ({run, mode}).
|
|
75
|
+
await emitFileContents(ctx.artifactWriter, "comparisonReport", assoc(ctx, { mode: ctx.config.mode }), reportPath);
|
|
74
76
|
// Build summary
|
|
75
77
|
const improved = report.improved.length;
|
|
76
78
|
const regressed = report.regressed.length;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Calls pure functions from pipeline/discovery-report.ts directly.
|
|
5
5
|
* Optional step — failure doesn't stop the pipeline.
|
|
6
6
|
*/
|
|
7
|
-
import type
|
|
7
|
+
import { type AppContext, type PipelineStep, type StepResult, type ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
|
|
8
8
|
export declare class DiscoveryReportStep implements PipelineStep {
|
|
9
9
|
readonly name = "discovery-report";
|
|
10
10
|
readonly optional = true;
|
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
8
8
|
import { resolve } from "path";
|
|
9
|
+
import { assoc, } from "../../_vendor/ailf-core/index.js";
|
|
10
|
+
import { emitFileContents } from "../../artifact-capture/emit-file.js";
|
|
9
11
|
import { formatDiscoveryMarkdown, generateDiscoveryReport, } from "../../pipeline/discovery-report.js";
|
|
10
12
|
export class DiscoveryReportStep {
|
|
11
13
|
name = "discovery-report";
|
|
@@ -38,7 +40,8 @@ export class DiscoveryReportStep {
|
|
|
38
40
|
mkdirSync(ctx.config.outputDir, { recursive: true });
|
|
39
41
|
const discoveryPath = resolve(ctx.config.outputDir, "discovery-report.md");
|
|
40
42
|
writeFileSync(discoveryPath, md);
|
|
41
|
-
|
|
43
|
+
// W0050 — discoveryReport is per-entry keyed by mode.
|
|
44
|
+
await emitFileContents(ctx.artifactWriter, "discoveryReport", assoc(ctx, { mode: ctx.config.mode }), discoveryPath);
|
|
42
45
|
console.log(md);
|
|
43
46
|
const invisible = report.invisibleDocs.length;
|
|
44
47
|
const f1 = report.overall.avgF1.toFixed(2);
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { existsSync, mkdirSync, writeFileSync } from "fs";
|
|
14
14
|
import { join } from "path";
|
|
15
|
-
import { isIdRef, isPathRef, isSlugRef, } from "../../_vendor/ailf-core/index.js";
|
|
15
|
+
import { assoc, isIdRef, isPathRef, isSlugRef, } from "../../_vendor/ailf-core/index.js";
|
|
16
|
+
import { emitFileContents } from "../../artifact-capture/emit-file.js";
|
|
16
17
|
import { getStepInputPaths } from "../../pipeline/cache.js";
|
|
17
18
|
import { buildCacheContext } from "../cache-context.js";
|
|
18
19
|
import { checkCanonicalContextsExist } from "../../pipeline/checks.js";
|
|
@@ -94,20 +95,13 @@ export class FetchDocsStep {
|
|
|
94
95
|
if (result.metadata) {
|
|
95
96
|
writeMetadataFiles(ctx.config.rootDir, result.metadata);
|
|
96
97
|
}
|
|
97
|
-
//
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
["url-fetch", "url-fetch.json"],
|
|
105
|
-
]) {
|
|
106
|
-
const filePath = join(contextsDir, filename);
|
|
107
|
-
if (existsSync(filePath)) {
|
|
108
|
-
ctx.collector.captureFile("fetch-docs", type, filePath);
|
|
109
|
-
}
|
|
110
|
-
}
|
|
98
|
+
// W0050 — documentManifest is run-scoped bulk JSON. The
|
|
99
|
+
// release-impact/document-overlay/url-fetch captures had no
|
|
100
|
+
// registered descriptors (they were extras-only); dropped per Q3
|
|
101
|
+
// ("producers always call emit; registered types only").
|
|
102
|
+
const documentManifestPath = join(ctx.config.rootDir, "contexts", "document-manifest.json");
|
|
103
|
+
if (existsSync(documentManifestPath)) {
|
|
104
|
+
await emitFileContents(ctx.artifactWriter, "documentManifest", assoc(ctx), documentManifestPath);
|
|
111
105
|
}
|
|
112
106
|
}
|
|
113
107
|
catch (err) {
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeline step: FinalizeRunStep — writes the run manifest at pipeline end.
|
|
3
|
+
*
|
|
4
|
+
* Inserts between `GapAnalysis` and `PublishReport`. Assembles a
|
|
5
|
+
* `RunManifest` from `state.artifactRefs` (populated by producer steps)
|
|
6
|
+
* and the shared `RunContext` (via `buildRunContext`), then writes it to
|
|
7
|
+
* `runs/{runId}/manifest.json`. The written manifest becomes the source
|
|
8
|
+
* of truth for artifact locations; `PublishReportStep` snapshots the
|
|
9
|
+
* `artifacts` slice into `Report.artifactManifest` (D0032).
|
|
10
|
+
*
|
|
11
|
+
* Design principles:
|
|
12
|
+
* - Single writer — one `writeManifest()` call per pipeline run.
|
|
13
|
+
* - Idempotent — retries produce the same manifest bytes for the same inputs.
|
|
14
|
+
* - Skipped when no writer is wired (local/air-gapped runs stay functional).
|
|
15
|
+
*
|
|
16
|
+
* @see docs/decisions/D0032-run-anchored-artifact-store.md
|
|
17
|
+
*/
|
|
18
|
+
import type { AppContext, PipelineState, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
|
|
19
|
+
export declare class FinalizeRunStep implements PipelineStep {
|
|
20
|
+
private readonly pipelineStart;
|
|
21
|
+
private readonly options;
|
|
22
|
+
readonly name = "finalize-run";
|
|
23
|
+
readonly optional = true;
|
|
24
|
+
constructor(pipelineStart: number, options?: {
|
|
25
|
+
evalFingerprint?: string;
|
|
26
|
+
});
|
|
27
|
+
check(): ValidationIssue[];
|
|
28
|
+
execute(ctx: AppContext, state: PipelineState): Promise<StepResult>;
|
|
29
|
+
}
|