@kici-dev/agent 0.1.27 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap/ensure-init-runner.d.ts +23 -22
- package/dist/bootstrap/payload-source.d.ts +32 -0
- package/dist/bootstrap/probe-platform.d.ts +21 -0
- package/dist/bootstrap/restage-agent.d.ts +43 -0
- package/dist/bootstrap/run-restage.d.ts +12 -0
- package/dist/bootstrap/s3-payload-source.d.ts +35 -0
- package/dist/bootstrap/ssh-exec.d.ts +14 -0
- package/dist/bootstrap/stage-agent-payload.d.ts +41 -0
- package/dist/checkout/changed-files.d.ts +34 -0
- package/dist/config.d.ts +36 -14
- package/dist/container-ts-loader-hook.js +147710 -0
- package/dist/execution/artifacts/artifact-engine.d.ts +51 -0
- package/dist/execution/dep-installer.d.ts +3 -3
- package/dist/execution/job-runner.d.ts +24 -6
- package/dist/execution/log-streamer.d.ts +17 -2
- package/dist/execution/rule-evaluator.d.ts +1 -12
- package/dist/execution/sandbox/container-hardening.d.ts +80 -0
- package/dist/execution/sandbox/container-sandbox.d.ts +54 -0
- package/dist/execution/sandbox/container-ts-loader-hook.d.ts +26 -0
- package/dist/execution/sandbox/fork-runner.d.ts +14 -0
- package/dist/execution/sandbox/index.d.ts +2 -1
- package/dist/execution/sandbox/ipc-protocol.d.ts +79 -3
- package/dist/execution/sandbox/step-loop.d.ts +4 -0
- package/dist/execution/sandbox/types.d.ts +25 -4
- package/dist/execution/sandbox/workflow-runner.d.ts +111 -5
- package/dist/execution/streaming-zx-log.d.ts +11 -3
- package/dist/execution/tmp-gc.d.ts +22 -7
- package/dist/execution/workflow-loader.d.ts +32 -1
- package/dist/index.js +44 -45
- package/dist/provenance/statement-builder.d.ts +3 -2
- package/dist/server.d.ts +1 -1
- package/dist/server.js +1256 -171
- package/dist/workflow-runner-bundle.js +215393 -0
- package/dist/workflow-runner.js +886 -244
- package/dist/ws/orchestrator-client.d.ts +105 -1
- package/package.json +14 -12
- package/sbom.spdx.json +1090 -1721
|
@@ -13,12 +13,15 @@
|
|
|
13
13
|
* This file is compiled alongside the agent by rolldown (existing build), but
|
|
14
14
|
* runs as a SEPARATE process spawned by the sandbox backend.
|
|
15
15
|
*/
|
|
16
|
+
import { $ } from 'zx';
|
|
17
|
+
import { type TempScope } from '@kici-dev/core/tmp';
|
|
16
18
|
import { ExecutionJobStatus } from '@kici-dev/engine';
|
|
17
|
-
import type { StepContext } from '@kici-dev/sdk';
|
|
18
|
-
import type { NeedsContext, FanoutPosition } from '@kici-dev/sdk';
|
|
19
|
+
import type { Step, StepInput, StepContext } from '@kici-dev/sdk';
|
|
20
|
+
import type { NeedsContext, FanoutPosition, EventDefinition } from '@kici-dev/sdk';
|
|
19
21
|
import type { OutputsMap, StepRefMap, TrackedStepSecrets } from '@kici-dev/sdk';
|
|
20
22
|
import type { RunnerToAgentMessage, JobExecutionRequest } from './ipc-protocol.js';
|
|
21
23
|
import { LogMasker } from './log-masker.js';
|
|
24
|
+
import { type RuleEvaluationResult } from '../rule-evaluator.js';
|
|
22
25
|
/**
|
|
23
26
|
* Build `ctx.needs` for a job's steps from the dispatch envelope. Reconstructs
|
|
24
27
|
* an {@link UpstreamSnapshot} from `upstreamJobOutputs` (flat per single job;
|
|
@@ -28,6 +31,62 @@ import { LogMasker } from './log-masker.js';
|
|
|
28
31
|
* undefined when the job declares no needs.
|
|
29
32
|
*/
|
|
30
33
|
export declare function buildStepNeedsContext(declaredNeeds: readonly unknown[] | undefined, upstreamJobOutputs: Record<string, Record<string, unknown>> | undefined, upstreamJobStatuses: Record<string, ExecutionJobStatus> | undefined): NeedsContext | undefined;
|
|
34
|
+
/**
|
|
35
|
+
* Build a fresh zx `$` shell bound to the sandbox working directory and the
|
|
36
|
+
* sanitized environment (process.env was set by the parent via env-sanitizer
|
|
37
|
+
* before spawning this process). This is the single shell-construction code
|
|
38
|
+
* path shared by step execution (`createSandboxStepContext`) and the per-job
|
|
39
|
+
* init phase (`runInitPhase`), so init commands run through the identical shell
|
|
40
|
+
* steps use — same cwd, same live sanitized env, same masked log streaming.
|
|
41
|
+
*
|
|
42
|
+
* The shell binds `process.env` by reference (not a copy), so a same-step
|
|
43
|
+
* ctx.setEnv / ctx.addPath mutation — which applyEnvDelta writes to live
|
|
44
|
+
* process.env — is visible to that step's own subprocesses, matching the
|
|
45
|
+
* "visible to this step" SDK contract.
|
|
46
|
+
*
|
|
47
|
+
* Intercept zx subprocess output via the log callback: zx does NOT write child
|
|
48
|
+
* stdout/stderr to process.stdout — it pipes to an internal VoidStream and only
|
|
49
|
+
* calls $.log() with { kind: 'stdout'|'stderr', verbose }. We override the log
|
|
50
|
+
* function (`makeStreamingZxLog`) to capture both kinds and send them as masked
|
|
51
|
+
* IPC log.line messages tagged with `stepIndex`.
|
|
52
|
+
*
|
|
53
|
+
* The shell is built with `verbose: true` so ordinary subprocess output is
|
|
54
|
+
* flagged `verbose: true` (captured) while a step that opts into
|
|
55
|
+
* `$({ quiet: true })` — e.g. a `sops -d` credential decrypt — is flagged
|
|
56
|
+
* `verbose: false` and dropped by `makeStreamingZxLog`. That gate is what keeps
|
|
57
|
+
* a decrypted secret out of the streamed run log; a `verbose: false` base would
|
|
58
|
+
* flag ordinary output `verbose: false` too and drop every line.
|
|
59
|
+
*
|
|
60
|
+
* IMPORTANT: The log function must be passed in the zx$() config, not set on the
|
|
61
|
+
* returned function. zx$() returns a plain function (not the proxy $), so setting
|
|
62
|
+
* step$.log would only set it on the function object and NOT propagate to the
|
|
63
|
+
* AsyncLocalStorage store that zx uses for ProcessPromise snapshots.
|
|
64
|
+
*/
|
|
65
|
+
export declare function buildSandboxShell(cwd: string, stepIndex: number, maskedSendFn: (msg: RunnerToAgentMessage) => void): typeof $;
|
|
66
|
+
/**
|
|
67
|
+
* Resolve the on-the-wire event name for `ctx.emit`. Accepts either an ad-hoc
|
|
68
|
+
* event-name string or a `defineEvent()` definition object (passed by the typed
|
|
69
|
+
* emit overload); a definition resolves to its `.name`, a string passes through.
|
|
70
|
+
*/
|
|
71
|
+
export declare function resolveEmitEventName(nameOrDefinition: string | EventDefinition): string;
|
|
72
|
+
/**
|
|
73
|
+
* Sanitize a raw identifier into a valid temp label: lowercase, every
|
|
74
|
+
* non-`[a-z0-9-]` char to `-`, falling back to `'step'` when the result is
|
|
75
|
+
* empty. Applied to both the caller-supplied `ctx.mktemp(label)` and the
|
|
76
|
+
* default step-id label so a friendly-but-irregular label never rejects at the
|
|
77
|
+
* scope. Mirrors the SDK test builder's `sanitizeTempLabel`.
|
|
78
|
+
*/
|
|
79
|
+
export declare function sanitizeTempLabel(raw: string): string;
|
|
80
|
+
/**
|
|
81
|
+
* Drain the job-scoped temp allocator once at job end.
|
|
82
|
+
*
|
|
83
|
+
* Called after the step loop and cancel-path hooks, immediately before the
|
|
84
|
+
* terminal `job.complete` emit, so a single call reclaims every `ctx.mktemp` /
|
|
85
|
+
* `ctx.mktempFile` allocation on success, failure, and cancel/timeout alike. A
|
|
86
|
+
* cleanup failure must NEVER change the job's terminal status, so any error is
|
|
87
|
+
* swallowed into `warn` rather than thrown out of `main()`.
|
|
88
|
+
*/
|
|
89
|
+
export declare function drainJobTempScope(scope: TempScope, warn: (message: string) => void): Promise<void>;
|
|
31
90
|
/**
|
|
32
91
|
* Create a StepContext natively inside the workflow runner.
|
|
33
92
|
*
|
|
@@ -35,7 +94,7 @@ export declare function buildStepNeedsContext(declaredNeeds: readonly unknown[]
|
|
|
35
94
|
* NOT serialized across the process boundary. This means zx $ runs natively
|
|
36
95
|
* inside this process with full shell access.
|
|
37
96
|
*/
|
|
38
|
-
export declare function createSandboxStepContext(workDir: string, stepIndex: number, stepName: string, request: JobExecutionRequest, maskedSendFn: (msg: RunnerToAgentMessage) => void, outputsMap: OutputsMap, refMap: StepRefMap, operatorSecretKeys: Set<string>, secretOutputs: Map<string, string>, jobOutputsMap: OutputsMap, secrets: TrackedStepSecrets, masker: LogMasker, signal: AbortSignal): StepContext;
|
|
97
|
+
export declare function createSandboxStepContext(workDir: string, stepIndex: number, stepName: string, request: JobExecutionRequest, maskedSendFn: (msg: RunnerToAgentMessage) => void, outputsMap: OutputsMap, refMap: StepRefMap, operatorSecretKeys: Set<string>, secretOutputs: Map<string, string>, jobOutputsMap: OutputsMap, secrets: TrackedStepSecrets, masker: LogMasker, signal: AbortSignal, jobTempScope: TempScope): StepContext;
|
|
39
98
|
/**
|
|
40
99
|
* Derive the fan-out position (`ctx.fanout`) from a dispatch request. Returns
|
|
41
100
|
* `undefined` for a non-fan-out job (no `fanoutTotal`), so `ctx.fanout` is only
|
|
@@ -44,12 +103,59 @@ export declare function createSandboxStepContext(workDir: string, stepIndex: num
|
|
|
44
103
|
export declare function deriveFanout(request: JobExecutionRequest): FanoutPosition | undefined;
|
|
45
104
|
/** Raw provider webhook body for ctx.rawPayload — nested in the envelope. */
|
|
46
105
|
export declare function rawPayloadFromEvent(event: Record<string, unknown> | undefined): Record<string, unknown> | undefined;
|
|
106
|
+
/**
|
|
107
|
+
* Mutable state threaded through {@link coerceStep} for one job normalization
|
|
108
|
+
* pass: the shared `step-N` counter, the bare-function → name ref map, and the
|
|
109
|
+
* set of step objects this pass has already auto-named.
|
|
110
|
+
*/
|
|
111
|
+
export interface CoerceState {
|
|
112
|
+
counter: number;
|
|
113
|
+
refMap: StepRefMap;
|
|
114
|
+
autoNamed: WeakSet<object>;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Coerce one raw entry (bare function or Step) into a normalized Step, naming
|
|
118
|
+
* anonymous steps `step-N` with a counter shared across the whole flattened
|
|
119
|
+
* sequence (parallel children inline) — matching the compiler's transformSteps
|
|
120
|
+
* enumeration so the flat-stepIndex invariant holds agent ↔ orchestrator.
|
|
121
|
+
*
|
|
122
|
+
* Unnamed steps are named by MUTATING the original object so the SDK's
|
|
123
|
+
* late-bound `.result` proxy (which reads `step.name` at access time) resolves.
|
|
124
|
+
* A step object reused twice in one pass gets a clone with a fresh name on the
|
|
125
|
+
* repeat encounter — mirroring the compiler's per-occurrence naming so lock
|
|
126
|
+
* parity holds; the original keeps its first binding.
|
|
127
|
+
*/
|
|
128
|
+
export declare function coerceStep(stepOrFn: StepInput, state: CoerceState): Step<any>;
|
|
129
|
+
/**
|
|
130
|
+
* Resolve `event.changedFiles` for rule evaluation before job/step rules run.
|
|
131
|
+
* Ground truth is the agent's own clone (`computeChangedFiles`); the
|
|
132
|
+
* orchestrator's already-fetched list (status `'fetched'`) is a free fast-path.
|
|
133
|
+
* Only runs when a rule could read `ctx.changedFiles` — rule-less jobs skip the
|
|
134
|
+
* git cost. Diff-less events (schedule/tag/manual) resolve to `'unavailable'`.
|
|
135
|
+
*/
|
|
136
|
+
export declare function resolveChangedFilesForRules(request: JobExecutionRequest, workDir: string, hasRules: boolean): Promise<void>;
|
|
137
|
+
/**
|
|
138
|
+
* Decide the terminal `job.complete` for job-level rule evaluation, or `null`
|
|
139
|
+
* when the job should proceed to step execution.
|
|
140
|
+
*
|
|
141
|
+
* - `allPassed` → `null` (run the steps).
|
|
142
|
+
* - a rule's `check()` **threw** (`evaluationError`) → FAIL: the gate could not
|
|
143
|
+
* be evaluated, so treating "couldn't decide" as "don't run" would be a false
|
|
144
|
+
* green. Report `status: failed` with the error surfaced.
|
|
145
|
+
* - a rule cleanly returned `false` → clean skip: `status: success` with every
|
|
146
|
+
* step marked skipped (a gate that evaluated and said "don't run").
|
|
147
|
+
*
|
|
148
|
+
* Pure and exported so the decision is unit-testable without `process.exit`.
|
|
149
|
+
*/
|
|
150
|
+
export declare function buildJobRuleCompletion(ruleResult: RuleEvaluationResult, normalizedSteps: Step[]): (RunnerToAgentMessage & {
|
|
151
|
+
type: 'job.complete';
|
|
152
|
+
}) | null;
|
|
47
153
|
/**
|
|
48
154
|
* Build the step loop's KICI_ENV/KICI_PATH callbacks with a per-step delta-file
|
|
49
155
|
* pair (keyed by step index). `beforeStepEnvFiles(stepIndex)` lazily creates the
|
|
50
156
|
* step's pair and points the runner's process.env at it (each step's zx $
|
|
51
|
-
*
|
|
52
|
-
*
|
|
157
|
+
* reads live process.env at spawn, which happens AFTER this before-hook, so
|
|
158
|
+
* the shell sees them; the pre-fork env allowlist does not
|
|
53
159
|
* re-filter runtime-set vars). `afterStepApplyEnvFiles(stepIndex)` applies that
|
|
54
160
|
* step's delta and releases the pair.
|
|
55
161
|
*
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { LogStream } from '@kici-dev/engine';
|
|
1
2
|
/**
|
|
2
3
|
* Shared factory for the zx `log` callback that streams a subprocess's
|
|
3
4
|
* stdout/stderr into the captured/streamed run log, line by line, via `emit`.
|
|
@@ -23,8 +24,15 @@
|
|
|
23
24
|
* `verbose: false` (suppressed). A `verbose: false` base would flag ordinary
|
|
24
25
|
* output `verbose: false` too, and this gate would then drop every line.
|
|
25
26
|
*
|
|
26
|
-
* The returned callback owns
|
|
27
|
-
* coalesced into whole lines before `emit` is called.
|
|
27
|
+
* The returned callback owns one line buffer PER STREAM, so partial chunks are
|
|
28
|
+
* coalesced into whole lines before `emit` is called. The buffers are separate
|
|
29
|
+
* because the two streams are independent pipes: a stdout chunk ending mid-line
|
|
30
|
+
* and a stderr chunk arriving next would otherwise concatenate into a single
|
|
31
|
+
* spliced line attributed to whichever kind completed it.
|
|
32
|
+
*
|
|
33
|
+
* `emit` receives the originating stream alongside the line so a diagnostic
|
|
34
|
+
* written to stderr stays distinguishable from ordinary progress output all the
|
|
35
|
+
* way to the persisted run log.
|
|
28
36
|
*/
|
|
29
|
-
export declare function makeStreamingZxLog(emit: (line: string) => void): (entry: unknown) => void;
|
|
37
|
+
export declare function makeStreamingZxLog(emit: (line: string, stream: LogStream) => void): (entry: unknown) => void;
|
|
30
38
|
//# sourceMappingURL=streaming-zx-log.d.ts.map
|
|
@@ -1,17 +1,32 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Startup garbage collection for this agent's own temp-directory families.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
4
|
+
* Bare job workdirs (`kici-<6 random chars>`, see job-runner.ts), labeled
|
|
5
|
+
* allocator dirs (`kici-<label>-<6 random chars>`, minted by the global temp
|
|
6
|
+
* allocator `@kici-dev/core/tmp`, which guarantees every allocation carries
|
|
7
|
+
* the `kici-` prefix), and isolated pnpm stores (`kici-pnpm-store-*`, see
|
|
8
|
+
* dep-installer.ts) clean themselves up in `finally` blocks — but a hard
|
|
9
|
+
* process death (SIGKILL, OOM kill) skips those, and on a long-lived
|
|
10
|
+
* bare-metal agent the leftovers then accumulate forever. Collecting anything
|
|
11
|
+
* older than a day at startup is safe on shared hosts: no job lives remotely
|
|
12
|
+
* that long (job timeouts are minutes), so a concurrent agent's in-flight
|
|
13
|
+
* dirs are never eligible.
|
|
14
|
+
*
|
|
15
|
+
* The deterministic persistent caches (`kici-agent-payloads`, `kici-data`,
|
|
16
|
+
* `kici-scaler-ledger`) also live directly under the temp root and can be far
|
|
17
|
+
* older than a day, so they are explicitly excluded — `kici-scaler-ledger`
|
|
18
|
+
* even structurally matches the allocator pattern (`ledger` is a 6-char label
|
|
19
|
+
* suffix), which a regex alone cannot distinguish.
|
|
11
20
|
*/
|
|
12
21
|
/**
|
|
13
22
|
* Collect this agent's stale temp dirs. `base` is overridable for tests;
|
|
14
23
|
* production callers use the default temp root. Never throws.
|
|
24
|
+
*
|
|
25
|
+
* The default base is `kiciTmpBase()`, not the OS temp root, so it scans the
|
|
26
|
+
* exact directory the agent now writes payloads/clones/pnpm stores into: the
|
|
27
|
+
* global temp allocator and the payload cache both honor `KICI_TMPDIR` via the
|
|
28
|
+
* same helper. If the GC scanned a different root, stale-temp reaping would
|
|
29
|
+
* silently stop working whenever `KICI_TMPDIR` is set.
|
|
15
30
|
*/
|
|
16
31
|
export declare function gcStaleAgentTmpDirs(base?: string): Promise<string[]>;
|
|
17
32
|
//# sourceMappingURL=tmp-gc.d.ts.map
|
|
@@ -6,7 +6,37 @@
|
|
|
6
6
|
* no Rolldown step at runtime. `@kici-dev/sdk` and host-repo deps resolve via
|
|
7
7
|
* Node's normal ESM lookup against `.kici/node_modules/`.
|
|
8
8
|
*/
|
|
9
|
-
import type { Workflow, StepInput, DynamicJobFn } from '@kici-dev/sdk';
|
|
9
|
+
import type { Workflow, StepInput, DynamicJobFn, OutputsMap, StepRefMap } from '@kici-dev/sdk';
|
|
10
|
+
/**
|
|
11
|
+
* SDK output-map setters resolved from a specific `@kici-dev/sdk` instance.
|
|
12
|
+
* The agent uses these to wire the workflow module's OWN SDK copy (a different
|
|
13
|
+
* physical module than the agent's bundled SDK) to the same output maps the
|
|
14
|
+
* step loop mutates, so within-job `.result` proxies — which read that copy's
|
|
15
|
+
* module-global `_stepOutputsMap` — resolve at access time.
|
|
16
|
+
*/
|
|
17
|
+
export interface SdkOutputSetters {
|
|
18
|
+
setStepOutputsMap: (map: OutputsMap) => void;
|
|
19
|
+
setStepRefMap: (map: StepRefMap) => void;
|
|
20
|
+
setJobOutputsMap: (map: OutputsMap) => void;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Resolve the `@kici-dev/sdk` instance the workflow module itself imports.
|
|
24
|
+
*
|
|
25
|
+
* The workflow's `.result` proxies read the module-global step-outputs map of
|
|
26
|
+
* whichever SDK copy the workflow file resolves — which is generally a
|
|
27
|
+
* different physical module than the agent's bundled SDK (the workflow is
|
|
28
|
+
* imported from the cloned source tree and resolves its deps against that
|
|
29
|
+
* tree's `node_modules`). Resolving via `createRequire(workflowFilePath)` walks
|
|
30
|
+
* `node_modules` from the workflow file exactly the way the workflow's own
|
|
31
|
+
* `import '@kici-dev/sdk'` does — including any hoisted copy — so the returned
|
|
32
|
+
* setters mutate the SAME module-global map object the proxies read. Node caches
|
|
33
|
+
* ESM modules by resolved URL, so importing that path yields the workflow's live
|
|
34
|
+
* singleton, not a fresh copy.
|
|
35
|
+
*
|
|
36
|
+
* Falls back to the agent's bundled setters when resolution fails (mirrors
|
|
37
|
+
* `resolveSdkSetters` in the compiler's test runner).
|
|
38
|
+
*/
|
|
39
|
+
export declare function resolveWorkflowSdkSetters(workflowFilePath: string): Promise<SdkOutputSetters>;
|
|
10
40
|
/**
|
|
11
41
|
* Compile schema version — must match `@kici-dev/compiler` lockfile/hasher.ts.
|
|
12
42
|
* Mixed into the content hash so compilation-approach changes produce different
|
|
@@ -33,6 +63,7 @@ export declare function ensureLoaderHookRegistered(): void;
|
|
|
33
63
|
*/
|
|
34
64
|
export declare function loadWorkflowSource(workDir: string, sourceFile: string, expectedContentHash?: string, resolvedHashFiles?: string[]): Promise<{
|
|
35
65
|
module: Record<string, unknown>;
|
|
66
|
+
sdkSetters: SdkOutputSetters;
|
|
36
67
|
}>;
|
|
37
68
|
/**
|
|
38
69
|
* Extract a workflow by name from a module's exports.
|
package/dist/index.js
CHANGED
|
@@ -5,7 +5,8 @@ import { z } from "zod";
|
|
|
5
5
|
import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/shared/env";
|
|
6
6
|
import { KNOWN_ROLES, parseHostPropertyAssignments, validateNoReservedLabels } from "@kici-dev/engine";
|
|
7
7
|
import { execFile } from "node:child_process";
|
|
8
|
-
import { access, cp, lstat, mkdir,
|
|
8
|
+
import { access, cp, lstat, mkdir, readFile, readdir, realpath, rename, rm, unlink, writeFile } from "node:fs/promises";
|
|
9
|
+
import { makeTempDir } from "@kici-dev/core/tmp";
|
|
9
10
|
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
10
11
|
import { promisify } from "node:util";
|
|
11
12
|
import { createLogger, sha256, toErrorMessage } from "@kici-dev/shared";
|
|
@@ -15,6 +16,7 @@ import { parse, stringify } from "yaml";
|
|
|
15
16
|
import { Readable, Transform } from "node:stream";
|
|
16
17
|
import { pipeline } from "node:stream/promises";
|
|
17
18
|
import { createGunzip } from "node:zlib";
|
|
19
|
+
import { HOME_ANCHOR, REPO_ANCHOR } from "@kici-dev/core";
|
|
18
20
|
import { c, x } from "tar";
|
|
19
21
|
import https from "node:https";
|
|
20
22
|
import http from "node:http";
|
|
@@ -85,10 +87,18 @@ const envDef = defineEnv({
|
|
|
85
87
|
dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
|
|
86
88
|
jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
|
|
87
89
|
backpressureMode: z.enum(["pause", "drop"]).default("pause"),
|
|
90
|
+
agentPayloadDir: z.string().optional(),
|
|
91
|
+
agentCommand: z.string().optional(),
|
|
88
92
|
sandbox: z.string().default("false").transform((s) => s === "true"),
|
|
89
93
|
trustedEnv: z.string().default("false").transform((s) => s === "true"),
|
|
90
94
|
inPlace: z.string().default("false").transform((s) => s === "true"),
|
|
91
95
|
sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
|
|
96
|
+
sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
|
|
97
|
+
sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
|
|
98
|
+
sandboxUser: z.string().optional(),
|
|
99
|
+
sandboxPidsLimit: z.coerce.number().int().positive().default(512),
|
|
100
|
+
sandboxMemoryBytes: z.coerce.number().int().positive().default(2 * 1024 * 1024 * 1024),
|
|
101
|
+
sandboxNanoCpus: z.coerce.number().int().positive().default(2 * 1e9),
|
|
92
102
|
scalerManaged: z.string().optional().transform((s) => s === "1"),
|
|
93
103
|
scalerIdleTimeoutMs: z.coerce.number().default(5e3),
|
|
94
104
|
scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
|
|
@@ -113,10 +123,18 @@ const envDef = defineEnv({
|
|
|
113
123
|
dockerKeepFailed: "KICI_DOCKER_KEEP_FAILED",
|
|
114
124
|
jobHeartbeatIntervalMs: "KICI_JOB_HEARTBEAT_INTERVAL_MS",
|
|
115
125
|
backpressureMode: "KICI_BACKPRESSURE_MODE",
|
|
126
|
+
agentPayloadDir: "KICI_AGENT_PAYLOAD_DIR",
|
|
127
|
+
agentCommand: "KICI_AGENT_COMMAND",
|
|
116
128
|
sandbox: "KICI_SANDBOX",
|
|
117
129
|
trustedEnv: "KICI_TRUSTED_ENV",
|
|
118
130
|
inPlace: "KICI_IN_PLACE",
|
|
119
131
|
sandboxNetwork: "KICI_SANDBOX_NETWORK",
|
|
132
|
+
sandboxHardened: "KICI_SANDBOX_HARDENED",
|
|
133
|
+
sandboxReadonlyRootfs: "KICI_SANDBOX_READONLY_ROOTFS",
|
|
134
|
+
sandboxUser: "KICI_SANDBOX_USER",
|
|
135
|
+
sandboxPidsLimit: "KICI_SANDBOX_PIDS_LIMIT",
|
|
136
|
+
sandboxMemoryBytes: "KICI_SANDBOX_MEMORY_BYTES",
|
|
137
|
+
sandboxNanoCpus: "KICI_SANDBOX_NANO_CPUS",
|
|
120
138
|
scalerManaged: "KICI_SCALER_MANAGED",
|
|
121
139
|
scalerIdleTimeoutMs: "KICI_SCALER_IDLE_TIMEOUT",
|
|
122
140
|
scalerPendingDispatchTimeoutMs: "KICI_SCALER_PENDING_DISPATCH_TIMEOUT",
|
|
@@ -146,7 +164,13 @@ const envDef = defineEnv({
|
|
|
146
164
|
* - KICI_SANDBOX (default: false) — enable bubblewrap (bwrap) namespace isolation for bare-metal execution
|
|
147
165
|
* - KICI_TRUSTED_ENV (default: false) — trusted fleet-agent profile: pass the ambient host env (minus the agent's own KiCI identity secrets) through to steps
|
|
148
166
|
* - KICI_IN_PLACE (default: false) — in-place no-clone profile: for a file:// source, use the real repo path as workDir and skip the clone (the routed deploy:stg profile)
|
|
149
|
-
* - KICI_SANDBOX_NETWORK (default: isolated, options: isolated | host) — when sandbox=true
|
|
167
|
+
* - KICI_SANDBOX_NETWORK (default: isolated, options: isolated | host) — sandbox network posture for BOTH backends: the bwrap network namespace (when sandbox=true) and the container job network. `host` shares the host network (container backend also binds host /etc/hosts read-only for name resolution); applies to the container backend under the default hardened posture (KICI_SANDBOX_HARDENED=true)
|
|
168
|
+
* - KICI_SANDBOX_HARDENED (default: true) — hardened-by-default job containers (CapDrop ALL, no-new-privileges, cgroup caps, tmpfs /tmp); set false to roll back to the legacy unhardened posture
|
|
169
|
+
* - KICI_SANDBOX_READONLY_ROOTFS (default: false) — opt-in read-only container rootfs (/tmp stays a writable tmpfs)
|
|
170
|
+
* - KICI_SANDBOX_USER (optional) — container user override (uid, uid:gid, or name); honors the image user when unset
|
|
171
|
+
* - KICI_SANDBOX_PIDS_LIMIT (default: 512) — max PIDs in the job container cgroup
|
|
172
|
+
* - KICI_SANDBOX_MEMORY_BYTES (default: 2 GiB) — memory cap in bytes for the job container cgroup
|
|
173
|
+
* - KICI_SANDBOX_NANO_CPUS (default: 2 CPUs) — CPU cap in nano-CPUs for the job container cgroup
|
|
150
174
|
* - KICI_SCALER_MANAGED (set to "1" by the orchestrator's auto-scaler — agent self-shuts down on idle)
|
|
151
175
|
* - KICI_SCALER_IDLE_TIMEOUT (ms, default 5000) — how long a scaler-managed agent waits before shutdown after going idle
|
|
152
176
|
* - KICI_SCALER_PENDING_DISPATCH_TIMEOUT (ms, default 60000) — extended idle window when register.ack signals a queued bound job
|
|
@@ -155,7 +179,7 @@ const envDef = defineEnv({
|
|
|
155
179
|
*/
|
|
156
180
|
function loadConfig() {
|
|
157
181
|
const data = envDef.parse();
|
|
158
|
-
if (!data.scalerManaged) validateNoReservedLabels(data.labels, "KICI_LABELS");
|
|
182
|
+
if (!data.scalerManaged && !data.agentToken) validateNoReservedLabels(data.labels, "KICI_LABELS");
|
|
159
183
|
validateUnknownKiciVars([...envDef.listKnownEnvVars(), ...LOGGER_ENV_VARS]);
|
|
160
184
|
return {
|
|
161
185
|
...data,
|
|
@@ -364,7 +388,7 @@ async function applyYarnrcBerryConfig(args) {
|
|
|
364
388
|
const hasPrivateRegistry = registries.length > 0 || Object.keys(installEnvSecrets).length > 0;
|
|
365
389
|
const yarnrcPath = join(args.kiciDir, ".yarnrc.yml");
|
|
366
390
|
const { raw: original, doc } = await readOriginalYarnrc(yarnrcPath);
|
|
367
|
-
const cacheFolder = await
|
|
391
|
+
const { path: cacheFolder, cleanup: cleanupCache } = await makeTempDir("yarn-berry-cache");
|
|
368
392
|
const merged = {
|
|
369
393
|
...doc,
|
|
370
394
|
nodeLinker: "node-modules",
|
|
@@ -401,10 +425,7 @@ async function applyYarnrcBerryConfig(args) {
|
|
|
401
425
|
if (original === null) await unlink(yarnrcPath).catch(() => {});
|
|
402
426
|
else await writeFile(yarnrcPath, original, { encoding: "utf8" });
|
|
403
427
|
} catch {}
|
|
404
|
-
await
|
|
405
|
-
recursive: true,
|
|
406
|
-
force: true
|
|
407
|
-
}).catch(() => {});
|
|
428
|
+
await cleanupCache().catch(() => {});
|
|
408
429
|
};
|
|
409
430
|
return {
|
|
410
431
|
extraEnv: {
|
|
@@ -756,9 +777,9 @@ async function detectKiciYarnFlavor(repoRoot, kiciDir) {
|
|
|
756
777
|
* Install `.kici/` dependencies inline with the repo's package manager.
|
|
757
778
|
*
|
|
758
779
|
* Falls back to this when the dep cache is unavailable or a download fails.
|
|
759
|
-
* The install runs with an isolated cache/store directory (
|
|
760
|
-
* `
|
|
761
|
-
* is removed after installation.
|
|
780
|
+
* The install runs with an isolated cache/store directory (allocated under the
|
|
781
|
+
* global temp base, which honors `KICI_TMPDIR`) to prevent cache poisoning
|
|
782
|
+
* between build jobs; the directory is removed after installation.
|
|
762
783
|
*
|
|
763
784
|
* If `opts.npmRegistries` / `opts.installEnvSecrets` is provided, a job-scoped
|
|
764
785
|
* `.kici/.npmrc` overlay is synthesized for the install, restored in `finally`,
|
|
@@ -847,7 +868,7 @@ function envWithNodeOnPath(extraEnv, nodeDir) {
|
|
|
847
868
|
/** Run `npm install` in `.kici/` with an isolated cache directory. */
|
|
848
869
|
async function runNpmInstall(args) {
|
|
849
870
|
const { npmCliPath, nodeExe, nodeDir } = resolveNpm();
|
|
850
|
-
const cacheDir = await
|
|
871
|
+
const { path: cacheDir, cleanup } = await makeTempDir("npm-cache");
|
|
851
872
|
const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
|
|
852
873
|
const buildArgs = (...prefix) => {
|
|
853
874
|
const a = [
|
|
@@ -872,10 +893,7 @@ async function runNpmInstall(args) {
|
|
|
872
893
|
maxBuffer: INSTALL_MAX_BUFFER
|
|
873
894
|
});
|
|
874
895
|
} finally {
|
|
875
|
-
await
|
|
876
|
-
recursive: true,
|
|
877
|
-
force: true
|
|
878
|
-
}).catch(() => {});
|
|
896
|
+
await cleanup().catch(() => {});
|
|
879
897
|
}
|
|
880
898
|
}
|
|
881
899
|
/**
|
|
@@ -889,7 +907,7 @@ async function runNpmInstall(args) {
|
|
|
889
907
|
async function runPnpmInstall(args) {
|
|
890
908
|
await assertPnpmAvailable();
|
|
891
909
|
const { nodeDir } = resolveNpm();
|
|
892
|
-
const storeDir = await
|
|
910
|
+
const { path: storeDir, cleanup } = await makeTempDir("pnpm-store");
|
|
893
911
|
const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
|
|
894
912
|
const argv = [
|
|
895
913
|
"install",
|
|
@@ -909,10 +927,7 @@ async function runPnpmInstall(args) {
|
|
|
909
927
|
maxBuffer: INSTALL_MAX_BUFFER
|
|
910
928
|
});
|
|
911
929
|
} finally {
|
|
912
|
-
await
|
|
913
|
-
recursive: true,
|
|
914
|
-
force: true
|
|
915
|
-
}).catch(() => {});
|
|
930
|
+
await cleanup().catch(() => {});
|
|
916
931
|
}
|
|
917
932
|
}
|
|
918
933
|
/** Pure: argv for `yarn install` with an isolated cache folder. */
|
|
@@ -938,7 +953,7 @@ function buildYarnInstallArgs(cacheDir, hasPrivateRegistry) {
|
|
|
938
953
|
async function runYarnInstall(args) {
|
|
939
954
|
await assertYarnAvailable();
|
|
940
955
|
const { nodeDir } = resolveNpm();
|
|
941
|
-
const cacheDir = await
|
|
956
|
+
const { path: cacheDir, cleanup } = await makeTempDir("yarn-cache");
|
|
942
957
|
const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
|
|
943
958
|
const argv = buildYarnInstallArgs(cacheDir, args.hasPrivateRegistry);
|
|
944
959
|
try {
|
|
@@ -950,10 +965,7 @@ async function runYarnInstall(args) {
|
|
|
950
965
|
maxBuffer: INSTALL_MAX_BUFFER
|
|
951
966
|
});
|
|
952
967
|
} finally {
|
|
953
|
-
await
|
|
954
|
-
recursive: true,
|
|
955
|
-
force: true
|
|
956
|
-
}).catch(() => {});
|
|
968
|
+
await cleanup().catch(() => {});
|
|
957
969
|
}
|
|
958
970
|
}
|
|
959
971
|
/** Pure: argv for a berry `yarn install`. Cache + linker live in .yarnrc.yml. */
|
|
@@ -1224,10 +1236,6 @@ var init_download = __esmMin((() => {
|
|
|
1224
1236
|
const logger = createLogger({ prefix: "cache-engine" });
|
|
1225
1237
|
/** Download timeout for a presigned cache GET: 5 minutes. */
|
|
1226
1238
|
const DOWNLOAD_TIMEOUT_MS = 300 * 1e3;
|
|
1227
|
-
/** Anchor prefix for repo-root-relative cache entries inside the tar. */
|
|
1228
|
-
const REPO_ANCHOR = "__repo__";
|
|
1229
|
-
/** Anchor prefix for home-relative (`~`) cache entries inside the tar. */
|
|
1230
|
-
const HOME_ANCHOR = "__home__";
|
|
1231
1239
|
/**
|
|
1232
1240
|
* Resolve a cache path. `~`-prefixed -> home root; otherwise repo-root-relative.
|
|
1233
1241
|
* Rejects absolute paths and `..` escapes so a workflow cannot read or clobber
|
|
@@ -1268,7 +1276,7 @@ function anchorEntries(workDir, paths, roots) {
|
|
|
1268
1276
|
*/
|
|
1269
1277
|
async function packCachePaths(workDir, paths, roots) {
|
|
1270
1278
|
const entries = anchorEntries(workDir, paths, roots);
|
|
1271
|
-
const staging = await
|
|
1279
|
+
const { path: staging, cleanup } = await makeTempDir("cache-pack");
|
|
1272
1280
|
try {
|
|
1273
1281
|
const topLevel = /* @__PURE__ */ new Set();
|
|
1274
1282
|
for (const e of entries) {
|
|
@@ -1299,10 +1307,7 @@ async function packCachePaths(workDir, paths, roots) {
|
|
|
1299
1307
|
hash
|
|
1300
1308
|
};
|
|
1301
1309
|
} finally {
|
|
1302
|
-
await
|
|
1303
|
-
recursive: true,
|
|
1304
|
-
force: true
|
|
1305
|
-
});
|
|
1310
|
+
await cleanup();
|
|
1306
1311
|
}
|
|
1307
1312
|
}
|
|
1308
1313
|
/** Move the extracted `__repo__` / `__home__` groups from a scratch dir into place. */
|
|
@@ -1333,7 +1338,7 @@ async function extractCacheTarball(tarball, workDir, expectedHash, roots) {
|
|
|
1333
1338
|
if (actual !== expectedHash) throw new Error(`Cache tarball checksum mismatch: expected ${expectedHash}, got ${actual}`);
|
|
1334
1339
|
const home = roots?.home ?? homedir();
|
|
1335
1340
|
await mkdir(workDir, { recursive: true });
|
|
1336
|
-
const scratch = await
|
|
1341
|
+
const { path: scratch, cleanup } = await makeTempDir("cache-extract");
|
|
1337
1342
|
try {
|
|
1338
1343
|
await new Promise((res, rej) => {
|
|
1339
1344
|
Readable.from(tarball).pipe(x({
|
|
@@ -1343,10 +1348,7 @@ async function extractCacheTarball(tarball, workDir, expectedHash, roots) {
|
|
|
1343
1348
|
});
|
|
1344
1349
|
await moveAnchoredGroups(scratch, workDir, home);
|
|
1345
1350
|
} finally {
|
|
1346
|
-
await
|
|
1347
|
-
recursive: true,
|
|
1348
|
-
force: true
|
|
1349
|
-
});
|
|
1351
|
+
await cleanup();
|
|
1350
1352
|
}
|
|
1351
1353
|
}
|
|
1352
1354
|
/**
|
|
@@ -1365,17 +1367,14 @@ async function downloadAndExtractCache(url, workDir, expectedHash, roots) {
|
|
|
1365
1367
|
cb(null, chunk);
|
|
1366
1368
|
} });
|
|
1367
1369
|
await mkdir(workDir, { recursive: true });
|
|
1368
|
-
const scratch = await
|
|
1370
|
+
const { path: scratch, cleanup } = await makeTempDir("cache-extract");
|
|
1369
1371
|
try {
|
|
1370
1372
|
await pipeline(Readable.fromWeb(response.body), hashTransform, createGunzip(), x({ cwd: scratch }));
|
|
1371
1373
|
const digest = hash.digest("hex");
|
|
1372
1374
|
if (digest !== expectedHash) throw new Error(`Cache tarball checksum mismatch on download: expected ${expectedHash}, got ${digest}`);
|
|
1373
1375
|
await moveAnchoredGroups(scratch, workDir, home);
|
|
1374
1376
|
} finally {
|
|
1375
|
-
await
|
|
1376
|
-
recursive: true,
|
|
1377
|
-
force: true
|
|
1378
|
-
});
|
|
1377
|
+
await cleanup();
|
|
1379
1378
|
}
|
|
1380
1379
|
}
|
|
1381
1380
|
/** Build the imperative `ctx.cache` API bound to a workDir + transport. */
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Build a SLSA v1.0 in-toto provenance statement from the server-truth identity
|
|
3
3
|
* token claims plus the caller-supplied subject. The build context comes
|
|
4
|
-
* entirely from the JWT claims (
|
|
4
|
+
* entirely from the JWT claims (minted server-side by the orchestrator,
|
|
5
|
+
* unforgeable), so the
|
|
5
6
|
* statement's identity equals the token's identity by construction.
|
|
6
7
|
*/
|
|
7
8
|
import { type KiciProvenanceStatement } from '@kici-dev/engine/provenance/schema';
|
|
8
9
|
import type { SourceOrigin } from '@kici-dev/engine';
|
|
9
|
-
/** The KiCI identity-token claims the builder reads (
|
|
10
|
+
/** The KiCI identity-token claims the builder reads (minter server-truth). */
|
|
10
11
|
export interface ProvenanceTokenClaims {
|
|
11
12
|
iss: string;
|
|
12
13
|
repository?: string | null;
|
package/dist/server.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Startup sequence:
|
|
5
5
|
* 1. Load config
|
|
6
6
|
* 2. Create logger
|
|
7
|
-
* 3. Create JobRunner with send
|
|
7
|
+
* 3. Create JobRunner with send callbacks
|
|
8
8
|
* 4. Create OrchestratorClient with dispatch and cancel handlers
|
|
9
9
|
* 5. Add WS log transport (if not scaler-managed)
|
|
10
10
|
* 6. Connect OrchestratorClient
|