@kici-dev/agent 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checkout/clone-job-repos.d.ts +53 -0
- package/dist/checkout/credential-helper-bin.d.ts +28 -0
- package/dist/checkout/credential-helper-host.d.ts +48 -0
- package/dist/checkout/credential-helper.d.ts +44 -0
- package/dist/checkout/git-clone.d.ts +26 -0
- package/dist/checkout/grant-table.d.ts +30 -0
- package/dist/checkout/job-git-credentials.d.ts +49 -0
- package/dist/checkout/write-elevation.d.ts +40 -0
- package/dist/config.d.ts +38 -0
- package/dist/container-ts-loader-hook.js +2047 -1814
- package/dist/execution/between-jobs-controller.d.ts +50 -0
- package/dist/execution/between-jobs-reset.d.ts +25 -0
- package/dist/execution/cleanup-rerun.d.ts +21 -0
- package/dist/execution/dynamic-job-serializer.d.ts +6 -2
- package/dist/execution/image-build/build-engine.d.ts +75 -0
- package/dist/execution/image-build/build-step.d.ts +57 -0
- package/dist/execution/image-build/resolve-build-spec.d.ts +41 -0
- package/dist/execution/image-build/runtime-facts.d.ts +31 -0
- package/dist/execution/job-runner.d.ts +48 -0
- package/dist/execution/sandbox/bare-metal-sandbox.d.ts +25 -1
- package/dist/execution/sandbox/container-sandbox.d.ts +97 -2
- package/dist/execution/sandbox/fork-runner.d.ts +55 -1
- package/dist/execution/sandbox/image-preflight.d.ts +38 -0
- package/dist/execution/sandbox/ipc-protocol.d.ts +96 -2
- package/dist/execution/sandbox/kici-runtime.d.ts +48 -0
- package/dist/execution/sandbox/step-loop.d.ts +7 -0
- package/dist/execution/sandbox/types.d.ts +55 -1
- package/dist/execution/sandbox/workflow-runner.d.ts +23 -3
- package/dist/idle-shutdown.d.ts +24 -0
- package/dist/index.js +133 -62
- package/dist/metrics/prometheus.d.ts +26 -0
- package/dist/server.js +2050 -312
- package/dist/workflow-runner-bundle.js +41961 -41319
- package/dist/workflow-runner.js +332 -136
- package/dist/ws/orchestrator-client.d.ts +95 -3
- package/package.json +10 -10
- package/sbom.spdx.json +460 -455
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { ExecutionMode, BetweenJobsRunOn } from '../config.js';
|
|
2
|
+
import { runBetweenJobsReset, type ResetStatus } from './between-jobs-reset.js';
|
|
3
|
+
import { runDeclaredCleanupOutOfBand, type CleanupRerunStatus } from './cleanup-rerun.js';
|
|
4
|
+
interface ControllerConfig {
|
|
5
|
+
betweenJobsResetCommand?: string;
|
|
6
|
+
betweenJobsResetTimeoutMs: number;
|
|
7
|
+
betweenJobsResetRunOn: BetweenJobsRunOn;
|
|
8
|
+
orphanCleanup: boolean;
|
|
9
|
+
drainOnResetFailure: boolean;
|
|
10
|
+
}
|
|
11
|
+
export interface AfterJobContext {
|
|
12
|
+
completionHooksRan: boolean;
|
|
13
|
+
jobFailed: boolean;
|
|
14
|
+
backend: ExecutionMode;
|
|
15
|
+
declaresCleanup: boolean;
|
|
16
|
+
workDir: string;
|
|
17
|
+
reap: () => Promise<number>;
|
|
18
|
+
deleteWorkdir: () => Promise<void>;
|
|
19
|
+
cleanupSpawn?: (workDir: string, signal: AbortSignal) => Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
export interface BetweenJobsOutcome {
|
|
22
|
+
rerun: CleanupRerunStatus;
|
|
23
|
+
reaped: number;
|
|
24
|
+
reset: ResetStatus;
|
|
25
|
+
consecutiveResetFailures: number;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Supervisor-owned between-jobs phase for a reused (bare-metal / in-place) agent.
|
|
29
|
+
* After every job it sequences: (1) out-of-band declared-cleanup re-run when the
|
|
30
|
+
* runner died before its completion hooks ran, (2) process-group reap of the
|
|
31
|
+
* job's descendant tree, (3) workdir deletion, (4) the operator reset command.
|
|
32
|
+
* Each phase is delegated so the class stays unit-testable, and a persistently
|
|
33
|
+
* failing reset can drain the agent via the consecutive-failure counter.
|
|
34
|
+
*/
|
|
35
|
+
export declare class BetweenJobsController {
|
|
36
|
+
private deps;
|
|
37
|
+
private _consecutiveResetFailures;
|
|
38
|
+
/** Consecutive between-jobs reset failures, for the supervisor's drain gate. */
|
|
39
|
+
get consecutiveResetFailures(): number;
|
|
40
|
+
constructor(deps: {
|
|
41
|
+
config: ControllerConfig;
|
|
42
|
+
rerun?: typeof runDeclaredCleanupOutOfBand;
|
|
43
|
+
reset?: typeof runBetweenJobsReset;
|
|
44
|
+
});
|
|
45
|
+
private runRerunPhase;
|
|
46
|
+
private runResetPhase;
|
|
47
|
+
afterJob(ctx: AfterJobContext): Promise<BetweenJobsOutcome>;
|
|
48
|
+
}
|
|
49
|
+
export {};
|
|
50
|
+
//# sourceMappingURL=between-jobs-controller.d.ts.map
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { BetweenJobsRunOn } from '../config.js';
|
|
2
|
+
export type ResetStatus = 'success' | 'failed' | 'timeout' | 'skipped';
|
|
3
|
+
export interface ExecResult {
|
|
4
|
+
code: number | null;
|
|
5
|
+
timedOut?: boolean;
|
|
6
|
+
}
|
|
7
|
+
export type ExecFn = (command: string, timeoutMs: number) => Promise<ExecResult>;
|
|
8
|
+
export interface ResetInput {
|
|
9
|
+
command?: string;
|
|
10
|
+
timeoutMs: number;
|
|
11
|
+
runOn: BetweenJobsRunOn;
|
|
12
|
+
jobFailed: boolean;
|
|
13
|
+
exec?: ExecFn;
|
|
14
|
+
}
|
|
15
|
+
export interface ResetResult {
|
|
16
|
+
status: ResetStatus;
|
|
17
|
+
durationMs: number;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Run the operator between-jobs reset command. Fail-open: never throws, always
|
|
21
|
+
* resolves to a status. Skips when unconfigured, or when runOn=on-failure and
|
|
22
|
+
* the job succeeded.
|
|
23
|
+
*/
|
|
24
|
+
export declare function runBetweenJobsReset(input: ResetInput): Promise<ResetResult>;
|
|
25
|
+
//# sourceMappingURL=between-jobs-reset.d.ts.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { ExecutionMode } from '../config.js';
|
|
2
|
+
export type CleanupRerunStatus = 'success' | 'failed' | 'timeout' | 'skipped';
|
|
3
|
+
export interface CleanupRerunInput {
|
|
4
|
+
workDir: string;
|
|
5
|
+
backend: ExecutionMode;
|
|
6
|
+
declaresCleanup: boolean;
|
|
7
|
+
timeoutMs: number;
|
|
8
|
+
spawn: (workDir: string, signal: AbortSignal) => Promise<void>;
|
|
9
|
+
}
|
|
10
|
+
export interface CleanupRerunResult {
|
|
11
|
+
status: CleanupRerunStatus;
|
|
12
|
+
durationMs: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Re-run a hard-killed job's declared cleanup/onFailure hooks out-of-band,
|
|
16
|
+
* against its preserved workdir, in a fresh bounded child. Returns 'skipped'
|
|
17
|
+
* when there is nothing to do (no declared cleanup, wrong backend, or a
|
|
18
|
+
* non-positive timeout). Never throws — a failure is reported as a status.
|
|
19
|
+
*/
|
|
20
|
+
export declare function runDeclaredCleanupOutOfBand(input: CleanupRerunInput): Promise<CleanupRerunResult>;
|
|
21
|
+
//# sourceMappingURL=cleanup-rerun.d.ts.map
|
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
*
|
|
14
14
|
* Constraints:
|
|
15
15
|
* - Generated jobs are limited to MAX_DYNAMIC_JOBS per DynamicJobFn invocation
|
|
16
|
-
* - Generated job names must be unique within the same DynamicJobFn output
|
|
16
|
+
* - Generated job names must be unique within the same DynamicJobFn output, and
|
|
17
|
+
* across a whole eval round when the caller threads a shared `seenNames` set
|
|
17
18
|
*/
|
|
18
19
|
import type { $ as Shell } from 'zx';
|
|
19
20
|
import type { Job, Logger } from '@kici-dev/sdk';
|
|
@@ -54,9 +55,12 @@ export interface SerializerContext {
|
|
|
54
55
|
*
|
|
55
56
|
* @param jobs - Jobs returned by a DynamicJobFn
|
|
56
57
|
* @param ctx - Eval-time context used to resolve dynamic fields on generated jobs
|
|
58
|
+
* @param seenNames - Optional accumulator of job names already emitted earlier
|
|
59
|
+
* in the same eval round; when supplied, a name already present throws and
|
|
60
|
+
* every generated name is added so later generators in the round see it
|
|
57
61
|
* @returns Serialized LockJob array ready for orchestrator dispatch
|
|
58
62
|
* @throws Error if validation fails (duplicates, limit exceeded) or if a user-supplied
|
|
59
63
|
* dynamic function throws / times out / returns an unsupported value
|
|
60
64
|
*/
|
|
61
|
-
export declare function serializeJobsToLock(jobs: Job[], ctx: SerializerContext, staticNames?: Set<string>, allowedGroups?: Set<string>): Promise<LockJob[]>;
|
|
65
|
+
export declare function serializeJobsToLock(jobs: Job[], ctx: SerializerContext, staticNames?: Set<string>, allowedGroups?: Set<string>, seenNames?: Set<string>): Promise<LockJob[]>;
|
|
62
66
|
//# sourceMappingURL=dynamic-job-serializer.d.ts.map
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build a job's container image with the host's build CLI.
|
|
3
|
+
*
|
|
4
|
+
* The CLI is REQUIRED — there is deliberately no socket-API fallback. One build
|
|
5
|
+
* path means one set of Dockerfile semantics: `.dockerignore`, BuildKit and
|
|
6
|
+
* every directive behave as they do on the author's own machine, instead of
|
|
7
|
+
* depending on which agent happened to pick the job up. A host without a CLI
|
|
8
|
+
* cannot run a Dockerfile job, and says so in as many words.
|
|
9
|
+
*/
|
|
10
|
+
import { z } from 'zod';
|
|
11
|
+
import type { JobImageBuildSpec } from './resolve-build-spec.js';
|
|
12
|
+
/** The build CLIs an agent host may provide. */
|
|
13
|
+
export declare const ContainerBuildCli: z.ZodEnum<{
|
|
14
|
+
docker: "docker";
|
|
15
|
+
podman: "podman";
|
|
16
|
+
}>;
|
|
17
|
+
export type ContainerBuildCli = z.infer<typeof ContainerBuildCli>;
|
|
18
|
+
/** Is `bin` executable somewhere on `PATH`? */
|
|
19
|
+
export declare function binaryOnPath(bin: string): boolean;
|
|
20
|
+
/**
|
|
21
|
+
* The container socket the SANDBOX will use.
|
|
22
|
+
*
|
|
23
|
+
* Mirrors what dockerode's `new Docker()` resolves, and exists so build and run
|
|
24
|
+
* provably agree. A host with both runtimes whose sandbox socket points at
|
|
25
|
+
* podman would otherwise build with docker and then start the job container on
|
|
26
|
+
* a daemon that has never heard of that image — a failure that surfaces as
|
|
27
|
+
* "no such image" and names nothing.
|
|
28
|
+
*/
|
|
29
|
+
export declare function sandboxSocketPath(): string;
|
|
30
|
+
export declare function resolveBuildCli(args: {
|
|
31
|
+
configured?: ContainerBuildCli | undefined;
|
|
32
|
+
onPath?: (bin: string) => boolean;
|
|
33
|
+
}): ContainerBuildCli;
|
|
34
|
+
export declare function buildArgv(args: {
|
|
35
|
+
cli: ContainerBuildCli;
|
|
36
|
+
spec: JobImageBuildSpec;
|
|
37
|
+
socketPath?: string | undefined;
|
|
38
|
+
}): string[];
|
|
39
|
+
/** Registry credentials in the shape a build CLI's config file expects. */
|
|
40
|
+
export interface BuildAuthconfig {
|
|
41
|
+
username: string;
|
|
42
|
+
password: string;
|
|
43
|
+
serveraddress: string;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Split a chunk stream into whole lines, carrying the remainder between chunks.
|
|
47
|
+
*
|
|
48
|
+
* `flush` matters: a builder whose last line has no trailing newline — which is
|
|
49
|
+
* exactly the shape of an error written just before exit — would otherwise leave
|
|
50
|
+
* that line in the carry, and the one line the author most needs would be the
|
|
51
|
+
* one that never reaches the run log.
|
|
52
|
+
*/
|
|
53
|
+
export declare function makeLineSplitter(onLine: (line: string) => void): {
|
|
54
|
+
write: (chunk: Buffer) => void;
|
|
55
|
+
flush: () => void;
|
|
56
|
+
};
|
|
57
|
+
export interface BuildJobImageArgs {
|
|
58
|
+
spec: JobImageBuildSpec;
|
|
59
|
+
cli: ContainerBuildCli;
|
|
60
|
+
socketPath?: string | undefined;
|
|
61
|
+
authconfig?: BuildAuthconfig | undefined;
|
|
62
|
+
/** Every line of build output, in order, for the run log. */
|
|
63
|
+
onLog: (line: string) => void;
|
|
64
|
+
signal?: AbortSignal | undefined;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Run the build. Resolves when the image is tagged; rejects with the builder's
|
|
68
|
+
* own last words otherwise.
|
|
69
|
+
*
|
|
70
|
+
* The rejection carries real output rather than an exit code because the author
|
|
71
|
+
* is the one who has to act on it, and "exit 1" tells them nothing about which
|
|
72
|
+
* `RUN` failed.
|
|
73
|
+
*/
|
|
74
|
+
export declare function buildJobImage(args: BuildJobImageArgs): Promise<void>;
|
|
75
|
+
//# sourceMappingURL=build-engine.d.ts.map
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run a job's image build and render it as a step in the run.
|
|
3
|
+
*
|
|
4
|
+
* The build happens on the AGENT, before the sandbox exists, so it cannot use
|
|
5
|
+
* the runner's IPC pseudo-step channel that the cache phase uses — that channel
|
|
6
|
+
* belongs to a process which has not started yet. It uses the agent-side seam
|
|
7
|
+
* the synthetic `build` / `init` / `global-eval` jobs already use: a step-status
|
|
8
|
+
* pair plus a log streamer bound to a step index.
|
|
9
|
+
*/
|
|
10
|
+
import { ExecutionStepStatus } from '@kici-dev/engine';
|
|
11
|
+
import type { LockJob } from '@kici-dev/engine';
|
|
12
|
+
import { type JobImageBuildSpec } from './resolve-build-spec.js';
|
|
13
|
+
/** Step name the run timeline shows for a job's image build. */
|
|
14
|
+
export declare const CONTAINER_BUILD_STEP_NAME = "container:build";
|
|
15
|
+
/**
|
|
16
|
+
* Step index the image build reports under.
|
|
17
|
+
*
|
|
18
|
+
* A large positive constant, for two reasons that are easy to trip over:
|
|
19
|
+
*
|
|
20
|
+
* - `step.status` carries `z.number().int().nonnegative()`, so a negative index
|
|
21
|
+
* is not available. An agent that sent one to an OLDER orchestrator would be
|
|
22
|
+
* disconnected mid-job (an invalid message closes the socket), and two
|
|
23
|
+
* consumers read a negative index as absent — the check-run reporter writes
|
|
24
|
+
* `steps[stepIndex]`, and the dashboard gates log fetching on `stepIndex >= 0`.
|
|
25
|
+
* - It must clear every index the in-sandbox allocators reach. The cache phase
|
|
26
|
+
* tops out around `stepCount * 3 + 100 + (stepCount + 1) * 1000`, so this
|
|
27
|
+
* clears a job of several hundred steps with room to spare.
|
|
28
|
+
*
|
|
29
|
+
* DISPLAY ORDER does not come from this index. The step is tagged with the
|
|
30
|
+
* `container:build` {@link SetupStepType}, and readers sort setup pseudo-steps
|
|
31
|
+
* ahead of the real steps by type — so the build renders where it ran, first,
|
|
32
|
+
* without a negative index or a protocol change.
|
|
33
|
+
*/
|
|
34
|
+
export declare const CONTAINER_BUILD_STEP_INDEX = 1000000;
|
|
35
|
+
export interface RunJobImageBuildArgs {
|
|
36
|
+
container: LockJob['container'];
|
|
37
|
+
workDir: string;
|
|
38
|
+
jobId: string;
|
|
39
|
+
jobName: string;
|
|
40
|
+
/** Perform the build. Injected so the step logic is testable without a host. */
|
|
41
|
+
build: (spec: JobImageBuildSpec, onLog: (line: string) => void) => Promise<void>;
|
|
42
|
+
/** Every line of build output, in order, for the run log. */
|
|
43
|
+
onLog: (line: string) => void;
|
|
44
|
+
sendStepStatus: (name: string, state: ExecutionStepStatus, data?: Record<string, unknown>) => void;
|
|
45
|
+
fileExists?: (p: string) => boolean;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Build the job's image when it declared a Dockerfile, and return the tag the
|
|
49
|
+
* sandbox should run.
|
|
50
|
+
*
|
|
51
|
+
* Returns `undefined` when there is nothing to build — a job with no container,
|
|
52
|
+
* or one that names a finalized image. That is the common case and emits no
|
|
53
|
+
* step at all: a run timeline should not grow an empty entry for work that did
|
|
54
|
+
* not happen.
|
|
55
|
+
*/
|
|
56
|
+
export declare function runJobImageBuild(args: RunJobImageBuildArgs): Promise<string | undefined>;
|
|
57
|
+
//# sourceMappingURL=build-step.d.ts.map
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turn a job's `container.dockerfile` into an absolute, checked build spec.
|
|
3
|
+
*
|
|
4
|
+
* Pure on purpose. Every rule that decides what gets built — the anchoring, the
|
|
5
|
+
* escape refusal, the tag shape — is decided here and unit-tested without a
|
|
6
|
+
* container runtime anywhere near it. The half that needs a host lives in
|
|
7
|
+
* `build-engine.ts`.
|
|
8
|
+
*/
|
|
9
|
+
import type { LockJob } from '@kici-dev/engine';
|
|
10
|
+
/** Everything the build CLI needs, resolved and checked. */
|
|
11
|
+
export interface JobImageBuildSpec {
|
|
12
|
+
/** Absolute path to the Dockerfile. */
|
|
13
|
+
dockerfilePath: string;
|
|
14
|
+
/** Absolute build-context directory. */
|
|
15
|
+
contextDir: string;
|
|
16
|
+
/** Build stage to stop at, when the job named one. */
|
|
17
|
+
target?: string;
|
|
18
|
+
/** Build arguments. Plain strings — never secret material. */
|
|
19
|
+
args: Record<string, string>;
|
|
20
|
+
/** Tag the built image gets, and the sandbox then runs. */
|
|
21
|
+
tag: string;
|
|
22
|
+
/** Labels that put the image in reach of the leak sweep. */
|
|
23
|
+
labels: Record<string, string>;
|
|
24
|
+
}
|
|
25
|
+
export interface ResolveJobImageBuildSpecArgs {
|
|
26
|
+
container: LockJob['container'];
|
|
27
|
+
/** The cloned tree. Every path resolves against this and must stay inside it. */
|
|
28
|
+
workDir: string;
|
|
29
|
+
jobId: string;
|
|
30
|
+
jobName: string;
|
|
31
|
+
/** Injected for tests; defaults to a real filesystem check. */
|
|
32
|
+
fileExists?: (p: string) => boolean;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Resolve the build spec for a job, or `undefined` when the job builds nothing.
|
|
36
|
+
*
|
|
37
|
+
* `undefined` is the common case and is not a failure: a job with no container,
|
|
38
|
+
* or one that names a finalized image, has nothing to build.
|
|
39
|
+
*/
|
|
40
|
+
export declare function resolveJobImageBuildSpec(args: ResolveJobImageBuildSpecArgs): JobImageBuildSpec | undefined;
|
|
41
|
+
//# sourceMappingURL=resolve-build-spec.d.ts.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What this agent's own host can do with containers, discovered at startup.
|
|
3
|
+
*
|
|
4
|
+
* The orchestrator cannot answer this. An agent runs on its own machine, and
|
|
5
|
+
* whether that machine has a container runtime is the agent's fact — probing
|
|
6
|
+
* the orchestrator's filesystem answers a different question, and doing so once
|
|
7
|
+
* stranded container jobs that had been running fine, because the probe and the
|
|
8
|
+
* job ran in different places.
|
|
9
|
+
*
|
|
10
|
+
* Reported as `kici:runtime:*` labels, which the register-time scope gate
|
|
11
|
+
* accepts unchallenged as self-reported facts. Deliberately NOT
|
|
12
|
+
* `kici:capability:*` — that prefix grants a privilege and stays token-bound.
|
|
13
|
+
*/
|
|
14
|
+
import { RuntimeFact } from '@kici-dev/engine';
|
|
15
|
+
export interface DetectRuntimeFactsDeps {
|
|
16
|
+
/** Injected for tests; defaults to a real filesystem check. */
|
|
17
|
+
pathExists?: (p: string) => boolean;
|
|
18
|
+
binaryOnPath?: (bin: string) => boolean;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Discover this host's runtime facts.
|
|
22
|
+
*
|
|
23
|
+
* Presence of the socket FILE, not a handshake: registration must not block on
|
|
24
|
+
* a daemon that is slow or wedged, and a job that reaches a broken runtime
|
|
25
|
+
* still fails with the runtime's own error. The label answers "is there a
|
|
26
|
+
* runtime here at all", which is the routing question.
|
|
27
|
+
*/
|
|
28
|
+
export declare function detectRuntimeFacts(deps?: DetectRuntimeFactsDeps): RuntimeFact[];
|
|
29
|
+
/** The `kici:runtime:*` labels this host should register with. */
|
|
30
|
+
export declare function runtimeFactLabels(deps?: DetectRuntimeFactsDeps): string[];
|
|
31
|
+
//# sourceMappingURL=runtime-facts.d.ts.map
|
|
@@ -5,6 +5,7 @@ import { type FilterEvalInput } from './init-runner.js';
|
|
|
5
5
|
import { buildNeedsContext } from '@kici-dev/sdk';
|
|
6
6
|
import type { $ as Shell } from 'zx';
|
|
7
7
|
import type { CacheRequestIpc, CacheResponseIpc, ProvenanceRequestIpc, ProvenanceResponseIpc, ArtifactRequestIpc, ArtifactResponseIpc, StepApprovalRequestIpc, StepApprovalResolvedIpc } from './sandbox/index.js';
|
|
8
|
+
import { BetweenJobsController } from './between-jobs-controller.js';
|
|
8
9
|
/**
|
|
9
10
|
* Materialize the source tree a non-global workflow's `filter` reads through
|
|
10
11
|
* `ctx.sourceRepo.path`.
|
|
@@ -72,10 +73,17 @@ export interface JobRunnerDeps {
|
|
|
72
73
|
send: (msg: AgentToOrchestratorMessage) => void;
|
|
73
74
|
/** Agent config */
|
|
74
75
|
config: AppConfig;
|
|
76
|
+
/**
|
|
77
|
+
* Supervisor-owned between-jobs phase, run after every job to reap the
|
|
78
|
+
* process tree, re-run declared cleanup out-of-band, delete the workdir, and
|
|
79
|
+
* run the operator reset command. Optional so unit harnesses can omit it.
|
|
80
|
+
*/
|
|
81
|
+
betweenJobsController?: BetweenJobsController;
|
|
75
82
|
/** Request a pre-signed S3 upload URL from the orchestrator via WS request-response. */
|
|
76
83
|
requestUploadUrl: (jobId: string, cacheType: 'source' | 'deps', key: {
|
|
77
84
|
contentHash?: string;
|
|
78
85
|
lockfileHash?: string;
|
|
86
|
+
depsHash?: string;
|
|
79
87
|
platform: string;
|
|
80
88
|
arch: string;
|
|
81
89
|
}) => Promise<string>;
|
|
@@ -244,6 +252,8 @@ export declare class JobRunner {
|
|
|
244
252
|
private readonly _sendRunEvent;
|
|
245
253
|
private readonly _sendConcurrencyReport;
|
|
246
254
|
private readonly _sendApiRequest?;
|
|
255
|
+
/** Live git credentials per job, so the sandbox can reach the grant table. */
|
|
256
|
+
private readonly jobGitCredentials;
|
|
247
257
|
private readonly _requestUserCache?;
|
|
248
258
|
private readonly _relayProvenance?;
|
|
249
259
|
private readonly _requestUserArtifact?;
|
|
@@ -252,6 +262,15 @@ export declare class JobRunner {
|
|
|
252
262
|
readonly activeJobs: Map<string, ActiveJob>;
|
|
253
263
|
/** Active sandbox for the current job (used for abort). */
|
|
254
264
|
private activeSandbox;
|
|
265
|
+
/** Supervisor-owned between-jobs phase (reap / cleanup re-run / reset). */
|
|
266
|
+
private readonly betweenJobsController?;
|
|
267
|
+
/**
|
|
268
|
+
* Facts about the just-finished standard job, captured before sandbox
|
|
269
|
+
* teardown so the between-jobs controller (run in `execute`'s `.finally`) can
|
|
270
|
+
* reach them. Null for special job types (init / build / dynamic), which run
|
|
271
|
+
* no sandbox — the controller then only deletes the workdir and resets.
|
|
272
|
+
*/
|
|
273
|
+
private betweenJobsFacts;
|
|
255
274
|
constructor(deps: JobRunnerDeps);
|
|
256
275
|
/**
|
|
257
276
|
* Execute a dispatched job through its full lifecycle.
|
|
@@ -260,6 +279,23 @@ export declare class JobRunner {
|
|
|
260
279
|
* reports status, and cleans up.
|
|
261
280
|
*/
|
|
262
281
|
execute(dispatch: JobDispatch): Promise<void>;
|
|
282
|
+
/**
|
|
283
|
+
* Stand up per-job git credentials, or `undefined` when the agent has no
|
|
284
|
+
* orchestrator relay (unit harnesses, offline local runs).
|
|
285
|
+
*
|
|
286
|
+
* A failure here must not fail the job: without a helper, git falls back to
|
|
287
|
+
* its own mechanisms exactly as it did before this existed. The job simply
|
|
288
|
+
* cannot push.
|
|
289
|
+
*/
|
|
290
|
+
private startGitCredentials;
|
|
291
|
+
/**
|
|
292
|
+
* Run the supervisor-owned between-jobs phase after a job finishes. Delegates
|
|
293
|
+
* to the injected `BetweenJobsController` (out-of-band cleanup → reap →
|
|
294
|
+
* workdir delete → operator reset) with facts captured before sandbox
|
|
295
|
+
* teardown. When no controller is wired (unit harnesses) it just deletes the
|
|
296
|
+
* workdir, preserving the historical behavior. Never throws.
|
|
297
|
+
*/
|
|
298
|
+
private runBetweenJobsPhase;
|
|
263
299
|
/**
|
|
264
300
|
* Cancel a running job by signaling its abort controller
|
|
265
301
|
* and aborting the active sandbox.
|
|
@@ -407,6 +443,18 @@ export declare class JobRunner {
|
|
|
407
443
|
* and sends the result back to the orchestrator.
|
|
408
444
|
*/
|
|
409
445
|
private handleDynamicJobFn;
|
|
446
|
+
/**
|
|
447
|
+
* Build the job's container image when it declared a Dockerfile, and return
|
|
448
|
+
* the tag the sandbox must run.
|
|
449
|
+
*
|
|
450
|
+
* Returns `undefined` for every other job — one with no container, or one
|
|
451
|
+
* naming a finalized image — which is the common case and costs nothing.
|
|
452
|
+
*
|
|
453
|
+
* Extracted from `setupSandboxForExecution` so that function stays inside the
|
|
454
|
+
* 200-line ceiling, and so the build's streamer lifecycle is visibly bounded
|
|
455
|
+
* by one `finally`.
|
|
456
|
+
*/
|
|
457
|
+
private buildJobImageIfDeclared;
|
|
410
458
|
/**
|
|
411
459
|
* Create the appropriate sandbox backend based on execution mode.
|
|
412
460
|
*/
|
|
@@ -31,6 +31,11 @@ interface BareMetalSandboxOptions {
|
|
|
31
31
|
sandboxNetwork?: 'isolated' | 'host';
|
|
32
32
|
/** Pre-sanitized environment variables (system allowlist + user env). */
|
|
33
33
|
env: Record<string, string>;
|
|
34
|
+
/**
|
|
35
|
+
* When true (and bwrap is off), spawn the runner in its own process group so
|
|
36
|
+
* the between-jobs phase can reap a backgrounded daemon. Default true.
|
|
37
|
+
*/
|
|
38
|
+
orphanCleanup?: boolean;
|
|
34
39
|
}
|
|
35
40
|
/**
|
|
36
41
|
* Bare-metal execution sandbox implementation.
|
|
@@ -43,8 +48,10 @@ export declare class BareMetalSandbox implements ExecutionSandbox {
|
|
|
43
48
|
private readonly useBwrap;
|
|
44
49
|
private readonly sandboxNetwork;
|
|
45
50
|
private readonly env;
|
|
51
|
+
private readonly orphanCleanup;
|
|
46
52
|
private runner;
|
|
47
53
|
private workDir;
|
|
54
|
+
private lastOptions;
|
|
48
55
|
constructor(options: BareMetalSandboxOptions);
|
|
49
56
|
/**
|
|
50
57
|
* Validate that the runner path exists and bwrap is available (if needed).
|
|
@@ -61,9 +68,26 @@ export declare class BareMetalSandbox implements ExecutionSandbox {
|
|
|
61
68
|
*/
|
|
62
69
|
abort(): Promise<void>;
|
|
63
70
|
/**
|
|
64
|
-
* Clean up the child process if still running.
|
|
71
|
+
* Clean up the child process if still running. The handle reference is kept
|
|
72
|
+
* (not nulled) so the between-jobs phase can still read `completionHooksRan` /
|
|
73
|
+
* `declaresCleanup` and reap the process group after teardown — the group
|
|
74
|
+
* survives the single child's death. The next `executeJob` overwrites it.
|
|
65
75
|
*/
|
|
66
76
|
teardown(): Promise<void>;
|
|
77
|
+
/** Whether the runner signalled its completion hooks ran. */
|
|
78
|
+
get completionHooksRan(): boolean;
|
|
79
|
+
/** Whether the job declared an onFailure / cleanup hook. */
|
|
80
|
+
get declaresCleanup(): boolean;
|
|
81
|
+
/** Reap the finished job's process group. */
|
|
82
|
+
reap(): Promise<number>;
|
|
83
|
+
/**
|
|
84
|
+
* Re-run the finished job's declared cleanup / onFailure hooks against the
|
|
85
|
+
* preserved workdir, in a fresh cleanup-only child. Reuses the last job's
|
|
86
|
+
* dispatch + orchestrator relays but suppresses step/log callbacks (the
|
|
87
|
+
* original job already reported and its log streamers are gone). Rejects when
|
|
88
|
+
* the cleanup-only child fails so the caller can time it out.
|
|
89
|
+
*/
|
|
90
|
+
runCleanupOnly(workDir: string, signal: AbortSignal): Promise<void>;
|
|
67
91
|
}
|
|
68
92
|
export {};
|
|
69
93
|
//# sourceMappingURL=bare-metal-sandbox.d.ts.map
|
|
@@ -12,8 +12,11 @@
|
|
|
12
12
|
* - Agent-internal credentials (KICI_*, KICI_DATABASE_URL, etc.) NEVER enter the container
|
|
13
13
|
* - IPC uses demuxed Docker stream with JSON-line parsing on stdout
|
|
14
14
|
*
|
|
15
|
-
* The container image
|
|
16
|
-
*
|
|
15
|
+
* The container image does NOT need Node.js or git. KiCI provisions its own
|
|
16
|
+
* runtime — a pinned, official glibc-2.17 Node plus the runner bundle, mounted
|
|
17
|
+
* read-only at /opt/kici — and launches the runner with THAT node. The image
|
|
18
|
+
* needs only a glibc and a shell, which the preflight asserts before the
|
|
19
|
+
* container is created.
|
|
17
20
|
*/
|
|
18
21
|
import Docker from 'dockerode';
|
|
19
22
|
import type { ExecutionSandbox, SandboxSetupOptions, JobExecutionOptions, JobExecutionResult } from './types.js';
|
|
@@ -27,6 +30,51 @@ interface ContainerSandboxOptions {
|
|
|
27
30
|
runnerPath: string;
|
|
28
31
|
/** Mount target inside container (default: /opt/kici/workflow-runner.js). */
|
|
29
32
|
runnerMountPath?: string;
|
|
33
|
+
/**
|
|
34
|
+
* Pre-provisioned KiCI Node tree (the directory whose `bin/node` is the
|
|
35
|
+
* injected runtime), bind-mounted read-only at `/opt/kici/node`. A HOST
|
|
36
|
+
* path, or the name of a volume already holding the tree.
|
|
37
|
+
*
|
|
38
|
+
* When supplied, the runner launches on THAT node and the image needs no
|
|
39
|
+
* Node of its own. Wins over `runtimeImage` — a caller that already has the
|
|
40
|
+
* tree should not pay a materialization to get the same one.
|
|
41
|
+
*/
|
|
42
|
+
runtimeNodePath?: string;
|
|
43
|
+
/**
|
|
44
|
+
* KiCI agent image carrying `/opt/kici`, from which the Node tree is
|
|
45
|
+
* materialized into a named volume during setup.
|
|
46
|
+
*
|
|
47
|
+
* This is how an agent nesting a job container gets a runtime: a bind mount
|
|
48
|
+
* needs a HOST path, and the agent may itself be containerized, so it cannot
|
|
49
|
+
* assume `/opt/kici` exists on the host filesystem. Copying the tree out of
|
|
50
|
+
* the image into a volume once, then mounting that volume, works either way.
|
|
51
|
+
*
|
|
52
|
+
* Absent (and no `runtimeNodePath`) means no injection: the runner falls
|
|
53
|
+
* back to the image's own `node`, which is the historical contract and still
|
|
54
|
+
* correct for an image that ships one. Both modes are correct; the fallback
|
|
55
|
+
* is not a workaround for a missing mount but the mode a caller that has no
|
|
56
|
+
* runtime to inject is in.
|
|
57
|
+
*/
|
|
58
|
+
runtimeImage?: string;
|
|
59
|
+
/**
|
|
60
|
+
* Tag this sandbox's image was BUILT under, when the job declared a
|
|
61
|
+
* Dockerfile rather than naming an image.
|
|
62
|
+
*
|
|
63
|
+
* Present only for a built image, and removed at teardown — a tag per run
|
|
64
|
+
* would otherwise accumulate on the host forever. Removing the tag leaves the
|
|
65
|
+
* layer cache untouched, which is what makes the next build fast, so this
|
|
66
|
+
* costs nothing but the name.
|
|
67
|
+
*/
|
|
68
|
+
buildTag?: string;
|
|
69
|
+
/**
|
|
70
|
+
* Registry credentials for pulling `image`, already resolved by the
|
|
71
|
+
* orchestrator. Absent means an anonymous pull.
|
|
72
|
+
*/
|
|
73
|
+
registryAuth?: {
|
|
74
|
+
username: string;
|
|
75
|
+
password: string;
|
|
76
|
+
serveraddress: string;
|
|
77
|
+
};
|
|
30
78
|
/**
|
|
31
79
|
* Path to the pure-JS container loader-hook bundle on the HOST (bind-mounted
|
|
32
80
|
* read-only). Defaults to `container-ts-loader-hook.js` next to `runnerPath`.
|
|
@@ -51,6 +99,17 @@ export declare class ContainerSandbox implements ExecutionSandbox {
|
|
|
51
99
|
private readonly docker;
|
|
52
100
|
private readonly image;
|
|
53
101
|
private readonly runnerPath;
|
|
102
|
+
private readonly runtimeNodePath;
|
|
103
|
+
private readonly runtimeImage;
|
|
104
|
+
private readonly buildTag;
|
|
105
|
+
/**
|
|
106
|
+
* The runtime actually injected into this job's container — the configured
|
|
107
|
+
* path, or the volume materialized during setup. Resolved once in setup()
|
|
108
|
+
* because materialization needs the container runtime, and read by both the
|
|
109
|
+
* bind list and the runner launch.
|
|
110
|
+
*/
|
|
111
|
+
private resolvedRuntimeNode;
|
|
112
|
+
private readonly registryAuth;
|
|
54
113
|
private readonly runnerMountPath;
|
|
55
114
|
/** Host path to the pure-JS container loader-hook bundle (bind-mounted :ro). */
|
|
56
115
|
private readonly hookHostPath;
|
|
@@ -81,7 +140,35 @@ export declare class ContainerSandbox implements ExecutionSandbox {
|
|
|
81
140
|
* private images pre-pulled with registry auth stay working) skip the pull.
|
|
82
141
|
*/
|
|
83
142
|
private ensureImagePresent;
|
|
143
|
+
/**
|
|
144
|
+
* Resolve the Node tree to inject into the job container, materializing it
|
|
145
|
+
* when only an image was configured.
|
|
146
|
+
*
|
|
147
|
+
* A configured path wins: a caller that already provisioned the tree should
|
|
148
|
+
* not pay a copy to arrive at the same one. Returning `undefined` is a real
|
|
149
|
+
* outcome, not a failure — an agent with no runtime source runs the job on
|
|
150
|
+
* the image's own `node`, which is what a `node:*` image was always doing.
|
|
151
|
+
*
|
|
152
|
+
* A materialization that FAILS is not softened into that outcome. Continuing
|
|
153
|
+
* would start the job against an image the operator never claimed ships Node,
|
|
154
|
+
* and the resulting "node: not found" says nothing about the runtime that was
|
|
155
|
+
* supposed to be there.
|
|
156
|
+
*/
|
|
157
|
+
private resolveRuntimeNode;
|
|
84
158
|
setup(options: SandboxSetupOptions): Promise<void>;
|
|
159
|
+
/**
|
|
160
|
+
* Populate the container's `/workspace` volume from the host working tree.
|
|
161
|
+
*
|
|
162
|
+
* `/workspace` is a container-owned anonymous volume rather than a host bind,
|
|
163
|
+
* which is what dissolves the host-uid vs container-uid conflict once
|
|
164
|
+
* `CapDrop: ['ALL']` removes CAP_DAC_OVERRIDE — so the tree is streamed in as
|
|
165
|
+
* a tar rather than mounted.
|
|
166
|
+
*
|
|
167
|
+
* A failure here is fatal on purpose. Swallowing it would start the job
|
|
168
|
+
* against an EMPTY workspace, which surfaces as a baffling "file not found"
|
|
169
|
+
* in whichever step happens to touch the repo first.
|
|
170
|
+
*/
|
|
171
|
+
private copyWorkspaceIn;
|
|
85
172
|
/**
|
|
86
173
|
* Build the container's read-only bind list.
|
|
87
174
|
*
|
|
@@ -133,6 +220,14 @@ export declare class ContainerSandbox implements ExecutionSandbox {
|
|
|
133
220
|
*/
|
|
134
221
|
private buildExecutionResult;
|
|
135
222
|
abort(): Promise<void>;
|
|
223
|
+
/**
|
|
224
|
+
* Drop the tag a `container.dockerfile` build produced.
|
|
225
|
+
*
|
|
226
|
+
* Best-effort: teardown must not fail a job that already finished. The LAYER
|
|
227
|
+
* cache — the thing that makes the next build fast — is not a tag and is
|
|
228
|
+
* untouched by this.
|
|
229
|
+
*/
|
|
230
|
+
private reclaimBuiltImage;
|
|
136
231
|
teardown(): Promise<void>;
|
|
137
232
|
/**
|
|
138
233
|
* Send the execute request to the workflow runner via the exec's stdin.
|