@inflexa-ai/harness 0.30.2 → 0.30.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/sandbox/cpu-files.d.ts +37 -0
- package/dist/sandbox/cpu-files.js +55 -0
- package/dist/sandbox/docker-client.d.ts +15 -0
- package/dist/sandbox/docker-client.js +100 -11
- package/dist/sandbox/k8s-client.d.ts +5 -0
- package/dist/sandbox/k8s-client.js +128 -19
- package/dist/sandbox/thread-env.d.ts +29 -0
- package/dist/sandbox/thread-env.js +49 -0
- package/dist/state/data-profile.d.ts +35 -0
- package/dist/state/data-profile.js +56 -0
- package/dist/state/index.d.ts +1 -1
- package/dist/state/index.js +1 -1
- package/dist/tasks/data-profile.d.ts +16 -1
- package/dist/tasks/data-profile.js +31 -10
- package/package.json +1 -1
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two cpu files that a sandbox reads for its core count.
|
|
3
|
+
*
|
|
4
|
+
* The thread env (`thread-env.ts`) reaches each library that reads a thread
|
|
5
|
+
* count from the environment. Two base calls read a kernel file instead:
|
|
6
|
+
* `parallel::detectCores()` greps `/proc/cpuinfo`, and `os.cpu_count()` of
|
|
7
|
+
* Python 3.12 reads `/sys/devices/system/cpu/online` through glibc. Both files
|
|
8
|
+
* describe the host, not the cgroup. A read-only bind of a small file over each
|
|
9
|
+
* path makes both calls report the quota.
|
|
10
|
+
*
|
|
11
|
+
* This module makes the content of the two files, and it reads the
|
|
12
|
+
* `/proc/cpuinfo` of the harness host that the content is cut from. The Docker
|
|
13
|
+
* backend writes the content to the host and binds it. The K8s backend puts
|
|
14
|
+
* the content in a ConfigMap and mounts each key with `subPath`.
|
|
15
|
+
*/
|
|
16
|
+
/** The two container paths that the cpu files cover. */
|
|
17
|
+
export declare const CPU_ONLINE_PATH = "/sys/devices/system/cpu/online";
|
|
18
|
+
export declare const CPUINFO_PATH = "/proc/cpuinfo";
|
|
19
|
+
export declare function readHostCpuinfoOnce(): Promise<string | undefined>;
|
|
20
|
+
export interface CpuFiles {
|
|
21
|
+
/** Content of `/sys/devices/system/cpu/online`: `0` for one thread, else the range `0-<n-1>`. */
|
|
22
|
+
readonly online: string;
|
|
23
|
+
/**
|
|
24
|
+
* Content of `/proc/cpuinfo`: the first n `processor` blocks of the real file
|
|
25
|
+
* of the host. `undefined` when the host gives no `/proc/cpuinfo`.
|
|
26
|
+
*/
|
|
27
|
+
readonly cpuinfo: string | undefined;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* The content of the two cpu files for `threads` cpus.
|
|
31
|
+
*
|
|
32
|
+
* `threads` is a positive integer. The caller applies the same floor as
|
|
33
|
+
* `threadLimitEnv`: `Math.max(1, Math.floor(spec.cpu))`. When the host has fewer
|
|
34
|
+
* processor blocks than `threads`, all blocks stay. Each block keeps its own
|
|
35
|
+
* text, thus the `processor` numbers stay `0` to `n-1` as in the real file.
|
|
36
|
+
*/
|
|
37
|
+
export declare function cpuFiles(threads: number, hostCpuinfo: string | undefined): CpuFiles;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two cpu files that a sandbox reads for its core count.
|
|
3
|
+
*
|
|
4
|
+
* The thread env (`thread-env.ts`) reaches each library that reads a thread
|
|
5
|
+
* count from the environment. Two base calls read a kernel file instead:
|
|
6
|
+
* `parallel::detectCores()` greps `/proc/cpuinfo`, and `os.cpu_count()` of
|
|
7
|
+
* Python 3.12 reads `/sys/devices/system/cpu/online` through glibc. Both files
|
|
8
|
+
* describe the host, not the cgroup. A read-only bind of a small file over each
|
|
9
|
+
* path makes both calls report the quota.
|
|
10
|
+
*
|
|
11
|
+
* This module makes the content of the two files, and it reads the
|
|
12
|
+
* `/proc/cpuinfo` of the harness host that the content is cut from. The Docker
|
|
13
|
+
* backend writes the content to the host and binds it. The K8s backend puts
|
|
14
|
+
* the content in a ConfigMap and mounts each key with `subPath`.
|
|
15
|
+
*/
|
|
16
|
+
import { readFile } from "node:fs/promises";
|
|
17
|
+
/** The two container paths that the cpu files cover. */
|
|
18
|
+
export const CPU_ONLINE_PATH = "/sys/devices/system/cpu/online";
|
|
19
|
+
export const CPUINFO_PATH = "/proc/cpuinfo";
|
|
20
|
+
/**
|
|
21
|
+
* The `/proc/cpuinfo` of the harness host, read once per process. The content
|
|
22
|
+
* does not change while the process runs. A host with no such file, for
|
|
23
|
+
* example macOS, gives `undefined`, and the caller covers only `online`.
|
|
24
|
+
*/
|
|
25
|
+
let hostCpuinfo;
|
|
26
|
+
export function readHostCpuinfoOnce() {
|
|
27
|
+
hostCpuinfo ??= readFile(CPUINFO_PATH, "utf8").catch(() => undefined);
|
|
28
|
+
return hostCpuinfo;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Split the text of `/proc/cpuinfo` into its `processor` blocks. A blank line
|
|
32
|
+
* ends a block. A block with no `processor` line, for example the `Hardware`
|
|
33
|
+
* trailer of an ARM kernel, is not a processor and is dropped.
|
|
34
|
+
*/
|
|
35
|
+
function processorBlocks(cpuinfo) {
|
|
36
|
+
return cpuinfo
|
|
37
|
+
.split(/\n\s*\n/)
|
|
38
|
+
.map((block) => block.trim())
|
|
39
|
+
.filter((block) => /^processor\b/m.test(block));
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The content of the two cpu files for `threads` cpus.
|
|
43
|
+
*
|
|
44
|
+
* `threads` is a positive integer. The caller applies the same floor as
|
|
45
|
+
* `threadLimitEnv`: `Math.max(1, Math.floor(spec.cpu))`. When the host has fewer
|
|
46
|
+
* processor blocks than `threads`, all blocks stay. Each block keeps its own
|
|
47
|
+
* text, thus the `processor` numbers stay `0` to `n-1` as in the real file.
|
|
48
|
+
*/
|
|
49
|
+
export function cpuFiles(threads, hostCpuinfo) {
|
|
50
|
+
const online = `${threads === 1 ? "0" : `0-${threads - 1}`}\n`;
|
|
51
|
+
if (hostCpuinfo === undefined)
|
|
52
|
+
return { online, cpuinfo: undefined };
|
|
53
|
+
const blocks = processorBlocks(hostCpuinfo).slice(0, threads);
|
|
54
|
+
return { online, cpuinfo: blocks.map((block) => `${block}\n\n`).join("") };
|
|
55
|
+
}
|
|
@@ -7,6 +7,15 @@
|
|
|
7
7
|
* `/mnt/libs` / `/mnt/refs` when their host paths are configured. Container
|
|
8
8
|
* paths and lib-store env come from the shared mount plan (`mount-plan.ts`).
|
|
9
9
|
*
|
|
10
|
+
* ## The cpu files
|
|
11
|
+
*
|
|
12
|
+
* Two more binds put a small file over `/sys/devices/system/cpu/online` and
|
|
13
|
+
* over `/proc/cpuinfo` (`cpu-files.ts`). Each file describes `spec.cpu` cpus,
|
|
14
|
+
* not the host. The host side of each bind is a file under
|
|
15
|
+
* `<parent of the workspace root>/.cpu/<sandboxId>/`, thus outside the tree
|
|
16
|
+
* that the sandbox sees. `teardown` removes the directory when this process
|
|
17
|
+
* wrote it. A directory from an earlier process stays.
|
|
18
|
+
*
|
|
10
19
|
* ## Transport and confinement
|
|
11
20
|
*
|
|
12
21
|
* The container joins the default bridge and publishes sandbox-server's port to
|
|
@@ -69,6 +78,12 @@ export interface DockerClientConfig {
|
|
|
69
78
|
docker?: Docker;
|
|
70
79
|
/** Injected for tests so `/health` polling can be stubbed. */
|
|
71
80
|
fetch?: typeof fetch;
|
|
81
|
+
/**
|
|
82
|
+
* Injected for tests so the `/proc/cpuinfo` of the host can be stubbed. The
|
|
83
|
+
* default reads the file once per process. `undefined` means that the host
|
|
84
|
+
* has no such file, for example macOS.
|
|
85
|
+
*/
|
|
86
|
+
readHostCpuinfo?: () => Promise<string | undefined>;
|
|
72
87
|
/**
|
|
73
88
|
* Optional logger so a lib-store degradation (a configured store whose `current`
|
|
74
89
|
* vanished or went incomplete by sandbox-create time) is observable instead of a
|
|
@@ -7,6 +7,15 @@
|
|
|
7
7
|
* `/mnt/libs` / `/mnt/refs` when their host paths are configured. Container
|
|
8
8
|
* paths and lib-store env come from the shared mount plan (`mount-plan.ts`).
|
|
9
9
|
*
|
|
10
|
+
* ## The cpu files
|
|
11
|
+
*
|
|
12
|
+
* Two more binds put a small file over `/sys/devices/system/cpu/online` and
|
|
13
|
+
* over `/proc/cpuinfo` (`cpu-files.ts`). Each file describes `spec.cpu` cpus,
|
|
14
|
+
* not the host. The host side of each bind is a file under
|
|
15
|
+
* `<parent of the workspace root>/.cpu/<sandboxId>/`, thus outside the tree
|
|
16
|
+
* that the sandbox sees. `teardown` removes the directory when this process
|
|
17
|
+
* wrote it. A directory from an earlier process stays.
|
|
18
|
+
*
|
|
10
19
|
* ## Transport and confinement
|
|
11
20
|
*
|
|
12
21
|
* The container joins the default bridge and publishes sandbox-server's port to
|
|
@@ -29,13 +38,16 @@
|
|
|
29
38
|
* effective capabilities.
|
|
30
39
|
*/
|
|
31
40
|
import { existsSync, lstatSync, statSync } from "node:fs";
|
|
32
|
-
import {
|
|
41
|
+
import { mkdir, rm, writeFile } from "node:fs/promises";
|
|
42
|
+
import { dirname, join } from "node:path";
|
|
33
43
|
import Docker from "dockerode";
|
|
34
44
|
import { ResultAsync, err, ok } from "neverthrow";
|
|
35
45
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
36
46
|
import { tailWritePrefix } from "../workspace/paths.js";
|
|
37
47
|
import { trySandbox } from "./sandbox-error.js";
|
|
38
48
|
import { buildMountPlan, sandboxWriteTail } from "./mount-plan.js";
|
|
49
|
+
import { CPU_ONLINE_PATH, CPUINFO_PATH, cpuFiles, readHostCpuinfoOnce } from "./cpu-files.js";
|
|
50
|
+
import { threadLimitEnv } from "./thread-env.js";
|
|
39
51
|
/** Read the originating HTTP status off any `SandboxError` variant that carries one. */
|
|
40
52
|
function statusOf(e) {
|
|
41
53
|
return "status" in e ? e.status : undefined;
|
|
@@ -60,6 +72,8 @@ const MANAGED_BY_VALUE = "cortex";
|
|
|
60
72
|
const OWNER_WORKFLOW_LABEL = "cortex/owner-workflow-id";
|
|
61
73
|
const RUN_ID_LABEL = "cortex/run-id";
|
|
62
74
|
const STEP_ID_LABEL = "cortex/step-id";
|
|
75
|
+
/** The directory next to the workspace root that holds the cpu files of each sandbox. */
|
|
76
|
+
const CPU_FILES_DIR = ".cpu";
|
|
63
77
|
/**
|
|
64
78
|
* Whether the lib store's `current` resolves to a COMPLETE, usable version, not merely a
|
|
65
79
|
* present symlink. By sandbox-create time `current` may be gone (concurrent prune/`rm`), a
|
|
@@ -104,6 +118,39 @@ function refStoreUsable(refStorePath) {
|
|
|
104
118
|
return false;
|
|
105
119
|
}
|
|
106
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* Write the cpu files of one sandbox into `dir` and return their binds.
|
|
123
|
+
*
|
|
124
|
+
* The directory `.cpu` is made with no `recursive` option on purpose. The
|
|
125
|
+
* parent of the workspace root must exist already. Thus a root that is not on
|
|
126
|
+
* disk gives no directory at an unexpected place, and the sandbox starts with
|
|
127
|
+
* no cpu files. The failure is a degradation, not an error: the thread env
|
|
128
|
+
* still limits each library, but `detectCores()` and `os.cpu_count()` report
|
|
129
|
+
* the host.
|
|
130
|
+
*/
|
|
131
|
+
async function writeCpuFiles(dir, files, logger, sandboxId) {
|
|
132
|
+
const ignoreExists = (cause) => {
|
|
133
|
+
if (cause.code !== "EEXIST")
|
|
134
|
+
throw cause;
|
|
135
|
+
};
|
|
136
|
+
try {
|
|
137
|
+
await mkdir(dirname(dir)).catch(ignoreExists);
|
|
138
|
+
await mkdir(dir).catch(ignoreExists);
|
|
139
|
+
const online = join(dir, "online");
|
|
140
|
+
await writeFile(online, files.online);
|
|
141
|
+
const binds = [`${online}:${CPU_ONLINE_PATH}:ro`];
|
|
142
|
+
if (files.cpuinfo !== undefined) {
|
|
143
|
+
const cpuinfo = join(dir, "cpuinfo");
|
|
144
|
+
await writeFile(cpuinfo, files.cpuinfo);
|
|
145
|
+
binds.push(`${cpuinfo}:${CPUINFO_PATH}:ro`);
|
|
146
|
+
}
|
|
147
|
+
return binds;
|
|
148
|
+
}
|
|
149
|
+
catch (cause) {
|
|
150
|
+
logger.warn("cpu files not written — the sandbox reports the cores of the host", { sandboxId, dir, cause });
|
|
151
|
+
return [];
|
|
152
|
+
}
|
|
153
|
+
}
|
|
107
154
|
async function pollHealth(fetchImpl, url, timeoutMs) {
|
|
108
155
|
const deadline = Date.now() + timeoutMs;
|
|
109
156
|
let lastErr;
|
|
@@ -199,6 +246,9 @@ export function createDockerSandboxOps(config) {
|
|
|
199
246
|
const fetchImpl = config.fetch ?? fetch;
|
|
200
247
|
const transport = config.transport ?? "poll";
|
|
201
248
|
const pollMode = transport === "poll";
|
|
249
|
+
const readHostCpuinfo = config.readHostCpuinfo ?? readHostCpuinfoOnce;
|
|
250
|
+
/** The cpu directory of each sandbox that this process wrote, for `teardown`. */
|
|
251
|
+
const cpuDirs = new Map();
|
|
202
252
|
return {
|
|
203
253
|
createSandbox(meta, identity) {
|
|
204
254
|
return new ResultAsync((async () => {
|
|
@@ -228,6 +278,18 @@ export function createDockerSandboxOps(config) {
|
|
|
228
278
|
// a crafted tail cannot escape the resolved root into the container. The
|
|
229
279
|
// tail is the step directory, or the one the caller declared.
|
|
230
280
|
const write = sandboxWriteTail(meta);
|
|
281
|
+
// The same floor as `threadLimitEnv`: a fractional cpu request is one cpu.
|
|
282
|
+
const threads = Math.max(1, Math.floor(meta.resources.cpu));
|
|
283
|
+
const cpuinfo = await readHostCpuinfo();
|
|
284
|
+
if (cpuinfo === undefined) {
|
|
285
|
+
logger.debug("no host /proc/cpuinfo — the sandbox gets the online file only", { sandboxId });
|
|
286
|
+
}
|
|
287
|
+
// Next to the workspace root, not under it: the root is bound read-only
|
|
288
|
+
// at `/{analysisId}`, and a file under it is visible in the sandbox.
|
|
289
|
+
const cpuDir = join(dirname(hostTreePath), CPU_FILES_DIR, sandboxId);
|
|
290
|
+
const cpuBinds = await writeCpuFiles(cpuDir, cpuFiles(threads, cpuinfo), logger, sandboxId);
|
|
291
|
+
if (cpuBinds.length > 0)
|
|
292
|
+
cpuDirs.set(sandboxId, cpuDir);
|
|
231
293
|
const binds = [
|
|
232
294
|
`${hostTreePath}:${plan.readonlyTreePath}:ro`,
|
|
233
295
|
...(write && plan.writableStepPath
|
|
@@ -235,6 +297,7 @@ export function createDockerSandboxOps(config) {
|
|
|
235
297
|
: []),
|
|
236
298
|
...(libsMounted && config.libStorePath ? [`${config.libStorePath}:${plan.libsPath}:ro`] : []),
|
|
237
299
|
...(refsMounted && config.refStorePath ? [`${config.refStorePath}:${plan.refsPath}:ro`] : []),
|
|
300
|
+
...cpuBinds,
|
|
238
301
|
];
|
|
239
302
|
const createFailed = (status, cause) => ({
|
|
240
303
|
type: "container_create_failed",
|
|
@@ -244,17 +307,22 @@ export function createDockerSandboxOps(config) {
|
|
|
244
307
|
cause,
|
|
245
308
|
});
|
|
246
309
|
const image = meta.image ?? config.image;
|
|
310
|
+
const spec = meta.resources;
|
|
247
311
|
// Poll mode never dials out and carries no CORTEX_BASE_URL; it sets the
|
|
248
312
|
// firewall flag so the root entrypoint installs the egress block before
|
|
249
313
|
// dropping to uid 1000. Callback mode is the inverse.
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
314
|
+
//
|
|
315
|
+
// Composed as one record, not as a list of `K=V`. A duplicate key in
|
|
316
|
+
// the Env array resolves at the libc that scans `environ`, thus the
|
|
317
|
+
// later spread must win here, not there.
|
|
318
|
+
const env = Object.entries({
|
|
319
|
+
SANDBOX_TRANSPORT: transport,
|
|
320
|
+
SANDBOX_CALLBACK_SECRET: callbackSecret,
|
|
321
|
+
...(pollMode ? { SANDBOX_EGRESS_FIREWALL: "1" } : { CORTEX_BASE_URL: config.cortexBaseUrl }),
|
|
322
|
+
...threadLimitEnv(spec),
|
|
323
|
+
...plan.env,
|
|
324
|
+
...(meta.extraEnv ?? {}),
|
|
325
|
+
}).map(([k, v]) => `${k}=${v}`);
|
|
258
326
|
const createOpts = {
|
|
259
327
|
name: sandboxId,
|
|
260
328
|
...(config.platform !== undefined && { platform: config.platform }),
|
|
@@ -289,6 +357,11 @@ export function createDockerSandboxOps(config) {
|
|
|
289
357
|
SecurityOpt: ["no-new-privileges"],
|
|
290
358
|
NanoCpus: Math.round(spec.cpu * 1e9),
|
|
291
359
|
Memory: spec.memoryGb * 1024 ** 3,
|
|
360
|
+
// `Memory` alone lets the container swap as much again. A fork
|
|
361
|
+
// storm then thrashes for minutes, and sandbox-server stops to
|
|
362
|
+
// answer. With no swap the OOM killer removes the largest fork in
|
|
363
|
+
// seconds, and sandbox-server survives. K8s runs with no swap.
|
|
364
|
+
MemorySwap: spec.memoryGb * 1024 ** 3,
|
|
292
365
|
AutoRemove: false,
|
|
293
366
|
},
|
|
294
367
|
};
|
|
@@ -329,10 +402,10 @@ export function createDockerSandboxOps(config) {
|
|
|
329
402
|
})());
|
|
330
403
|
},
|
|
331
404
|
teardown(ref) {
|
|
332
|
-
return
|
|
405
|
+
return teardownSandbox(ref.sandboxId);
|
|
333
406
|
},
|
|
334
407
|
teardownById(sandboxId) {
|
|
335
|
-
return
|
|
408
|
+
return teardownSandbox(sandboxId);
|
|
336
409
|
},
|
|
337
410
|
listManagedSandboxes() {
|
|
338
411
|
return trySandbox(() => docker.listContainers({
|
|
@@ -361,6 +434,22 @@ export function createDockerSandboxOps(config) {
|
|
|
361
434
|
},
|
|
362
435
|
isAliveById,
|
|
363
436
|
};
|
|
437
|
+
/**
|
|
438
|
+
* Remove the container, then the cpu files that this process wrote for it.
|
|
439
|
+
* A failure of the file removal does not fail the teardown: the container is
|
|
440
|
+
* gone, and a stale small directory is not a risk to a later sandbox.
|
|
441
|
+
*/
|
|
442
|
+
function teardownSandbox(sandboxId) {
|
|
443
|
+
return removeContainerIgnoreMissing(docker, sandboxId).map(async () => {
|
|
444
|
+
const cpuDir = cpuDirs.get(sandboxId);
|
|
445
|
+
if (cpuDir === undefined)
|
|
446
|
+
return;
|
|
447
|
+
cpuDirs.delete(sandboxId);
|
|
448
|
+
await rm(cpuDir, { recursive: true, force: true }).catch((cause) => {
|
|
449
|
+
logger.debug("cpu files not removed", { sandboxId, cpuDir, cause });
|
|
450
|
+
});
|
|
451
|
+
});
|
|
452
|
+
}
|
|
364
453
|
function isAliveById(sandboxId) {
|
|
365
454
|
return trySandbox(async () => {
|
|
366
455
|
const info = await docker.getContainer(sandboxId).inspect();
|
|
@@ -60,6 +60,11 @@ export interface K8sClientConfig {
|
|
|
60
60
|
tolerations?: V1Toleration[];
|
|
61
61
|
/** RuntimeClass for runtime isolation (e.g. `gvisor`). Omit for the default runtime. */
|
|
62
62
|
runtimeClassName?: string;
|
|
63
|
+
/**
|
|
64
|
+
* The `/proc/cpuinfo` text that the cpu ConfigMap of each Job is cut from.
|
|
65
|
+
* Defaults to the file of the host of this process, read once. Injected for tests.
|
|
66
|
+
*/
|
|
67
|
+
readHostCpuinfo?: () => Promise<string | undefined>;
|
|
63
68
|
/** Injected for tests. */
|
|
64
69
|
batchApi?: BatchV1Api;
|
|
65
70
|
coreApi?: CoreV1Api;
|
|
@@ -16,6 +16,8 @@ import { BatchV1Api, CoreV1Api, KubeConfig, } from "@kubernetes/client-node";
|
|
|
16
16
|
import { ResultAsync, err, ok } from "neverthrow";
|
|
17
17
|
import { trySandbox } from "./sandbox-error.js";
|
|
18
18
|
import { buildMountPlan, buildSessionSubPaths } from "./mount-plan.js";
|
|
19
|
+
import { threadLimitEnv } from "./thread-env.js";
|
|
20
|
+
import { CPU_ONLINE_PATH, CPUINFO_PATH, cpuFiles, readHostCpuinfoOnce } from "./cpu-files.js";
|
|
19
21
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
20
22
|
/** Read the originating HTTP status off any `SandboxError` variant that carries one. */
|
|
21
23
|
function statusOf(e) {
|
|
@@ -24,6 +26,8 @@ function statusOf(e) {
|
|
|
24
26
|
const SESSION_VOLUME_NAME = "session";
|
|
25
27
|
const LIBS_VOLUME_NAME = "libs";
|
|
26
28
|
const REFS_VOLUME_NAME = "refs";
|
|
29
|
+
/** The ConfigMap volume that holds the two cpu files of the sandbox (`cpu-files.ts`). */
|
|
30
|
+
const CPU_VOLUME_NAME = "cpu";
|
|
27
31
|
const SANDBOX_SERVER_PORT = 8765;
|
|
28
32
|
const POD_READY_TIMEOUT_MS = 5 * 60_000;
|
|
29
33
|
const POD_POLL_INTERVAL_MS = 1_000;
|
|
@@ -59,6 +63,19 @@ export function sanitizeLabelValue(value) {
|
|
|
59
63
|
function readOwnerWorkflowId(metadata) {
|
|
60
64
|
return metadata?.annotations?.[OWNER_WORKFLOW_ANNOTATION] ?? null;
|
|
61
65
|
}
|
|
66
|
+
function deleteConfigMapIgnoreMissing(coreApi, namespace, name) {
|
|
67
|
+
return trySandbox(async () => {
|
|
68
|
+
await coreApi.deleteNamespacedConfigMap({ namespace, name });
|
|
69
|
+
}, (status, cause) => ({
|
|
70
|
+
type: "teardown_failed",
|
|
71
|
+
op: "k8s.teardown",
|
|
72
|
+
sandboxId: name,
|
|
73
|
+
status,
|
|
74
|
+
cause,
|
|
75
|
+
})).orElse((e) =>
|
|
76
|
+
// A 404 means the owner reference removed it with the Job already.
|
|
77
|
+
statusOf(e) === 404 ? ok(undefined) : err(e));
|
|
78
|
+
}
|
|
62
79
|
function deleteJobIgnoreMissing(batchApi, namespace, name) {
|
|
63
80
|
return trySandbox(async () => {
|
|
64
81
|
await batchApi.deleteNamespacedJob({
|
|
@@ -107,31 +124,32 @@ function workspaceSubPathFor(config, analysisId) {
|
|
|
107
124
|
}
|
|
108
125
|
return posix;
|
|
109
126
|
}
|
|
110
|
-
function buildJobSpec(meta, config, identity) {
|
|
127
|
+
function buildJobSpec(meta, config, identity, cpu) {
|
|
111
128
|
const { sandboxId } = identity;
|
|
112
129
|
const plan = buildMountPlan(meta, {
|
|
113
130
|
libs: !!config.libStorePvc,
|
|
114
131
|
refs: !!config.refStorePvc,
|
|
115
132
|
});
|
|
116
133
|
const transport = config.transport ?? "poll";
|
|
117
|
-
const
|
|
118
|
-
|
|
134
|
+
const spec = meta.resources;
|
|
135
|
+
// Composed as one record, not as a list of `{name, value}`. A duplicate name
|
|
136
|
+
// in the env of a container resolves at the kubelet, thus the later spread
|
|
137
|
+
// must win here, not there.
|
|
138
|
+
const env = Object.entries({
|
|
139
|
+
SANDBOX_TRANSPORT: transport,
|
|
119
140
|
// Poll mode never dials out, so the URL is omitted (matching the Docker
|
|
120
141
|
// backend) — the pod spec itself then documents that no callback egress
|
|
121
142
|
// is expected. sandbox-server neither reads nor requires it in poll mode.
|
|
122
|
-
...(transport === "callback" ?
|
|
123
|
-
|
|
124
|
-
...
|
|
125
|
-
...
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
})),
|
|
129
|
-
];
|
|
143
|
+
...(transport === "callback" ? { CORTEX_BASE_URL: config.cortexBaseUrl } : {}),
|
|
144
|
+
SANDBOX_CALLBACK_SECRET: identity.callbackSecret,
|
|
145
|
+
...threadLimitEnv(spec),
|
|
146
|
+
...plan.env,
|
|
147
|
+
...(meta.extraEnv ?? {}),
|
|
148
|
+
}).map(([name, value]) => ({ name, value }));
|
|
130
149
|
// requests == limits → Guaranteed QoS and a predictable OOM bound; also
|
|
131
150
|
// satisfies the namespace ResourceQuota, which rejects any pod that omits
|
|
132
151
|
// requests.cpu/memory. The CPU limit can throttle a hot step, but a throttle
|
|
133
152
|
// beats an eviction.
|
|
134
|
-
const spec = meta.resources;
|
|
135
153
|
const quantities = {
|
|
136
154
|
cpu: String(spec.cpu),
|
|
137
155
|
memory: `${spec.memoryGb}Gi`,
|
|
@@ -187,6 +205,16 @@ function buildJobSpec(meta, config, identity) {
|
|
|
187
205
|
readOnly: true,
|
|
188
206
|
});
|
|
189
207
|
}
|
|
208
|
+
// The cpu files ride in a ConfigMap named after the Job (`ensureCpuConfigMap`).
|
|
209
|
+
// A `subPath` mount of one key over one kernel path is accepted by the API
|
|
210
|
+
// server, the kubelet, and gVisor. Under gVisor the sandbox sees no cgroup
|
|
211
|
+
// at all, thus these two files are the only source of the quota for
|
|
212
|
+
// `parallel::detectCores()` and `os.cpu_count()`.
|
|
213
|
+
volumes.push({ name: CPU_VOLUME_NAME, configMap: { name: sandboxId } });
|
|
214
|
+
volumeMounts.push({ name: CPU_VOLUME_NAME, mountPath: CPU_ONLINE_PATH, subPath: "online", readOnly: true });
|
|
215
|
+
if (cpu.hasCpuinfo) {
|
|
216
|
+
volumeMounts.push({ name: CPU_VOLUME_NAME, mountPath: CPUINFO_PATH, subPath: "cpuinfo", readOnly: true });
|
|
217
|
+
}
|
|
190
218
|
const podSpec = {
|
|
191
219
|
automountServiceAccountToken: false,
|
|
192
220
|
restartPolicy: "Never",
|
|
@@ -357,6 +385,9 @@ function waitForJobGone(batchApi, namespace, name) {
|
|
|
357
385
|
* means an (astronomically rare) name collision with a *different* step —
|
|
358
386
|
* adopting its pod would HMAC-fail every callback, and deleting its Job would
|
|
359
387
|
* kill a live sibling. Refuse loudly instead.
|
|
388
|
+
*
|
|
389
|
+
* The result is the uid of the Job that the pod belongs to, for the owner
|
|
390
|
+
* reference of the cpu ConfigMap.
|
|
360
391
|
*/
|
|
361
392
|
function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflowId, job) {
|
|
362
393
|
const createFailed = (status, cause) => ({
|
|
@@ -369,7 +400,7 @@ function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflow
|
|
|
369
400
|
return new ResultAsync((async () => {
|
|
370
401
|
const created = await trySandbox(() => batchApi.createNamespacedJob({ namespace, body: job }), createFailed);
|
|
371
402
|
if (created.isOk())
|
|
372
|
-
return ok(
|
|
403
|
+
return ok(created.value?.metadata?.uid);
|
|
373
404
|
if (statusOf(created.error) !== 409)
|
|
374
405
|
return err(created.error);
|
|
375
406
|
const existingRead = await trySandbox(() => batchApi.readNamespacedJob({ namespace, name: sandboxId }), createFailed);
|
|
@@ -397,20 +428,92 @@ function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflow
|
|
|
397
428
|
const recreated = await trySandbox(() => batchApi.createNamespacedJob({ namespace, body: job }), createFailed);
|
|
398
429
|
if (recreated.isErr())
|
|
399
430
|
return err(recreated.error);
|
|
431
|
+
return ok(recreated.value?.metadata?.uid);
|
|
400
432
|
}
|
|
433
|
+
return ok(existingRead.value.metadata?.uid);
|
|
434
|
+
})());
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* Make the cpu ConfigMap of a Job, or bring an existing one up to date.
|
|
438
|
+
*
|
|
439
|
+
* The ConfigMap comes after the Job, because the Job is its owner. Thus the
|
|
440
|
+
* TTL of the Job and a Foreground delete both remove the ConfigMap with it.
|
|
441
|
+
* The pod waits at its mount until the ConfigMap exists, and that wait is
|
|
442
|
+
* shorter than the schedule of the pod.
|
|
443
|
+
*
|
|
444
|
+
* A `409 AlreadyExists` is the recovery re-run of the spawn step, or a
|
|
445
|
+
* ConfigMap whose prior owner Job is not yet collected. Both cases get the
|
|
446
|
+
* same data and the current owner through a replace.
|
|
447
|
+
*/
|
|
448
|
+
function ensureCpuConfigMap(coreApi, namespace, sandboxId, jobUid, files) {
|
|
449
|
+
const createFailed = (status, cause) => ({
|
|
450
|
+
type: "container_create_failed",
|
|
451
|
+
op: "k8s.createNamespacedConfigMap",
|
|
452
|
+
sandboxId,
|
|
453
|
+
status,
|
|
454
|
+
cause,
|
|
455
|
+
});
|
|
456
|
+
const body = {
|
|
457
|
+
apiVersion: "v1",
|
|
458
|
+
kind: "ConfigMap",
|
|
459
|
+
metadata: {
|
|
460
|
+
name: sandboxId,
|
|
461
|
+
namespace,
|
|
462
|
+
labels: {
|
|
463
|
+
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
|
|
464
|
+
role: "sandbox",
|
|
465
|
+
"cortex/sandbox-id": sandboxId,
|
|
466
|
+
},
|
|
467
|
+
...(jobUid !== undefined && {
|
|
468
|
+
ownerReferences: [{ apiVersion: "batch/v1", kind: "Job", name: sandboxId, uid: jobUid }],
|
|
469
|
+
}),
|
|
470
|
+
},
|
|
471
|
+
data: {
|
|
472
|
+
online: files.online,
|
|
473
|
+
...(files.cpuinfo !== undefined && { cpuinfo: files.cpuinfo }),
|
|
474
|
+
},
|
|
475
|
+
};
|
|
476
|
+
return new ResultAsync((async () => {
|
|
477
|
+
const created = await trySandbox(() => coreApi.createNamespacedConfigMap({ namespace, body }), createFailed);
|
|
478
|
+
if (created.isOk())
|
|
479
|
+
return ok(undefined);
|
|
480
|
+
if (statusOf(created.error) !== 409)
|
|
481
|
+
return err(created.error);
|
|
482
|
+
const existing = await trySandbox(() => coreApi.readNamespacedConfigMap({ namespace, name: sandboxId }), createFailed);
|
|
483
|
+
if (existing.isErr())
|
|
484
|
+
return err(existing.error);
|
|
485
|
+
const replaced = await trySandbox(() => coreApi.replaceNamespacedConfigMap({
|
|
486
|
+
namespace,
|
|
487
|
+
name: sandboxId,
|
|
488
|
+
body: { ...body, metadata: { ...body.metadata, resourceVersion: existing.value.metadata?.resourceVersion } },
|
|
489
|
+
}), createFailed);
|
|
490
|
+
if (replaced.isErr())
|
|
491
|
+
return err(replaced.error);
|
|
401
492
|
return ok(undefined);
|
|
402
493
|
})());
|
|
403
494
|
}
|
|
404
495
|
export function createK8sSandboxOps(config) {
|
|
405
496
|
const { batchApi, coreApi } = config.batchApi && config.coreApi ? { batchApi: config.batchApi, coreApi: config.coreApi } : buildKubeApi();
|
|
497
|
+
const readHostCpuinfo = config.readHostCpuinfo ?? readHostCpuinfoOnce;
|
|
406
498
|
return {
|
|
407
499
|
createSandbox(meta, identity) {
|
|
408
500
|
return new ResultAsync((async () => {
|
|
409
501
|
const { sandboxId } = identity;
|
|
410
|
-
|
|
502
|
+
// The same floor as `threadLimitEnv`: a fraction of a core still
|
|
503
|
+
// keeps one thread busy, and a count of zero is not representable.
|
|
504
|
+
const threads = Math.max(1, Math.floor(meta.resources.cpu));
|
|
505
|
+
const files = cpuFiles(threads, await readHostCpuinfo());
|
|
506
|
+
const job = buildJobSpec(meta, config, identity, { hasCpuinfo: files.cpuinfo !== undefined });
|
|
411
507
|
const adopted = await createOrAdoptJob(batchApi, coreApi, config.namespace, sandboxId, meta.childWorkflowId, job);
|
|
412
508
|
if (adopted.isErr())
|
|
413
509
|
return err(adopted.error);
|
|
510
|
+
const cpuMap = await ensureCpuConfigMap(coreApi, config.namespace, sandboxId, adopted.value, files);
|
|
511
|
+
if (cpuMap.isErr()) {
|
|
512
|
+
// A pod whose ConfigMap never comes stays at ContainerCreating
|
|
513
|
+
// until the ready wait expires. Fail now, with the real cause.
|
|
514
|
+
await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
|
|
515
|
+
return err(cpuMap.error);
|
|
516
|
+
}
|
|
414
517
|
// A Job whose pod never schedules (quota rejection, no nodes, spot
|
|
415
518
|
// reclaim) is reaped by neither backoffLimit (an admission rejection
|
|
416
519
|
// creates no pod to count) nor ttlSecondsAfterFinished (the Job never
|
|
@@ -434,20 +537,22 @@ export function createK8sSandboxOps(config) {
|
|
|
434
537
|
cause,
|
|
435
538
|
}));
|
|
436
539
|
if (registered.isErr()) {
|
|
437
|
-
await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
|
|
540
|
+
await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
|
|
438
541
|
return err(registered.error);
|
|
439
542
|
}
|
|
440
543
|
return ok(ref);
|
|
441
544
|
}
|
|
442
|
-
await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
|
|
545
|
+
await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
|
|
443
546
|
return err(ready.error);
|
|
444
547
|
})());
|
|
445
548
|
},
|
|
549
|
+
// The owner reference removes the cpu ConfigMap with the Job. The explicit
|
|
550
|
+
// delete covers a ConfigMap that got no owner, and it is a no-op on 404.
|
|
446
551
|
teardown(ref) {
|
|
447
|
-
return deleteJobIgnoreMissing(batchApi, config.namespace, ref.sandboxId);
|
|
552
|
+
return deleteJobIgnoreMissing(batchApi, config.namespace, ref.sandboxId).andThen(() => deleteConfigMapIgnoreMissing(coreApi, config.namespace, ref.sandboxId));
|
|
448
553
|
},
|
|
449
554
|
teardownById(sandboxId) {
|
|
450
|
-
return deleteJobIgnoreMissing(batchApi, config.namespace, sandboxId);
|
|
555
|
+
return deleteJobIgnoreMissing(batchApi, config.namespace, sandboxId).andThen(() => deleteConfigMapIgnoreMissing(coreApi, config.namespace, sandboxId));
|
|
451
556
|
},
|
|
452
557
|
listManagedSandboxes() {
|
|
453
558
|
return trySandbox(() => batchApi.listNamespacedJob({
|
|
@@ -508,9 +613,13 @@ export function createK8sSandboxOps(config) {
|
|
|
508
613
|
* matters; a delete error here is logged and swallowed so it never masks the
|
|
509
614
|
* create error returned to the caller.
|
|
510
615
|
*/
|
|
511
|
-
async function cleanupFailedJob(batchApi, namespace, sandboxId, logger) {
|
|
616
|
+
async function cleanupFailedJob(batchApi, coreApi, namespace, sandboxId, logger) {
|
|
512
617
|
const deleted = await deleteJobIgnoreMissing(batchApi, namespace, sandboxId);
|
|
513
618
|
if (deleted.isErr()) {
|
|
514
619
|
logger.named("k8s-client").error("failed to delete Job after startup failure", { sandboxId, err: deleted.error });
|
|
515
620
|
}
|
|
621
|
+
const deletedMap = await deleteConfigMapIgnoreMissing(coreApi, namespace, sandboxId);
|
|
622
|
+
if (deletedMap.isErr()) {
|
|
623
|
+
logger.named("k8s-client").error("failed to delete cpu ConfigMap after startup failure", { sandboxId, err: deletedMap.error });
|
|
624
|
+
}
|
|
516
625
|
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Thread-count env for a sandbox container.
|
|
3
|
+
*
|
|
4
|
+
* Each backend turns the `ResourceSpec` of a step into a hard cgroup limit:
|
|
5
|
+
* `NanoCpus` and `Memory` on Docker, `limits.cpu` and `limits.memory` on K8s. A
|
|
6
|
+
* cgroup quota is invisible to most runtimes inside it. `parallel::detectCores()`
|
|
7
|
+
* greps `/proc/cpuinfo`, and `os.cpu_count()` reads
|
|
8
|
+
* `/sys/devices/system/cpu/online`. Both files describe the host. Thus a step
|
|
9
|
+
* with 2 CPUs on a host of 12 cores forks 12 workers into a cgroup of 2 CPUs,
|
|
10
|
+
* and each fork copies its working set until the memory limit stops it.
|
|
11
|
+
*
|
|
12
|
+
* The env below publishes the quota to each library that reads a thread count
|
|
13
|
+
* from the environment. `parallelly::availableCores()` and `joblib.cpu_count()`
|
|
14
|
+
* read the cgroup quota on their own, thus they need no value here.
|
|
15
|
+
*
|
|
16
|
+
* `MC_CORES` is absent on purpose. R defaults `mclapply` to 2 workers, not to
|
|
17
|
+
* the core count. A value there raises the fork count instead of a decrease.
|
|
18
|
+
*/
|
|
19
|
+
import type { ResourceSpec } from "../config/resource-limits.js";
|
|
20
|
+
/**
|
|
21
|
+
* Env that binds each parallel library in the container to `spec.cpu`.
|
|
22
|
+
*
|
|
23
|
+
* A fractional CPU request floors to one thread. A pool of zero is not
|
|
24
|
+
* representable, and two threads on half a core only add context switches.
|
|
25
|
+
*
|
|
26
|
+
* The caller puts these before `plan.env` and before the `extraEnv` of the
|
|
27
|
+
* embedder, thus a host that knows more about a step keeps the last word.
|
|
28
|
+
*/
|
|
29
|
+
export declare function threadLimitEnv(spec: ResourceSpec): Record<string, string>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Thread-count env for a sandbox container.
|
|
3
|
+
*
|
|
4
|
+
* Each backend turns the `ResourceSpec` of a step into a hard cgroup limit:
|
|
5
|
+
* `NanoCpus` and `Memory` on Docker, `limits.cpu` and `limits.memory` on K8s. A
|
|
6
|
+
* cgroup quota is invisible to most runtimes inside it. `parallel::detectCores()`
|
|
7
|
+
* greps `/proc/cpuinfo`, and `os.cpu_count()` reads
|
|
8
|
+
* `/sys/devices/system/cpu/online`. Both files describe the host. Thus a step
|
|
9
|
+
* with 2 CPUs on a host of 12 cores forks 12 workers into a cgroup of 2 CPUs,
|
|
10
|
+
* and each fork copies its working set until the memory limit stops it.
|
|
11
|
+
*
|
|
12
|
+
* The env below publishes the quota to each library that reads a thread count
|
|
13
|
+
* from the environment. `parallelly::availableCores()` and `joblib.cpu_count()`
|
|
14
|
+
* read the cgroup quota on their own, thus they need no value here.
|
|
15
|
+
*
|
|
16
|
+
* `MC_CORES` is absent on purpose. R defaults `mclapply` to 2 workers, not to
|
|
17
|
+
* the core count. A value there raises the fork count instead of a decrease.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Env that binds each parallel library in the container to `spec.cpu`.
|
|
21
|
+
*
|
|
22
|
+
* A fractional CPU request floors to one thread. A pool of zero is not
|
|
23
|
+
* representable, and two threads on half a core only add context switches.
|
|
24
|
+
*
|
|
25
|
+
* The caller puts these before `plan.env` and before the `extraEnv` of the
|
|
26
|
+
* embedder, thus a host that knows more about a step keeps the last word.
|
|
27
|
+
*/
|
|
28
|
+
export function threadLimitEnv(spec) {
|
|
29
|
+
const threads = String(Math.max(1, Math.floor(spec.cpu)));
|
|
30
|
+
return {
|
|
31
|
+
// OpenMP: the compiled code of R and Python packages.
|
|
32
|
+
OMP_NUM_THREADS: threads,
|
|
33
|
+
// OpenBLAS keeps a private pool. A pthread build reads this name first.
|
|
34
|
+
OPENBLAS_NUM_THREADS: threads,
|
|
35
|
+
// MKL reads this name before OMP_NUM_THREADS.
|
|
36
|
+
MKL_NUM_THREADS: threads,
|
|
37
|
+
// BiocParallel sizes `bpparam()` from this name. DESeq2 `parallel=TRUE`
|
|
38
|
+
// and the other Bioconductor packages size their workers from `bpparam()`.
|
|
39
|
+
BIOCPARALLEL_WORKER_NUMBER: threads,
|
|
40
|
+
// data.table defaults to half of the host cores.
|
|
41
|
+
R_DATATABLE_NUM_THREADS: threads,
|
|
42
|
+
// The Python numerical stack.
|
|
43
|
+
NUMBA_NUM_THREADS: threads,
|
|
44
|
+
NUMEXPR_MAX_THREADS: threads,
|
|
45
|
+
LOKY_MAX_CPU_COUNT: threads,
|
|
46
|
+
POLARS_MAX_THREADS: threads,
|
|
47
|
+
RAYON_NUM_THREADS: threads,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
@@ -171,6 +171,41 @@ export declare function reconcileOrphanedDataProfile(pool: Querier, analysisId:
|
|
|
171
171
|
* finds no running row to stamp and no-ops instead of re-seeding the cleared one.
|
|
172
172
|
*/
|
|
173
173
|
export declare function clearDataProfile(pool: Querier, analysisId: string): ResultAsync<boolean, DbError>;
|
|
174
|
+
/**
|
|
175
|
+
* Stamp a row `completed` with no result, because the analysis has no input files.
|
|
176
|
+
*
|
|
177
|
+
* A profile describes files. An analysis with no files has nothing to describe, and
|
|
178
|
+
* thus the profile is complete at once: no claim into `running`, no workflow, and no
|
|
179
|
+
* sandbox. The write is a terminal write like {@link completeDataProfile}, but it starts
|
|
180
|
+
* from a settled row or an empty row, not from a `running` one. As a result
|
|
181
|
+
* `data_profile_started_at` and `data_profile_completed_at` carry the same instant, and
|
|
182
|
+
* `data_profile_workflow_id` stays NULL, because no stream is addressable (see
|
|
183
|
+
* {@link recordDataProfileWorkflowId}).
|
|
184
|
+
*
|
|
185
|
+
* The seed is written as `[]`, so the completed row names the set that it covers: the
|
|
186
|
+
* empty set. A `completed` row with a NULL seed stays forbidden (see
|
|
187
|
+
* {@link completeDataProfile}). A prior `data_profile_result` is replaced with NULL,
|
|
188
|
+
* because it describes files that the analysis no longer has. A prior `failed` row is
|
|
189
|
+
* replaced too, because the set that failed is gone.
|
|
190
|
+
*
|
|
191
|
+
* Two guards ride in the CAS:
|
|
192
|
+
* - `IS DISTINCT FROM 'running'`: a live workflow owns the row, and its terminal write
|
|
193
|
+
* must not land on a row that this write moved. The same reason as
|
|
194
|
+
* {@link clearDataProfile}.
|
|
195
|
+
* - {@link UNSEEDED}: the row names no file. A seed upsert that lands between a caller's
|
|
196
|
+
* pre-read and this write makes the row a seeded, startable row. A `completed` stamp
|
|
197
|
+
* over it would hide those files behind a finished profile. The CAS refuses that row,
|
|
198
|
+
* and the caller reads the status again to name the cause.
|
|
199
|
+
*
|
|
200
|
+
* The rerun claim carries {@link SEEDED}, so a completed-empty row is not claimable until
|
|
201
|
+
* a later seed upsert names files. Then `tryRerunDataProfile` takes it, the same as any
|
|
202
|
+
* completed row.
|
|
203
|
+
*
|
|
204
|
+
* `ok(true)` when this call stamped the row. `ok(false)` when the guard refused it: no
|
|
205
|
+
* such analysis, a live run, or a seed that names files. A refusal stays in the ok
|
|
206
|
+
* channel, the same as the sibling CAS ops.
|
|
207
|
+
*/
|
|
208
|
+
export declare function completeEmptyDataProfile(pool: Querier, analysisId: string): ResultAsync<boolean, DbError>;
|
|
174
209
|
/**
|
|
175
210
|
* Load an analysis's data-profile ledger state. Returns `null` for BOTH miss
|
|
176
211
|
* conditions, deliberately indistinguishable to consumers:
|
|
@@ -20,6 +20,13 @@ import { tryMutation, tryQuery } from "../lib/db-result.js";
|
|
|
20
20
|
* and surfacing a hand-corrupted row as a `DbError` beats silently claiming it.
|
|
21
21
|
*/
|
|
22
22
|
const SEEDED = "seed_input_file_ids IS NOT NULL AND jsonb_array_length(seed_input_file_ids) > 0";
|
|
23
|
+
/**
|
|
24
|
+
* The negation of {@link SEEDED}: the row names no input file. A NULL seed and `[]`
|
|
25
|
+
* both name zero files, and {@link completeEmptyDataProfile} treats them the same.
|
|
26
|
+
* The two predicates are disjoint, so a claim into `running` and an empty-set
|
|
27
|
+
* completion can never both win on one row.
|
|
28
|
+
*/
|
|
29
|
+
const UNSEEDED = "(seed_input_file_ids IS NULL OR jsonb_array_length(seed_input_file_ids) = 0)";
|
|
23
30
|
/**
|
|
24
31
|
* Try to claim a startable row into `running`. Startable means `'pending'` (seeded,
|
|
25
32
|
* not yet run) or NULL (no profile: never profiled, or cleared by `clearDataProfile`).
|
|
@@ -279,6 +286,55 @@ export function clearDataProfile(pool, analysisId) {
|
|
|
279
286
|
return (result.rowCount ?? 0) > 0;
|
|
280
287
|
});
|
|
281
288
|
}
|
|
289
|
+
/**
|
|
290
|
+
* Stamp a row `completed` with no result, because the analysis has no input files.
|
|
291
|
+
*
|
|
292
|
+
* A profile describes files. An analysis with no files has nothing to describe, and
|
|
293
|
+
* thus the profile is complete at once: no claim into `running`, no workflow, and no
|
|
294
|
+
* sandbox. The write is a terminal write like {@link completeDataProfile}, but it starts
|
|
295
|
+
* from a settled row or an empty row, not from a `running` one. As a result
|
|
296
|
+
* `data_profile_started_at` and `data_profile_completed_at` carry the same instant, and
|
|
297
|
+
* `data_profile_workflow_id` stays NULL, because no stream is addressable (see
|
|
298
|
+
* {@link recordDataProfileWorkflowId}).
|
|
299
|
+
*
|
|
300
|
+
* The seed is written as `[]`, so the completed row names the set that it covers: the
|
|
301
|
+
* empty set. A `completed` row with a NULL seed stays forbidden (see
|
|
302
|
+
* {@link completeDataProfile}). A prior `data_profile_result` is replaced with NULL,
|
|
303
|
+
* because it describes files that the analysis no longer has. A prior `failed` row is
|
|
304
|
+
* replaced too, because the set that failed is gone.
|
|
305
|
+
*
|
|
306
|
+
* Two guards ride in the CAS:
|
|
307
|
+
* - `IS DISTINCT FROM 'running'`: a live workflow owns the row, and its terminal write
|
|
308
|
+
* must not land on a row that this write moved. The same reason as
|
|
309
|
+
* {@link clearDataProfile}.
|
|
310
|
+
* - {@link UNSEEDED}: the row names no file. A seed upsert that lands between a caller's
|
|
311
|
+
* pre-read and this write makes the row a seeded, startable row. A `completed` stamp
|
|
312
|
+
* over it would hide those files behind a finished profile. The CAS refuses that row,
|
|
313
|
+
* and the caller reads the status again to name the cause.
|
|
314
|
+
*
|
|
315
|
+
* The rerun claim carries {@link SEEDED}, so a completed-empty row is not claimable until
|
|
316
|
+
* a later seed upsert names files. Then `tryRerunDataProfile` takes it, the same as any
|
|
317
|
+
* completed row.
|
|
318
|
+
*
|
|
319
|
+
* `ok(true)` when this call stamped the row. `ok(false)` when the guard refused it: no
|
|
320
|
+
* such analysis, a live run, or a seed that names files. A refusal stays in the ok
|
|
321
|
+
* channel, the same as the sibling CAS ops.
|
|
322
|
+
*/
|
|
323
|
+
export function completeEmptyDataProfile(pool, analysisId) {
|
|
324
|
+
const now = new Date().toISOString();
|
|
325
|
+
return tryMutation("dataProfile.completeEmptyDataProfile", async () => {
|
|
326
|
+
const result = await pool.query({
|
|
327
|
+
text: `UPDATE cortex_analysis_state
|
|
328
|
+
SET data_profile_status = 'completed', data_profile_error = NULL,
|
|
329
|
+
data_profile_started_at = $1, data_profile_completed_at = $1,
|
|
330
|
+
data_profile_result = NULL, data_profile_workflow_id = NULL,
|
|
331
|
+
seed_input_file_ids = '[]'::jsonb
|
|
332
|
+
WHERE analysis_id = $2 AND data_profile_status IS DISTINCT FROM 'running' AND ${UNSEEDED}`,
|
|
333
|
+
values: [now, analysisId],
|
|
334
|
+
});
|
|
335
|
+
return (result.rowCount ?? 0) > 0;
|
|
336
|
+
});
|
|
337
|
+
}
|
|
282
338
|
/**
|
|
283
339
|
* Load an analysis's data-profile ledger state. Returns `null` for BOTH miss
|
|
284
340
|
* conditions, deliberately indistinguishable to consumers:
|
package/dist/state/index.d.ts
CHANGED
|
@@ -20,7 +20,7 @@ export { setSandboxRef, setActiveExecId, clearSandboxRef, queryActiveSandboxes,
|
|
|
20
20
|
export type { ActiveSandboxRow } from "./active-sandboxes.js";
|
|
21
21
|
export { insertPlan, upsertPlan, loadPlan } from "./plans.js";
|
|
22
22
|
export type { InsertPlanInput, UpsertPlanInput } from "./plans.js";
|
|
23
|
-
export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
|
|
23
|
+
export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, completeEmptyDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
|
|
24
24
|
export type { DataProfileStatus, DataProfileResult, DataProfileChecked, DataProfileCompanionCompleteness, DataProfileDimension, DataProfileGroup, DataProfileGroupSlot, DataProfileFile, DataProfileMemberAnnotation, DataProfileObservation, DataProfilePartition, DataProfileProbeReport, DataProfileQualityAssessment, DataProfileQuarantine, DataProfileRecipeStep, DataProfileKind, DataProfileAxis, DataProfileInputSignature, DataProfileCoverage, } from "./data-profile.js";
|
|
25
25
|
export { queryRunCountsByAnalyses, queryThreadCountsByAnalyses, queryDataProfileStatusByAnalyses } from "./analyses-metrics.js";
|
|
26
26
|
export { createAnalysisPurge } from "./purge-analysis.js";
|
package/dist/state/index.js
CHANGED
|
@@ -14,7 +14,7 @@ setRunSynthesisOutcome, queryRun, queryActiveRun, queryNonTerminalRunsByAnalysis
|
|
|
14
14
|
export { insertStepExecution, seedStepExecutions, sweepPendingStepExecutions, updateStepExecution, queryStepsByRun } from "./step-executions.js";
|
|
15
15
|
export { setSandboxRef, setActiveExecId, clearSandboxRef, queryActiveSandboxes, reconcileReapedSandbox } from "./active-sandboxes.js";
|
|
16
16
|
export { insertPlan, upsertPlan, loadPlan } from "./plans.js";
|
|
17
|
-
export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
|
|
17
|
+
export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, completeEmptyDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
|
|
18
18
|
export { queryRunCountsByAnalyses, queryThreadCountsByAnalyses, queryDataProfileStatusByAnalyses } from "./analyses-metrics.js";
|
|
19
19
|
// Not per-entity: one analysis's whole Postgres footprint, reached from its id
|
|
20
20
|
// alone, with the workflow ledger behind an injected seam.
|
|
@@ -132,7 +132,16 @@ export declare function registerDataProfileWorkflow(deps: DataProfileDeps): (inp
|
|
|
132
132
|
* 'failed' on error, and revokes the run authorization on every terminal path.
|
|
133
133
|
*/
|
|
134
134
|
export declare function runDataProfileBody(input: DataProfileWorkflowInput, deps: DataProfileDeps): Promise<void>;
|
|
135
|
-
|
|
135
|
+
/**
|
|
136
|
+
* What the trigger did:
|
|
137
|
+
* - `"started"`: a first profile claimed the row and a workflow runs.
|
|
138
|
+
* - `"restarted"`: a re-profile claimed a `completed` row and a workflow runs.
|
|
139
|
+
* - `"already_running"`: a workflow already owns the row, and nothing new ran.
|
|
140
|
+
* - `"completed"`: the analysis has no input files, so the row is `completed` at once
|
|
141
|
+
* with no result. No workflow ran (see {@link completeEmptyDataProfile}).
|
|
142
|
+
* - `"failed"`: the trigger refused the call or faulted, and the row is unchanged.
|
|
143
|
+
*/
|
|
144
|
+
export type DataProfileTriggerResult = "started" | "restarted" | "already_running" | "completed" | "failed";
|
|
136
145
|
/**
|
|
137
146
|
* Route-side deps for triggering the data-profile workflow: the ledger pool,
|
|
138
147
|
* the run authorizer, and the registered workflow callable. The body's
|
|
@@ -188,6 +197,12 @@ export declare function dataProfileWorkflowId(analysisId: string, nonce: string)
|
|
|
188
197
|
* row is claimed by neither: retrying a failure is a deliberate act the caller drives
|
|
189
198
|
* through `tryRetryDataProfile` + `runDataProfile`.
|
|
190
199
|
*
|
|
200
|
+
* An analysis with no input files takes none of the claims. An empty manifest against
|
|
201
|
+
* a seed that names no file stamps the row `completed` at once, with no result and no
|
|
202
|
+
* workflow (see {@link completeEmptyDataProfile}), and the trigger returns
|
|
203
|
+
* `"completed"`. An empty manifest against a seed that names files is a divergence
|
|
204
|
+
* between the caller and the ledger, and the trigger refuses it.
|
|
205
|
+
*
|
|
191
206
|
* Returns what happened, so the caller can surface it (e.g. in the seed response).
|
|
192
207
|
*/
|
|
193
208
|
export declare function triggerDataProfile(deps: DataProfileTriggerDeps, params: DataProfileTriggerParams): Promise<DataProfileTriggerResult>;
|
|
@@ -44,7 +44,7 @@ import { scanInputTree } from "../input-scan/scan.js";
|
|
|
44
44
|
import { PROFILE_INDEX_TYPES, buildProfileIndexEntries } from "./data-profile-index.js";
|
|
45
45
|
import { absorbRecipe, renderAbsorbDelta } from "./data-profile-absorb.js";
|
|
46
46
|
import { formatResolutionErrors, resolveProfileSubmission } from "./data-profile-resolve.js";
|
|
47
|
-
import { completeDataProfile, failDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, tryRerunDataProfile, tryStartDataProfile, upsertArtifacts, } from "../state/index.js";
|
|
47
|
+
import { completeDataProfile, completeEmptyDataProfile, failDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, tryRerunDataProfile, tryStartDataProfile, upsertArtifacts, } from "../state/index.js";
|
|
48
48
|
import { createProfileActivityEmitter } from "./data-profile-activity.js";
|
|
49
49
|
import { ensureSearchIndex, searchIndexName } from "../workspace/search-config.js";
|
|
50
50
|
// The run literal is declared in `contracts/` — a consumer reads it back off recorded usage, and this
|
|
@@ -668,6 +668,12 @@ async function startDataProfileWorkflow(deps, params) {
|
|
|
668
668
|
* row is claimed by neither: retrying a failure is a deliberate act the caller drives
|
|
669
669
|
* through `tryRetryDataProfile` + `runDataProfile`.
|
|
670
670
|
*
|
|
671
|
+
* An analysis with no input files takes none of the claims. An empty manifest against
|
|
672
|
+
* a seed that names no file stamps the row `completed` at once, with no result and no
|
|
673
|
+
* workflow (see {@link completeEmptyDataProfile}), and the trigger returns
|
|
674
|
+
* `"completed"`. An empty manifest against a seed that names files is a divergence
|
|
675
|
+
* between the caller and the ledger, and the trigger refuses it.
|
|
676
|
+
*
|
|
671
677
|
* Returns what happened, so the caller can surface it (e.g. in the seed response).
|
|
672
678
|
*/
|
|
673
679
|
export async function triggerDataProfile(deps, params) {
|
|
@@ -682,17 +688,32 @@ export async function triggerDataProfile(deps, params) {
|
|
|
682
688
|
// `loadSeedInputFileIds`, which returns null for BOTH a missing analysis row and
|
|
683
689
|
// a NULL-seed row (`loadDataProfileStatus` would hide the seed of a cleared row).
|
|
684
690
|
const seeded = unwrapOrThrow(await loadSeedInputFileIds(deps.pool, analysisId));
|
|
685
|
-
if (
|
|
686
|
-
|
|
691
|
+
if (params.stagedInputs.length === 0) {
|
|
692
|
+
// The trigger validates the manifest it is about to dispatch against the ledger
|
|
693
|
+
// seed. A seeded row profiled against an EMPTY manifest would claim `running` and
|
|
694
|
+
// then hit the body's empty-manifest path, and complete with a NULL result while
|
|
695
|
+
// the seed still names files. Refuse the divergence before any claim.
|
|
696
|
+
if (seeded !== null && seeded.length > 0) {
|
|
697
|
+
logger.error("trigger rejected: empty manifest dispatched against a non-empty seed", { seededCount: seeded.length });
|
|
698
|
+
return "failed";
|
|
699
|
+
}
|
|
700
|
+
// The analysis has no input files, so there is nothing to profile. The row is
|
|
701
|
+
// `completed` at once, with no workflow. The CAS carries the same seed
|
|
702
|
+
// predicate as this pre-read, so a seed upsert that lands in between cannot be
|
|
703
|
+
// hidden behind a finished profile: the stamp refuses, and the status read
|
|
704
|
+
// below names the cause.
|
|
705
|
+
if (unwrapOrThrow(await completeEmptyDataProfile(deps.pool, analysisId)))
|
|
706
|
+
return "completed";
|
|
707
|
+
const current = unwrapOrThrow(await loadDataProfileStatus(deps.pool, analysisId));
|
|
708
|
+
if (current?.status === "running")
|
|
709
|
+
return "already_running";
|
|
710
|
+
logger.error("trigger rejected: empty-set completion refused (no analysis row, or the seed now names files)");
|
|
687
711
|
return "failed";
|
|
688
712
|
}
|
|
689
|
-
//
|
|
690
|
-
//
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
// then loops on re-triggering. Refuse the divergence before any claim.
|
|
694
|
-
if (params.stagedInputs.length === 0) {
|
|
695
|
-
logger.error("trigger rejected: empty manifest dispatched against a non-empty seed", { seededCount: seeded.length });
|
|
713
|
+
// A non-empty manifest against a seed that names no file: the caller skipped the
|
|
714
|
+
// seed, or the analysis row is missing. Name the cause before any claim.
|
|
715
|
+
if (seeded === null || seeded.length === 0) {
|
|
716
|
+
logger.error("trigger rejected: no seeded input set (missing analysis row or caller skipped seeding)");
|
|
696
717
|
return "failed";
|
|
697
718
|
}
|
|
698
719
|
const started = unwrapOrThrow(await tryStartDataProfile(deps.pool, analysisId));
|
package/package.json
CHANGED