@inflexa-ai/harness 0.30.1 → 0.30.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { generateText, streamText, wrapLanguageModel, } from "ai";
1
+ import { streamText, wrapLanguageModel, } from "ai";
2
2
  import { createAnthropic } from "@ai-sdk/anthropic";
3
3
  import { createOpenAI } from "@ai-sdk/openai";
4
4
  import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
@@ -255,10 +255,10 @@ function requestedModelIdOf(model) {
255
255
  }
256
256
  /**
257
257
  * Capture the served model id at the raw model boundary, because that is the
258
- * only place it is still distinguishable from the requested one: `generateText`
259
- * and `streamText` both coalesce an unreported response model id to the bound
260
- * model's own id, so reading `result.response.modelId` would silently republish
261
- * the requested id as an endpoint's claim. A middleware sees the provider's
258
+ * only place it is still distinguishable from the requested one: `streamText`
259
+ * coalesces an unreported response model id to the bound model's own id, so
260
+ * reading `result.response.modelId` would silently republish the requested id
261
+ * as an endpoint's claim. A middleware sees the provider's
262
262
  * result before that fallback applies, so "reported nothing" stays absent.
263
263
  *
264
264
  * Built per call, never per provider: one provider instance serves concurrent
@@ -303,16 +303,6 @@ function responseFromMessages(input) {
303
303
  }
304
304
  return { message, finishReason, rawFinishReason, usage, requestedModelId, servedModelId };
305
305
  }
306
- function responseFromGenerate(result, identity) {
307
- return responseFromMessages({
308
- messages: result.responseMessages,
309
- fallbackText: result.text,
310
- finishReason: result.finishReason,
311
- rawFinishReason: result.rawFinishReason,
312
- usage: result.usage,
313
- ...identity,
314
- });
315
- }
316
306
  /**
317
307
  * Drop empty text parts — and messages left with no content at all — before
318
308
  * the wire call. The Anthropic Messages API rejects any request containing an
@@ -356,9 +346,57 @@ async function pullNextDelta(iterator) {
356
346
  return { kind: "delta", text: part.text };
357
347
  if (part.type === "abort")
358
348
  return { kind: "aborted", reason: part.reason };
349
+ // The SDK reports a wire failure as a part, not as a rejection, and it
350
+ // then closes the stream. Skipping it would turn a failed call into an
351
+ // empty assistant turn that the loop reads as a real answer.
352
+ if (part.type === "error")
353
+ return { kind: "failed", error: part.error };
359
354
  }
360
355
  return { kind: "end" };
361
356
  }
357
+ /**
358
+ * The HTTP status that each Anthropic stream error type stands for.
359
+ *
360
+ * A provider names a mid-stream failure in the payload, not in the status line:
361
+ * a stream that carries `overloaded_error` reports the condition that a 529
362
+ * names on the non-streaming wire. The classifier reads the status alone, thus
363
+ * the part has to reach it as an error that names one, or the same condition
364
+ * would classify one way on `chat` and another way on `chatStream`.
365
+ */
366
+ const STREAM_ERROR_STATUS = {
367
+ invalid_request_error: 400,
368
+ authentication_error: 401,
369
+ permission_error: 403,
370
+ not_found_error: 404,
371
+ request_too_large: 413,
372
+ rate_limit_error: 429,
373
+ timeout_error: 408,
374
+ api_error: 500,
375
+ overloaded_error: 529,
376
+ };
377
+ /**
378
+ * The failure that an `error` part of the SDK stream stands for.
379
+ *
380
+ * The part carries whatever the provider put on the wire, and that is often a
381
+ * bare object rather than an `Error`. Left as it is, it reaches the message as
382
+ * `[object Object]` and reaches the classifier with no status at all.
383
+ */
384
+ function streamPartFailure(error) {
385
+ if (error instanceof Error)
386
+ return error;
387
+ if (typeof error !== "object" || error === null)
388
+ return new Error(String(error));
389
+ const reported = error;
390
+ const type = typeof reported.type === "string" ? reported.type : undefined;
391
+ const message = typeof reported.message === "string" ? reported.message : undefined;
392
+ const failure = new Error(message ?? type ?? "The provider reported a stream error.", { cause: error });
393
+ const status = type === undefined ? undefined : STREAM_ERROR_STATUS[type];
394
+ // `extractStatus` walks the chain for this field, thus the named condition
395
+ // classifies the same way it would from a status line.
396
+ if (status !== undefined)
397
+ Object.assign(failure, { status });
398
+ return failure;
399
+ }
362
400
  /**
363
401
  * The failure that an `abort` part of the SDK stream stands for.
364
402
  *
@@ -394,16 +432,46 @@ export function createAiSdkProvider(deps) {
394
432
  const requestTimeoutMs = deps.requestTimeoutMs;
395
433
  const requestedModelId = requestedModelIdOf(deps.model);
396
434
  routeSdkWarningsTo(logger);
435
+ /**
436
+ * Record a call that the envelope gave up on.
437
+ *
438
+ * The envelope reports one attempt at `debug`, thus a deployment that runs
439
+ * at `info` keeps no trace of a call that exhausted its retries: the reply
440
+ * carries the failure to the user and the operator sees nothing. This is the
441
+ * one record that names the outcome.
442
+ */
443
+ function logFailure(session, failure, thrown) {
444
+ logger.error("provider call failed", {
445
+ workload: workloadOf(session),
446
+ kind: failure.type,
447
+ retryable: failure.retryable,
448
+ ...logger.errorFields(thrown),
449
+ });
450
+ }
397
451
  function chat(req, session, signal) {
398
452
  const run = async () => {
399
453
  const retry = createRetry(signal, logger, maxRetries);
400
454
  const capture = captureServedModelId(deps.model);
401
455
  try {
402
- const result = await retry(async () => {
456
+ const collected = await retry(async () => {
403
457
  // Attribution headers are time-limited; resolving them inside the
404
458
  // retried closure keeps them fresh across a multi-minute window.
405
459
  const headers = await resolveBillingHeaders(deps.resolveBilling, session);
406
- return generateText({
460
+ // The call streams on the wire and collapses below.
461
+ //
462
+ // A non-streaming turn sends no header until the model completes,
463
+ // thus the `headersTimeout` of the host HTTP client bounds the whole
464
+ // generation. No option of the SDK lifts that bound, because it sits
465
+ // under the fetch. Node defaults the field to 300s, and a turn near
466
+ // 18k output tokens reaches it, while `DEFAULT_MAX_OUTPUT_TOKENS`
467
+ // asks for up to 64k. A cut turn then costs twice: the envelope
468
+ // starts the same generation again from zero.
469
+ //
470
+ // Anthropic states the rule for its own SDKs: a large `max_tokens`
471
+ // makes streaming necessary, to prevent an HTTP timeout. Thus a
472
+ // ceiling this size streams whether or not a consumer wants the
473
+ // deltas, and `chatStream` stays the seam that forwards them.
474
+ const result = streamText({
407
475
  model: capture.model,
408
476
  system: req.system,
409
477
  messages: sanitizeMessages(req.messages),
@@ -416,20 +484,53 @@ export function createAiSdkProvider(deps) {
416
484
  maxRetries: 0,
417
485
  headers,
418
486
  abortSignal: signal,
419
- // No SDK timeout rides here. `chunkMs` bounds a stream only,
420
- // thus it is inert on this call. The fetch guard is the full
421
- // bound, because the headers of a non-streaming response arrive
422
- // when the model completes.
487
+ // `firstChunkMs` bounds the wait for the first content chunk,
488
+ // and `chunkMs` bounds each later gap. Thus each silent interval
489
+ // carries the one configured bound, and a turn that writes
490
+ // steadily runs as long as it needs.
491
+ ...(requestTimeoutMs !== undefined ? { timeout: { firstChunkMs: requestTimeoutMs, chunkMs: requestTimeoutMs } } : {}),
423
492
  providerOptions: req.providerOptions,
424
493
  reasoning: req.reasoning,
425
494
  });
495
+ // The drain sits inside the retried closure, thus a failure at any
496
+ // point of the stream re-runs the whole attempt and the envelope
497
+ // keeps its whole-call scope. No delta reaches a consumer here —
498
+ // `chatStream` is the seam that forwards them.
499
+ const iterator = result.fullStream[Symbol.asyncIterator]();
500
+ let text = "";
501
+ for (let pull = await pullNextDelta(iterator); pull.kind !== "end"; pull = await pullNextDelta(iterator)) {
502
+ // A caller abort rides out as the reason of its own signal, thus
503
+ // it stays a cancellation and reaches the caller as a throw.
504
+ if (pull.kind === "aborted")
505
+ throw abortedStreamFailure(signal, pull.reason, requestTimeoutMs);
506
+ if (pull.kind === "failed")
507
+ throw streamPartFailure(pull.error);
508
+ text += pull.text;
509
+ }
510
+ // These promises settle only once the stream is fully drained, thus
511
+ // the loop above must reach its end before this point.
512
+ return {
513
+ messages: await result.responseMessages,
514
+ fallbackText: text,
515
+ finishReason: await result.finishReason,
516
+ rawFinishReason: await result.rawFinishReason,
517
+ usage: await result.usage,
518
+ };
426
519
  });
427
- return ok(responseFromGenerate(result, { requestedModelId, servedModelId: capture.servedModelId() }));
520
+ return ok(responseFromMessages({
521
+ ...collected,
522
+ requestedModelId,
523
+ // Read after the drain: the metadata chunk that carries the
524
+ // served id reaches the capture only as the stream is consumed.
525
+ servedModelId: capture.servedModelId(),
526
+ }));
428
527
  }
429
528
  catch (e) {
430
529
  if (isAbortError(e) || signal?.aborted)
431
530
  throw e;
432
- return err(toProviderError(unwrapForClassification(e), workloadOf(session)));
531
+ const failure = toProviderError(unwrapForClassification(e), workloadOf(session));
532
+ logFailure(session, failure, e);
533
+ return err(failure);
433
534
  }
434
535
  };
435
536
  return new ResultAsync(run());
@@ -477,6 +578,8 @@ export function createAiSdkProvider(deps) {
477
578
  // fresh window under the retry policy of a connection error.
478
579
  if (first.kind === "aborted")
479
580
  throw abortedStreamFailure(signal, first.reason, requestTimeoutMs);
581
+ if (first.kind === "failed")
582
+ throw streamPartFailure(first.error);
480
583
  if (first.kind === "end") {
481
584
  // A doStream promise rejection — the shape a real connection
482
585
  // failure takes — does NOT surface at this first pull: streamText
@@ -522,6 +625,8 @@ export function createAiSdkProvider(deps) {
522
625
  // terminal `ProviderError` from the throwable.
523
626
  if (pull.kind === "aborted")
524
627
  throw abortedStreamFailure(signal, pull.reason, requestTimeoutMs);
628
+ if (pull.kind === "failed")
629
+ throw streamPartFailure(pull.error);
525
630
  text += pull.text;
526
631
  yield { type: "text-delta", text: pull.text };
527
632
  }
@@ -541,7 +646,9 @@ export function createAiSdkProvider(deps) {
541
646
  catch (e) {
542
647
  if (isAbortError(e) || signal?.aborted)
543
648
  throw e;
544
- throw toProviderError(unwrapForClassification(e), workloadOf(session));
649
+ const failure = toProviderError(unwrapForClassification(e), workloadOf(session));
650
+ logFailure(session, failure, e);
651
+ throw failure;
545
652
  }
546
653
  }
547
654
  // The provider advertises the guard limit on its own instance, thus a
@@ -0,0 +1,37 @@
1
+ /**
2
+ * The two cpu files that a sandbox reads for its core count.
3
+ *
4
+ * The thread env (`thread-env.ts`) reaches each library that reads a thread
5
+ * count from the environment. Two base calls read a kernel file instead:
6
+ * `parallel::detectCores()` greps `/proc/cpuinfo`, and `os.cpu_count()` of
7
+ * Python 3.12 reads `/sys/devices/system/cpu/online` through glibc. Both files
8
+ * describe the host, not the cgroup. A read-only bind of a small file over each
9
+ * path makes both calls report the quota.
10
+ *
11
+ * This module makes the content of the two files, and it reads the
12
+ * `/proc/cpuinfo` of the harness host that the content is cut from. The Docker
13
+ * backend writes the content to the host and binds it. The K8s backend puts
14
+ * the content in a ConfigMap and mounts each key with `subPath`.
15
+ */
16
+ /** The two container paths that the cpu files cover. */
17
+ export declare const CPU_ONLINE_PATH = "/sys/devices/system/cpu/online";
18
+ export declare const CPUINFO_PATH = "/proc/cpuinfo";
19
+ export declare function readHostCpuinfoOnce(): Promise<string | undefined>;
20
+ export interface CpuFiles {
21
+ /** Content of `/sys/devices/system/cpu/online`: `0` for one thread, else the range `0-<n-1>`. */
22
+ readonly online: string;
23
+ /**
24
+ * Content of `/proc/cpuinfo`: the first n `processor` blocks of the real file
25
+ * of the host. `undefined` when the host gives no `/proc/cpuinfo`.
26
+ */
27
+ readonly cpuinfo: string | undefined;
28
+ }
29
+ /**
30
+ * The content of the two cpu files for `threads` cpus.
31
+ *
32
+ * `threads` is a positive integer. The caller applies the same floor as
33
+ * `threadLimitEnv`: `Math.max(1, Math.floor(spec.cpu))`. When the host has fewer
34
+ * processor blocks than `threads`, all blocks stay. Each block keeps its own
35
+ * text, thus the `processor` numbers stay `0` to `n-1` as in the real file.
36
+ */
37
+ export declare function cpuFiles(threads: number, hostCpuinfo: string | undefined): CpuFiles;
@@ -0,0 +1,55 @@
1
+ /**
2
+ * The two cpu files that a sandbox reads for its core count.
3
+ *
4
+ * The thread env (`thread-env.ts`) reaches each library that reads a thread
5
+ * count from the environment. Two base calls read a kernel file instead:
6
+ * `parallel::detectCores()` greps `/proc/cpuinfo`, and `os.cpu_count()` of
7
+ * Python 3.12 reads `/sys/devices/system/cpu/online` through glibc. Both files
8
+ * describe the host, not the cgroup. A read-only bind of a small file over each
9
+ * path makes both calls report the quota.
10
+ *
11
+ * This module makes the content of the two files, and it reads the
12
+ * `/proc/cpuinfo` of the harness host that the content is cut from. The Docker
13
+ * backend writes the content to the host and binds it. The K8s backend puts
14
+ * the content in a ConfigMap and mounts each key with `subPath`.
15
+ */
16
+ import { readFile } from "node:fs/promises";
17
+ /** The two container paths that the cpu files cover. */
18
+ export const CPU_ONLINE_PATH = "/sys/devices/system/cpu/online";
19
+ export const CPUINFO_PATH = "/proc/cpuinfo";
20
+ /**
21
+ * The `/proc/cpuinfo` of the harness host, read once per process. The content
22
+ * does not change while the process runs. A host with no such file, for
23
+ * example macOS, gives `undefined`, and the caller covers only `online`.
24
+ */
25
+ let hostCpuinfo;
26
+ export function readHostCpuinfoOnce() {
27
+ hostCpuinfo ??= readFile(CPUINFO_PATH, "utf8").catch(() => undefined);
28
+ return hostCpuinfo;
29
+ }
30
+ /**
31
+ * Split the text of `/proc/cpuinfo` into its `processor` blocks. A blank line
32
+ * ends a block. A block with no `processor` line, for example the `Hardware`
33
+ * trailer of an ARM kernel, is not a processor and is dropped.
34
+ */
35
+ function processorBlocks(cpuinfo) {
36
+ return cpuinfo
37
+ .split(/\n\s*\n/)
38
+ .map((block) => block.trim())
39
+ .filter((block) => /^processor\b/m.test(block));
40
+ }
41
+ /**
42
+ * The content of the two cpu files for `threads` cpus.
43
+ *
44
+ * `threads` is a positive integer. The caller applies the same floor as
45
+ * `threadLimitEnv`: `Math.max(1, Math.floor(spec.cpu))`. When the host has fewer
46
+ * processor blocks than `threads`, all blocks stay. Each block keeps its own
47
+ * text, thus the `processor` numbers stay `0` to `n-1` as in the real file.
48
+ */
49
+ export function cpuFiles(threads, hostCpuinfo) {
50
+ const online = `${threads === 1 ? "0" : `0-${threads - 1}`}\n`;
51
+ if (hostCpuinfo === undefined)
52
+ return { online, cpuinfo: undefined };
53
+ const blocks = processorBlocks(hostCpuinfo).slice(0, threads);
54
+ return { online, cpuinfo: blocks.map((block) => `${block}\n\n`).join("") };
55
+ }
@@ -7,6 +7,15 @@
7
7
  * `/mnt/libs` / `/mnt/refs` when their host paths are configured. Container
8
8
  * paths and lib-store env come from the shared mount plan (`mount-plan.ts`).
9
9
  *
10
+ * ## The cpu files
11
+ *
12
+ * Two more binds put a small file over `/sys/devices/system/cpu/online` and
13
+ * over `/proc/cpuinfo` (`cpu-files.ts`). Each file describes `spec.cpu` cpus,
14
+ * not the host. The host side of each bind is a file under
15
+ * `<parent of the workspace root>/.cpu/<sandboxId>/`, thus outside the tree
16
+ * that the sandbox sees. `teardown` removes the directory when this process
17
+ * wrote it. A directory from an earlier process stays.
18
+ *
10
19
  * ## Transport and confinement
11
20
  *
12
21
  * The container joins the default bridge and publishes sandbox-server's port to
@@ -69,6 +78,12 @@ export interface DockerClientConfig {
69
78
  docker?: Docker;
70
79
  /** Injected for tests so `/health` polling can be stubbed. */
71
80
  fetch?: typeof fetch;
81
+ /**
82
+ * Injected for tests so the `/proc/cpuinfo` of the host can be stubbed. The
83
+ * default reads the file once per process. `undefined` means that the host
84
+ * has no such file, for example macOS.
85
+ */
86
+ readHostCpuinfo?: () => Promise<string | undefined>;
72
87
  /**
73
88
  * Optional logger so a lib-store degradation (a configured store whose `current`
74
89
  * vanished or went incomplete by sandbox-create time) is observable instead of a
@@ -7,6 +7,15 @@
7
7
  * `/mnt/libs` / `/mnt/refs` when their host paths are configured. Container
8
8
  * paths and lib-store env come from the shared mount plan (`mount-plan.ts`).
9
9
  *
10
+ * ## The cpu files
11
+ *
12
+ * Two more binds put a small file over `/sys/devices/system/cpu/online` and
13
+ * over `/proc/cpuinfo` (`cpu-files.ts`). Each file describes `spec.cpu` cpus,
14
+ * not the host. The host side of each bind is a file under
15
+ * `<parent of the workspace root>/.cpu/<sandboxId>/`, thus outside the tree
16
+ * that the sandbox sees. `teardown` removes the directory when this process
17
+ * wrote it. A directory from an earlier process stays.
18
+ *
10
19
  * ## Transport and confinement
11
20
  *
12
21
  * The container joins the default bridge and publishes sandbox-server's port to
@@ -29,13 +38,16 @@
29
38
  * effective capabilities.
30
39
  */
31
40
  import { existsSync, lstatSync, statSync } from "node:fs";
32
- import { join } from "node:path";
41
+ import { mkdir, rm, writeFile } from "node:fs/promises";
42
+ import { dirname, join } from "node:path";
33
43
  import Docker from "dockerode";
34
44
  import { ResultAsync, err, ok } from "neverthrow";
35
45
  import { createNoopLogger } from "../lib/console-logger.js";
36
46
  import { tailWritePrefix } from "../workspace/paths.js";
37
47
  import { trySandbox } from "./sandbox-error.js";
38
48
  import { buildMountPlan, sandboxWriteTail } from "./mount-plan.js";
49
+ import { CPU_ONLINE_PATH, CPUINFO_PATH, cpuFiles, readHostCpuinfoOnce } from "./cpu-files.js";
50
+ import { threadLimitEnv } from "./thread-env.js";
39
51
  /** Read the originating HTTP status off any `SandboxError` variant that carries one. */
40
52
  function statusOf(e) {
41
53
  return "status" in e ? e.status : undefined;
@@ -60,6 +72,8 @@ const MANAGED_BY_VALUE = "cortex";
60
72
  const OWNER_WORKFLOW_LABEL = "cortex/owner-workflow-id";
61
73
  const RUN_ID_LABEL = "cortex/run-id";
62
74
  const STEP_ID_LABEL = "cortex/step-id";
75
+ /** The directory next to the workspace root that holds the cpu files of each sandbox. */
76
+ const CPU_FILES_DIR = ".cpu";
63
77
  /**
64
78
  * Whether the lib store's `current` resolves to a COMPLETE, usable version, not merely a
65
79
  * present symlink. By sandbox-create time `current` may be gone (concurrent prune/`rm`), a
@@ -104,6 +118,39 @@ function refStoreUsable(refStorePath) {
104
118
  return false;
105
119
  }
106
120
  }
121
+ /**
122
+ * Write the cpu files of one sandbox into `dir` and return their binds.
123
+ *
124
+ * The directory `.cpu` is made with no `recursive` option on purpose. The
125
+ * parent of the workspace root must exist already. Thus a root that is not on
126
+ * disk gives no directory at an unexpected place, and the sandbox starts with
127
+ * no cpu files. The failure is a degradation, not an error: the thread env
128
+ * still limits each library, but `detectCores()` and `os.cpu_count()` report
129
+ * the host.
130
+ */
131
+ async function writeCpuFiles(dir, files, logger, sandboxId) {
132
+ const ignoreExists = (cause) => {
133
+ if (cause.code !== "EEXIST")
134
+ throw cause;
135
+ };
136
+ try {
137
+ await mkdir(dirname(dir)).catch(ignoreExists);
138
+ await mkdir(dir).catch(ignoreExists);
139
+ const online = join(dir, "online");
140
+ await writeFile(online, files.online);
141
+ const binds = [`${online}:${CPU_ONLINE_PATH}:ro`];
142
+ if (files.cpuinfo !== undefined) {
143
+ const cpuinfo = join(dir, "cpuinfo");
144
+ await writeFile(cpuinfo, files.cpuinfo);
145
+ binds.push(`${cpuinfo}:${CPUINFO_PATH}:ro`);
146
+ }
147
+ return binds;
148
+ }
149
+ catch (cause) {
150
+ logger.warn("cpu files not written — the sandbox reports the cores of the host", { sandboxId, dir, cause });
151
+ return [];
152
+ }
153
+ }
107
154
  async function pollHealth(fetchImpl, url, timeoutMs) {
108
155
  const deadline = Date.now() + timeoutMs;
109
156
  let lastErr;
@@ -199,6 +246,9 @@ export function createDockerSandboxOps(config) {
199
246
  const fetchImpl = config.fetch ?? fetch;
200
247
  const transport = config.transport ?? "poll";
201
248
  const pollMode = transport === "poll";
249
+ const readHostCpuinfo = config.readHostCpuinfo ?? readHostCpuinfoOnce;
250
+ /** The cpu directory of each sandbox that this process wrote, for `teardown`. */
251
+ const cpuDirs = new Map();
202
252
  return {
203
253
  createSandbox(meta, identity) {
204
254
  return new ResultAsync((async () => {
@@ -228,6 +278,18 @@ export function createDockerSandboxOps(config) {
228
278
  // a crafted tail cannot escape the resolved root into the container. The
229
279
  // tail is the step directory, or the one the caller declared.
230
280
  const write = sandboxWriteTail(meta);
281
+ // The same floor as `threadLimitEnv`: a fractional cpu request is one cpu.
282
+ const threads = Math.max(1, Math.floor(meta.resources.cpu));
283
+ const cpuinfo = await readHostCpuinfo();
284
+ if (cpuinfo === undefined) {
285
+ logger.debug("no host /proc/cpuinfo — the sandbox gets the online file only", { sandboxId });
286
+ }
287
+ // Next to the workspace root, not under it: the root is bound read-only
288
+ // at `/{analysisId}`, and a file under it is visible in the sandbox.
289
+ const cpuDir = join(dirname(hostTreePath), CPU_FILES_DIR, sandboxId);
290
+ const cpuBinds = await writeCpuFiles(cpuDir, cpuFiles(threads, cpuinfo), logger, sandboxId);
291
+ if (cpuBinds.length > 0)
292
+ cpuDirs.set(sandboxId, cpuDir);
231
293
  const binds = [
232
294
  `${hostTreePath}:${plan.readonlyTreePath}:ro`,
233
295
  ...(write && plan.writableStepPath
@@ -235,6 +297,7 @@ export function createDockerSandboxOps(config) {
235
297
  : []),
236
298
  ...(libsMounted && config.libStorePath ? [`${config.libStorePath}:${plan.libsPath}:ro`] : []),
237
299
  ...(refsMounted && config.refStorePath ? [`${config.refStorePath}:${plan.refsPath}:ro`] : []),
300
+ ...cpuBinds,
238
301
  ];
239
302
  const createFailed = (status, cause) => ({
240
303
  type: "container_create_failed",
@@ -244,17 +307,22 @@ export function createDockerSandboxOps(config) {
244
307
  cause,
245
308
  });
246
309
  const image = meta.image ?? config.image;
310
+ const spec = meta.resources;
247
311
  // Poll mode never dials out and carries no CORTEX_BASE_URL; it sets the
248
312
  // firewall flag so the root entrypoint installs the egress block before
249
313
  // dropping to uid 1000. Callback mode is the inverse.
250
- const env = [
251
- `SANDBOX_TRANSPORT=${transport}`,
252
- `SANDBOX_CALLBACK_SECRET=${callbackSecret}`,
253
- ...(pollMode ? ["SANDBOX_EGRESS_FIREWALL=1"] : [`CORTEX_BASE_URL=${config.cortexBaseUrl}`]),
254
- ...Object.entries(plan.env).map(([k, v]) => `${k}=${v}`),
255
- ...Object.entries(meta.extraEnv ?? {}).map(([k, v]) => `${k}=${v}`),
256
- ];
257
- const spec = meta.resources;
314
+ //
315
+ // Composed as one record, not as a list of `K=V`. A duplicate key in
316
+ // the Env array resolves at the libc that scans `environ`, thus the
317
+ // later spread must win here, not there.
318
+ const env = Object.entries({
319
+ SANDBOX_TRANSPORT: transport,
320
+ SANDBOX_CALLBACK_SECRET: callbackSecret,
321
+ ...(pollMode ? { SANDBOX_EGRESS_FIREWALL: "1" } : { CORTEX_BASE_URL: config.cortexBaseUrl }),
322
+ ...threadLimitEnv(spec),
323
+ ...plan.env,
324
+ ...(meta.extraEnv ?? {}),
325
+ }).map(([k, v]) => `${k}=${v}`);
258
326
  const createOpts = {
259
327
  name: sandboxId,
260
328
  ...(config.platform !== undefined && { platform: config.platform }),
@@ -289,6 +357,11 @@ export function createDockerSandboxOps(config) {
289
357
  SecurityOpt: ["no-new-privileges"],
290
358
  NanoCpus: Math.round(spec.cpu * 1e9),
291
359
  Memory: spec.memoryGb * 1024 ** 3,
360
+ // `Memory` alone lets the container swap as much again. A fork
361
+ // storm then thrashes for minutes, and sandbox-server stops to
362
+ // answer. With no swap the OOM killer removes the largest fork in
363
+ // seconds, and sandbox-server survives. K8s runs with no swap.
364
+ MemorySwap: spec.memoryGb * 1024 ** 3,
292
365
  AutoRemove: false,
293
366
  },
294
367
  };
@@ -329,10 +402,10 @@ export function createDockerSandboxOps(config) {
329
402
  })());
330
403
  },
331
404
  teardown(ref) {
332
- return removeContainerIgnoreMissing(docker, ref.sandboxId);
405
+ return teardownSandbox(ref.sandboxId);
333
406
  },
334
407
  teardownById(sandboxId) {
335
- return removeContainerIgnoreMissing(docker, sandboxId);
408
+ return teardownSandbox(sandboxId);
336
409
  },
337
410
  listManagedSandboxes() {
338
411
  return trySandbox(() => docker.listContainers({
@@ -361,6 +434,22 @@ export function createDockerSandboxOps(config) {
361
434
  },
362
435
  isAliveById,
363
436
  };
437
+ /**
438
+ * Remove the container, then the cpu files that this process wrote for it.
439
+ * A failure of the file removal does not fail the teardown: the container is
440
+ * gone, and a stale small directory is not a risk to a later sandbox.
441
+ */
442
+ function teardownSandbox(sandboxId) {
443
+ return removeContainerIgnoreMissing(docker, sandboxId).map(async () => {
444
+ const cpuDir = cpuDirs.get(sandboxId);
445
+ if (cpuDir === undefined)
446
+ return;
447
+ cpuDirs.delete(sandboxId);
448
+ await rm(cpuDir, { recursive: true, force: true }).catch((cause) => {
449
+ logger.debug("cpu files not removed", { sandboxId, cpuDir, cause });
450
+ });
451
+ });
452
+ }
364
453
  function isAliveById(sandboxId) {
365
454
  return trySandbox(async () => {
366
455
  const info = await docker.getContainer(sandboxId).inspect();
@@ -60,6 +60,11 @@ export interface K8sClientConfig {
60
60
  tolerations?: V1Toleration[];
61
61
  /** RuntimeClass for runtime isolation (e.g. `gvisor`). Omit for the default runtime. */
62
62
  runtimeClassName?: string;
63
+ /**
64
+ * The `/proc/cpuinfo` text that the cpu ConfigMap of each Job is cut from.
65
+ * Defaults to the file of the host of this process, read once. Injected for tests.
66
+ */
67
+ readHostCpuinfo?: () => Promise<string | undefined>;
63
68
  /** Injected for tests. */
64
69
  batchApi?: BatchV1Api;
65
70
  coreApi?: CoreV1Api;
@@ -16,6 +16,8 @@ import { BatchV1Api, CoreV1Api, KubeConfig, } from "@kubernetes/client-node";
16
16
  import { ResultAsync, err, ok } from "neverthrow";
17
17
  import { trySandbox } from "./sandbox-error.js";
18
18
  import { buildMountPlan, buildSessionSubPaths } from "./mount-plan.js";
19
+ import { threadLimitEnv } from "./thread-env.js";
20
+ import { CPU_ONLINE_PATH, CPUINFO_PATH, cpuFiles, readHostCpuinfoOnce } from "./cpu-files.js";
19
21
  import { createNoopLogger } from "../lib/console-logger.js";
20
22
  /** Read the originating HTTP status off any `SandboxError` variant that carries one. */
21
23
  function statusOf(e) {
@@ -24,6 +26,8 @@ function statusOf(e) {
24
26
  const SESSION_VOLUME_NAME = "session";
25
27
  const LIBS_VOLUME_NAME = "libs";
26
28
  const REFS_VOLUME_NAME = "refs";
29
+ /** The ConfigMap volume that holds the two cpu files of the sandbox (`cpu-files.ts`). */
30
+ const CPU_VOLUME_NAME = "cpu";
27
31
  const SANDBOX_SERVER_PORT = 8765;
28
32
  const POD_READY_TIMEOUT_MS = 5 * 60_000;
29
33
  const POD_POLL_INTERVAL_MS = 1_000;
@@ -59,6 +63,19 @@ export function sanitizeLabelValue(value) {
59
63
  function readOwnerWorkflowId(metadata) {
60
64
  return metadata?.annotations?.[OWNER_WORKFLOW_ANNOTATION] ?? null;
61
65
  }
66
+ function deleteConfigMapIgnoreMissing(coreApi, namespace, name) {
67
+ return trySandbox(async () => {
68
+ await coreApi.deleteNamespacedConfigMap({ namespace, name });
69
+ }, (status, cause) => ({
70
+ type: "teardown_failed",
71
+ op: "k8s.teardown",
72
+ sandboxId: name,
73
+ status,
74
+ cause,
75
+ })).orElse((e) =>
76
+ // A 404 means the owner reference removed it with the Job already.
77
+ statusOf(e) === 404 ? ok(undefined) : err(e));
78
+ }
62
79
  function deleteJobIgnoreMissing(batchApi, namespace, name) {
63
80
  return trySandbox(async () => {
64
81
  await batchApi.deleteNamespacedJob({
@@ -107,31 +124,32 @@ function workspaceSubPathFor(config, analysisId) {
107
124
  }
108
125
  return posix;
109
126
  }
110
- function buildJobSpec(meta, config, identity) {
127
+ function buildJobSpec(meta, config, identity, cpu) {
111
128
  const { sandboxId } = identity;
112
129
  const plan = buildMountPlan(meta, {
113
130
  libs: !!config.libStorePvc,
114
131
  refs: !!config.refStorePvc,
115
132
  });
116
133
  const transport = config.transport ?? "poll";
117
- const env = [
118
- { name: "SANDBOX_TRANSPORT", value: transport },
134
+ const spec = meta.resources;
135
+ // Composed as one record, not as a list of `{name, value}`. A duplicate name
136
+ // in the env of a container resolves at the kubelet, thus the later spread
137
+ // must win here, not there.
138
+ const env = Object.entries({
139
+ SANDBOX_TRANSPORT: transport,
119
140
  // Poll mode never dials out, so the URL is omitted (matching the Docker
120
141
  // backend) — the pod spec itself then documents that no callback egress
121
142
  // is expected. sandbox-server neither reads nor requires it in poll mode.
122
- ...(transport === "callback" ? [{ name: "CORTEX_BASE_URL", value: config.cortexBaseUrl }] : []),
123
- { name: "SANDBOX_CALLBACK_SECRET", value: identity.callbackSecret },
124
- ...Object.entries(plan.env).map(([name, value]) => ({ name, value })),
125
- ...Object.entries(meta.extraEnv ?? {}).map(([k, v]) => ({
126
- name: k,
127
- value: v,
128
- })),
129
- ];
143
+ ...(transport === "callback" ? { CORTEX_BASE_URL: config.cortexBaseUrl } : {}),
144
+ SANDBOX_CALLBACK_SECRET: identity.callbackSecret,
145
+ ...threadLimitEnv(spec),
146
+ ...plan.env,
147
+ ...(meta.extraEnv ?? {}),
148
+ }).map(([name, value]) => ({ name, value }));
130
149
  // requests == limits → Guaranteed QoS and a predictable OOM bound; also
131
150
  // satisfies the namespace ResourceQuota, which rejects any pod that omits
132
151
  // requests.cpu/memory. The CPU limit can throttle a hot step, but a throttle
133
152
  // beats an eviction.
134
- const spec = meta.resources;
135
153
  const quantities = {
136
154
  cpu: String(spec.cpu),
137
155
  memory: `${spec.memoryGb}Gi`,
@@ -187,6 +205,16 @@ function buildJobSpec(meta, config, identity) {
187
205
  readOnly: true,
188
206
  });
189
207
  }
208
+ // The cpu files ride in a ConfigMap named after the Job (`ensureCpuConfigMap`).
209
+ // A `subPath` mount of one key over one kernel path is accepted by the API
210
+ // server, the kubelet, and gVisor. Under gVisor the sandbox sees no cgroup
211
+ // at all, thus these two files are the only source of the quota for
212
+ // `parallel::detectCores()` and `os.cpu_count()`.
213
+ volumes.push({ name: CPU_VOLUME_NAME, configMap: { name: sandboxId } });
214
+ volumeMounts.push({ name: CPU_VOLUME_NAME, mountPath: CPU_ONLINE_PATH, subPath: "online", readOnly: true });
215
+ if (cpu.hasCpuinfo) {
216
+ volumeMounts.push({ name: CPU_VOLUME_NAME, mountPath: CPUINFO_PATH, subPath: "cpuinfo", readOnly: true });
217
+ }
190
218
  const podSpec = {
191
219
  automountServiceAccountToken: false,
192
220
  restartPolicy: "Never",
@@ -357,6 +385,9 @@ function waitForJobGone(batchApi, namespace, name) {
357
385
  * means an (astronomically rare) name collision with a *different* step —
358
386
  * adopting its pod would HMAC-fail every callback, and deleting its Job would
359
387
  * kill a live sibling. Refuse loudly instead.
388
+ *
389
+ * The result is the uid of the Job that the pod belongs to, for the owner
390
+ * reference of the cpu ConfigMap.
360
391
  */
361
392
  function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflowId, job) {
362
393
  const createFailed = (status, cause) => ({
@@ -369,7 +400,7 @@ function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflow
369
400
  return new ResultAsync((async () => {
370
401
  const created = await trySandbox(() => batchApi.createNamespacedJob({ namespace, body: job }), createFailed);
371
402
  if (created.isOk())
372
- return ok(undefined);
403
+ return ok(created.value?.metadata?.uid);
373
404
  if (statusOf(created.error) !== 409)
374
405
  return err(created.error);
375
406
  const existingRead = await trySandbox(() => batchApi.readNamespacedJob({ namespace, name: sandboxId }), createFailed);
@@ -397,20 +428,92 @@ function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflow
397
428
  const recreated = await trySandbox(() => batchApi.createNamespacedJob({ namespace, body: job }), createFailed);
398
429
  if (recreated.isErr())
399
430
  return err(recreated.error);
431
+ return ok(recreated.value?.metadata?.uid);
400
432
  }
433
+ return ok(existingRead.value.metadata?.uid);
434
+ })());
435
+ }
436
+ /**
437
+ * Make the cpu ConfigMap of a Job, or bring an existing one up to date.
438
+ *
439
+ * The ConfigMap comes after the Job, because the Job is its owner. Thus the
440
+ * TTL of the Job and a Foreground delete both remove the ConfigMap with it.
441
+ * The pod waits at its mount until the ConfigMap exists, and that wait is
442
+ * shorter than the schedule of the pod.
443
+ *
444
+ * A `409 AlreadyExists` is the recovery re-run of the spawn step, or a
445
+ * ConfigMap whose prior owner Job is not yet collected. Both cases get the
446
+ * same data and the current owner through a replace.
447
+ */
448
+ function ensureCpuConfigMap(coreApi, namespace, sandboxId, jobUid, files) {
449
+ const createFailed = (status, cause) => ({
450
+ type: "container_create_failed",
451
+ op: "k8s.createNamespacedConfigMap",
452
+ sandboxId,
453
+ status,
454
+ cause,
455
+ });
456
+ const body = {
457
+ apiVersion: "v1",
458
+ kind: "ConfigMap",
459
+ metadata: {
460
+ name: sandboxId,
461
+ namespace,
462
+ labels: {
463
+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
464
+ role: "sandbox",
465
+ "cortex/sandbox-id": sandboxId,
466
+ },
467
+ ...(jobUid !== undefined && {
468
+ ownerReferences: [{ apiVersion: "batch/v1", kind: "Job", name: sandboxId, uid: jobUid }],
469
+ }),
470
+ },
471
+ data: {
472
+ online: files.online,
473
+ ...(files.cpuinfo !== undefined && { cpuinfo: files.cpuinfo }),
474
+ },
475
+ };
476
+ return new ResultAsync((async () => {
477
+ const created = await trySandbox(() => coreApi.createNamespacedConfigMap({ namespace, body }), createFailed);
478
+ if (created.isOk())
479
+ return ok(undefined);
480
+ if (statusOf(created.error) !== 409)
481
+ return err(created.error);
482
+ const existing = await trySandbox(() => coreApi.readNamespacedConfigMap({ namespace, name: sandboxId }), createFailed);
483
+ if (existing.isErr())
484
+ return err(existing.error);
485
+ const replaced = await trySandbox(() => coreApi.replaceNamespacedConfigMap({
486
+ namespace,
487
+ name: sandboxId,
488
+ body: { ...body, metadata: { ...body.metadata, resourceVersion: existing.value.metadata?.resourceVersion } },
489
+ }), createFailed);
490
+ if (replaced.isErr())
491
+ return err(replaced.error);
401
492
  return ok(undefined);
402
493
  })());
403
494
  }
404
495
  export function createK8sSandboxOps(config) {
405
496
  const { batchApi, coreApi } = config.batchApi && config.coreApi ? { batchApi: config.batchApi, coreApi: config.coreApi } : buildKubeApi();
497
+ const readHostCpuinfo = config.readHostCpuinfo ?? readHostCpuinfoOnce;
406
498
  return {
407
499
  createSandbox(meta, identity) {
408
500
  return new ResultAsync((async () => {
409
501
  const { sandboxId } = identity;
410
- const job = buildJobSpec(meta, config, identity);
502
+ // The same floor as `threadLimitEnv`: a fraction of a core still
503
+ // keeps one thread busy, and a count of zero is not representable.
504
+ const threads = Math.max(1, Math.floor(meta.resources.cpu));
505
+ const files = cpuFiles(threads, await readHostCpuinfo());
506
+ const job = buildJobSpec(meta, config, identity, { hasCpuinfo: files.cpuinfo !== undefined });
411
507
  const adopted = await createOrAdoptJob(batchApi, coreApi, config.namespace, sandboxId, meta.childWorkflowId, job);
412
508
  if (adopted.isErr())
413
509
  return err(adopted.error);
510
+ const cpuMap = await ensureCpuConfigMap(coreApi, config.namespace, sandboxId, adopted.value, files);
511
+ if (cpuMap.isErr()) {
512
+ // A pod whose ConfigMap never comes stays at ContainerCreating
513
+ // until the ready wait expires. Fail now, with the real cause.
514
+ await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
515
+ return err(cpuMap.error);
516
+ }
414
517
  // A Job whose pod never schedules (quota rejection, no nodes, spot
415
518
  // reclaim) is reaped by neither backoffLimit (an admission rejection
416
519
  // creates no pod to count) nor ttlSecondsAfterFinished (the Job never
@@ -434,20 +537,22 @@ export function createK8sSandboxOps(config) {
434
537
  cause,
435
538
  }));
436
539
  if (registered.isErr()) {
437
- await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
540
+ await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
438
541
  return err(registered.error);
439
542
  }
440
543
  return ok(ref);
441
544
  }
442
- await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
545
+ await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
443
546
  return err(ready.error);
444
547
  })());
445
548
  },
549
+ // The owner reference removes the cpu ConfigMap with the Job. The explicit
550
+ // delete covers a ConfigMap that got no owner, and it is a no-op on 404.
446
551
  teardown(ref) {
447
- return deleteJobIgnoreMissing(batchApi, config.namespace, ref.sandboxId);
552
+ return deleteJobIgnoreMissing(batchApi, config.namespace, ref.sandboxId).andThen(() => deleteConfigMapIgnoreMissing(coreApi, config.namespace, ref.sandboxId));
448
553
  },
449
554
  teardownById(sandboxId) {
450
- return deleteJobIgnoreMissing(batchApi, config.namespace, sandboxId);
555
+ return deleteJobIgnoreMissing(batchApi, config.namespace, sandboxId).andThen(() => deleteConfigMapIgnoreMissing(coreApi, config.namespace, sandboxId));
451
556
  },
452
557
  listManagedSandboxes() {
453
558
  return trySandbox(() => batchApi.listNamespacedJob({
@@ -508,9 +613,13 @@ export function createK8sSandboxOps(config) {
508
613
  * matters; a delete error here is logged and swallowed so it never masks the
509
614
  * create error returned to the caller.
510
615
  */
511
- async function cleanupFailedJob(batchApi, namespace, sandboxId, logger) {
616
+ async function cleanupFailedJob(batchApi, coreApi, namespace, sandboxId, logger) {
512
617
  const deleted = await deleteJobIgnoreMissing(batchApi, namespace, sandboxId);
513
618
  if (deleted.isErr()) {
514
619
  logger.named("k8s-client").error("failed to delete Job after startup failure", { sandboxId, err: deleted.error });
515
620
  }
621
+ const deletedMap = await deleteConfigMapIgnoreMissing(coreApi, namespace, sandboxId);
622
+ if (deletedMap.isErr()) {
623
+ logger.named("k8s-client").error("failed to delete cpu ConfigMap after startup failure", { sandboxId, err: deletedMap.error });
624
+ }
516
625
  }
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Thread-count env for a sandbox container.
3
+ *
4
+ * Each backend turns the `ResourceSpec` of a step into a hard cgroup limit:
5
+ * `NanoCpus` and `Memory` on Docker, `limits.cpu` and `limits.memory` on K8s. A
6
+ * cgroup quota is invisible to most runtimes inside it. `parallel::detectCores()`
7
+ * greps `/proc/cpuinfo`, and `os.cpu_count()` reads
8
+ * `/sys/devices/system/cpu/online`. Both files describe the host. Thus a step
9
+ * with 2 CPUs on a host of 12 cores forks 12 workers into a cgroup of 2 CPUs,
10
+ * and each fork copies its working set until the memory limit stops it.
11
+ *
12
+ * The env below publishes the quota to each library that reads a thread count
13
+ * from the environment. `parallelly::availableCores()` and `joblib.cpu_count()`
14
+ * read the cgroup quota on their own, thus they need no value here.
15
+ *
16
+ * `MC_CORES` is absent on purpose. R defaults `mclapply` to 2 workers, not to
17
+ * the core count. A value there raises the fork count instead of a decrease.
18
+ */
19
+ import type { ResourceSpec } from "../config/resource-limits.js";
20
+ /**
21
+ * Env that binds each parallel library in the container to `spec.cpu`.
22
+ *
23
+ * A fractional CPU request floors to one thread. A pool of zero is not
24
+ * representable, and two threads on half a core only add context switches.
25
+ *
26
+ * The caller puts these before `plan.env` and before the `extraEnv` of the
27
+ * embedder, thus a host that knows more about a step keeps the last word.
28
+ */
29
+ export declare function threadLimitEnv(spec: ResourceSpec): Record<string, string>;
@@ -0,0 +1,49 @@
1
+ /**
2
+ * Thread-count env for a sandbox container.
3
+ *
4
+ * Each backend turns the `ResourceSpec` of a step into a hard cgroup limit:
5
+ * `NanoCpus` and `Memory` on Docker, `limits.cpu` and `limits.memory` on K8s. A
6
+ * cgroup quota is invisible to most runtimes inside it. `parallel::detectCores()`
7
+ * greps `/proc/cpuinfo`, and `os.cpu_count()` reads
8
+ * `/sys/devices/system/cpu/online`. Both files describe the host. Thus a step
9
+ * with 2 CPUs on a host of 12 cores forks 12 workers into a cgroup of 2 CPUs,
10
+ * and each fork copies its working set until the memory limit stops it.
11
+ *
12
+ * The env below publishes the quota to each library that reads a thread count
13
+ * from the environment. `parallelly::availableCores()` and `joblib.cpu_count()`
14
+ * read the cgroup quota on their own, thus they need no value here.
15
+ *
16
+ * `MC_CORES` is absent on purpose. R defaults `mclapply` to 2 workers, not to
17
+ * the core count. A value there raises the fork count instead of a decrease.
18
+ */
19
+ /**
20
+ * Env that binds each parallel library in the container to `spec.cpu`.
21
+ *
22
+ * A fractional CPU request floors to one thread. A pool of zero is not
23
+ * representable, and two threads on half a core only add context switches.
24
+ *
25
+ * The caller puts these before `plan.env` and before the `extraEnv` of the
26
+ * embedder, thus a host that knows more about a step keeps the last word.
27
+ */
28
+ export function threadLimitEnv(spec) {
29
+ const threads = String(Math.max(1, Math.floor(spec.cpu)));
30
+ return {
31
+ // OpenMP: the compiled code of R and Python packages.
32
+ OMP_NUM_THREADS: threads,
33
+ // OpenBLAS keeps a private pool. A pthread build reads this name first.
34
+ OPENBLAS_NUM_THREADS: threads,
35
+ // MKL reads this name before OMP_NUM_THREADS.
36
+ MKL_NUM_THREADS: threads,
37
+ // BiocParallel sizes `bpparam()` from this name. DESeq2 `parallel=TRUE`
38
+ // and the other Bioconductor packages size their workers from `bpparam()`.
39
+ BIOCPARALLEL_WORKER_NUMBER: threads,
40
+ // data.table defaults to half of the host cores.
41
+ R_DATATABLE_NUM_THREADS: threads,
42
+ // The Python numerical stack.
43
+ NUMBA_NUM_THREADS: threads,
44
+ NUMEXPR_MAX_THREADS: threads,
45
+ LOKY_MAX_CPU_COUNT: threads,
46
+ POLARS_MAX_THREADS: threads,
47
+ RAYON_NUM_THREADS: threads,
48
+ };
49
+ }
@@ -171,6 +171,41 @@ export declare function reconcileOrphanedDataProfile(pool: Querier, analysisId:
171
171
  * finds no running row to stamp and no-ops instead of re-seeding the cleared one.
172
172
  */
173
173
  export declare function clearDataProfile(pool: Querier, analysisId: string): ResultAsync<boolean, DbError>;
174
+ /**
175
+ * Stamp a row `completed` with no result, because the analysis has no input files.
176
+ *
177
+ * A profile describes files. An analysis with no files has nothing to describe, and
178
+ * thus the profile is complete at once: no claim into `running`, no workflow, and no
179
+ * sandbox. The write is a terminal write like {@link completeDataProfile}, but it starts
180
+ * from a settled row or an empty row, not from a `running` one. As a result
181
+ * `data_profile_started_at` and `data_profile_completed_at` carry the same instant, and
182
+ * `data_profile_workflow_id` stays NULL, because no stream is addressable (see
183
+ * {@link recordDataProfileWorkflowId}).
184
+ *
185
+ * The seed is written as `[]`, so the completed row names the set that it covers: the
186
+ * empty set. A `completed` row with a NULL seed stays forbidden (see
187
+ * {@link completeDataProfile}). A prior `data_profile_result` is replaced with NULL,
188
+ * because it describes files that the analysis no longer has. A prior `failed` row is
189
+ * replaced too, because the set that failed is gone.
190
+ *
191
+ * Two guards ride in the CAS:
192
+ * - `IS DISTINCT FROM 'running'`: a live workflow owns the row, and its terminal write
193
+ * must not land on a row that this write moved. The same reason as
194
+ * {@link clearDataProfile}.
195
+ * - {@link UNSEEDED}: the row names no file. A seed upsert that lands between a caller's
196
+ * pre-read and this write makes the row a seeded, startable row. A `completed` stamp
197
+ * over it would hide those files behind a finished profile. The CAS refuses that row,
198
+ * and the caller reads the status again to name the cause.
199
+ *
200
+ * The rerun claim carries {@link SEEDED}, so a completed-empty row is not claimable until
201
+ * a later seed upsert names files. Then `tryRerunDataProfile` takes it, the same as any
202
+ * completed row.
203
+ *
204
+ * `ok(true)` when this call stamped the row. `ok(false)` when the guard refused it: no
205
+ * such analysis, a live run, or a seed that names files. A refusal stays in the ok
206
+ * channel, the same as the sibling CAS ops.
207
+ */
208
+ export declare function completeEmptyDataProfile(pool: Querier, analysisId: string): ResultAsync<boolean, DbError>;
174
209
  /**
175
210
  * Load an analysis's data-profile ledger state. Returns `null` for BOTH miss
176
211
  * conditions, deliberately indistinguishable to consumers:
@@ -20,6 +20,13 @@ import { tryMutation, tryQuery } from "../lib/db-result.js";
20
20
  * and surfacing a hand-corrupted row as a `DbError` beats silently claiming it.
21
21
  */
22
22
  const SEEDED = "seed_input_file_ids IS NOT NULL AND jsonb_array_length(seed_input_file_ids) > 0";
23
+ /**
24
+ * The negation of {@link SEEDED}: the row names no input file. A NULL seed and `[]`
25
+ * both name zero files, and {@link completeEmptyDataProfile} treats them the same.
26
+ * The two predicates are disjoint, so a claim into `running` and an empty-set
27
+ * completion can never both win on one row.
28
+ */
29
+ const UNSEEDED = "(seed_input_file_ids IS NULL OR jsonb_array_length(seed_input_file_ids) = 0)";
23
30
  /**
24
31
  * Try to claim a startable row into `running`. Startable means `'pending'` (seeded,
25
32
  * not yet run) or NULL (no profile: never profiled, or cleared by `clearDataProfile`).
@@ -279,6 +286,55 @@ export function clearDataProfile(pool, analysisId) {
279
286
  return (result.rowCount ?? 0) > 0;
280
287
  });
281
288
  }
289
+ /**
290
+ * Stamp a row `completed` with no result, because the analysis has no input files.
291
+ *
292
+ * A profile describes files. An analysis with no files has nothing to describe, and
293
+ * thus the profile is complete at once: no claim into `running`, no workflow, and no
294
+ * sandbox. The write is a terminal write like {@link completeDataProfile}, but it starts
295
+ * from a settled row or an empty row, not from a `running` one. As a result
296
+ * `data_profile_started_at` and `data_profile_completed_at` carry the same instant, and
297
+ * `data_profile_workflow_id` stays NULL, because no stream is addressable (see
298
+ * {@link recordDataProfileWorkflowId}).
299
+ *
300
+ * The seed is written as `[]`, so the completed row names the set that it covers: the
301
+ * empty set. A `completed` row with a NULL seed stays forbidden (see
302
+ * {@link completeDataProfile}). A prior `data_profile_result` is replaced with NULL,
303
+ * because it describes files that the analysis no longer has. A prior `failed` row is
304
+ * replaced too, because the set that failed is gone.
305
+ *
306
+ * Two guards ride in the CAS:
307
+ * - `IS DISTINCT FROM 'running'`: a live workflow owns the row, and its terminal write
308
+ * must not land on a row that this write moved. The same reason as
309
+ * {@link clearDataProfile}.
310
+ * - {@link UNSEEDED}: the row names no file. A seed upsert that lands between a caller's
311
+ * pre-read and this write makes the row a seeded, startable row. A `completed` stamp
312
+ * over it would hide those files behind a finished profile. The CAS refuses that row,
313
+ * and the caller reads the status again to name the cause.
314
+ *
315
+ * The rerun claim carries {@link SEEDED}, so a completed-empty row is not claimable until
316
+ * a later seed upsert names files. Then `tryRerunDataProfile` takes it, the same as any
317
+ * completed row.
318
+ *
319
+ * `ok(true)` when this call stamped the row. `ok(false)` when the guard refused it: no
320
+ * such analysis, a live run, or a seed that names files. A refusal stays in the ok
321
+ * channel, the same as the sibling CAS ops.
322
+ */
323
+ export function completeEmptyDataProfile(pool, analysisId) {
324
+ const now = new Date().toISOString();
325
+ return tryMutation("dataProfile.completeEmptyDataProfile", async () => {
326
+ const result = await pool.query({
327
+ text: `UPDATE cortex_analysis_state
328
+ SET data_profile_status = 'completed', data_profile_error = NULL,
329
+ data_profile_started_at = $1, data_profile_completed_at = $1,
330
+ data_profile_result = NULL, data_profile_workflow_id = NULL,
331
+ seed_input_file_ids = '[]'::jsonb
332
+ WHERE analysis_id = $2 AND data_profile_status IS DISTINCT FROM 'running' AND ${UNSEEDED}`,
333
+ values: [now, analysisId],
334
+ });
335
+ return (result.rowCount ?? 0) > 0;
336
+ });
337
+ }
282
338
  /**
283
339
  * Load an analysis's data-profile ledger state. Returns `null` for BOTH miss
284
340
  * conditions, deliberately indistinguishable to consumers:
@@ -20,7 +20,7 @@ export { setSandboxRef, setActiveExecId, clearSandboxRef, queryActiveSandboxes,
20
20
  export type { ActiveSandboxRow } from "./active-sandboxes.js";
21
21
  export { insertPlan, upsertPlan, loadPlan } from "./plans.js";
22
22
  export type { InsertPlanInput, UpsertPlanInput } from "./plans.js";
23
- export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
23
+ export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, completeEmptyDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
24
24
  export type { DataProfileStatus, DataProfileResult, DataProfileChecked, DataProfileCompanionCompleteness, DataProfileDimension, DataProfileGroup, DataProfileGroupSlot, DataProfileFile, DataProfileMemberAnnotation, DataProfileObservation, DataProfilePartition, DataProfileProbeReport, DataProfileQualityAssessment, DataProfileQuarantine, DataProfileRecipeStep, DataProfileKind, DataProfileAxis, DataProfileInputSignature, DataProfileCoverage, } from "./data-profile.js";
25
25
  export { queryRunCountsByAnalyses, queryThreadCountsByAnalyses, queryDataProfileStatusByAnalyses } from "./analyses-metrics.js";
26
26
  export { createAnalysisPurge } from "./purge-analysis.js";
@@ -14,7 +14,7 @@ setRunSynthesisOutcome, queryRun, queryActiveRun, queryNonTerminalRunsByAnalysis
14
14
  export { insertStepExecution, seedStepExecutions, sweepPendingStepExecutions, updateStepExecution, queryStepsByRun } from "./step-executions.js";
15
15
  export { setSandboxRef, setActiveExecId, clearSandboxRef, queryActiveSandboxes, reconcileReapedSandbox } from "./active-sandboxes.js";
16
16
  export { insertPlan, upsertPlan, loadPlan } from "./plans.js";
17
- export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
17
+ export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, completeEmptyDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
18
18
  export { queryRunCountsByAnalyses, queryThreadCountsByAnalyses, queryDataProfileStatusByAnalyses } from "./analyses-metrics.js";
19
19
  // Not per-entity: one analysis's whole Postgres footprint, reached from its id
20
20
  // alone, with the workflow ledger behind an injected seam.
@@ -132,7 +132,16 @@ export declare function registerDataProfileWorkflow(deps: DataProfileDeps): (inp
132
132
  * 'failed' on error, and revokes the run authorization on every terminal path.
133
133
  */
134
134
  export declare function runDataProfileBody(input: DataProfileWorkflowInput, deps: DataProfileDeps): Promise<void>;
135
- export type DataProfileTriggerResult = "started" | "restarted" | "already_running" | "failed";
135
+ /**
136
+ * What the trigger did:
137
+ * - `"started"`: a first profile claimed the row and a workflow runs.
138
+ * - `"restarted"`: a re-profile claimed a `completed` row and a workflow runs.
139
+ * - `"already_running"`: a workflow already owns the row, and nothing new ran.
140
+ * - `"completed"`: the analysis has no input files, so the row is `completed` at once
141
+ * with no result. No workflow ran (see {@link completeEmptyDataProfile}).
142
+ * - `"failed"`: the trigger refused the call or faulted, and the row is unchanged.
143
+ */
144
+ export type DataProfileTriggerResult = "started" | "restarted" | "already_running" | "completed" | "failed";
136
145
  /**
137
146
  * Route-side deps for triggering the data-profile workflow: the ledger pool,
138
147
  * the run authorizer, and the registered workflow callable. The body's
@@ -188,6 +197,12 @@ export declare function dataProfileWorkflowId(analysisId: string, nonce: string)
188
197
  * row is claimed by neither: retrying a failure is a deliberate act the caller drives
189
198
  * through `tryRetryDataProfile` + `runDataProfile`.
190
199
  *
200
+ * An analysis with no input files takes none of the claims. An empty manifest against
201
+ * a seed that names no file stamps the row `completed` at once, with no result and no
202
+ * workflow (see {@link completeEmptyDataProfile}), and the trigger returns
203
+ * `"completed"`. An empty manifest against a seed that names files is a divergence
204
+ * between the caller and the ledger, and the trigger refuses it.
205
+ *
191
206
  * Returns what happened, so the caller can surface it (e.g. in the seed response).
192
207
  */
193
208
  export declare function triggerDataProfile(deps: DataProfileTriggerDeps, params: DataProfileTriggerParams): Promise<DataProfileTriggerResult>;
@@ -44,7 +44,7 @@ import { scanInputTree } from "../input-scan/scan.js";
44
44
  import { PROFILE_INDEX_TYPES, buildProfileIndexEntries } from "./data-profile-index.js";
45
45
  import { absorbRecipe, renderAbsorbDelta } from "./data-profile-absorb.js";
46
46
  import { formatResolutionErrors, resolveProfileSubmission } from "./data-profile-resolve.js";
47
- import { completeDataProfile, failDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, tryRerunDataProfile, tryStartDataProfile, upsertArtifacts, } from "../state/index.js";
47
+ import { completeDataProfile, completeEmptyDataProfile, failDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, tryRerunDataProfile, tryStartDataProfile, upsertArtifacts, } from "../state/index.js";
48
48
  import { createProfileActivityEmitter } from "./data-profile-activity.js";
49
49
  import { ensureSearchIndex, searchIndexName } from "../workspace/search-config.js";
50
50
  // The run literal is declared in `contracts/` — a consumer reads it back off recorded usage, and this
@@ -668,6 +668,12 @@ async function startDataProfileWorkflow(deps, params) {
668
668
  * row is claimed by neither: retrying a failure is a deliberate act the caller drives
669
669
  * through `tryRetryDataProfile` + `runDataProfile`.
670
670
  *
671
+ * An analysis with no input files takes none of the claims. An empty manifest against
672
+ * a seed that names no file stamps the row `completed` at once, with no result and no
673
+ * workflow (see {@link completeEmptyDataProfile}), and the trigger returns
674
+ * `"completed"`. An empty manifest against a seed that names files is a divergence
675
+ * between the caller and the ledger, and the trigger refuses it.
676
+ *
671
677
  * Returns what happened, so the caller can surface it (e.g. in the seed response).
672
678
  */
673
679
  export async function triggerDataProfile(deps, params) {
@@ -682,17 +688,32 @@ export async function triggerDataProfile(deps, params) {
682
688
  // `loadSeedInputFileIds`, which returns null for BOTH a missing analysis row and
683
689
  // a NULL-seed row (`loadDataProfileStatus` would hide the seed of a cleared row).
684
690
  const seeded = unwrapOrThrow(await loadSeedInputFileIds(deps.pool, analysisId));
685
- if (seeded === null || seeded.length === 0) {
686
- logger.error("trigger rejected: no seeded input set (missing analysis row or caller skipped seeding)");
691
+ if (params.stagedInputs.length === 0) {
692
+ // The trigger validates the manifest it is about to dispatch against the ledger
693
+ // seed. A seeded row profiled against an EMPTY manifest would claim `running` and
694
+ // then hit the body's empty-manifest path, and complete with a NULL result while
695
+ // the seed still names files. Refuse the divergence before any claim.
696
+ if (seeded !== null && seeded.length > 0) {
697
+ logger.error("trigger rejected: empty manifest dispatched against a non-empty seed", { seededCount: seeded.length });
698
+ return "failed";
699
+ }
700
+ // The analysis has no input files, so there is nothing to profile. The row is
701
+ // `completed` at once, with no workflow. The CAS carries the same seed
702
+ // predicate as this pre-read, so a seed upsert that lands in between cannot be
703
+ // hidden behind a finished profile: the stamp refuses, and the status read
704
+ // below names the cause.
705
+ if (unwrapOrThrow(await completeEmptyDataProfile(deps.pool, analysisId)))
706
+ return "completed";
707
+ const current = unwrapOrThrow(await loadDataProfileStatus(deps.pool, analysisId));
708
+ if (current?.status === "running")
709
+ return "already_running";
710
+ logger.error("trigger rejected: empty-set completion refused (no analysis row, or the seed now names files)");
687
711
  return "failed";
688
712
  }
689
- // The trigger validates the ledger seed above; it must also validate the manifest
690
- // it is about to dispatch. A seeded row profiled against an EMPTY manifest would
691
- // claim `running` and then hit the body's empty-manifest path — completing with a
692
- // NULL result while the seed still names files, an incoherence a staleness policy
693
- // then loops on re-triggering. Refuse the divergence before any claim.
694
- if (params.stagedInputs.length === 0) {
695
- logger.error("trigger rejected: empty manifest dispatched against a non-empty seed", { seededCount: seeded.length });
713
+ // A non-empty manifest against a seed that names no file: the caller skipped the
714
+ // seed, or the analysis row is missing. Name the cause before any claim.
715
+ if (seeded === null || seeded.length === 0) {
716
+ logger.error("trigger rejected: no seeded input set (missing analysis row or caller skipped seeding)");
696
717
  return "failed";
697
718
  }
698
719
  const started = unwrapOrThrow(await tryStartDataProfile(deps.pool, analysisId));
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@inflexa-ai/harness",
3
- "version": "0.30.1",
3
+ "version": "0.30.3",
4
4
  "description": "Host-agnostic bioinformatics agent harness — hand-rolled agent loop, DBOS-durable workflows, sandboxed compute.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",