@capybara-hq/kernel 0.1.0-rc.3 → 0.1.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin/admin.controller.d.ts +9 -4
- package/dist/admin/admin.controller.js +13 -4
- package/dist/admin/extensions.controller.d.ts +8 -1
- package/dist/admin/extensions.controller.js +22 -2
- package/dist/agents/agent-lookup.d.ts +3 -0
- package/dist/agents/agent-lookup.js +10 -0
- package/dist/agents/agent-profile.service.d.ts +29 -0
- package/dist/agents/agent-profile.service.js +117 -0
- package/dist/agents/agents.controller.d.ts +93 -3
- package/dist/agents/agents.controller.js +114 -4
- package/dist/agents/agents.module.js +6 -2
- package/dist/agents/agents.service.d.ts +13 -5
- package/dist/agents/agents.service.js +30 -7
- package/dist/agents/definition.d.ts +4 -0
- package/dist/agents/definition.js +10 -0
- package/dist/agents/instruction-files.d.ts +60 -0
- package/dist/agents/instruction-files.js +111 -0
- package/dist/agents/models.controller.d.ts +9 -3
- package/dist/agents/models.controller.js +19 -4
- package/dist/agents/profile.d.ts +96 -0
- package/dist/agents/profile.js +51 -0
- package/dist/agents/readiness.controller.d.ts +26 -0
- package/dist/agents/readiness.controller.js +56 -0
- package/dist/agents/templates.js +1 -1
- package/dist/background/background.controller.d.ts +38 -0
- package/dist/background/background.controller.js +87 -0
- package/dist/background/background.module.d.ts +10 -0
- package/dist/background/background.module.js +32 -0
- package/dist/background/finish.d.ts +43 -0
- package/dist/background/finish.js +94 -0
- package/dist/background/launch-lock.d.ts +14 -0
- package/dist/background/launch-lock.js +39 -0
- package/dist/background/launch.d.ts +36 -0
- package/dist/background/launch.js +70 -0
- package/dist/background/poll.job.d.ts +53 -0
- package/dist/background/poll.job.js +182 -0
- package/dist/background/render.d.ts +23 -0
- package/dist/background/render.js +48 -0
- package/dist/background/session.d.ts +37 -0
- package/dist/background/session.js +66 -0
- package/dist/background/status.d.ts +35 -0
- package/dist/background/status.js +57 -0
- package/dist/background/stop.d.ts +62 -0
- package/dist/background/stop.js +225 -0
- package/dist/background/store.d.ts +150 -0
- package/dist/background/store.js +351 -0
- package/dist/background/store.module.d.ts +6 -0
- package/dist/background/store.module.js +22 -0
- package/dist/background/tasks.service.d.ts +67 -0
- package/dist/background/tasks.service.js +186 -0
- package/dist/background/wake-rules.d.ts +57 -0
- package/dist/background/wake-rules.js +83 -0
- package/dist/background/wake.d.ts +28 -0
- package/dist/background/wake.js +57 -0
- package/dist/catalog/claude/agents.js +2 -0
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/catalog/plugin-import.js +1 -1
- package/dist/catalog/plugin-upgrade.js +1 -1
- package/dist/channels/cards.js +1 -1
- package/dist/channels/connections.service.d.ts +16 -1
- package/dist/channels/connections.service.js +38 -4
- package/dist/channels/threads.d.ts +5 -0
- package/dist/channels/threads.js +8 -0
- package/dist/config/schema.d.ts +18 -0
- package/dist/config/schema.js +102 -0
- package/dist/conversations/access.js +2 -0
- package/dist/conversations/attachments.service.d.ts +30 -0
- package/dist/conversations/attachments.service.js +130 -5
- package/dist/conversations/conversations.controller.d.ts +36 -5
- package/dist/conversations/conversations.controller.js +62 -12
- package/dist/conversations/conversations.module.js +3 -0
- package/dist/conversations/conversations.service.d.ts +44 -2
- package/dist/conversations/conversations.service.js +140 -12
- package/dist/conversations/event-log.d.ts +1 -1
- package/dist/conversations/event-log.js +10 -1
- package/dist/conversations/events.d.ts +58 -0
- package/dist/conversations/events.js +57 -2
- package/dist/conversations/queue.d.ts +3 -1
- package/dist/conversations/queue.js +13 -2
- package/dist/conversations/queue.service.js +3 -1
- package/dist/conversations/titles.d.ts +7 -0
- package/dist/conversations/titles.js +15 -0
- package/dist/db/kernel-schema.d.ts +58 -2
- package/dist/db/pool.d.ts +5 -0
- package/dist/db/pool.js +3 -0
- package/dist/errors.d.ts +8 -0
- package/dist/errors.js +8 -0
- package/dist/evals/run-replay.d.ts +5 -2
- package/dist/evals/run-replay.js +30 -7
- package/dist/evals/step-replay.js +12 -6
- package/dist/evals/trajectory.js +1 -1
- package/dist/extensions/admin-ops.js +2 -0
- package/dist/extensions/admin-view.d.ts +3 -1
- package/dist/extensions/admin-view.js +10 -2
- package/dist/extensions/blobs.js +1 -0
- package/dist/extensions/config-issues.d.ts +13 -1
- package/dist/extensions/config-issues.js +22 -5
- package/dist/extensions/config-json-schema.d.ts +43 -7
- package/dist/extensions/config-json-schema.js +124 -58
- package/dist/extensions/deployment-config-store.d.ts +6 -0
- package/dist/extensions/deployment-config.d.ts +50 -3
- package/dist/extensions/deployment-config.js +125 -28
- package/dist/extensions/display-name.d.ts +22 -0
- package/dist/extensions/display-name.js +77 -0
- package/dist/extensions/extensions.module.js +6 -0
- package/dist/extensions/host.d.ts +26 -1
- package/dist/extensions/host.js +82 -9
- package/dist/extensions/org-config.controller.d.ts +11 -3
- package/dist/extensions/org-config.controller.js +14 -6
- package/dist/extensions/org-config.d.ts +6 -0
- package/dist/extensions/org-config.js +12 -13
- package/dist/extensions/package.d.ts +14 -1
- package/dist/extensions/package.js +51 -2
- package/dist/extensions/permission-view.d.ts +3 -0
- package/dist/extensions/point-defaults.controller.d.ts +34 -0
- package/dist/extensions/point-defaults.controller.js +71 -0
- package/dist/extensions/point-defaults.d.ts +64 -0
- package/dist/extensions/point-defaults.js +206 -0
- package/dist/extensions/registry.d.ts +1 -1
- package/dist/extensions/registry.js +2 -0
- package/dist/http/api.module.js +8 -0
- package/dist/http/nest-logger.d.ts +6 -5
- package/dist/http/nest-logger.js +24 -10
- package/dist/identity/auth.controller.js +9 -5
- package/dist/identity/cookies.d.ts +20 -5
- package/dist/identity/cookies.js +44 -6
- package/dist/identity/me.controller.d.ts +5 -1
- package/dist/identity/me.controller.js +12 -3
- package/dist/identity/sessions.d.ts +12 -0
- package/dist/identity/sessions.js +45 -0
- package/dist/index.d.ts +1 -1
- package/dist/infra/config/kernel-config.service.js +6 -0
- package/dist/infra/database/database.module.js +1 -0
- package/dist/infra/events/kernel-events.d.ts +8 -0
- package/dist/infra/lifecycle/process-guards.js +4 -0
- package/dist/infra/logging/logging.module.d.ts +4 -2
- package/dist/infra/logging/logging.module.js +24 -6
- package/dist/infra/queue/redis-connection.d.ts +6 -0
- package/dist/infra/queue/redis-connection.js +7 -0
- package/dist/infra/redis/redis.module.js +6 -2
- package/dist/infra/tokens.d.ts +6 -1
- package/dist/infra/tokens.js +6 -1
- package/dist/log.d.ts +13 -1
- package/dist/log.js +93 -17
- package/dist/models/purposes.d.ts +23 -0
- package/dist/models/purposes.js +31 -10
- package/dist/orgs/offboarding.d.ts +3 -1
- package/dist/orgs/offboarding.js +26 -16
- package/dist/orgs/settings.service.d.ts +5 -0
- package/dist/orgs/settings.service.js +13 -2
- package/dist/roles/worker.module.js +6 -0
- package/dist/runtime/context/build.d.ts +9 -7
- package/dist/runtime/context/build.js +27 -19
- package/dist/runtime/context/claude-tools.d.ts +2 -1
- package/dist/runtime/context/claude-tools.js +5 -4
- package/dist/runtime/context/compaction.js +4 -0
- package/dist/runtime/langgraph-loop.js +3 -2
- package/dist/runtime/loop.d.ts +4 -3
- package/dist/runtime/maintenance.d.ts +4 -2
- package/dist/runtime/maintenance.js +17 -5
- package/dist/runtime/model-call.d.ts +17 -1
- package/dist/runtime/model-call.js +3 -2
- package/dist/runtime/providers.d.ts +6 -0
- package/dist/runtime/providers.js +8 -0
- package/dist/runtime/run-queue.d.ts +25 -2
- package/dist/runtime/run-queue.js +129 -33
- package/dist/runtime/run-worker.js +16 -3
- package/dist/runtime/runtime.module.js +8 -3
- package/dist/runtime/step-log.d.ts +6 -0
- package/dist/runtime/step-log.js +16 -2
- package/dist/runtime/subagents/definition.js +3 -1
- package/dist/runtime/usage-report.d.ts +2 -1
- package/dist/runtime/usage-report.js +2 -1
- package/dist/runtime/waits.d.ts +8 -2
- package/dist/runtime/waits.js +37 -11
- package/dist/sandbox/sandbox.module.js +6 -1
- package/dist/sandbox/sandbox.service.d.ts +61 -3
- package/dist/sandbox/sandbox.service.js +141 -7
- package/dist/tools/approvals.service.js +2 -0
- package/dist/tools/builtin-names.d.ts +20 -2
- package/dist/tools/builtin-names.js +29 -2
- package/dist/tools/builtins/builtin-tools.d.ts +15 -1
- package/dist/tools/builtins/builtin-tools.js +34 -2
- package/dist/tools/builtins/sandbox/bash.d.ts +1 -1
- package/dist/tools/builtins/sandbox/bash.js +230 -3
- package/dist/tools/builtins/sandbox/common.d.ts +27 -4
- package/dist/tools/builtins/sandbox/common.js +7 -3
- package/dist/tools/builtins/sandbox/deliver.js +1 -1
- package/dist/tools/builtins/sandbox/index.d.ts +2 -0
- package/dist/tools/builtins/sandbox/index.js +6 -0
- package/dist/tools/builtins/sandbox/task-output.d.ts +63 -0
- package/dist/tools/builtins/sandbox/task-output.js +304 -0
- package/dist/tools/builtins/sandbox/task-stop.d.ts +10 -0
- package/dist/tools/builtins/sandbox/task-stop.js +112 -0
- package/dist/tools/builtins/web/errors.d.ts +8 -0
- package/dist/tools/builtins/web/errors.js +53 -0
- package/dist/tools/builtins/web/extract.d.ts +62 -0
- package/dist/tools/builtins/web/extract.js +267 -0
- package/dist/tools/builtins/web/fetch.d.ts +55 -0
- package/dist/tools/builtins/web/fetch.js +174 -0
- package/dist/tools/builtins/web/index.d.ts +13 -0
- package/dist/tools/builtins/web/index.js +12 -0
- package/dist/tools/builtins/web/page-cache.d.ts +25 -0
- package/dist/tools/builtins/web/page-cache.js +77 -0
- package/dist/tools/builtins/web/search.d.ts +25 -0
- package/dist/tools/builtins/web/search.js +209 -0
- package/dist/tools/builtins/web/url.d.ts +17 -0
- package/dist/tools/builtins/web/url.js +43 -0
- package/dist/tools/oauth/flow-paths.d.ts +1 -1
- package/dist/tools/oauth/flow-paths.js +1 -1
- package/dist/tools/pipeline.js +15 -0
- package/dist/tools/registry.d.ts +22 -3
- package/dist/tools/registry.js +26 -11
- package/dist/tools/tools.module.js +12 -0
- package/dist/tools/validate.js +1 -1
- package/package.json +3 -3
- package/src/migrations/0064_account_last_org.sql +7 -0
- package/src/migrations/0065_conversation_archive.sql +8 -0
- package/src/migrations/0066_extension_display_name.sql +4 -0
- package/src/migrations/0067_agent_profiles.sql +20 -0
- package/src/migrations/0068_agent_avatar_uploads.sql +7 -0
- package/src/migrations/0069_attachments_agent_idx.sql +3 -0
- package/src/migrations/0070_background_tasks.sql +37 -0
- package/src/migrations/0071_background_tasks_reserve.sql +6 -0
- package/src/migrations/0072_background_tasks_stopping.sql +5 -0
- package/src/migrations/0073_org_extension_point_defaults.sql +11 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
/**
|
|
3
|
+
* Launching a background Bash task (spec 2026-11-05-background-bash-design §4.2): the task id, the
|
|
4
|
+
* one short exec that starts the command detached from the exec, and the model-visible result.
|
|
5
|
+
* Every path is relative to `/workspace` (the launch exec's `cwd`), so a provider that maps
|
|
6
|
+
* `/workspace` needs no rewriting inside the script.
|
|
7
|
+
*/
|
|
8
|
+
/** The launch exec's own time limit: it only writes a file and forks. */
|
|
9
|
+
export const LAUNCH_EXEC_TIMEOUT_MS = 30_000;
|
|
10
|
+
/**
|
|
11
|
+
* The longest a background task may run, in seconds: `sandbox.background.maxSeconds`, at most
|
|
12
|
+
* `sandbox.limits.maxCommandSeconds`. It is also a task's default `timeout`.
|
|
13
|
+
*/
|
|
14
|
+
export const backgroundMaxSeconds = (cfg) => Math.min(cfg.sandbox.background.maxSeconds, cfg.sandbox.limits.maxCommandSeconds);
|
|
15
|
+
/** The directory of the task files, relative to `/workspace`. */
|
|
16
|
+
export const TASKS_DIR = ".capybara/tasks";
|
|
17
|
+
/** A task's output file, relative to `/workspace` (`output_path`). */
|
|
18
|
+
export const taskOutputPath = (id) => `${TASKS_DIR}/${id}.out`;
|
|
19
|
+
/** The file the launch writes the session id to, relative to `/workspace`. */
|
|
20
|
+
export const taskSidPath = (id) => `${TASKS_DIR}/${id}.sid`;
|
|
21
|
+
const BASE32 = "abcdefghijklmnopqrstuvwxyz234567";
|
|
22
|
+
/** A new task id: `bt_` plus 6 lowercase base32 characters. */
|
|
23
|
+
export function newTaskId() {
|
|
24
|
+
const bytes = randomBytes(6);
|
|
25
|
+
let out = "bt_";
|
|
26
|
+
for (const b of bytes)
|
|
27
|
+
out += BASE32[b & 31];
|
|
28
|
+
return out;
|
|
29
|
+
}
|
|
30
|
+
/** The programs the launch and the later polls and stops need inside the sandbox. */
|
|
31
|
+
const REQUIRED = ["bash", "setsid", "nohup", "timeout", "pgrep", "pkill", "base64"];
|
|
32
|
+
/** A task id ({@link newTaskId}). */
|
|
33
|
+
export const TASK_ID_RE = /^bt_[a-z2-7]{6}$/;
|
|
34
|
+
/**
|
|
35
|
+
* The shell of the launch exec: writes `command` (base64 in the script, so nothing is quoted by
|
|
36
|
+
* hand) to `.capybara/tasks/<id>.sh` and starts it with bash (the shell a foreground `Bash` call
|
|
37
|
+
* runs its command in) in a session of its own under `timeout -s KILL <seconds>`, with its exit
|
|
38
|
+
* status written to `<id>.exit` and its output to `<id>.out`; writes the session id (`$!`, the pid
|
|
39
|
+
* `setsid` started) to `<id>.sid` and prints it. A sandbox without one of the needed programs fails with a readable error on
|
|
40
|
+
* stderr and exit code 127, before anything starts. `id` must be a task id and `seconds` a
|
|
41
|
+
* positive integer (both go into the script unquoted).
|
|
42
|
+
*/
|
|
43
|
+
export function launchScript(id, command, seconds) {
|
|
44
|
+
if (!TASK_ID_RE.test(id))
|
|
45
|
+
throw new Error(`invalid background task id: ${JSON.stringify(id)}`);
|
|
46
|
+
if (!Number.isSafeInteger(seconds) || seconds < 1) {
|
|
47
|
+
throw new Error(`invalid background task seconds: ${seconds}`);
|
|
48
|
+
}
|
|
49
|
+
const b64 = Buffer.from(command, "utf8").toString("base64");
|
|
50
|
+
const t = `${TASKS_DIR}/${id}`;
|
|
51
|
+
return [
|
|
52
|
+
"set -e",
|
|
53
|
+
`for c in ${REQUIRED.join(" ")}; do command -v "$c" >/dev/null 2>&1 || { echo "Background tasks need ${REQUIRED.join(", ")} in the sandbox; $c is missing." >&2; exit 127; }; done`,
|
|
54
|
+
`mkdir -p ${TASKS_DIR}`,
|
|
55
|
+
`printf '%s' '${b64}' | base64 -d > ${t}.sh`,
|
|
56
|
+
`setsid nohup sh -c 'timeout -s KILL ${seconds} bash ${t}.sh; echo $? > ${t}.exit' > ${t}.out 2>&1 < /dev/null &`,
|
|
57
|
+
// the session id goes to a file first: a launch that ends before its output comes back
|
|
58
|
+
// (aborted, timed out) still leaves the id of what it started
|
|
59
|
+
`echo $! > ${t}.sid`,
|
|
60
|
+
`cat ${t}.sid`,
|
|
61
|
+
].join("\n");
|
|
62
|
+
}
|
|
63
|
+
/** The model-visible result of a launch (spec §2.1). */
|
|
64
|
+
export function startedText(id, description) {
|
|
65
|
+
return [
|
|
66
|
+
`Started background task ${id} ("${description}").`,
|
|
67
|
+
`Output: /workspace/${taskOutputPath(id)}`,
|
|
68
|
+
"Read it with TaskOutput (or Read), stop it with TaskStop. You will be told when it finishes.",
|
|
69
|
+
].join("\n");
|
|
70
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { KernelConfig } from "../config/schema.js";
|
|
2
|
+
import { EventLog } from "../conversations/event-log.js";
|
|
3
|
+
import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
4
|
+
import { LockService } from "../infra/lock/lock.service.js";
|
|
5
|
+
import type { KernelLogger } from "../log.js";
|
|
6
|
+
import { SandboxService } from "../sandbox/sandbox.service.js";
|
|
7
|
+
import { BackgroundTaskFinisher } from "./finish.js";
|
|
8
|
+
import { BackgroundTaskStore } from "./store.js";
|
|
9
|
+
/** The poll of the running background tasks (`@LockedInterval`, every `pollSeconds`). */
|
|
10
|
+
export declare const BACKGROUND_POLL_JOB = "background-task-poll";
|
|
11
|
+
/** The daily prune of finished background tasks older than `tools.outputRetentionDays`. */
|
|
12
|
+
export declare const BACKGROUND_PRUNE_JOB = "background-task-prune";
|
|
13
|
+
/**
|
|
14
|
+
* A reservation (`sid` null) older than this has lost its launch. The slowest legitimate launch
|
|
15
|
+
* is the pipeline's own limit for a background `Bash` call ({@link LAUNCH_EXEC_TIMEOUT_MS} plus
|
|
16
|
+
* {@link sandboxBaseTimeoutMs}: the tool timeout, `sandbox.lockWaitSeconds` and the setup margin),
|
|
17
|
+
* so twice that (never under 2 minutes) leaves a wide margin: only a launching process that died
|
|
18
|
+
* leaves a reservation this old.
|
|
19
|
+
*/
|
|
20
|
+
export declare const launchAbandonedAfterMs: (cfg: KernelConfig) => number;
|
|
21
|
+
/**
|
|
22
|
+
* The worker's background task jobs (spec 2026-11-05-background-bash-design §4.3), each once per
|
|
23
|
+
* tick across replicas:
|
|
24
|
+
* - `background-task-poll` (every `sandbox.background.pollSeconds`): for every sandbox with
|
|
25
|
+
* running tasks, one exec through `SandboxService.ifRunning` (which never creates or resumes a
|
|
26
|
+
* sandbox) reports each task's exit file and liveness; ended tasks finish through
|
|
27
|
+
* {@link BackgroundTaskFinisher}. Each poll takes and releases a lease, so `last_used_at` stays
|
|
28
|
+
* fresh and a sandbox with running tasks is never idle-suspended;
|
|
29
|
+
* - `background-task-prune` (daily): deletes finished tasks older than `tools.outputRetentionDays`.
|
|
30
|
+
*/
|
|
31
|
+
export declare class BackgroundTaskPollJob {
|
|
32
|
+
private readonly store;
|
|
33
|
+
private readonly finisher;
|
|
34
|
+
private readonly eventLog;
|
|
35
|
+
private readonly sandbox;
|
|
36
|
+
readonly config: KernelConfigService;
|
|
37
|
+
readonly lockService: LockService;
|
|
38
|
+
readonly log: KernelLogger;
|
|
39
|
+
constructor(store: Pick<BackgroundTaskStore, "runningStarted" | "pruneFinished" | "failAbandonedLaunches">, finisher: BackgroundTaskFinisher, eventLog: Pick<EventLog, "publishChanged">, sandbox: Pick<SandboxService, "available" | "ifRunning">, config: KernelConfigService, lockService: LockService, log: KernelLogger);
|
|
40
|
+
poll(signal?: AbortSignal): Promise<void>;
|
|
41
|
+
/**
|
|
42
|
+
* One poll of every running task (the body of {@link poll}); returns how many tasks finished.
|
|
43
|
+
* A sandbox whose poll fails (busy, a provider error) is logged and tried again next tick.
|
|
44
|
+
* First, reservations abandoned by a launch that died ({@link launchAbandonedAfterMs}) are
|
|
45
|
+
* failed, so they stop holding a slot of `sandbox.background.maxTasks` (not counted in the
|
|
46
|
+
* result: they never started), with a `changed: "background"` frame to their conversations.
|
|
47
|
+
*/
|
|
48
|
+
pollOnce(signal?: AbortSignal): Promise<number>;
|
|
49
|
+
/** Polls the tasks of one sandbox instance; returns how many finished. */
|
|
50
|
+
private pollSandbox;
|
|
51
|
+
/** Deletes the finished tasks older than `tools.outputRetentionDays` (their rows only). */
|
|
52
|
+
prune(): Promise<void>;
|
|
53
|
+
}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
8
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
|
+
};
|
|
10
|
+
import { Inject, Injectable } from "@nestjs/common";
|
|
11
|
+
import { EventLog } from "../conversations/event-log.js";
|
|
12
|
+
import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
13
|
+
import { LockService } from "../infra/lock/lock.service.js";
|
|
14
|
+
import { LockedCron } from "../infra/schedule/locked-cron.decorator.js";
|
|
15
|
+
import { LockedInterval } from "../infra/schedule/locked-interval.js";
|
|
16
|
+
import { KERNEL_LOGGER } from "../infra/tokens.js";
|
|
17
|
+
import { SandboxService } from "../sandbox/sandbox.service.js";
|
|
18
|
+
import { sandboxBaseTimeoutMs } from "../tools/builtins/sandbox/common.js";
|
|
19
|
+
import { BackgroundTaskFinisher } from "./finish.js";
|
|
20
|
+
import { LAUNCH_EXEC_TIMEOUT_MS } from "./launch.js";
|
|
21
|
+
import { parsePollOutput, polledAs, pollScript } from "./status.js";
|
|
22
|
+
import { notRunningAs, RECREATED } from "./stop.js";
|
|
23
|
+
import { BackgroundTaskStore } from "./store.js";
|
|
24
|
+
/** The poll of the running background tasks (`@LockedInterval`, every `pollSeconds`). */
|
|
25
|
+
export const BACKGROUND_POLL_JOB = "background-task-poll";
|
|
26
|
+
/** The daily prune of finished background tasks older than `tools.outputRetentionDays`. */
|
|
27
|
+
export const BACKGROUND_PRUNE_JOB = "background-task-prune";
|
|
28
|
+
/** The least age at which a reservation counts as abandoned, whatever the config. */
|
|
29
|
+
const LAUNCH_ABANDONED_FLOOR_MS = 120_000;
|
|
30
|
+
/**
|
|
31
|
+
* A reservation (`sid` null) older than this has lost its launch. The slowest legitimate launch
|
|
32
|
+
* is the pipeline's own limit for a background `Bash` call ({@link LAUNCH_EXEC_TIMEOUT_MS} plus
|
|
33
|
+
* {@link sandboxBaseTimeoutMs}: the tool timeout, `sandbox.lockWaitSeconds` and the setup margin),
|
|
34
|
+
* so twice that (never under 2 minutes) leaves a wide margin: only a launching process that died
|
|
35
|
+
* leaves a reservation this old.
|
|
36
|
+
*/
|
|
37
|
+
export const launchAbandonedAfterMs = (cfg) => Math.max(LAUNCH_ABANDONED_FLOOR_MS, 2 * (LAUNCH_EXEC_TIMEOUT_MS + sandboxBaseTimeoutMs(cfg)));
|
|
38
|
+
/** The time limit of one sandbox's poll exec. */
|
|
39
|
+
const POLL_EXEC_TIMEOUT_MS = 30_000;
|
|
40
|
+
/** The most output one poll exec keeps (a line per task, with a tail when it ended). */
|
|
41
|
+
const POLL_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
|
|
42
|
+
/**
|
|
43
|
+
* The worker's background task jobs (spec 2026-11-05-background-bash-design §4.3), each once per
|
|
44
|
+
* tick across replicas:
|
|
45
|
+
* - `background-task-poll` (every `sandbox.background.pollSeconds`): for every sandbox with
|
|
46
|
+
* running tasks, one exec through `SandboxService.ifRunning` (which never creates or resumes a
|
|
47
|
+
* sandbox) reports each task's exit file and liveness; ended tasks finish through
|
|
48
|
+
* {@link BackgroundTaskFinisher}. Each poll takes and releases a lease, so `last_used_at` stays
|
|
49
|
+
* fresh and a sandbox with running tasks is never idle-suspended;
|
|
50
|
+
* - `background-task-prune` (daily): deletes finished tasks older than `tools.outputRetentionDays`.
|
|
51
|
+
*/
|
|
52
|
+
let BackgroundTaskPollJob = class BackgroundTaskPollJob {
|
|
53
|
+
store;
|
|
54
|
+
finisher;
|
|
55
|
+
eventLog;
|
|
56
|
+
sandbox;
|
|
57
|
+
config;
|
|
58
|
+
lockService;
|
|
59
|
+
log;
|
|
60
|
+
constructor(store, finisher, eventLog, sandbox, config, lockService, log) {
|
|
61
|
+
this.store = store;
|
|
62
|
+
this.finisher = finisher;
|
|
63
|
+
this.eventLog = eventLog;
|
|
64
|
+
this.sandbox = sandbox;
|
|
65
|
+
this.config = config;
|
|
66
|
+
this.lockService = lockService;
|
|
67
|
+
this.log = log;
|
|
68
|
+
}
|
|
69
|
+
async poll(signal) {
|
|
70
|
+
await this.pollOnce(signal);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* One poll of every running task (the body of {@link poll}); returns how many tasks finished.
|
|
74
|
+
* A sandbox whose poll fails (busy, a provider error) is logged and tried again next tick.
|
|
75
|
+
* First, reservations abandoned by a launch that died ({@link launchAbandonedAfterMs}) are
|
|
76
|
+
* failed, so they stop holding a slot of `sandbox.background.maxTasks` (not counted in the
|
|
77
|
+
* result: they never started), with a `changed: "background"` frame to their conversations.
|
|
78
|
+
*/
|
|
79
|
+
async pollOnce(signal = new AbortController().signal) {
|
|
80
|
+
const abandoned = await this.store.failAbandonedLaunches(new Date(Date.now() - launchAbandonedAfterMs(this.config.get())));
|
|
81
|
+
if (abandoned.length > 0) {
|
|
82
|
+
this.log.warn("abandoned background launches failed", { count: abandoned.length });
|
|
83
|
+
// a client that listed the reservation as running refreshes (the owner and, for a
|
|
84
|
+
// sub-agent's task, the conversation that owns the sandbox); a failed publish only logs
|
|
85
|
+
const conversations = new Set(abandoned.flatMap((t) => [t.conversationId, t.sandboxConversationId]));
|
|
86
|
+
for (const c of conversations)
|
|
87
|
+
await this.eventLog.publishChanged(c, "background");
|
|
88
|
+
}
|
|
89
|
+
if (!this.sandbox.available)
|
|
90
|
+
return 0;
|
|
91
|
+
const groups = new Map();
|
|
92
|
+
for (const t of await this.store.runningStarted()) {
|
|
93
|
+
if (t.sid === null || t.sandboxCreatedAt === null)
|
|
94
|
+
continue;
|
|
95
|
+
const key = `${t.sandboxConversationId} ${t.sandboxCreatedAt.getTime()}`;
|
|
96
|
+
groups.set(key, [...(groups.get(key) ?? []), t]);
|
|
97
|
+
}
|
|
98
|
+
let finished = 0;
|
|
99
|
+
for (const tasks of groups.values()) {
|
|
100
|
+
if (signal.aborted)
|
|
101
|
+
break;
|
|
102
|
+
try {
|
|
103
|
+
finished += await this.pollSandbox(tasks, signal);
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
if (signal.aborted)
|
|
107
|
+
break;
|
|
108
|
+
this.log.warn("cannot poll background tasks; retrying next tick", {
|
|
109
|
+
conversationId: tasks[0]?.sandboxConversationId,
|
|
110
|
+
err,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (finished > 0)
|
|
115
|
+
this.log.info("background tasks finished", { count: finished });
|
|
116
|
+
return finished;
|
|
117
|
+
}
|
|
118
|
+
/** Polls the tasks of one sandbox instance; returns how many finished. */
|
|
119
|
+
async pollSandbox(tasks, signal) {
|
|
120
|
+
const [first] = tasks;
|
|
121
|
+
const r = await this.sandbox.ifRunning(first.sandboxConversationId, { createdAt: first.sandboxCreatedAt }, async (p, h, sig) => {
|
|
122
|
+
const res = await p.exec(h, {
|
|
123
|
+
command: pollScript(tasks.map((t) => ({ id: t.id, sid: t.sid }))),
|
|
124
|
+
cwd: "/workspace",
|
|
125
|
+
timeoutMs: POLL_EXEC_TIMEOUT_MS,
|
|
126
|
+
maxOutputBytes: POLL_MAX_OUTPUT_BYTES,
|
|
127
|
+
signal: sig,
|
|
128
|
+
});
|
|
129
|
+
if (res.exitCode !== 0 || res.truncated) {
|
|
130
|
+
throw new Error(`the poll exec failed (exit ${res.exitCode}${res.truncated ? ", truncated" : ""}): ${res.stderr.slice(0, 500)}`);
|
|
131
|
+
}
|
|
132
|
+
return parsePollOutput(res.stdout);
|
|
133
|
+
}, signal);
|
|
134
|
+
const now = new Date();
|
|
135
|
+
let n = 0;
|
|
136
|
+
for (const t of tasks) {
|
|
137
|
+
let to;
|
|
138
|
+
let lastLines = "";
|
|
139
|
+
if (r.kind === "ran") {
|
|
140
|
+
const report = r.value.get(t.id);
|
|
141
|
+
// a task the exec did not report is checked again next tick
|
|
142
|
+
to = report ? polledAs(t, report, now) : null;
|
|
143
|
+
lastLines = report?.lastLines ?? "";
|
|
144
|
+
}
|
|
145
|
+
else if (r.kind === "not_running") {
|
|
146
|
+
// a suspended sandbox may keep the processes (E2B's pause): killed after its next resume
|
|
147
|
+
to = notRunningAs(r.status, r.status === "suspended");
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
to = RECREATED;
|
|
151
|
+
}
|
|
152
|
+
// a stop that claimed the task after the read above finishes it itself
|
|
153
|
+
if (to && (await this.finisher.finish(t, to, lastLines, false, { unclaimed: true })))
|
|
154
|
+
n++;
|
|
155
|
+
}
|
|
156
|
+
return n;
|
|
157
|
+
}
|
|
158
|
+
/** Deletes the finished tasks older than `tools.outputRetentionDays` (their rows only). */
|
|
159
|
+
async prune() {
|
|
160
|
+
const days = this.config.get().tools.outputRetentionDays;
|
|
161
|
+
const n = await this.store.pruneFinished(new Date(Date.now() - days * 86_400_000));
|
|
162
|
+
if (n > 0)
|
|
163
|
+
this.log.info("finished background tasks pruned", { count: n });
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
__decorate([
|
|
167
|
+
LockedInterval(BACKGROUND_POLL_JOB, (cfg) => cfg.sandbox.background.pollSeconds * 1000)
|
|
168
|
+
], BackgroundTaskPollJob.prototype, "poll", null);
|
|
169
|
+
__decorate([
|
|
170
|
+
LockedCron("30 3 * * *", { name: BACKGROUND_PRUNE_JOB })
|
|
171
|
+
], BackgroundTaskPollJob.prototype, "prune", null);
|
|
172
|
+
BackgroundTaskPollJob = __decorate([
|
|
173
|
+
Injectable(),
|
|
174
|
+
__param(0, Inject(BackgroundTaskStore)),
|
|
175
|
+
__param(1, Inject(BackgroundTaskFinisher)),
|
|
176
|
+
__param(2, Inject(EventLog)),
|
|
177
|
+
__param(3, Inject(SandboxService)),
|
|
178
|
+
__param(4, Inject(KernelConfigService)),
|
|
179
|
+
__param(5, Inject(LockService)),
|
|
180
|
+
__param(6, Inject(KERNEL_LOGGER))
|
|
181
|
+
], BackgroundTaskPollJob);
|
|
182
|
+
export { BackgroundTaskPollJob };
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { EventPayload } from "../conversations/events.js";
|
|
2
|
+
/** `42s`, `2m05s`, `1h02m05s`. */
|
|
3
|
+
export declare function formatElapsed(ms: number): string;
|
|
4
|
+
/** Why a task was lost, as the model reads it (an unknown reason as is). */
|
|
5
|
+
export declare const lostReasonText: (reason: string) => string;
|
|
6
|
+
type Finished = EventPayload<"background_task_finished">;
|
|
7
|
+
/**
|
|
8
|
+
* The model-visible text of a `background_task_finished` event, rendered as a system message (spec
|
|
9
|
+
* 2026-11-05-background-bash-design §2.4):
|
|
10
|
+
*
|
|
11
|
+
* ```
|
|
12
|
+
* Background task bt_k3f9a2 ("Run the test suite") completed with exit code 1 after 4m12s.
|
|
13
|
+
* Last lines:
|
|
14
|
+
* …
|
|
15
|
+
* Full output: /workspace/.capybara/tasks/bt_k3f9a2.out
|
|
16
|
+
* ```
|
|
17
|
+
*
|
|
18
|
+
* Without an exit code the first line has no "with exit code"; without output the last lines are
|
|
19
|
+
* `Last lines: (none)`. A finish a tool call already reported (`reported`) is one short line.
|
|
20
|
+
* Read only from the payload, so a replay renders it identically.
|
|
21
|
+
*/
|
|
22
|
+
export declare function renderBackgroundFinished(p: Finished): string;
|
|
23
|
+
export {};
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/** `42s`, `2m05s`, `1h02m05s`. */
|
|
2
|
+
export function formatElapsed(ms) {
|
|
3
|
+
const s = Math.max(0, Math.floor(ms / 1000));
|
|
4
|
+
const pad = (n) => String(n).padStart(2, "0");
|
|
5
|
+
if (s < 60)
|
|
6
|
+
return `${s}s`;
|
|
7
|
+
if (s < 3600)
|
|
8
|
+
return `${Math.floor(s / 60)}m${pad(s % 60)}s`;
|
|
9
|
+
return `${Math.floor(s / 3600)}h${pad(Math.floor((s % 3600) / 60))}m${pad(s % 60)}s`;
|
|
10
|
+
}
|
|
11
|
+
const LOST_REASONS = {
|
|
12
|
+
sandbox_suspended: "the sandbox was suspended",
|
|
13
|
+
sandbox_gone: "the sandbox is gone",
|
|
14
|
+
sandbox_recreated: "the sandbox was recreated",
|
|
15
|
+
process_gone: "its processes ended without an exit code",
|
|
16
|
+
launch_abandoned: "its launch was interrupted",
|
|
17
|
+
};
|
|
18
|
+
/** Why a task was lost, as the model reads it (an unknown reason as is). */
|
|
19
|
+
export const lostReasonText = (reason) => LOST_REASONS[reason] ?? reason;
|
|
20
|
+
const statusWord = (status) => status === "timed_out" ? "timed out" : status;
|
|
21
|
+
/**
|
|
22
|
+
* The model-visible text of a `background_task_finished` event, rendered as a system message (spec
|
|
23
|
+
* 2026-11-05-background-bash-design §2.4):
|
|
24
|
+
*
|
|
25
|
+
* ```
|
|
26
|
+
* Background task bt_k3f9a2 ("Run the test suite") completed with exit code 1 after 4m12s.
|
|
27
|
+
* Last lines:
|
|
28
|
+
* …
|
|
29
|
+
* Full output: /workspace/.capybara/tasks/bt_k3f9a2.out
|
|
30
|
+
* ```
|
|
31
|
+
*
|
|
32
|
+
* Without an exit code the first line has no "with exit code"; without output the last lines are
|
|
33
|
+
* `Last lines: (none)`. A finish a tool call already reported (`reported`) is one short line.
|
|
34
|
+
* Read only from the payload, so a replay renders it identically.
|
|
35
|
+
*/
|
|
36
|
+
export function renderBackgroundFinished(p) {
|
|
37
|
+
const head = `Background task ${p.taskId} (${JSON.stringify(p.description)}) ${statusWord(p.status)}`;
|
|
38
|
+
if (p.reported)
|
|
39
|
+
return `${head}; already reported.`;
|
|
40
|
+
const lost = p.lostReason === undefined ? "" : ` (${lostReasonText(p.lostReason)})`;
|
|
41
|
+
const exit = p.exitCode === null ? "" : ` with exit code ${p.exitCode}`;
|
|
42
|
+
const lines = p.lastLines.replace(/\n+$/, "");
|
|
43
|
+
return [
|
|
44
|
+
`${head}${lost}${exit} after ${formatElapsed(p.elapsedMs)}.`,
|
|
45
|
+
lines === "" ? "Last lines: (none)" : `Last lines:\n${lines}`,
|
|
46
|
+
`Full output: ${p.outputPath}`,
|
|
47
|
+
].join("\n");
|
|
48
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { SandboxHandle, SandboxProvider } from "@capybara-hq/plugin-sdk";
|
|
2
|
+
/**
|
|
3
|
+
* A background task's session inside the sandbox (spec 2026-11-05-background-bash-design §4.2):
|
|
4
|
+
* every process of a task is in the session `setsid` started, so liveness is `pgrep -s <sid>` and
|
|
5
|
+
* stopping is `pkill -s <sid>` (never `kill -0` or `/proc`, which disagree between sandbox-fake and
|
|
6
|
+
* Linux).
|
|
7
|
+
*/
|
|
8
|
+
/** A session id (the pid `setsid` started). */
|
|
9
|
+
export declare const SID_RE: RegExp;
|
|
10
|
+
/** The time limit of one kill of a session ({@link killSession}). */
|
|
11
|
+
export declare const KILL_SESSION_TIMEOUT_MS = 10000;
|
|
12
|
+
/** Throws unless `id` is a task id and `sid` a session id (both go into scripts unquoted). */
|
|
13
|
+
export declare function assertTaskRef(id: string, sid: string): void;
|
|
14
|
+
/**
|
|
15
|
+
* A shell fragment that kills every process of the session `sid` with SIGKILL, repeated until the
|
|
16
|
+
* session is empty (pkill exits 1: no process left): a process may fork between pkill's listing
|
|
17
|
+
* and its signals, and the child would live on. Sets `ks` to 0 when the session was confirmed
|
|
18
|
+
* empty, else to pkill's failing status (3 when it was still not empty after 10 tries). Uses only
|
|
19
|
+
* the variables `ks`, `kc` and `ki`. `sid` must be a session id.
|
|
20
|
+
*/
|
|
21
|
+
export declare function killSessionScript(sid: string): string;
|
|
22
|
+
/**
|
|
23
|
+
* Kills every process of the session `sid` ({@link killSessionScript}) in an exec of its own. Any
|
|
24
|
+
* pkill status other than 0 and 1 is a failure. Runs with a fresh signal of its own (a command
|
|
25
|
+
* must be killed even when the caller was just aborted). True when the session was confirmed
|
|
26
|
+
* empty; false when it could not be (an error is never thrown).
|
|
27
|
+
*/
|
|
28
|
+
export declare function killSession(p: SandboxProvider, h: SandboxHandle, sid: string): Promise<boolean>;
|
|
29
|
+
/** The longest `lastLines` of a finished task (spec §2.4). */
|
|
30
|
+
export declare const LAST_LINES_MAX_CHARS = 4000;
|
|
31
|
+
/**
|
|
32
|
+
* A shell pipeline printing the last 20 lines of task `id`'s output (at most {@link TAIL_BYTES}),
|
|
33
|
+
* base64 on one line (empty when there is no output file). Relative to `/workspace`.
|
|
34
|
+
*/
|
|
35
|
+
export declare function tailCommand(id: string): string;
|
|
36
|
+
/** Decodes a {@link tailCommand} result into `lastLines`: its end, at most 4 000 characters. */
|
|
37
|
+
export declare function decodeTail(b64: string): string;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { TASK_ID_RE, TASKS_DIR } from "./launch.js";
|
|
2
|
+
/**
|
|
3
|
+
* A background task's session inside the sandbox (spec 2026-11-05-background-bash-design §4.2):
|
|
4
|
+
* every process of a task is in the session `setsid` started, so liveness is `pgrep -s <sid>` and
|
|
5
|
+
* stopping is `pkill -s <sid>` (never `kill -0` or `/proc`, which disagree between sandbox-fake and
|
|
6
|
+
* Linux).
|
|
7
|
+
*/
|
|
8
|
+
/** A session id (the pid `setsid` started). */
|
|
9
|
+
export const SID_RE = /^[1-9][0-9]*$/;
|
|
10
|
+
/** The time limit of one kill of a session ({@link killSession}). */
|
|
11
|
+
export const KILL_SESSION_TIMEOUT_MS = 10_000;
|
|
12
|
+
/** Throws unless `id` is a task id and `sid` a session id (both go into scripts unquoted). */
|
|
13
|
+
export function assertTaskRef(id, sid) {
|
|
14
|
+
if (!TASK_ID_RE.test(id))
|
|
15
|
+
throw new Error(`invalid background task id: ${JSON.stringify(id)}`);
|
|
16
|
+
if (!SID_RE.test(sid))
|
|
17
|
+
throw new Error(`invalid background task session: ${JSON.stringify(sid)}`);
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* A shell fragment that kills every process of the session `sid` with SIGKILL, repeated until the
|
|
21
|
+
* session is empty (pkill exits 1: no process left): a process may fork between pkill's listing
|
|
22
|
+
* and its signals, and the child would live on. Sets `ks` to 0 when the session was confirmed
|
|
23
|
+
* empty, else to pkill's failing status (3 when it was still not empty after 10 tries). Uses only
|
|
24
|
+
* the variables `ks`, `kc` and `ki`. `sid` must be a session id.
|
|
25
|
+
*/
|
|
26
|
+
export function killSessionScript(sid) {
|
|
27
|
+
if (!SID_RE.test(sid))
|
|
28
|
+
throw new Error(`invalid background task session: ${JSON.stringify(sid)}`);
|
|
29
|
+
return `ks=3; for ki in 1 2 3 4 5 6 7 8 9 10; do pkill -KILL -s ${sid}; kc=$?; if [ "$kc" -eq 1 ]; then ks=0; break; fi; if [ "$kc" -ne 0 ]; then ks=$kc; break; fi; sleep 0.2; done`;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Kills every process of the session `sid` ({@link killSessionScript}) in an exec of its own. Any
|
|
33
|
+
* pkill status other than 0 and 1 is a failure. Runs with a fresh signal of its own (a command
|
|
34
|
+
* must be killed even when the caller was just aborted). True when the session was confirmed
|
|
35
|
+
* empty; false when it could not be (an error is never thrown).
|
|
36
|
+
*/
|
|
37
|
+
export async function killSession(p, h, sid) {
|
|
38
|
+
if (!SID_RE.test(sid))
|
|
39
|
+
return false;
|
|
40
|
+
const r = await p
|
|
41
|
+
.exec(h, {
|
|
42
|
+
command: `${killSessionScript(sid)}; exit "$ks"`,
|
|
43
|
+
cwd: "/workspace",
|
|
44
|
+
timeoutMs: KILL_SESSION_TIMEOUT_MS,
|
|
45
|
+
maxOutputBytes: 4096,
|
|
46
|
+
signal: AbortSignal.timeout(KILL_SESSION_TIMEOUT_MS),
|
|
47
|
+
})
|
|
48
|
+
.catch(() => undefined);
|
|
49
|
+
return r?.exitCode === 0;
|
|
50
|
+
}
|
|
51
|
+
/** The most bytes of a task's output end a tail reads (the event keeps at most 4 000 characters). */
|
|
52
|
+
const TAIL_BYTES = 12_000;
|
|
53
|
+
/** The longest `lastLines` of a finished task (spec §2.4). */
|
|
54
|
+
export const LAST_LINES_MAX_CHARS = 4000;
|
|
55
|
+
/**
|
|
56
|
+
* A shell pipeline printing the last 20 lines of task `id`'s output (at most {@link TAIL_BYTES}),
|
|
57
|
+
* base64 on one line (empty when there is no output file). Relative to `/workspace`.
|
|
58
|
+
*/
|
|
59
|
+
export function tailCommand(id) {
|
|
60
|
+
return `{ tail -n 20 ${TASKS_DIR}/${id}.out 2>/dev/null | tail -c ${TAIL_BYTES} | base64 | tr -d '\\n'; }`;
|
|
61
|
+
}
|
|
62
|
+
/** Decodes a {@link tailCommand} result into `lastLines`: its end, at most 4 000 characters. */
|
|
63
|
+
export function decodeTail(b64) {
|
|
64
|
+
const text = Buffer.from(b64.trim(), "base64").toString("utf8");
|
|
65
|
+
return text.length > LAST_LINES_MAX_CHARS ? text.slice(-LAST_LINES_MAX_CHARS) : text;
|
|
66
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { BackgroundFinish } from "./finish.js";
|
|
2
|
+
import type { BackgroundTaskRow } from "./store.js";
|
|
3
|
+
/** What the poll exec reports for one task. */
|
|
4
|
+
export interface TaskReport {
|
|
5
|
+
/** The exit file's code; null while there is none. */
|
|
6
|
+
exitCode: number | null;
|
|
7
|
+
/**
|
|
8
|
+
* Whether the session still has a process: `pgrep -s <sid>`, or, for a task whose command
|
|
9
|
+
* exited, whether its leftover processes could not be confirmed killed.
|
|
10
|
+
*/
|
|
11
|
+
alive: boolean;
|
|
12
|
+
/** The last lines (decoded), read when the task exited or no process was left. */
|
|
13
|
+
lastLines: string;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* The poll exec of one sandbox: for every task one line `<id> <exit code or -> <1|0> <tail or ->`.
|
|
17
|
+
* Liveness is asked before the exit file: a task ending in between is seen by its exit file, never
|
|
18
|
+
* as a task without process and exit file. A task whose command exited while its session still has
|
|
19
|
+
* processes (it left `server &` running) has the session killed in the same exec (so under the
|
|
20
|
+
* caller's lease), and is reported without a process once the session was confirmed empty: the
|
|
21
|
+
* task ends when its command exits, and nothing of it runs on untracked. The tail (base64) is read
|
|
22
|
+
* only for a task that exited or has no process left. Relative to `/workspace`.
|
|
23
|
+
*/
|
|
24
|
+
export declare function pollScript(tasks: {
|
|
25
|
+
id: string;
|
|
26
|
+
sid: string;
|
|
27
|
+
}[]): string;
|
|
28
|
+
/** Parses {@link pollScript}'s output; a task missing from it is absent from the map. */
|
|
29
|
+
export declare function parsePollOutput(stdout: string): Map<string, TaskReport>;
|
|
30
|
+
/**
|
|
31
|
+
* How a polled task ends, or null while it runs (spec §4.3). A task whose command exited but whose
|
|
32
|
+
* leftover processes could not be confirmed killed ({@link pollScript}) stays running: the next
|
|
33
|
+
* poll kills them again before it finishes.
|
|
34
|
+
*/
|
|
35
|
+
export declare function polledAs(task: BackgroundTaskRow, r: TaskReport, now: Date): BackgroundFinish | null;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { TASK_ID_RE, TASKS_DIR } from "./launch.js";
|
|
2
|
+
import { decodeTail, killSessionScript, SID_RE, tailCommand } from "./session.js";
|
|
3
|
+
import { exitedAs } from "./stop.js";
|
|
4
|
+
/**
|
|
5
|
+
* The poll exec of one sandbox: for every task one line `<id> <exit code or -> <1|0> <tail or ->`.
|
|
6
|
+
* Liveness is asked before the exit file: a task ending in between is seen by its exit file, never
|
|
7
|
+
* as a task without process and exit file. A task whose command exited while its session still has
|
|
8
|
+
* processes (it left `server &` running) has the session killed in the same exec (so under the
|
|
9
|
+
* caller's lease), and is reported without a process once the session was confirmed empty: the
|
|
10
|
+
* task ends when its command exits, and nothing of it runs on untracked. The tail (base64) is read
|
|
11
|
+
* only for a task that exited or has no process left. Relative to `/workspace`.
|
|
12
|
+
*/
|
|
13
|
+
export function pollScript(tasks) {
|
|
14
|
+
const lines = tasks.map(({ id, sid }) => {
|
|
15
|
+
if (!TASK_ID_RE.test(id) || !SID_RE.test(sid)) {
|
|
16
|
+
throw new Error(`invalid background task ${JSON.stringify(id)} / ${JSON.stringify(sid)}`);
|
|
17
|
+
}
|
|
18
|
+
const f = `${TASKS_DIR}/${id}`;
|
|
19
|
+
return [
|
|
20
|
+
`if pgrep -s ${sid} >/dev/null 2>&1; then a=1; else a=0; fi`,
|
|
21
|
+
`if [ -s ${f}.exit ]; then c=$(tr -dc '0-9-' < ${f}.exit); else c=-; fi`,
|
|
22
|
+
`[ -n "$c" ] || c=-`,
|
|
23
|
+
`if [ "$c" != - ] && [ "$a" = 1 ]; then ${killSessionScript(sid)}; [ "$ks" = 0 ] && a=0; fi`,
|
|
24
|
+
`if [ "$c" != - ] || [ "$a" = 0 ]; then t=$(${tailCommand(id)}); else t=; fi`,
|
|
25
|
+
`echo "${id} $c $a \${t:--}"`,
|
|
26
|
+
].join("\n");
|
|
27
|
+
});
|
|
28
|
+
return lines.join("\n");
|
|
29
|
+
}
|
|
30
|
+
/** Parses {@link pollScript}'s output; a task missing from it is absent from the map. */
|
|
31
|
+
export function parsePollOutput(stdout) {
|
|
32
|
+
const out = new Map();
|
|
33
|
+
for (const line of stdout.split("\n")) {
|
|
34
|
+
const m = /^(bt_[a-z2-7]{6}) (-?[0-9]+|-) ([01]) (\S+)$/.exec(line.trim());
|
|
35
|
+
if (!m)
|
|
36
|
+
continue;
|
|
37
|
+
const [, id, code, alive, tail] = m;
|
|
38
|
+
out.set(id, {
|
|
39
|
+
exitCode: code === "-" ? null : Number(code),
|
|
40
|
+
alive: alive === "1",
|
|
41
|
+
lastLines: tail === "-" ? "" : decodeTail(tail),
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* How a polled task ends, or null while it runs (spec §4.3). A task whose command exited but whose
|
|
48
|
+
* leftover processes could not be confirmed killed ({@link pollScript}) stays running: the next
|
|
49
|
+
* poll kills them again before it finishes.
|
|
50
|
+
*/
|
|
51
|
+
export function polledAs(task, r, now) {
|
|
52
|
+
if (r.exitCode !== null)
|
|
53
|
+
return r.alive ? null : exitedAs(task, r.exitCode, now);
|
|
54
|
+
if (r.alive)
|
|
55
|
+
return null;
|
|
56
|
+
return { status: "lost", lostReason: "process_gone", killPending: false };
|
|
57
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
2
|
+
import type { KernelLogger } from "../log.js";
|
|
3
|
+
import { SandboxService } from "../sandbox/sandbox.service.js";
|
|
4
|
+
import { type BackgroundFinish, BackgroundTaskFinisher } from "./finish.js";
|
|
5
|
+
import { type BackgroundTaskRow, BackgroundTaskStore } from "./store.js";
|
|
6
|
+
/** Why a task is stopped. */
|
|
7
|
+
export type StopReason = "tool" | "person" | "subagent_end" | "conversation_deleted";
|
|
8
|
+
/** How long a stop waits after SIGTERM before SIGKILL (spec §2.3). */
|
|
9
|
+
export declare const STOP_GRACE_MS = 5000;
|
|
10
|
+
/** How a task that exited by itself ends (spec §4.3): exit 137 past its deadline timed out. */
|
|
11
|
+
export declare function exitedAs(task: BackgroundTaskRow, code: number, now: Date): BackgroundFinish;
|
|
12
|
+
/** How a task whose sandbox is not running ends (`ifRunning`'s `not_running` status). */
|
|
13
|
+
export declare function notRunningAs(status: string, killPending: boolean): BackgroundFinish;
|
|
14
|
+
/** A task found in another sandbox instance (a rebuild): its processes are gone with the old one. */
|
|
15
|
+
export declare const RECREATED: BackgroundFinish;
|
|
16
|
+
/**
|
|
17
|
+
* Stops background tasks (spec 2026-11-05-background-bash-design §2.3, §4.3): SIGTERM to the
|
|
18
|
+
* task's session, up to 5 s for it to empty, then SIGKILL until it is empty, all through
|
|
19
|
+
* `SandboxService.ifRunning` (it never creates or resumes a sandbox); the task then finishes as
|
|
20
|
+
* `stopped` with its last 20 lines. A task that had already exited finishes as its exit says. When
|
|
21
|
+
* the sandbox is not running the task finishes as `lost` with `kill_pending` (killed right after a
|
|
22
|
+
* resume); in a rebuilt sandbox as `lost` (`sandbox_recreated`). Processes a task that had
|
|
23
|
+
* already exited left running are killed as well.
|
|
24
|
+
*/
|
|
25
|
+
export declare class BackgroundTaskStopper {
|
|
26
|
+
private readonly store;
|
|
27
|
+
private readonly finisher;
|
|
28
|
+
private readonly sandbox;
|
|
29
|
+
private readonly db;
|
|
30
|
+
private readonly log;
|
|
31
|
+
constructor(store: Pick<BackgroundTaskStore, "get" | "runningOwnedBy" | "runningIn" | "claimStop" | "releaseStop">, finisher: BackgroundTaskFinisher, sandbox: Pick<SandboxService, "ifRunning">, db: KernelDb, log: Pick<KernelLogger, "warn">);
|
|
32
|
+
/**
|
|
33
|
+
* Stops `task` and returns its row afterwards. A task that is no longer running, or that another
|
|
34
|
+
* live stop already claimed, is returned as it is now.
|
|
35
|
+
*
|
|
36
|
+
* The stop first claims the task (`stopping_at`), so the poller leaves it alone while the
|
|
37
|
+
* signals take effect and the stop's finish is its only one. A reservation whose launch has not
|
|
38
|
+
* started the command yet (no `sid`) is only claimed: the launch then kills what it starts and
|
|
39
|
+
* finishes the row `stopped`. When the stop fails while the sandbox runs (busy, a provider
|
|
40
|
+
* error, a session that could not be confirmed empty), the claim is given up, the task stays
|
|
41
|
+
* `running` and the error is thrown.
|
|
42
|
+
*/
|
|
43
|
+
stop(task: BackgroundTaskRow, why: StopReason, signal?: AbortSignal): Promise<BackgroundTaskRow>;
|
|
44
|
+
/**
|
|
45
|
+
* After the sub-agent run `runId` ended (spec §4.3): stops the running tasks that run started in
|
|
46
|
+
* `conversationId` when it is a sub-agent's conversation (nobody can read or stop them through
|
|
47
|
+
* that run's model after that); any other conversation is left alone. Only that run's tasks: a
|
|
48
|
+
* follow-up run of the same child keeps its own when an older run's end is handled late. Never
|
|
49
|
+
* throws (see {@link stopAll}).
|
|
50
|
+
*/
|
|
51
|
+
stopSubagentTasks(conversationId: string, runId: string): Promise<void>;
|
|
52
|
+
/**
|
|
53
|
+
* Before a conversation is deleted: stops every running task in its sandbox (its own and its
|
|
54
|
+
* sub-agents'). Never throws (see {@link stopAll}).
|
|
55
|
+
*/
|
|
56
|
+
stopForDeletedConversation(conversationId: string): Promise<void>;
|
|
57
|
+
/**
|
|
58
|
+
* Stops `tasks` side by side. A stop that fails while the sandbox runs leaves its task running
|
|
59
|
+
* (logged): the poller still checks it and it ends at its own deadline at the latest.
|
|
60
|
+
*/
|
|
61
|
+
private stopAll;
|
|
62
|
+
}
|