@north-light/api 0.3.580 → 0.3.587

package/dist/core/bash-job-supervisor.js

@@ -0,0 +1,144 @@
+export const MAX_BACKGROUND_BASH_LOG_BYTES = 10 * 1024 * 1024;
+export const BACKGROUND_BASH_LOG_TRUNCATION_MARKER = '\n[background bash job output truncated at 10 MiB; remaining output discarded]\n';
+const LOG_COLLECTOR_SCRIPT = [
+    'const fs = require("node:fs");',
+    `const cap = ${MAX_BACKGROUND_BASH_LOG_BYTES};`,
+    `const marker = Buffer.from(${JSON.stringify(BACKGROUND_BASH_LOG_TRUNCATION_MARKER)});`,
+    'const prefixCap = cap - marker.length;',
+    'const fd = fs.openSync(process.argv[1], "w");',
+    'let written = 0;',
+    'let truncated = false;',
+    'process.stdin.on("data", (chunk) => {',
+    '  if (truncated) return;',
+    '  const available = prefixCap - written;',
+    '  if (available > 0) {',
+    '    const prefix = chunk.subarray(0, available);',
+    '    fs.writeSync(fd, prefix);',
+    '    written += prefix.length;',
+    '    if (prefix.length === chunk.length) return;',
+    '  }',
+    '  fs.writeSync(fd, marker);',
+    '  truncated = true;',
+    '});',
+    'process.stdin.on("end", () => fs.closeSync(fd));',
+].join(' ');
+/** Runs beside the job (a sibling of the log collector, in the supervisor's process group) and, every
+ * CRTR_BASH_ACTIVITY_SAMPLE_MS (default 60 s), records when the job last showed life: its process tree
+ * used CPU, or a regular file one of its processes holds open for writing changed. It writes
+ * `job.activity`: line 1 `<epoch-ms of the last activity> <epoch-ms of this sample>`, then one watched file path per line.
+ * Self-contained (Node built-ins only) so a sandboxed job host can run it unchanged. Process tree and open
+ * files come from /proc where it exists, else `ps` and `lsof`; a missing `lsof` degrades to CPU and the job
+ * log alone. Arguments: supervisor pid, job log path, activity file path. */
+const ACTIVITY_SAMPLER_SCRIPT = [
+    'const fs = require("node:fs"), cp = require("node:child_process");',
+    'const rootPid = Number(process.argv[1]), out = process.argv[3];',
+    'let logPath = process.argv[2];',
+    'try { logPath = fs.realpathSync(logPath); } catch {}',
+    'const interval = Number(process.env.CRTR_BASH_ACTIVITY_SAMPLE_MS) || 60000;',
+    'const linux = fs.existsSync("/proc/self/stat");',
+    'function processes() {',
+    '  const procs = new Map();',
+    '  if (linux) {',
+    '    for (const e of fs.readdirSync("/proc")) {',
+    '      if (!/^[0-9]+$/.test(e)) continue;',
+    '      try {',
+    '        const s = fs.readFileSync("/proc/" + e + "/stat", "utf8");',
+    '        const r = s.slice(s.lastIndexOf(")") + 2).split(" ");',
+    '        procs.set(Number(e), { ppid: Number(r[1]), cpu: (Number(r[11]) + Number(r[12])) / 100 });',
+    '      } catch {}',
+    '    }',
+    '    return procs;',
+    '  }',
+    '  const ps = cp.execFileSync("ps", ["-ax", "-o", "pid=,ppid=,time="], { encoding: "utf8", timeout: 5000 });',
+    '  for (const line of ps.split("\\n")) {',
+    '    const f = line.trim().split(/\\s+/);',
+    '    if (f.length < 3) continue;',
+    '    const [days, clock] = f[2].includes("-") ? f[2].split("-") : ["0", f[2]];',
+    '    const cpu = clock.split(":").reduce((t, x) => t * 60 + Number(x), 0) + Number(days) * 86400;',
+    '    if (Number.isFinite(cpu)) procs.set(Number(f[0]), { ppid: Number(f[1]), cpu });',
+    '  }',
+    '  return procs;',
+    '}',
+    'function tree(procs) {',
+    '  const cpus = new Map(), stack = [rootPid];',
+    '  while (stack.length > 0) {',
+    '    const pid = stack.pop();',
+    '    if (pid === process.pid || cpus.has(pid) || !procs.has(pid)) continue;',
+    '    cpus.set(pid, procs.get(pid).cpu);',
+    '    for (const [child, p] of procs) if (p.ppid === pid) stack.push(child);',
+    '  }',
+    '  return cpus;',
+    '}',
+    'function writableFiles(pids) {',
+    '  const files = new Set();',
+    '  if (linux) {',
+    '    for (const pid of pids) {',
+    '      let fds = [];',
+    '      try { fds = fs.readdirSync("/proc/" + pid + "/fd"); } catch {}',
+    '      for (const fd of fds) {',
+    '        try {',
+    '          const info = fs.readFileSync("/proc/" + pid + "/fdinfo/" + fd, "utf8").match(/flags:\\s*([0-7]+)/);',
+    '          if (info === null || (parseInt(info[1], 8) & 3) === 0) continue;',
+    '          files.add(fs.readlinkSync("/proc/" + pid + "/fd/" + fd));',
+    '        } catch {}',
+    '      }',
+    '    }',
+    '    return files;',
+    '  }',
+    '  const r = cp.spawnSync("lsof", ["-nP", "-w", "-F", "pfatn", "-p", pids.join(",")], { encoding: "utf8", timeout: 10000 });',
+    '  if (r.error || typeof r.stdout !== "string") return files;',
+    '  let access = "", type = "";',
+    '  for (const line of r.stdout.split("\\n")) {',
+    '    const key = line[0], value = line.slice(1);',
+    '    if (key === "f") { access = ""; type = ""; }',
+    '    else if (key === "a") access = value;',
+    '    else if (key === "t") type = value;',
+    '    else if (key === "n" && (access === "w" || access === "u") && type === "REG") files.add(value);',
+    '  }',
+    '  return files;',
+    '}',
+    'let lastActivity = Date.now(), prevAt = lastActivity, prevCpu = new Map(), watched = [logPath];',
+    'try { prevCpu = tree(processes()); } catch {}',
+    'function record() {',
+    '  fs.writeFileSync(out + ".tmp", lastActivity + " " + prevAt + "\\n" + watched.join("\\n") + "\\n");',
+    '  fs.renameSync(out + ".tmp", out);',
+    '}',
+    'function sample() {',
+    '  try { process.kill(rootPid, 0); } catch (e) { if (e.code !== "EPERM") process.exit(0); }',
+    '  const at = Date.now();',
+    '  const cpus = tree(processes());',
+    '  let used = 0;',
+    '  for (const [pid, cpu] of cpus) used += Math.max(0, cpu - (prevCpu.get(pid) || 0));',
+    '  const paths = new Set([logPath]);',
+    '  let changed = false;',
+    '  for (const file of writableFiles([...cpus.keys()])) {',
+    '    try {',
+    '      const st = fs.statSync(file);',
+    '      if (!st.isFile()) continue;',
+    '      paths.add(fs.realpathSync(file));',
+    '      if (st.mtimeMs > prevAt) changed = true;',
+    '    } catch {}',
+    '  }',
+    '  try { if (fs.statSync(logPath).mtimeMs > prevAt) changed = true; } catch {}',
+    '  if (at - prevAt > 3 * interval) changed = true;',
+    '  if (changed || used > 0.02 * (at - prevAt) / 1000) lastActivity = at;',
+    '  watched = [...paths];',
+    '  prevAt = at;',
+    '  prevCpu = cpus;',
+    '  record();',
+    '}',
+    'record();',
+    'setInterval(() => { try { sample(); } catch {} }, interval);',
+].join(' ');
+/** File-backed command supervisor. Beside the job it runs the activity sampler. It writes the exit code to job.exit and, for a foreground
+ * command, claims job.done; the daemon reads job.exit of a backgrounded job and publishes `exited`. */
+export const BASH_JOB_SUPERVISOR_SCRIPT = [
+    '"$8" -e ' + JSON.stringify(ACTIVITY_SAMPLER_SCRIPT) + ' "$$" "$2" "$9" &',
+    'sampler=$!',
+    'n=$(command -v nice)',
+    '${n:+"$n" -n "$7"} "$6" "$1" 2>&1 | "$8" -e ' + JSON.stringify(LOG_COLLECTOR_SCRIPT) + ' "$2"',
+    'status=${PIPESTATUS[0]}',
+    'kill "$sampler" 2>/dev/null',
+    'echo "$status" > "$3"',
+    'mv "$4" "$5" 2>/dev/null || true',
+].join('\n');

package/dist/core/bash-jobs.d.ts

@@ -0,0 +1,168 @@
+export interface BashJobPaths {
+    jobId: string;
+    /** The node's agent folder (the parent of its context folder): every job
+     * file resolves beneath it, so a symlink the agent plants is never followed. */
+    root: string;
+    dir: string;
+    cmdSh: string;
+    jobLog: string;
+    jobExit: string;
+    jobRun: string;
+    jobBg: string;
+    jobDone: string;
+    /** Process-group id of the job's detached supervisor, written at spawn. It is
+     * what makes a job stoppable from outside the agent that started it (the
+     * Inspector's cancel) — without it on disk, only the agent's own handoff
+     * notice knows the group to signal. */
+    jobPgid: string;
+    /** Launch-time identity of the detached supervisor, used to refuse a PID that
+     * has since been reused by an unrelated process. */
+    jobPgidIdentity: string;
+    /** Written beside the job by its supervisor's activity sampler: when the job last
+     * showed life and which files it was watched through (see `readJobActivity`). */
+    jobActivity: string;
+    /** The auto-handoff cancel deadline (epoch ms) the valve leaves until its job row exists with the deadline on it;
+     * the daemon moves it into the row (see `readJobSupervisorState`) and removes the file. */
+    jobDeadline: string;
+    /** Written by a sandbox broker's job reporter: how much of the output and whether the exit reached the control plane. */
+    jobReported: string;
+}
+/** A purpose is display text, never shell input. Keep the file-backed control
+ * plane to one bounded, visible line so old or hand-written job directories
+ * cannot inject control content into terminal or JSON presenters. */
+export declare function normalizeBashJobPurpose(value: unknown): string | null;
+export declare function bashJobsDir(contextDir: string): string;
+/** Whether a job id can safely be used as one path segment. Generated ids are
+ * numeric epoch prefixes plus a hex suffix; this broader safe alphabet keeps
+ * older file-backed ids readable without allowing path or request delimiters. */
+export declare function isSafeBashJobId(jobId: string): boolean;
+export declare function bashJobPaths(contextDir: string, jobId: string): BashJobPaths;
+/** The job's supervisor process group, or undefined for a job started before
+ * pgid was persisted (or a partially-written dir). */
+export declare function readJobPgid(contextDir: string, jobId: string): number | undefined;
+/** Launch-time identity of the supervisor. Missing for jobs created before
+ * process-identity persistence, which are deliberately not safe to signal. */
+export declare function readJobPgidIdentity(contextDir: string, jobId: string): string | undefined;
+/** What the job's detached supervisor wrote beside the job: its process identity, the activity sample
+ *  and, once it ended, the exit code. Each field is absent until written. */
+export interface JobSupervisorState {
+    pgid?: number;
+    pgidIdentity?: string;
+    exitCode?: number;
+    activity?: {
+        lastActivityMs: number;
+        sampledAtMs: number;
+        files: string[];
+    };
+    /** A cancel deadline still on disk: left by the valve before its job row existed, or by a job started before deadlines lived on rows. */
+    deadlineMs?: number;
+}
+export declare function readJobSupervisorState(contextDir: string, jobId: string): JobSupervisorState;
+/** Write the auto-handoff cancel deadline beside the job; `clearJobDeadlineFile` removes it once the job row carries it. */
+export declare function writeJobDeadlineFile(paths: BashJobPaths, deadlineMs: number): void;
+export declare function clearJobDeadlineFile(paths: BashJobPaths): void;
+/** Ids of this node's job directories whose command has not exited — foreground (job.run) or backgrounded (job.bg),
+ *  with or without a job row. What a node stop must signal. */
+export declare function unfinishedBashJobIds(contextDir: string): string[];
+/** Scheduler niceness every command a node runs is started with. Agent work —
+ * builds, test suites, typechecks — sits below the daemon, the brokers, the
+ * viewers, and the person's own shell, so a fan-out of release builds cannot
+ * starve them: the scheduler only honors this under contention and it costs
+ * nothing on an idle host. */
+export declare const DEFAULT_BASH_NICENESS = 10;
+/** The politest priority the scheduler accepts. A NEGATIVE niceness (more
+ * priority than the broker itself) needs root: `nice` prints a permission error
+ * into the command's own log and runs it at the inherited priority anyway, so
+ * the override range stops at 0. */
+export declare const MAX_BASH_NICENESS = 19;
+/** Per-node override of the bash niceness, written by `crtr node config --nice`
+ * and read by the valve when it starts each command. Node-local like the job
+ * dirs beside it, so a change applies to the node's very next command without
+ * waiting for a revive. */
+export declare function bashNicenessPath(contextDir: string): string;
+/** The niceness this node's commands run at. Any missing, malformed, or
+ * out-of-range file reads as the default rather than an odd priority. */
+export declare function readBashNiceness(contextDir: string): number;
+/** Set the node's bash niceness. The caller owns validation; the store keeps
+ * one integer so the valve's read stays a single file read per command. */
+export declare function writeBashNiceness(contextDir: string, niceness: number): void;
+/** Launcher jobs have a root-owned bwrap pid, not a process group; direct
+ *  jobs keep their detached supervisor process group. */
+export declare function isBashJobProcessGroupAlive(pgid: number, identity?: string): boolean;
+/** Whether the job's process is CONFIRMED gone: a direct job's process group has no member
+ *  (signal 0 answers ESRCH), a launcher job's pid is dead. EPERM, any other error, and a missing
+ *  pgid or identity are not confirmation, so a job that cannot be read is never declared gone. */
+export declare function isBashJobProcessConfirmedGone(pgid: number | undefined, identity: string | undefined): boolean;
+/** Best-effort cancellation for a detached bash supervisor. It snapshots the
+ * observable descendant tree while the supervisor is still alive, signals its
+ * leaves before their parents, then uses the shared identity-guarded KILL
+ * escalation. A descendant that detached and reparented before this snapshot
+ * is outside this guarantee. */
+export declare function cancelBashJobProcessTree(pgid: number, expectedIdentity: string): Promise<boolean>;
+/** Stop a job's process: a launcher job through the launcher, a direct job by its supervisor's process tree.
+ *  False when no process was left to signal. */
+export declare function signalBashJob(pgid: number, identity: string, nodeId: string | undefined): Promise<boolean>;
+export type StopBackgroundBashJobResult = {
+    kind: 'stopped';
+    signaled: boolean;
+    pgid: number;
+    logPath: string;
+} | {
+    kind: 'missing';
+} | {
+    kind: 'retired';
+    logPath: string;
+} | {
+    kind: 'not-stoppable';
+    logPath: string;
+};
+/** Stop one live background job using the same best-effort tree cancellation
+ * semantics as inline abort and timeout. The caller owns the user-facing
+ * notification; this primitive owns only the job control-plane mutation.
+ *
+ * Only a job in the live-background state the roster projects (job.bg present,
+ * job.exit absent) may be signaled. Job directories persist after the
+ * supervisor exits, so a retired, still-foreground, or already-stopped job
+ * would otherwise have its recorded pgid signaled long after the OS could have
+ * reused that id for an unrelated process group. */
+export declare function stopBackgroundBashJob(contextDir: string, jobId: string, allowForeground?: boolean, nodeId?: string, reason?: string): Promise<StopBackgroundBashJobResult>;
+/** Last `count` lines of a job log, cheaply: read at most the trailing 64 KiB
+ * rather than the whole file, which for a long-running job can be enormous. */
+export declare function tailJobLog(logPath: string, count: number): string[];
+/** Lines written to a job log so far, counting a trailing unterminated line as
+ * one. Reads the whole file in fixed-size chunks, so even a log at the 10 MiB
+ * cap costs one buffer rather than its full size in memory. */
+export declare function countJobLogLines(logPath: string): number;
+/** Mint a fresh job id — `${Date.now()}-${hex}`. The sole minting site; the
+ * valve calls this instead of building the literal inline so the epoch-ms
+ * prefix is a real contract shared with its parser (jobStartedAtMs), not an
+ * incidental string format. */
+export declare function newBashJobId(): string;
+/** Parse the start epoch (ms) out of a job id's prefix. undefined unless the
+ * id is exactly `<finite-number>-<nonempty-suffix>` — defensive against a
+ * hand-created directory that doesn't follow the newBashJobId contract,
+ * including one with no separator at all (`indexOf('-') === -1` must not
+ * silently parse as a truncated prefix). */
+export declare function jobStartedAtMs(jobId: string): number | undefined;
+/** Render an elapsed duration the way every bash-handoff surface (the valve's
+ * notice, the `node bash background` result/toast, and the attach cockpit)
+ * shows it: `<60s -> "12s"`; `60s-59m -> "3m 40s"`; `>=60m -> "1h 04m"`. */
+export declare function formatBashElapsed(ms: number): string;
+/** The command line worth SHOWING for a job: the first non-empty line of
+ * `cmd.sh`, whitespace-collapsed to one line. */
+export declare function bashJobCommandLine(command: string): string;
+export type BackgroundBashJobResult = 'backgrounded' | 'already-backgrounded' | 'finished' | 'missing';
+/** Atomically hand one foreground job to its detached supervisor. The rename
+ * is the ownership decision: exactly one caller can move job.run to job.bg,
+ * while the supervisor races to move the same source to job.done. */
+export declare function backgroundBashJob(paths: BashJobPaths): BackgroundBashJobResult;
+/** The start time (ms epoch) of the oldest still-foreground bash command for
+ * this node, or undefined when none is running. This is the authoritative
+ * source for the attach viewer's background hint: a fresh or reconnected
+ * attach reads the same job.run state that backgrounding and completion race
+ * to claim, without viewer-local call tracking that can go stale. */
+export declare function oldestRunningBashJobStartedAtMs(contextDir: string): number | undefined;
+/** Request immediate handoff for every bash command still waiting on pi. Only
+ * the caller that wins the job.run → job.bg rename reports the job as handed
+ * off; another backgrounder or the foreground-completion path may win the race. */
+export declare function backgroundRunningBashJobs(contextDir: string): BashJobPaths[];