@rallycry/conveyor-agent 10.13.18 → 10.13.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/{boot-YZCOIV3Q.js → boot-BZE36LT6.js} +156 -125
  2. package/dist/boot-BZE36LT6.js.map +1 -0
  3. package/dist/chunk-5OQQSDVT.js +142 -0
  4. package/dist/chunk-5OQQSDVT.js.map +1 -0
  5. package/dist/{chunk-3GDXX3OX.js → chunk-EQO7QFGN.js} +6201 -2125
  6. package/dist/chunk-EQO7QFGN.js.map +1 -0
  7. package/dist/{chunk-AGGVNADW.js → chunk-HVSPADKP.js} +59 -13
  8. package/dist/chunk-HVSPADKP.js.map +1 -0
  9. package/dist/{chunk-UZTJJD7Y.js → chunk-KCB7CSWJ.js} +43 -2
  10. package/dist/chunk-KCB7CSWJ.js.map +1 -0
  11. package/dist/{chunk-72AEN6LB.js → chunk-KZQBYSMN.js} +137 -2
  12. package/dist/chunk-KZQBYSMN.js.map +1 -0
  13. package/dist/{chunk-UDSRAHKP.js → chunk-QOJTJCYZ.js} +67 -19
  14. package/dist/chunk-QOJTJCYZ.js.map +1 -0
  15. package/dist/chunk-ULS4QPRE.js +138 -0
  16. package/dist/chunk-ULS4QPRE.js.map +1 -0
  17. package/dist/cli.js +245 -551
  18. package/dist/cli.js.map +1 -1
  19. package/dist/{client-BU4XA7CV.js → client-LRVVHTNG.js} +2 -2
  20. package/dist/heartbeat-worker.js +10 -12
  21. package/dist/heartbeat-worker.js.map +1 -1
  22. package/dist/index.d.ts +153 -8
  23. package/dist/index.js +5 -5
  24. package/dist/oom-watchdog-U7JERHA2.js +11 -0
  25. package/dist/server-PPTELJWY.js +10 -0
  26. package/dist/server-PPTELJWY.js.map +1 -0
  27. package/package.json +3 -3
  28. package/runtime/entrypoint.sh +5 -4
  29. package/dist/boot-YZCOIV3Q.js.map +0 -1
  30. package/dist/chunk-3GDXX3OX.js.map +0 -1
  31. package/dist/chunk-72AEN6LB.js.map +0 -1
  32. package/dist/chunk-7TQO4ZF4.js +0 -60
  33. package/dist/chunk-7TQO4ZF4.js.map +0 -1
  34. package/dist/chunk-AGGVNADW.js.map +0 -1
  35. package/dist/chunk-UDSRAHKP.js.map +0 -1
  36. package/dist/chunk-UZTJJD7Y.js.map +0 -1
  37. package/dist/server-BXAFMYDM.js +0 -9
  38. /package/dist/{client-BU4XA7CV.js.map → client-LRVVHTNG.js.map} +0 -0
  39. /package/dist/{server-BXAFMYDM.js.map → oom-watchdog-U7JERHA2.js.map} +0 -0
@@ -41,6 +41,38 @@ var RemoteProcessHandle = class extends EventEmitter {
41
41
  function frameError(frame) {
42
42
  return new WorkbenchError(frame.message, frame.code);
43
43
  }
44
+ function execFailure(message, fields) {
45
+ const failure = new Error(message);
46
+ failure.stdout = fields.stdout;
47
+ failure.stderr = fields.stderr;
48
+ if (fields.code !== void 0) failure.code = fields.code;
49
+ if (fields.signal !== void 0) failure.signal = fields.signal;
50
+ return failure;
51
+ }
52
+ function settleExecOutcome(state, resolve, reject) {
53
+ const out = Buffer.concat(state.stdout).toString("utf8");
54
+ const errText = Buffer.concat(state.stderr).toString("utf8");
55
+ if (state.overflowed) {
56
+ reject(
57
+ execFailure(
58
+ `maxBuffer length exceeded (${state.maxBuffer} bytes): ${state.file} ${state.args.join(" ")}`,
59
+ { stdout: out, stderr: errText }
60
+ )
61
+ );
62
+ } else if (!state.exited) {
63
+ reject(state.error ?? new WorkbenchError("connection closed before exit"));
64
+ } else if (state.exitCode === 0) {
65
+ resolve({ stdout: out, stderr: errText });
66
+ } else {
67
+ reject(
68
+ execFailure(
69
+ state.timedOut ? `Command timed out: ${state.file}` : `Command failed: ${state.file} ${state.args.join(" ")}
70
+ ${errText}`,
71
+ { stdout: out, stderr: errText, code: state.exitCode, signal: state.exitSignal }
72
+ )
73
+ );
74
+ }
75
+ }
44
76
  var WorkbenchClient = class {
45
77
  port;
46
78
  host;
@@ -191,6 +223,10 @@ var WorkbenchClient = class {
191
223
  return new Promise((resolve, reject) => {
192
224
  const stdout = [];
193
225
  const stderr = [];
226
+ let stdoutBytes = 0;
227
+ let stderrBytes = 0;
228
+ const maxBuffer = opts.maxBuffer;
229
+ let overflowed = false;
194
230
  let exitCode = null;
195
231
  let exitSignal = null;
196
232
  let exited = false;
@@ -199,7 +235,19 @@ var WorkbenchClient = class {
199
235
  { op: "exec", argv: [file, ...args], cwd: opts.cwd ?? process.cwd() },
200
236
  (frame) => {
201
237
  if (frame.t === "out") {
202
- (frame.s === "stdout" ? stdout : stderr).push(Buffer.from(frame.d, "base64"));
238
+ const buf = Buffer.from(frame.d, "base64");
239
+ if (frame.s === "stdout") {
240
+ stdout.push(buf);
241
+ stdoutBytes += buf.length;
242
+ } else {
243
+ stderr.push(buf);
244
+ stderrBytes += buf.length;
245
+ }
246
+ if (maxBuffer !== void 0 && !overflowed && (stdoutBytes > maxBuffer || stderrBytes > maxBuffer)) {
247
+ overflowed = true;
248
+ if (timer) clearTimeout(timer);
249
+ writeFrame(socket, { t: "signal", mode: "term-group" });
250
+ }
203
251
  } else if (frame.t === "exit") {
204
252
  exited = true;
205
253
  exitCode = frame.code;
@@ -208,23 +256,23 @@ var WorkbenchClient = class {
208
256
  },
209
257
  (error) => {
210
258
  if (timer) clearTimeout(timer);
211
- const out = Buffer.concat(stdout).toString("utf8");
212
- const errText = Buffer.concat(stderr).toString("utf8");
213
- if (!exited) {
214
- reject(error ?? new WorkbenchError("connection closed before exit"));
215
- } else if (exitCode === 0) {
216
- resolve({ stdout: out, stderr: errText });
217
- } else {
218
- const failure = new Error(
219
- timedOut ? `Command timed out: ${file}` : `Command failed: ${file} ${args.join(" ")}
220
- ${errText}`
221
- );
222
- failure.code = exitCode;
223
- failure.signal = exitSignal;
224
- failure.stdout = out;
225
- failure.stderr = errText;
226
- reject(failure);
227
- }
259
+ settleExecOutcome(
260
+ {
261
+ file,
262
+ args,
263
+ stdout,
264
+ stderr,
265
+ overflowed,
266
+ exited,
267
+ exitCode,
268
+ exitSignal,
269
+ timedOut,
270
+ maxBuffer,
271
+ error: error ?? null
272
+ },
273
+ resolve,
274
+ reject
275
+ );
228
276
  }
229
277
  );
230
278
  const timer = opts.timeout ? setTimeout(() => {
@@ -361,4 +409,4 @@ export {
361
409
  getWorkbenchClient,
362
410
  resetWorkbenchClient
363
411
  };
364
- //# sourceMappingURL=chunk-UDSRAHKP.js.map
412
+ //# sourceMappingURL=chunk-QOJTJCYZ.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/workbench/client.ts","../src/workbench/errors.ts","../src/workbench/remote-process.ts"],"sourcesContent":["/**\n * Client half of the workbench launcher protocol. Runs in the protected agent\n * container; every method opens one loopback connection, sends the request\n * frame, and adapts the stream back to the local signature it replaces\n * (runSetupCommand, runStartCommand, execFile, PtySpawn, workspace file\n * reads). See server.ts / protocol.ts.\n */\n\nimport { connect, type Socket } from \"node:net\";\nimport type { PtyProcess, PtySpawnOptions } from \"../harness/pty/pty-support.js\";\nimport { WorkbenchError } from \"./errors.js\";\nimport { RemoteProcessHandle } from \"./remote-process.js\";\n\nexport { WorkbenchError } from \"./errors.js\";\nexport { RemoteProcessHandle } from \"./remote-process.js\";\nimport { workbenchPort, workbenchToken } from \"./mode.js\";\nimport {\n DEFAULT_WORKBENCH_PORT,\n FrameReader,\n writeFrame,\n type GitStatusFrame,\n type WorkbenchFrame,\n type WorkbenchRequest,\n} from \"./protocol.js\";\n\nexport interface WorkbenchClientOptions {\n port?: number;\n token?: string;\n host?: string;\n}\n\n/** Omit that distributes over a discriminated union (plain Omit collapses\n * the union to its common properties). */\ntype DistributiveOmit<T, K extends PropertyKey> = T extends unknown ? Omit<T, K> : never;\n\nfunction frameError(frame: Extract<WorkbenchFrame, { t: \"error\" }>): WorkbenchError {\n return new WorkbenchError(frame.message, frame.code);\n}\n\ntype ExecFailure = Error & {\n code?: number | null;\n signal?: string | null;\n stdout: string;\n stderr: string;\n};\n\n/** Build a rejection carrying the same stdout/stderr/code/signal fields node's\n * promisified execFile attaches, so callers can inspect them. */\nfunction execFailure(\n message: string,\n fields: { stdout: string; stderr: string; code?: number | null; signal?: string | null },\n): ExecFailure {\n const failure = new Error(message) as ExecFailure;\n failure.stdout = fields.stdout;\n failure.stderr = fields.stderr;\n if (fields.code !== undefined) failure.code = fields.code;\n if (fields.signal !== undefined) failure.signal = fields.signal;\n return failure;\n}\n\n/** Settle an execFile promise from the collected stream + exit state. Extracted\n * from `execFile` so the method body stays within the per-function line cap;\n * behavior is identical to the inline close handler it replaces. */\nfunction settleExecOutcome(\n state: {\n file: string;\n args: string[];\n stdout: Buffer[];\n stderr: Buffer[];\n overflowed: boolean;\n exited: boolean;\n exitCode: number | null;\n exitSignal: string | null;\n timedOut: boolean;\n maxBuffer: number | undefined;\n error: Error | null;\n },\n resolve: (value: { stdout: string; stderr: string }) => void,\n reject: (reason: unknown) => void,\n): void {\n const out = Buffer.concat(state.stdout).toString(\"utf8\");\n const errText = Buffer.concat(state.stderr).toString(\"utf8\");\n if (state.overflowed) {\n reject(\n execFailure(\n `maxBuffer length exceeded (${state.maxBuffer} bytes): ${state.file} ${state.args.join(\" \")}`,\n { stdout: out, stderr: errText },\n ),\n );\n } else if (!state.exited) {\n reject(state.error ?? new WorkbenchError(\"connection closed before exit\"));\n } else if (state.exitCode === 0) {\n resolve({ stdout: out, stderr: errText });\n } else {\n reject(\n execFailure(\n state.timedOut\n ? `Command timed out: ${state.file}`\n : `Command failed: ${state.file} ${state.args.join(\" \")}\\n${errText}`,\n { stdout: out, stderr: errText, code: state.exitCode, signal: state.exitSignal },\n ),\n );\n }\n}\n\nexport class WorkbenchClient {\n private readonly port: number;\n private readonly host: string;\n private readonly token: string;\n\n constructor(options: WorkbenchClientOptions = {}) {\n this.port = options.port ?? workbenchPort() ?? DEFAULT_WORKBENCH_PORT;\n this.host = options.host ?? \"127.0.0.1\";\n this.token = options.token ?? workbenchToken();\n }\n\n /** Open a connection, send the request, route response frames. */\n private open(\n request: DistributiveOmit<WorkbenchRequest, \"token\">,\n onFrame: (frame: WorkbenchFrame, socket: Socket) => void,\n onClose: (error?: Error) => void,\n ): Socket {\n const socket = connect(this.port, this.host);\n let sawError: Error | undefined;\n const reader = new FrameReader((raw) => {\n const frame = raw as unknown as WorkbenchFrame;\n if (frame.t === \"error\") {\n sawError = frameError(frame as Extract<WorkbenchFrame, { t: \"error\" }>);\n }\n onFrame(frame, socket);\n });\n socket.on(\"connect\", () => {\n writeFrame(socket, { ...request, token: this.token } as WorkbenchRequest);\n });\n socket.on(\"data\", (chunk) => reader.push(chunk));\n socket.on(\"error\", (err) => {\n sawError ??= err;\n });\n socket.on(\"close\", () => onClose(sawError));\n return socket;\n }\n\n ping(): Promise<string> {\n return new Promise((resolve, reject) => {\n let version: string | null = null;\n this.open(\n { op: \"ping\" },\n (frame) => {\n if (frame.t === \"pong\") version = frame.version;\n },\n (error) => {\n if (version === null) {\n reject(error ?? new WorkbenchError(\"connection closed before pong\"));\n } else {\n resolve(version);\n }\n },\n );\n });\n }\n\n /** One-shot poll of the workbench's git-prep state. A daemon without a\n * provider answers `ready` (non-pod servers must not wedge callers). */\n gitStatus(): Promise<GitStatusFrame> {\n return new Promise((resolve, reject) => {\n let status: GitStatusFrame | null = null;\n this.open(\n { op: \"gitStatus\" },\n (frame) => {\n if (frame.t === \"gitStatus\") status = frame;\n },\n (error) => {\n if (status) resolve(status);\n else reject(error ?? new WorkbenchError(\"connection closed before gitStatus\"));\n },\n );\n });\n }\n\n /** Mirrors setup/commands.ts runSetupCommand (shell form, streamed output,\n * abort → graceful term-group kill, non-zero exit rejects). */\n runSetupCommand(\n cmd: string,\n cwd: string,\n onOutput: (stream: \"stdout\" | \"stderr\", data: string) => void,\n signal?: AbortSignal,\n ): Promise<void> {\n return new Promise((resolve, reject) => {\n if (signal?.aborted) {\n const error = new Error(\"Operation aborted\");\n error.name = \"AbortError\";\n reject(error);\n return;\n }\n let exitCode: number | null = null;\n let exited = false;\n let aborting = false;\n const socket = this.open(\n { op: \"exec\", command: cmd, cwd },\n (frame) => {\n if (frame.t === \"out\" && !aborting) {\n onOutput(frame.s, Buffer.from(frame.d, \"base64\").toString(\"utf8\"));\n } else if (frame.t === \"exit\") {\n exited = true;\n exitCode = frame.code;\n }\n },\n (error) => {\n signal?.removeEventListener(\"abort\", onAbort);\n if (aborting) {\n const abortError = new Error(\"Operation aborted\");\n abortError.name = \"AbortError\";\n reject(abortError);\n } else if (!exited) {\n reject(error ?? new WorkbenchError(\"connection closed before exit\"));\n } else if (exitCode === 0) {\n resolve();\n } else {\n reject(new Error(`Setup command exited with code ${exitCode}`));\n }\n },\n );\n const onAbort = (): void => {\n if (exited || aborting) return;\n aborting = true;\n writeFrame(socket, { t: \"signal\", mode: \"term-group\" });\n };\n signal?.addEventListener(\"abort\", onAbort, { once: true });\n });\n }\n\n /** Mirrors setup/commands.ts runStartCommand: returns a live handle that\n * emits exit/error and whose kill() runs the graceful term-group stop. */\n runStartCommand(\n cmd: string,\n cwd: string,\n onOutput: (stream: \"stdout\" | \"stderr\", data: string) => void,\n ): RemoteProcessHandle {\n let socket: Socket | undefined;\n const handle = new RemoteProcessHandle(() => {\n if (socket) writeFrame(socket, { t: \"signal\", mode: \"term-group\" });\n });\n let exited = false;\n socket = this.open(\n { op: \"exec\", command: cmd, cwd },\n (frame) => {\n if (frame.t === \"out\") {\n onOutput(frame.s, Buffer.from(frame.d, \"base64\").toString(\"utf8\"));\n } else if (frame.t === \"exit\") {\n exited = true;\n handle.exitCode = frame.code ?? (frame.signal ? null : 0);\n handle.emit(\"exit\", frame.code, frame.signal);\n }\n },\n (error) => {\n if (!exited) {\n // Connection died without an exit frame (launcher restart, network\n // teardown): surface as an error + synthetic exit so the supervisor\n // observes the process ending rather than hanging on it forever.\n exited = true;\n handle.exitCode = handle.exitCode ?? 1;\n handle.emit(\n \"error\",\n error ?? new WorkbenchError(\"workbench connection lost\", \"workbench_gone\"),\n );\n handle.emit(\"exit\", handle.exitCode, null);\n }\n },\n );\n return handle;\n }\n\n /** Mirrors promisified execFile (argv form, no shell): resolves stdout,\n * rejects with stderr-bearing error on non-zero exit. */\n execFile(\n file: string,\n args: string[],\n opts: { cwd?: string; timeout?: number; maxBuffer?: number } = {},\n ): Promise<{ stdout: string; stderr: string }> {\n return new Promise((resolve, reject) => {\n const stdout: Buffer[] = [];\n const stderr: Buffer[] = [];\n let stdoutBytes = 0;\n let stderrBytes = 0;\n const maxBuffer = opts.maxBuffer;\n let overflowed = false;\n let exitCode: number | null = null;\n let exitSignal: string | null = null;\n let exited = false;\n let timedOut = false;\n const socket = this.open(\n { op: \"exec\", argv: [file, ...args], cwd: opts.cwd ?? process.cwd() },\n (frame) => {\n if (frame.t === \"out\") {\n const buf = Buffer.from(frame.d, \"base64\");\n if (frame.s === \"stdout\") {\n stdout.push(buf);\n stdoutBytes += buf.length;\n } else {\n stderr.push(buf);\n stderrBytes += buf.length;\n }\n // Enforce maxBuffer per-stream, mirroring node's execFile: kill the\n // command group once either stream overflows so a runaway process\n // can't buffer unbounded memory in-process.\n if (\n maxBuffer !== undefined &&\n !overflowed &&\n (stdoutBytes > maxBuffer || stderrBytes > maxBuffer)\n ) {\n overflowed = true;\n if (timer) clearTimeout(timer);\n writeFrame(socket, { t: \"signal\", mode: \"term-group\" });\n }\n } else if (frame.t === \"exit\") {\n exited = true;\n exitCode = frame.code;\n exitSignal = frame.signal;\n }\n },\n (error) => {\n if (timer) clearTimeout(timer);\n settleExecOutcome(\n {\n file,\n args,\n stdout,\n stderr,\n overflowed,\n exited,\n exitCode,\n exitSignal,\n timedOut,\n maxBuffer,\n error: error ?? null,\n },\n resolve,\n reject,\n );\n },\n );\n const timer = opts.timeout\n ? setTimeout(() => {\n timedOut = true;\n writeFrame(socket, { t: \"signal\", mode: \"term-group\" });\n }, opts.timeout)\n : null;\n timer?.unref();\n });\n }\n\n /**\n * Synchronous PtySpawn adapter: returns a PtyProcess immediately; writes\n * and resizes are queued until the connection opens. Exit surfaces through\n * onExit exactly as node-pty's would (a lost connection = exit code 1 —\n * the session's finalizeOnExit path treats it as a crashed CLI).\n */\n spawnPty(file: string, args: string[], options: PtySpawnOptions): PtyProcess {\n const dataListeners: Array<(data: string) => void> = [];\n const exitListeners: Array<(event: { exitCode: number }) => void> = [];\n let connected = false;\n let exited = false;\n const preConnectQueue: WorkbenchFrame[] = [];\n\n const socket = this.open(\n {\n op: \"pty\",\n file,\n args,\n cwd: options.cwd,\n env: options.env,\n cols: options.cols,\n rows: options.rows,\n },\n (frame) => {\n if (frame.t === \"data\") {\n const text = Buffer.from(frame.d, \"base64\").toString(\"utf8\");\n for (const listener of dataListeners) listener(text);\n } else if (frame.t === \"exit\") {\n exited = true;\n for (const listener of exitListeners) listener({ exitCode: frame.code ?? 1 });\n }\n },\n () => {\n if (!exited) {\n exited = true;\n for (const listener of exitListeners) listener({ exitCode: 1 });\n }\n },\n );\n socket.on(\"connect\", () => {\n connected = true;\n for (const frame of preConnectQueue.splice(0)) writeFrame(socket, frame);\n });\n\n const send = (frame: WorkbenchFrame): void => {\n if (exited) return;\n if (connected) writeFrame(socket, frame);\n else preConnectQueue.push(frame);\n };\n\n return {\n onData: (listener) => dataListeners.push(listener),\n onExit: (listener) => exitListeners.push(listener),\n write: (data) => send({ t: \"input\", d: Buffer.from(data, \"utf8\").toString(\"base64\") }),\n resize: (cols, rows) => send({ t: \"resize\", cols, rows }),\n kill: (sig) => send({ t: \"kill\", ...(sig ? { sig } : {}) }),\n };\n }\n\n readFile(path: string): Promise<Buffer> {\n return new Promise((resolve, reject) => {\n const chunks: Buffer[] = [];\n let ended = false;\n this.open(\n { op: \"readFile\", path },\n (frame) => {\n if (frame.t === \"data\") chunks.push(Buffer.from(frame.d, \"base64\"));\n else if (frame.t === \"end\") ended = true;\n },\n (error) => {\n if (ended) resolve(Buffer.concat(chunks));\n else reject(error ?? new WorkbenchError(\"connection closed before end\"));\n },\n );\n });\n }\n\n stat(path: string): Promise<{\n exists: boolean;\n isFile: boolean;\n isDirectory: boolean;\n size: number;\n mtimeMs: number;\n }> {\n return new Promise((resolve, reject) => {\n let stat: {\n exists: boolean;\n isFile: boolean;\n isDirectory: boolean;\n size: number;\n mtimeMs: number;\n } | null = null;\n this.open(\n { op: \"stat\", path },\n (frame) => {\n if (frame.t === \"stat\") {\n stat = {\n exists: frame.exists,\n isFile: frame.isFile,\n isDirectory: frame.isDirectory,\n size: frame.size,\n mtimeMs: frame.mtimeMs,\n };\n }\n },\n (error) => {\n if (stat) resolve(stat);\n else reject(error ?? new WorkbenchError(\"connection closed before stat\"));\n },\n );\n });\n }\n\n readdir(path: string): Promise<string[]> {\n return new Promise((resolve, reject) => {\n let entries: string[] | null = null;\n this.open(\n { op: \"readdir\", path },\n (frame) => {\n if (frame.t === \"entries\") entries = frame.names;\n },\n (error) => {\n if (entries) resolve(entries);\n else reject(error ?? new WorkbenchError(\"connection closed before entries\"));\n },\n );\n });\n }\n}\n\nlet singleton: WorkbenchClient | null = null;\n\n/** Lazy env-configured client (split-mode pods). */\nexport function getWorkbenchClient(): WorkbenchClient {\n singleton ??= new WorkbenchClient();\n return singleton;\n}\n\n/** Test seam. */\nexport function resetWorkbenchClient(): void {\n singleton = null;\n}\n","/** Error surfaced by the workbench client for launcher-side failures; `code`\n * carries the remote error code (e.g. \"ENOENT\", \"unauthorized\") so callers'\n * existing errno handling keeps working across the container boundary. */\nexport class WorkbenchError extends Error {\n code?: string;\n constructor(message: string, code?: string) {\n super(message);\n this.name = \"WorkbenchError\";\n if (code) this.code = code;\n }\n}\n","import { EventEmitter } from \"node:events\";\n\n/**\n * Structural stand-in for the ChildProcess surface the workspace-command\n * supervisor actually uses (pid/exitCode/kill + exit/error events) — see\n * ManagedChildProcess in setup/commands.ts. kill() requests the launcher's\n * graceful term-group sequence regardless of the signal argument.\n */\nexport class RemoteProcessHandle extends EventEmitter {\n pid: number | undefined = undefined;\n exitCode: number | null = null;\n\n constructor(private readonly sendSignal: () => void) {\n super();\n }\n\n kill(_signal?: NodeJS.Signals | number): boolean {\n this.sendSignal();\n return true;\n }\n}\n"],"mappings":";;;;;;;;;;;AAQA,SAAS,eAA4B;;;ACL9B,IAAM,iBAAN,cAA6B,MAAM;AAAA,EACxC;AAAA,EACA,YAAY,SAAiB,MAAe;AAC1C,UAAM,OAAO;AACb,SAAK,OAAO;AACZ,QAAI,KAAM,MAAK,OAAO;AAAA,EACxB;AACF;;;ACVA,SAAS,oBAAoB;AAQtB,IAAM,sBAAN,cAAkC,aAAa;AAAA,EAIpD,YAA6B,YAAwB;AACnD,UAAM;AADqB;AAAA,EAE7B;AAAA,EAF6B;AAAA,EAH7B,MAA0B;AAAA,EAC1B,WAA0B;AAAA,EAM1B,KAAK,SAA4C;AAC/C,SAAK,WAAW;AAChB,WAAO;AAAA,EACT;AACF;;;AFeA,SAAS,WAAW,OAAgE;AAClF,SAAO,IAAI,eAAe,MAAM,SAAS,MAAM,IAAI;AACrD;AAWA,SAAS,YACP,SACA,QACa;AACb,QAAM,UAAU,IAAI,MAAM,OAAO;AACjC,UAAQ,SAAS,OAAO;AACxB,UAAQ,SAAS,OAAO;AACxB,MAAI,OAAO,SAAS,OAAW,SAAQ,OAAO,OAAO;AACrD,MAAI,OAAO,WAAW,OAAW,SAAQ,SAAS,OAAO;AACzD,SAAO;AACT;AAKA,SAAS,kBACP,OAaA,SACA,QACM;AACN,QAAM,MAAM,OAAO,OAAO,MAAM,MAAM,EAAE,SAAS,MAAM;AACvD,QAAM,UAAU,OAAO,OAAO,MAAM,MAAM,EAAE,SAAS,MAAM;AAC3D,MAAI,MAAM,YAAY;AACpB;AAAA,MACE;AAAA,QACE,8BAA8B,MAAM,SAAS,YAAY,MAAM,IAAI,IAAI,MAAM,KAAK,KAAK,GAAG,CAAC;AAAA,QAC3F,EAAE,QAAQ,KAAK,QAAQ,QAAQ;AAAA,MACjC;AAAA,IACF;AAAA,EACF,WAAW,CAAC,MAAM,QAAQ;AACxB,WAAO,MAAM,SAAS,IAAI,eAAe,+BAA+B,CAAC;AAAA,EAC3E,WAAW,MAAM,aAAa,GAAG;AAC/B,YAAQ,EAAE,QAAQ,KAAK,QAAQ,QAAQ,CAAC;AAAA,EAC1C,OAAO;AACL;AAAA,MACE;AAAA,QACE,MAAM,WACF,sBAAsB,MAAM,IAAI,KAChC,mBAAmB,MAAM,IAAI,IAAI,MAAM,KAAK,KAAK,GAAG,CAAC;AAAA,EAAK,OAAO;AAAA,QACrE,EAAE,QAAQ,KAAK,QAAQ,SAAS,MAAM,MAAM,UAAU,QAAQ,MAAM,WAAW;AAAA,MACjF;AAAA,IACF;AAAA,EACF;AACF;AAEO,IAAM,kBAAN,MAAsB;AAAA,EACV;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,UAAkC,CAAC,GAAG;AAChD,SAAK,OAAO,QAAQ,QAAQ,cAAc,KAAK;AAC/C,SAAK,OAAO,QAAQ,QAAQ;AAC5B,SAAK,QAAQ,QAAQ,SAAS,eAAe;AAAA,EAC/C;AAAA;AAAA,EAGQ,KACN,SACA,SACA,SACQ;AACR,UAAM,SAAS,QAAQ,KAAK,MAAM,KAAK,IAAI;AAC3C,QAAI;AACJ,UAAM,SAAS,IAAI,YAAY,CAAC,QAAQ;AACtC,YAAM,QAAQ;AACd,UAAI,MAAM,MAAM,SAAS;AACvB,mBAAW,WAAW,KAAgD;AAAA,MACxE;AACA,cAAQ,OAAO,MAAM;AAAA,IACvB,CAAC;AACD,WAAO,GAAG,WAAW,MAAM;AACzB,iBAAW,QAAQ,EAAE,GAAG,SAAS,OAAO,KAAK,MAAM,CAAqB;AAAA,IAC1E,CAAC;AACD,WAAO,GAAG,QAAQ,CAAC,UAAU,OAAO,KAAK,KAAK,CAAC;AAC/C,WAAO,GAAG,SAAS,CAAC,QAAQ;AAC1B,mBAAa;AAAA,IACf,CAAC;AACD,WAAO,GAAG,SAAS,MAAM,QAAQ,QAAQ,CAAC;AAC1C,WAAO;AAAA,EACT;AAAA,EAEA,OAAwB;AACtB,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAI,UAAyB;AAC7B,WAAK;AAAA,QACH,EAAE,IAAI,OAAO;AAAA,QACb,CAAC,UAAU;AACT,cAAI,MAAM,MAAM,OAAQ,WAAU,MAAM;AAAA,QAC1C;AAAA,QACA,CAAC,UAAU;AACT,cAAI,YAAY,MAAM;AACpB,mBAAO,SAAS,IAAI,eAAe,+BAA+B,CAAC;AAAA,UACrE,OAAO;AACL,oBAAQ,OAAO;AAAA,UACjB;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA,EAIA,YAAqC;AACnC,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAI,SAAgC;AACpC,WAAK;AAAA,QACH,EAAE,IAAI,YAAY;AAAA,QAClB,CAAC,UAAU;AACT,cAAI,MAAM,MAAM,YAAa,UAAS;AAAA,QACxC;AAAA,QACA,CAAC,UAAU;AACT,cAAI,OAAQ,SAAQ,MAAM;AAAA,cACrB,QAAO,SAAS,IAAI,eAAe,oCAAoC,CAAC;AAAA,QAC/E;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA,EAIA,gBACE,KACA,KACA,UACA,QACe;AACf,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAI,QAAQ,SAAS;AACnB,cAAM,QAAQ,IAAI,MAAM,mBAAmB;AAC3C,cAAM,OAAO;AACb,eAAO,KAAK;AACZ;AAAA,MACF;AACA,UAAI,WAA0B;AAC9B,UAAI,SAAS;AACb,UAAI,WAAW;AACf,YAAM,SAAS,KAAK;AAAA,QAClB,EAAE,IAAI,QAAQ,SAAS,KAAK,IAAI;AAAA,QAChC,CAAC,UAAU;AACT,cAAI,MAAM,MAAM,SAAS,CAAC,UAAU;AAClC,qBAAS,MAAM,GAAG,OAAO,KAAK,MAAM,GAAG,QAAQ,EAAE,SAAS,MAAM,CAAC;AAAA,UACnE,WAAW,MAAM,MAAM,QAAQ;AAC7B,qBAAS;AACT,uBAAW,MAAM;AAAA,UACnB;AAAA,QACF;AAAA,QACA,CAAC,UAAU;AACT,kBAAQ,oBAAoB,SAAS,OAAO;AAC5C,cAAI,UAAU;AACZ,kBAAM,aAAa,IAAI,MAAM,mBAAmB;AAChD,uBAAW,OAAO;AAClB,mBAAO,UAAU;AAAA,UACnB,WAAW,CAAC,QAAQ;AAClB,mBAAO,SAAS,IAAI,eAAe,+BAA+B,CAAC;AAAA,UACrE,WAAW,aAAa,GAAG;AACzB,oBAAQ;AAAA,UACV,OAAO;AACL,mBAAO,IAAI,MAAM,kCAAkC,QAAQ,EAAE,CAAC;AAAA,UAChE;AAAA,QACF;AAAA,MACF;AACA,YAAM,UAAU,MAAY;AAC1B,YAAI,UAAU,SAAU;AACxB,mBAAW;AACX,mBAAW,QAAQ,EAAE,GAAG,UAAU,MAAM,aAAa,CAAC;AAAA,MACxD;AACA,cAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAAA,IAC3D,CAAC;AAAA,EACH;AAAA;AAAA;AAAA,EAIA,gBACE,KACA,KACA,UACqB;AACrB,QAAI;AACJ,UAAM,SAAS,IAAI,oBAAoB,MAAM;AAC3C,UAAI,OAAQ,YAAW,QAAQ,EAAE,GAAG,UAAU,MAAM,aAAa,CAAC;AAAA,IACpE,CAAC;AACD,QAAI,SAAS;AACb,aAAS,KAAK;AAAA,MACZ,EAAE,IAAI,QAAQ,SAAS,KAAK,IAAI;AAAA,MAChC,CAAC,UAAU;AACT,YAAI,MAAM,MAAM,OAAO;AACrB,mBAAS,MAAM,GAAG,OAAO,KAAK,MAAM,GAAG,QAAQ,EAAE,SAAS,MAAM,CAAC;AAAA,QACnE,WAAW,MAAM,MAAM,QAAQ;AAC7B,mBAAS;AACT,iBAAO,WAAW,MAAM,SAAS,MAAM,SAAS,OAAO;AACvD,iBAAO,KAAK,QAAQ,MAAM,MAAM,MAAM,MAAM;AAAA,QAC9C;AAAA,MACF;AAAA,MACA,CAAC,UAAU;AACT,YAAI,CAAC,QAAQ;AAIX,mBAAS;AACT,iBAAO,WAAW,OAAO,YAAY;AACrC,iBAAO;AAAA,YACL;AAAA,YACA,SAAS,IAAI,eAAe,6BAA6B,gBAAgB;AAAA,UAC3E;AACA,iBAAO,KAAK,QAAQ,OAAO,UAAU,IAAI;AAAA,QAC3C;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA,EAIA,SACE,MACA,MACA,OAA+D,CAAC,GACnB;AAC7C,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,SAAmB,CAAC;AAC1B,YAAM,SAAmB,CAAC;AAC1B,UAAI,cAAc;AAClB,UAAI,cAAc;AAClB,YAAM,YAAY,KAAK;AACvB,UAAI,aAAa;AACjB,UAAI,WAA0B;AAC9B,UAAI,aAA4B;AAChC,UAAI,SAAS;AACb,UAAI,WAAW;AACf,YAAM,SAAS,KAAK;AAAA,QAClB,EAAE,IAAI,QAAQ,MAAM,CAAC,MAAM,GAAG,IAAI,GAAG,KAAK,KAAK,OAAO,QAAQ,IAAI,EAAE;AAAA,QACpE,CAAC,UAAU;AACT,cAAI,MAAM,MAAM,OAAO;AACrB,kBAAM,MAAM,OAAO,KAAK,MAAM,GAAG,QAAQ;AACzC,gBAAI,MAAM,MAAM,UAAU;AACxB,qBAAO,KAAK,GAAG;AACf,6BAAe,IAAI;AAAA,YACrB,OAAO;AACL,qBAAO,KAAK,GAAG;AACf,6BAAe,IAAI;AAAA,YACrB;AAIA,gBACE,cAAc,UACd,CAAC,eACA,cAAc,aAAa,cAAc,YAC1C;AACA,2BAAa;AACb,kBAAI,MAAO,cAAa,KAAK;AAC7B,yBAAW,QAAQ,EAAE,GAAG,UAAU,MAAM,aAAa,CAAC;AAAA,YACxD;AAAA,UACF,WAAW,MAAM,MAAM,QAAQ;AAC7B,qBAAS;AACT,uBAAW,MAAM;AACjB,yBAAa,MAAM;AAAA,UACrB;AAAA,QACF;AAAA,QACA,CAAC,UAAU;AACT,cAAI,MAAO,cAAa,KAAK;AAC7B;AAAA,YACE;AAAA,cACE;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA,OAAO,SAAS;AAAA,YAClB;AAAA,YACA;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAAA,MACF;AACA,YAAM,QAAQ,KAAK,UACf,WAAW,MAAM;AACf,mBAAW;AACX,mBAAW,QAAQ,EAAE,GAAG,UAAU,MAAM,aAAa,CAAC;AAAA,MACxD,GAAG,KAAK,OAAO,IACf;AACJ,aAAO,MAAM;AAAA,IACf,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,SAAS,MAAc,MAAgB,SAAsC;AAC3E,UAAM,gBAA+C,CAAC;AACtD,UAAM,gBAA8D,CAAC;AACrE,QAAI,YAAY;AAChB,QAAI,SAAS;AACb,UAAM,kBAAoC,CAAC;AAE3C,UAAM,SAAS,KAAK;AAAA,MAClB;AAAA,QACE,IAAI;AAAA,QACJ;AAAA,QACA;AAAA,QACA,KAAK,QAAQ;AAAA,QACb,KAAK,QAAQ;AAAA,QACb,MAAM,QAAQ;AAAA,QACd,MAAM,QAAQ;AAAA,MAChB;AAAA,MACA,CAAC,UAAU;AACT,YAAI,MAAM,MAAM,QAAQ;AACtB,gBAAM,OAAO,OAAO,KAAK,MAAM,GAAG,QAAQ,EAAE,SAAS,MAAM;AAC3D,qBAAW,YAAY,cAAe,UAAS,IAAI;AAAA,QACrD,WAAW,MAAM,MAAM,QAAQ;AAC7B,mBAAS;AACT,qBAAW,YAAY,cAAe,UAAS,EAAE,UAAU,MAAM,QAAQ,EAAE,CAAC;AAAA,QAC9E;AAAA,MACF;AAAA,MACA,MAAM;AACJ,YAAI,CAAC,QAAQ;AACX,mBAAS;AACT,qBAAW,YAAY,cAAe,UAAS,EAAE,UAAU,EAAE,CAAC;AAAA,QAChE;AAAA,MACF;AAAA,IACF;AACA,WAAO,GAAG,WAAW,MAAM;AACzB,kBAAY;AACZ,iBAAW,SAAS,gBAAgB,OAAO,CAAC,EAAG,YAAW,QAAQ,KAAK;AAAA,IACzE,CAAC;AAED,UAAM,OAAO,CAAC,UAAgC;AAC5C,UAAI,OAAQ;AACZ,UAAI,UAAW,YAAW,QAAQ,KAAK;AAAA,UAClC,iBAAgB,KAAK,KAAK;AAAA,IACjC;AAEA,WAAO;AAAA,MACL,QAAQ,CAAC,aAAa,cAAc,KAAK,QAAQ;AAAA,MACjD,QAAQ,CAAC,aAAa,cAAc,KAAK,QAAQ;AAAA,MACjD,OAAO,CAAC,SAAS,KAAK,EAAE,GAAG,SAAS,GAAG,OAAO,KAAK,MAAM,MAAM,EAAE,SAAS,QAAQ,EAAE,CAAC;AAAA,MACrF,QAAQ,CAAC,MAAM,SAAS,KAAK,EAAE,GAAG,UAAU,MAAM,KAAK,CAAC;AAAA,MACxD,MAAM,CAAC,QAAQ,KAAK,EAAE,GAAG,QAAQ,GAAI,MAAM,EAAE,IAAI,IAAI,CAAC,EAAG,CAAC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEA,SAAS,MAA+B;AACtC,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,SAAmB,CAAC;AAC1B,UAAI,QAAQ;AACZ,WAAK;AAAA,QACH,EAAE,IAAI,YAAY,KAAK;AAAA,QACvB,CAAC,UAAU;AACT,cAAI,MAAM,MAAM,OAAQ,QAAO,KAAK,OAAO,KAAK,MAAM,GAAG,QAAQ,CAAC;AAAA,mBACzD,MAAM,MAAM,MAAO,SAAQ;AAAA,QACtC;AAAA,QACA,CAAC,UAAU;AACT,cAAI,MAAO,SAAQ,OAAO,OAAO,MAAM,CAAC;AAAA,cACnC,QAAO,SAAS,IAAI,eAAe,8BAA8B,CAAC;AAAA,QACzE;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,KAAK,MAMF;AACD,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAI,OAMO;AACX,WAAK;AAAA,QACH,EAAE,IAAI,QAAQ,KAAK;AAAA,QACnB,CAAC,UAAU;AACT,cAAI,MAAM,MAAM,QAAQ;AACtB,mBAAO;AAAA,cACL,QAAQ,MAAM;AAAA,cACd,QAAQ,MAAM;AAAA,cACd,aAAa,MAAM;AAAA,cACnB,MAAM,MAAM;AAAA,cACZ,SAAS,MAAM;AAAA,YACjB;AAAA,UACF;AAAA,QACF;AAAA,QACA,CAAC,UAAU;AACT,cAAI,KAAM,SAAQ,IAAI;AAAA,cACjB,QAAO,SAAS,IAAI,eAAe,+BAA+B,CAAC;AAAA,QAC1E;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ,MAAiC;AACvC,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAI,UAA2B;AAC/B,WAAK;AAAA,QACH,EAAE,IAAI,WAAW,KAAK;AAAA,QACtB,CAAC,UAAU;AACT,cAAI,MAAM,MAAM,UAAW,WAAU,MAAM;AAAA,QAC7C;AAAA,QACA,CAAC,UAAU;AACT,cAAI,QAAS,SAAQ,OAAO;AAAA,cACvB,QAAO,SAAS,IAAI,eAAe,kCAAkC,CAAC;AAAA,QAC7E;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEA,IAAI,YAAoC;AAGjC,SAAS,qBAAsC;AACpD,gBAAc,IAAI,gBAAgB;AAClC,SAAO;AACT;AAGO,SAAS,uBAA6B;AAC3C,cAAY;AACd;","names":[]}
@@ -0,0 +1,138 @@
1
+ // src/workbench/oom-watchdog.ts
2
+ import { readdirSync, readFileSync } from "fs";
3
+ import { join } from "path";
4
+ function oomWatchdogOptionsFromEnv(log, env = process.env) {
5
+ if (env.CONVEYOR_OOM_WATCHDOG === "0") {
6
+ log.info("[oom-watchdog] disabled via CONVEYOR_OOM_WATCHDOG=0");
7
+ return void 0;
8
+ }
9
+ const headroomMb = Number(env.CONVEYOR_OOM_HEADROOM_MB);
10
+ return {
11
+ log,
12
+ ...Number.isFinite(headroomMb) && headroomMb > 0 ? { headroomBytes: headroomMb * 1024 * 1024 } : {}
13
+ };
14
+ }
15
+ var DEFAULT_POLL_MS = 100;
16
+ var DEFAULT_HEADROOM_BYTES = 256 * 1024 * 1024;
17
+ var DEFAULT_COOLDOWN_MS = 1e3;
18
+ var DEFAULT_MIN_VICTIM_RSS_BYTES = 64 * 1024 * 1024;
19
+ var PAGE_BYTES = 4096;
20
+ function readTrimmed(path) {
21
+ try {
22
+ return readFileSync(path, "utf8").trim();
23
+ } catch {
24
+ return null;
25
+ }
26
+ }
27
+ function parseProcStat(raw) {
28
+ const rest = raw.slice(raw.lastIndexOf(")") + 2).split(" ");
29
+ const pgid = Number(rest[2]);
30
+ const rssPages = Number(rest[21]);
31
+ if (!Number.isFinite(pgid) || !Number.isFinite(rssPages)) return null;
32
+ return { pgid, rssBytes: rssPages * PAGE_BYTES };
33
+ }
34
+ function biggestForeignProcessGroup(procDir, selfPgid) {
35
+ const groups = /* @__PURE__ */ new Map();
36
+ let entries;
37
+ try {
38
+ entries = readdirSync(procDir);
39
+ } catch {
40
+ return null;
41
+ }
42
+ for (const entry of entries) {
43
+ if (!/^\d+$/.test(entry)) continue;
44
+ const raw = readTrimmed(join(procDir, entry, "stat"));
45
+ if (!raw) continue;
46
+ const stat = parseProcStat(raw);
47
+ if (!stat || stat.pgid === selfPgid) continue;
48
+ groups.set(stat.pgid, (groups.get(stat.pgid) ?? 0) + stat.rssBytes);
49
+ }
50
+ let best = null;
51
+ for (const [pgid, rssBytes] of groups) {
52
+ if (!best || rssBytes > best.rssBytes) best = { pgid, rssBytes };
53
+ }
54
+ return best;
55
+ }
56
+ function readUsedBytes(cgroupDir) {
57
+ const current = Number(readTrimmed(join(cgroupDir, "memory.current")));
58
+ if (!Number.isFinite(current)) return null;
59
+ const stat = readTrimmed(join(cgroupDir, "memory.stat"));
60
+ const inactiveFile = Number(/^inactive_file (\d+)$/m.exec(stat ?? "")?.[1] ?? 0);
61
+ return Math.max(0, current - inactiveFile);
62
+ }
63
+ function readSelfPgid(procDir) {
64
+ const raw = readTrimmed(join(procDir, "self", "stat"));
65
+ const stat = raw ? parseProcStat(raw) : null;
66
+ return stat?.pgid ?? process.pid;
67
+ }
68
+ function startOomWatchdog(opts) {
69
+ const cgroupDir = opts.cgroupDir ?? "/sys/fs/cgroup";
70
+ const procDir = opts.procDir ?? "/proc";
71
+ const pollMs = opts.pollMs ?? DEFAULT_POLL_MS;
72
+ const headroomBytes = opts.headroomBytes ?? DEFAULT_HEADROOM_BYTES;
73
+ const cooldownMs = opts.cooldownMs ?? DEFAULT_COOLDOWN_MS;
74
+ const minVictimRssBytes = opts.minVictimRssBytes ?? DEFAULT_MIN_VICTIM_RSS_BYTES;
75
+ const kill = opts.kill ?? ((pgid, signal) => process.kill(-pgid, signal));
76
+ const limitRaw = readTrimmed(join(cgroupDir, "memory.max"));
77
+ const limitBytes = Number(limitRaw);
78
+ if (!limitRaw || limitRaw === "max" || !Number.isFinite(limitBytes) || limitBytes <= 0) {
79
+ opts.log.info(`[oom-watchdog] disabled \u2014 no finite memory limit (memory.max=${limitRaw})`);
80
+ return null;
81
+ }
82
+ const selfPgid = readSelfPgid(procDir);
83
+ let prevUsed = null;
84
+ let quietUntil = 0;
85
+ const tick = () => {
86
+ const now = Date.now();
87
+ if (now < quietUntil) return;
88
+ const used = readUsedBytes(cgroupDir);
89
+ if (used === null) return;
90
+ const delta = prevUsed === null ? 0 : Math.max(0, used - prevUsed);
91
+ prevUsed = used;
92
+ const headroomHit = limitBytes - used <= headroomBytes;
93
+ if (!headroomHit && used + delta * 2 < limitBytes) return;
94
+ const victim = biggestForeignProcessGroup(procDir, selfPgid);
95
+ if (!victim || victim.rssBytes < minVictimRssBytes) {
96
+ quietUntil = now + cooldownMs;
97
+ prevUsed = null;
98
+ opts.log.warn(
99
+ `[oom-watchdog] memory pressure (used=${used} limit=${limitBytes}) but no eligible victim group`
100
+ );
101
+ return;
102
+ }
103
+ const event = {
104
+ pgid: victim.pgid,
105
+ victimRssBytes: victim.rssBytes,
106
+ usedBytes: used,
107
+ limitBytes,
108
+ reason: headroomHit ? "headroom" : "projection"
109
+ };
110
+ opts.log.warn(
111
+ `[oom-watchdog] killing pgid=${event.pgid} rss=${Math.round(event.victimRssBytes / 1048576)}MiB used=${Math.round(used / 1048576)}/${Math.round(limitBytes / 1048576)}MiB reason=${event.reason}`
112
+ );
113
+ try {
114
+ opts.onKill?.(event);
115
+ } catch {
116
+ }
117
+ try {
118
+ kill(victim.pgid, "SIGKILL");
119
+ } catch (err) {
120
+ opts.log.warn(`[oom-watchdog] kill pgid=${victim.pgid} failed: ${String(err)}`);
121
+ }
122
+ quietUntil = now + cooldownMs;
123
+ prevUsed = null;
124
+ };
125
+ const interval = setInterval(tick, pollMs);
126
+ interval.unref();
127
+ opts.log.info(
128
+ `[oom-watchdog] armed limit=${Math.round(limitBytes / 1048576)}MiB headroom=${Math.round(headroomBytes / 1048576)}MiB poll=${pollMs}ms`
129
+ );
130
+ return { stop: () => clearInterval(interval) };
131
+ }
132
+
133
+ export {
134
+ oomWatchdogOptionsFromEnv,
135
+ biggestForeignProcessGroup,
136
+ startOomWatchdog
137
+ };
138
+ //# sourceMappingURL=chunk-ULS4QPRE.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/workbench/oom-watchdog.ts"],"sourcesContent":["/**\n * Early-OOM watchdog for the workbench container.\n *\n * On cgroup v2, Kubernetes sets `memory.oom.group=1` on every container:\n * when the workbench hits its memory limit the kernel kills EVERY process in\n * the container — dev servers, sshd, the claude PTY, the launcher itself —\n * then kubelet restart backoff and janitor restart-strikes compound the\n * damage. Autopilot exposes no kubelet knob to soften this and the cgroup\n * subtree is read-only in-container, so the fix is userspace: watch our own\n * cgroup's memory and kill the hungriest workload process group before the\n * kernel takes the whole container.\n *\n * Trigger design (validated on the real cluster, wb-oom-canary 2026-07-20):\n * a flat high threshold loses the race against fast allocators (a 2.5 GB/s\n * hog beat a 90%/250ms poll to the limit), while a flat low threshold wastes\n * memory for well-behaved workloads. So two conditions, checked every poll:\n * - headroom: less than `headroomBytes` left in the cgroup, or\n * - projection: current + 2×(last poll's growth) would cross the limit.\n * Fast spikes trip the projection early (~83% in the canary); slow growth\n * runs to high utilization (~88%) before the headroom floor fires.\n *\n * Victim selection: the largest-RSS process group other than the launcher's\n * own — the launcher already runs every workload as its own detached process\n * group precisely so it can be killed as a unit (see server.ts).\n */\n\nimport { readdirSync, readFileSync } from \"node:fs\";\nimport { join } from \"node:path\";\nimport type { BootLogger } from \"../boot/types.js\";\n\nexport interface OomKillEvent {\n pgid: number;\n victimRssBytes: number;\n usedBytes: number;\n limitBytes: number;\n reason: \"headroom\" | \"projection\";\n}\n\nexport interface OomWatchdogOptions {\n log: BootLogger;\n /** Fired synchronously just before the SIGKILL, so a report frame can be\n * written to the victim's connection ahead of its exit frame. */\n onKill?: (event: OomKillEvent) => void;\n pollMs?: number;\n headroomBytes?: number;\n /** Post-kill quiet period — lets memory.current fall before re-evaluating. */\n cooldownMs?: number;\n /** Groups below this RSS are never killed: reaping them can't meaningfully\n * relieve pressure, and it protects sshd/idle shells from a kill spiral\n * when the pressure is actually page cache or the launcher itself. */\n minVictimRssBytes?: number;\n /** Injection points for tests. */\n cgroupDir?: string;\n procDir?: string;\n kill?: (pgid: number, signal: NodeJS.Signals) => void;\n}\n\nexport interface OomWatchdogHandle {\n stop(): void;\n}\n\n/**\n * Production config for the boot paths: armed by default in workbench pods,\n * `CONVEYOR_OOM_WATCHDOG=0` is the ops kill-switch (no republish needed to\n * turn it off), `CONVEYOR_OOM_HEADROOM_MB` tunes the floor. Returns undefined\n * when disabled so callers can pass it straight to WorkbenchServerOptions.\n */\nexport function oomWatchdogOptionsFromEnv(\n log: BootLogger,\n env: NodeJS.ProcessEnv = process.env,\n): Omit<OomWatchdogOptions, \"onKill\"> | undefined {\n if (env.CONVEYOR_OOM_WATCHDOG === \"0\") {\n log.info(\"[oom-watchdog] disabled via CONVEYOR_OOM_WATCHDOG=0\");\n return undefined;\n }\n const headroomMb = Number(env.CONVEYOR_OOM_HEADROOM_MB);\n return {\n log,\n ...(Number.isFinite(headroomMb) && headroomMb > 0\n ? { headroomBytes: headroomMb * 1024 * 1024 }\n : {}),\n };\n}\n\nconst DEFAULT_POLL_MS = 100;\nconst DEFAULT_HEADROOM_BYTES = 256 * 1024 * 1024;\nconst DEFAULT_COOLDOWN_MS = 1000;\nconst DEFAULT_MIN_VICTIM_RSS_BYTES = 64 * 1024 * 1024;\n// Reading page size at runtime needs a syscall binding node doesn't expose;\n// every linux target we deploy to (and the canary validated on) is 4KiB.\nconst PAGE_BYTES = 4096;\n\nfunction readTrimmed(path: string): string | null {\n try {\n return readFileSync(path, \"utf8\").trim();\n } catch {\n return null;\n }\n}\n\n/** Fields after the `(comm)` in /proc/<pid>/stat: [2]=pgrp, [21]=rss pages.\n * comm can contain spaces/parens, so split after the LAST ')'. */\nfunction parseProcStat(raw: string): { pgid: number; rssBytes: number } | null {\n const rest = raw.slice(raw.lastIndexOf(\")\") + 2).split(\" \");\n const pgid = Number(rest[2]);\n const rssPages = Number(rest[21]);\n if (!Number.isFinite(pgid) || !Number.isFinite(rssPages)) return null;\n return { pgid, rssBytes: rssPages * PAGE_BYTES };\n}\n\n/** Sum RSS per process group across the container, excluding `selfPgid` —\n * grandchildren that re-setsid (dev servers under a start command shell)\n * show up as their own groups and are eligible victims individually. */\nexport function biggestForeignProcessGroup(\n procDir: string,\n selfPgid: number,\n): { pgid: number; rssBytes: number } | null {\n const groups = new Map<number, number>();\n let entries: string[];\n try {\n entries = readdirSync(procDir);\n } catch {\n return null;\n }\n for (const entry of entries) {\n if (!/^\\d+$/.test(entry)) continue;\n const raw = readTrimmed(join(procDir, entry, \"stat\"));\n // Missing stat file = the process exited mid-scan.\n if (!raw) continue;\n const stat = parseProcStat(raw);\n if (!stat || stat.pgid === selfPgid) continue;\n groups.set(stat.pgid, (groups.get(stat.pgid) ?? 0) + stat.rssBytes);\n }\n let best: { pgid: number; rssBytes: number } | null = null;\n for (const [pgid, rssBytes] of groups) {\n if (!best || rssBytes > best.rssBytes) best = { pgid, rssBytes };\n }\n return best;\n}\n\n/** `memory.current` counts reclaimable page cache the kernel would evict\n * before OOMing; subtract inactive file cache so a git/build IO burst can't\n * read as memory pressure and trigger a false kill. */\nfunction readUsedBytes(cgroupDir: string): number | null {\n const current = Number(readTrimmed(join(cgroupDir, \"memory.current\")));\n if (!Number.isFinite(current)) return null;\n const stat = readTrimmed(join(cgroupDir, \"memory.stat\"));\n const inactiveFile = Number(/^inactive_file (\\d+)$/m.exec(stat ?? \"\")?.[1] ?? 0);\n return Math.max(0, current - inactiveFile);\n}\n\nfunction readSelfPgid(procDir: string): number {\n const raw = readTrimmed(join(procDir, \"self\", \"stat\"));\n const stat = raw ? parseProcStat(raw) : null;\n return stat?.pgid ?? process.pid;\n}\n\n/**\n * Arm the watchdog. Returns null (disabled) when the cgroup has no finite\n * memory limit — local dev, tests, and non-container embedders have nothing\n * to defend against and no meaningful `memory.max` to poll.\n */\nexport function startOomWatchdog(opts: OomWatchdogOptions): OomWatchdogHandle | null {\n const cgroupDir = opts.cgroupDir ?? \"/sys/fs/cgroup\";\n const procDir = opts.procDir ?? \"/proc\";\n const pollMs = opts.pollMs ?? DEFAULT_POLL_MS;\n const headroomBytes = opts.headroomBytes ?? DEFAULT_HEADROOM_BYTES;\n const cooldownMs = opts.cooldownMs ?? DEFAULT_COOLDOWN_MS;\n const minVictimRssBytes = opts.minVictimRssBytes ?? DEFAULT_MIN_VICTIM_RSS_BYTES;\n const kill = opts.kill ?? ((pgid, signal) => process.kill(-pgid, signal));\n\n const limitRaw = readTrimmed(join(cgroupDir, \"memory.max\"));\n const limitBytes = Number(limitRaw);\n if (!limitRaw || limitRaw === \"max\" || !Number.isFinite(limitBytes) || limitBytes <= 0) {\n opts.log.info(`[oom-watchdog] disabled — no finite memory limit (memory.max=${limitRaw})`);\n return null;\n }\n const selfPgid = readSelfPgid(procDir);\n\n // First poll is baseline-only for the projection: seeding prevUsed with 0\n // would read all existing memory as one poll's growth and false-kill any\n // container that arms while already warm.\n let prevUsed: number | null = null;\n let quietUntil = 0;\n const tick = (): void => {\n const now = Date.now();\n if (now < quietUntil) return;\n const used = readUsedBytes(cgroupDir);\n if (used === null) return;\n const delta = prevUsed === null ? 0 : Math.max(0, used - prevUsed);\n prevUsed = used;\n const headroomHit = limitBytes - used <= headroomBytes;\n if (!headroomHit && used + delta * 2 < limitBytes) return;\n\n const victim = biggestForeignProcessGroup(procDir, selfPgid);\n if (!victim || victim.rssBytes < minVictimRssBytes) {\n // Pressure without a killable workload (launcher-owned memory, page\n // cache churn). Nothing safe to do — back off so this doesn't spam.\n quietUntil = now + cooldownMs;\n prevUsed = null;\n opts.log.warn(\n `[oom-watchdog] memory pressure (used=${used} limit=${limitBytes}) but no eligible victim group`,\n );\n return;\n }\n const event: OomKillEvent = {\n pgid: victim.pgid,\n victimRssBytes: victim.rssBytes,\n usedBytes: used,\n limitBytes,\n reason: headroomHit ? \"headroom\" : \"projection\",\n };\n opts.log.warn(\n `[oom-watchdog] killing pgid=${event.pgid} rss=${Math.round(event.victimRssBytes / 1048576)}MiB ` +\n `used=${Math.round(used / 1048576)}/${Math.round(limitBytes / 1048576)}MiB reason=${event.reason}`,\n );\n try {\n opts.onKill?.(event);\n } catch {\n /* reporting must never block the kill */\n }\n try {\n kill(victim.pgid, \"SIGKILL\");\n } catch (err) {\n opts.log.warn(`[oom-watchdog] kill pgid=${victim.pgid} failed: ${String(err)}`);\n }\n quietUntil = now + cooldownMs;\n prevUsed = null;\n };\n\n const interval = setInterval(tick, pollMs);\n interval.unref();\n opts.log.info(\n `[oom-watchdog] armed limit=${Math.round(limitBytes / 1048576)}MiB ` +\n `headroom=${Math.round(headroomBytes / 1048576)}MiB poll=${pollMs}ms`,\n );\n return { stop: () => clearInterval(interval) };\n}\n"],"mappings":";AA0BA,SAAS,aAAa,oBAAoB;AAC1C,SAAS,YAAY;AAwCd,SAAS,0BACd,KACA,MAAyB,QAAQ,KACe;AAChD,MAAI,IAAI,0BAA0B,KAAK;AACrC,QAAI,KAAK,qDAAqD;AAC9D,WAAO;AAAA,EACT;AACA,QAAM,aAAa,OAAO,IAAI,wBAAwB;AACtD,SAAO;AAAA,IACL;AAAA,IACA,GAAI,OAAO,SAAS,UAAU,KAAK,aAAa,IAC5C,EAAE,eAAe,aAAa,OAAO,KAAK,IAC1C,CAAC;AAAA,EACP;AACF;AAEA,IAAM,kBAAkB;AACxB,IAAM,yBAAyB,MAAM,OAAO;AAC5C,IAAM,sBAAsB;AAC5B,IAAM,+BAA+B,KAAK,OAAO;AAGjD,IAAM,aAAa;AAEnB,SAAS,YAAY,MAA6B;AAChD,MAAI;AACF,WAAO,aAAa,MAAM,MAAM,EAAE,KAAK;AAAA,EACzC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAIA,SAAS,cAAc,KAAwD;AAC7E,QAAM,OAAO,IAAI,MAAM,IAAI,YAAY,GAAG,IAAI,CAAC,EAAE,MAAM,GAAG;AAC1D,QAAM,OAAO,OAAO,KAAK,CAAC,CAAC;AAC3B,QAAM,WAAW,OAAO,KAAK,EAAE,CAAC;AAChC,MAAI,CAAC,OAAO,SAAS,IAAI,KAAK,CAAC,OAAO,SAAS,QAAQ,EAAG,QAAO;AACjE,SAAO,EAAE,MAAM,UAAU,WAAW,WAAW;AACjD;AAKO,SAAS,2BACd,SACA,UAC2C;AAC3C,QAAM,SAAS,oBAAI,IAAoB;AACvC,MAAI;AACJ,MAAI;AACF,cAAU,YAAY,OAAO;AAAA,EAC/B,QAAQ;AACN,WAAO;AAAA,EACT;AACA,aAAW,SAAS,SAAS;AAC3B,QAAI,CAAC,QAAQ,KAAK,KAAK,EAAG;AAC1B,UAAM,MAAM,YAAY,KAAK,SAAS,OAAO,MAAM,CAAC;AAEpD,QAAI,CAAC,IAAK;AACV,UAAM,OAAO,cAAc,GAAG;AAC9B,QAAI,CAAC,QAAQ,KAAK,SAAS,SAAU;AACrC,WAAO,IAAI,KAAK,OAAO,OAAO,IAAI,KAAK,IAAI,KAAK,KAAK,KAAK,QAAQ;AAAA,EACpE;AACA,MAAI,OAAkD;AACtD,aAAW,CAAC,MAAM,QAAQ,KAAK,QAAQ;AACrC,QAAI,CAAC,QAAQ,WAAW,KAAK,SAAU,QAAO,EAAE,MAAM,SAAS;AAAA,EACjE;AACA,SAAO;AACT;AAKA,SAAS,cAAc,WAAkC;AACvD,QAAM,UAAU,OAAO,YAAY,KAAK,WAAW,gBAAgB,CAAC,CAAC;AACrE,MAAI,CAAC,OAAO,SAAS,OAAO,EAAG,QAAO;AACtC,QAAM,OAAO,YAAY,KAAK,WAAW,aAAa,CAAC;AACvD,QAAM,eAAe,OAAO,yBAAyB,KAAK,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC;AAC/E,SAAO,KAAK,IAAI,GAAG,UAAU,YAAY;AAC3C;AAEA,SAAS,aAAa,SAAyB;AAC7C,QAAM,MAAM,YAAY,KAAK,SAAS,QAAQ,MAAM,CAAC;AACrD,QAAM,OAAO,MAAM,cAAc,GAAG,IAAI;AACxC,SAAO,MAAM,QAAQ,QAAQ;AAC/B;AAOO,SAAS,iBAAiB,MAAoD;AACnF,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,UAAU,KAAK,WAAW;AAChC,QAAM,SAAS,KAAK,UAAU;AAC9B,QAAM,gBAAgB,KAAK,iBAAiB;AAC5C,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,oBAAoB,KAAK,qBAAqB;AACpD,QAAM,OAAO,KAAK,SAAS,CAAC,MAAM,WAAW,QAAQ,KAAK,CAAC,MAAM,MAAM;AAEvE,QAAM,WAAW,YAAY,KAAK,WAAW,YAAY,CAAC;AAC1D,QAAM,aAAa,OAAO,QAAQ;AAClC,MAAI,CAAC,YAAY,aAAa,SAAS,CAAC,OAAO,SAAS,UAAU,KAAK,cAAc,GAAG;AACtF,SAAK,IAAI,KAAK,qEAAgE,QAAQ,GAAG;AACzF,WAAO;AAAA,EACT;AACA,QAAM,WAAW,aAAa,OAAO;AAKrC,MAAI,WAA0B;AAC9B,MAAI,aAAa;AACjB,QAAM,OAAO,MAAY;AACvB,UAAM,MAAM,KAAK,IAAI;AACrB,QAAI,MAAM,WAAY;AACtB,UAAM,OAAO,cAAc,SAAS;AACpC,QAAI,SAAS,KAAM;AACnB,UAAM,QAAQ,aAAa,OAAO,IAAI,KAAK,IAAI,GAAG,OAAO,QAAQ;AACjE,eAAW;AACX,UAAM,cAAc,aAAa,QAAQ;AACzC,QAAI,CAAC,eAAe,OAAO,QAAQ,IAAI,WAAY;AAEnD,UAAM,SAAS,2BAA2B,SAAS,QAAQ;AAC3D,QAAI,CAAC,UAAU,OAAO,WAAW,mBAAmB;AAGlD,mBAAa,MAAM;AACnB,iBAAW;AACX,WAAK,IAAI;AAAA,QACP,wCAAwC,IAAI,UAAU,UAAU;AAAA,MAClE;AACA;AAAA,IACF;AACA,UAAM,QAAsB;AAAA,MAC1B,MAAM,OAAO;AAAA,MACb,gBAAgB,OAAO;AAAA,MACvB,WAAW;AAAA,MACX;AAAA,MACA,QAAQ,cAAc,aAAa;AAAA,IACrC;AACA,SAAK,IAAI;AAAA,MACP,+BAA+B,MAAM,IAAI,QAAQ,KAAK,MAAM,MAAM,iBAAiB,OAAO,CAAC,YACjF,KAAK,MAAM,OAAO,OAAO,CAAC,IAAI,KAAK,MAAM,aAAa,OAAO,CAAC,cAAc,MAAM,MAAM;AAAA,IACpG;AACA,QAAI;AACF,WAAK,SAAS,KAAK;AAAA,IACrB,QAAQ;AAAA,IAER;AACA,QAAI;AACF,WAAK,OAAO,MAAM,SAAS;AAAA,IAC7B,SAAS,KAAK;AACZ,WAAK,IAAI,KAAK,4BAA4B,OAAO,IAAI,YAAY,OAAO,GAAG,CAAC,EAAE;AAAA,IAChF;AACA,iBAAa,MAAM;AACnB,eAAW;AAAA,EACb;AAEA,QAAM,WAAW,YAAY,MAAM,MAAM;AACzC,WAAS,MAAM;AACf,OAAK,IAAI;AAAA,IACP,8BAA8B,KAAK,MAAM,aAAa,OAAO,CAAC,gBAChD,KAAK,MAAM,gBAAgB,OAAO,CAAC,YAAY,MAAM;AAAA,EACrE;AACA,SAAO,EAAE,MAAM,MAAM,cAAc,QAAQ,EAAE;AAC/C;","names":[]}