@vellumai/assistant 0.12.5-dev.202609252115.c8e1713 → 0.12.5-dev.202609252215.b94bc6c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/ARCHITECTURE.md CHANGED
@@ -49,6 +49,8 @@ Resource pressure monitoring reports sustained CPU/memory pressure on platform-h
49
49
 
50
50
  When the container hits its memory limit, the kernel kills the process with the highest `oom_score` (memory share plus `oom_score_adj`). `src/util/oom-priority.ts` steers that choice so the assistant is never the victim of a runaway tool: the daemon sets itself to -700 as the first thing in `runDaemon`, before any spawn. Children inherit the parent's value at fork, so every spawn path resets its child: shell children get +1000 from the Linux prefix in `buildShellInvocation` (`packages/environments`), desktop children and ACP agents and terminals get +1000 from `withOomScoreAdj`, an `sh -c ... exec` wrapper that keeps pid and process group, and workers, the embedding worker, and Qdrant get 0 from a parent-side write right after spawn (the resource monitor gets -500 so it outlives everything but the daemon and records the kill). The monitor also sweeps the daemon's descendants every few seconds (`src/monitoring/oom-inheritance-sweep.ts`) and writes 0 over any negative value it finds, logging the comm, so a spawn path that forgets the reset degrades to a few seconds of inherited protection instead of a protected runaway. Bundled plugins (the memory plugin's rerank worker, for one) rely on the sweep rather than importing the helper, since the plugin import boundary forbids reaching outside the plugin. Raising never needs a capability; lowering needs `CAP_SYS_RESOURCE`, which the platform grants and self-hosted Docker does not, where the daemon simply stays at 0. Nothing here caps a single process or acts early: it only decides who dies once the kernel has to kill.
51
51
 
52
+ **Verification telemetry:** the resource monitor reports kernel OOM kills in the container as a `watchdog` event with `check_name` `oom_kill` (`src/monitoring/oom-kill-reporter.ts`). It reads the kernel log through `dmesg` (the `/dev/kmsg` device is not mounted in the container; `dmesg` uses the syslog syscall, which the Kata profile permits) when the cgroup `memory.events` kill counter moves, on its first scan after start (the kill that took the daemon down restarts the monitor with it), and once a minute as a fallback. The kernel log is shared by every container on that kernel and cgroup paths read `/` from inside, so attribution is by the counter: a move of N claims the N most recent new kills; the first scan and hosts without the counter claim every new kill and say so with `attribution: "kernel_log"`. The `detail` bag lists each victim's comm, `oom_score_adj` and resident size, the cgroup memory figures at the time, and `killed_daemon`, detected by the daemon's -700 score rather than a pid the restart has since replaced; where the log is unreadable a counter move still yields a victimless event. A cursor (kernel boot id plus last log timestamp) on the monitor data dir advances only after the event is queued, so a telemetry store outage delays a report rather than losing one, and resets when the kernel reboots.
53
+
52
54
  ### Single-Header JWT Auth Model
53
55
 
54
56
  All HTTP API requests use a single `Authorization: Bearer <jwt>` header for authentication. The JWT carries identity, permissions, and policy versioning in a unified token.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vellumai/assistant",
3
- "version": "0.12.5-dev.202609252115.c8e1713",
3
+ "version": "0.12.5-dev.202609252215.b94bc6c",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "exports": {
@@ -214,8 +214,7 @@ function attachOperationAction(
214
214
  opts.output &&
215
215
  result.screenshots?.length
216
216
  ) {
217
- const screenshot = result.screenshots[0];
218
- const buffer = Buffer.from(screenshot.data, "base64");
217
+ const buffer = Buffer.from(result.screenshots[0].data, "base64");
219
218
  try {
220
219
  writeFileSync(String(opts.output), buffer);
221
220
  } catch (err) {
@@ -89,8 +89,9 @@ export function parsePluginArtifact(
89
89
  if (typeof vellum !== "object" || vellum === null || Array.isArray(vellum)) {
90
90
  return null;
91
91
  }
92
- const artifact = (vellum as Record<string, unknown>).artifact;
93
- const parsed = PluginArtifactSchema.safeParse(artifact);
92
+ const parsed = PluginArtifactSchema.safeParse(
93
+ (vellum as Record<string, unknown>).artifact,
94
+ );
94
95
  if (!parsed.success) {
95
96
  return null;
96
97
  }
@@ -0,0 +1,262 @@
1
+ import { mkdtempSync, rmSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import { join } from "node:path";
4
+ import { afterEach, beforeEach, describe, expect, test } from "bun:test";
5
+
6
+ import {
7
+ createOomKillReporter,
8
+ type KernelLogEntry,
9
+ oomKillsFromEntries,
10
+ parseDmesgJson,
11
+ parseDmesgText,
12
+ } from "../oom-kill-reporter.js";
13
+ import type { ResourceSample } from "../resource-sample-types.js";
14
+
15
+ const TOOL_KILL =
16
+ "Memory cgroup out of memory: Killed process 4242 (python3) total-vm:1263184kB, anon-rss:409600kB, file-rss:1024kB, shmem-rss:0kB, UID:0 pgtables:900kB oom_score_adj:1000";
17
+ const DAEMON_KILL =
18
+ "Memory cgroup out of memory: Killed process 1 (bun) total-vm:3000000kB, anon-rss:2000000kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:5000kB oom_score_adj:-700";
19
+ const LEGACY_KILL =
20
+ "Out of memory: Killed process 77 (node) total-vm:500000kB, anon-rss:300000kB, file-rss:0kB, shmem-rss:0kB";
21
+ const NOISE = [
22
+ "bun invoked oom-killer: gfp_mask=0xcc0(GFP_KERNEL), order=0, oom_score_adj=-700",
23
+ "oom_reaper: reaped process 4242 (python3), now anon-rss:0kB, file-rss:0kB, shmem-rss:0kB",
24
+ ];
25
+
26
+ const entry = (time: number, msg: string): KernelLogEntry => ({ time, msg });
27
+
28
+ function sample(
29
+ oomKillDelta: number | null,
30
+ countersAvailable = true,
31
+ ): ResourceSample {
32
+ const events = { low: 0, high: 0, max: 0, oom: 0, oomKill: 0 };
33
+ return {
34
+ ts: 0,
35
+ memory: {
36
+ currentBytes: 3_000_000_000,
37
+ limitBytes: 3_221_225_472,
38
+ peakBytes: 3_221_000_000,
39
+ ratio: 0.93,
40
+ },
41
+ memoryStat: null,
42
+ reclaim: null,
43
+ cpu: null,
44
+ events: countersAvailable ? events : null,
45
+ deltas:
46
+ oomKillDelta == null
47
+ ? null
48
+ : {
49
+ events: { ...events, oomKill: oomKillDelta },
50
+ reclaim: null,
51
+ cpu: null,
52
+ },
53
+ disk: null,
54
+ activeConversations: null,
55
+ };
56
+ }
57
+
58
+ describe("kernel log parsing", () => {
59
+ test("parses dmesg --json and plain dmesg into the same entries", () => {
60
+ const json = `{"dmesg":[{"pri":3,"time": 44170.542000,"msg":${JSON.stringify(TOOL_KILL)}},{"pri":6,"time":44171.0,"msg":"x"}]}`;
61
+ const text = `[44170.542000] ${TOOL_KILL}\n[ 44171.000000] x\n`;
62
+ expect(parseDmesgJson(json)).toEqual(parseDmesgText(text));
63
+ expect(parseDmesgJson("not json")).toBeNull();
64
+ });
65
+
66
+ test("extracts victims, sizes and oom_score_adj, oldest first, ignoring noise", () => {
67
+ const kills = oomKillsFromEntries([
68
+ entry(30, NOISE[0]),
69
+ entry(50, LEGACY_KILL),
70
+ entry(31, TOOL_KILL),
71
+ entry(32, NOISE[1]),
72
+ ]);
73
+ expect(kills).toEqual([
74
+ {
75
+ time: 31,
76
+ pid: 4242,
77
+ comm: "python3",
78
+ oomScoreAdj: 1000,
79
+ anonRssKb: 409600,
80
+ totalVmKb: 1263184,
81
+ },
82
+ {
83
+ time: 50,
84
+ pid: 77,
85
+ comm: "node",
86
+ oomScoreAdj: null,
87
+ anonRssKb: 300000,
88
+ totalVmKb: 500000,
89
+ },
90
+ ]);
91
+ });
92
+ });
93
+
94
+ describe("createOomKillReporter", () => {
95
+ let dir: string;
96
+ let recorded: Array<{ value?: number | null; detail?: unknown }>;
97
+ let reads: number;
98
+ let recordOk: boolean;
99
+ let consent: boolean | null;
100
+
101
+ beforeEach(() => {
102
+ dir = mkdtempSync(join(tmpdir(), "oom-kill-reporter-"));
103
+ recorded = [];
104
+ reads = 0;
105
+ recordOk = true;
106
+ consent = true;
107
+ });
108
+
109
+ afterEach(() => {
110
+ rmSync(dir, { recursive: true, force: true });
111
+ });
112
+
113
+ function reporter(
114
+ log: () => KernelLogEntry[] | null,
115
+ bootId: string | null = "boot-a",
116
+ ) {
117
+ return createOomKillReporter({
118
+ readKernelLog: async () => {
119
+ reads++;
120
+ return log();
121
+ },
122
+ readBootId: () => bootId,
123
+ shareAnalytics: () => consent,
124
+ cursorPath: join(dir, "cursor.json"),
125
+ record: (record) => {
126
+ if (!recordOk) {
127
+ return null;
128
+ }
129
+ recorded.push(record);
130
+ return { id: "evt", createdAt: 0 };
131
+ },
132
+ });
133
+ }
134
+
135
+ test("attributes the last N new kills to a counter move of N", async () => {
136
+ const log = () => [
137
+ entry(10, LEGACY_KILL), // a neighbour container's, before ours
138
+ entry(20, TOOL_KILL),
139
+ entry(21, TOOL_KILL),
140
+ ];
141
+ const r = reporter(log);
142
+ await r.check(sample(0), 0, 1); // first scan on an empty-counter container reports what it finds
143
+ recorded.length = 0;
144
+
145
+ const r2 = reporter(() => [
146
+ ...log(),
147
+ entry(30, TOOL_KILL),
148
+ entry(31, TOOL_KILL),
149
+ ]);
150
+ // Fresh reporter, same cursor: only the two new kills are fresh, counter says 2.
151
+ const report = await r2.check(sample(2), 1_000, 1);
152
+ expect(report).toMatchObject({ attribution: "cgroup_counter", unnamed: 0 });
153
+ expect(report?.victims.map((v) => v.time)).toEqual([30, 31]);
154
+ expect(recorded).toHaveLength(1);
155
+ expect(recorded[0].value).toBe(2);
156
+ });
157
+
158
+ test("first scan reports the kill that preceded the monitor and flags the daemon by score", async () => {
159
+ const r = reporter(() => [entry(5, DAEMON_KILL)]);
160
+ const report = await r.check(sample(0), 0, 999);
161
+ expect(report?.attribution).toBe("kernel_log");
162
+ expect(recorded[0].detail).toMatchObject({
163
+ killed_daemon: true,
164
+ victim_count: 1,
165
+ memory_limit_bytes: 3_221_225_472,
166
+ });
167
+
168
+ // A second monitor incarnation on the same kernel sees the same log: nothing new.
169
+ expect(
170
+ await reporter(() => [entry(5, DAEMON_KILL)]).check(sample(0), 0, 999),
171
+ ).toBeNull();
172
+ expect(recorded).toHaveLength(1);
173
+ });
174
+
175
+ test("ignores neighbours' kills when the local counter did not move", async () => {
176
+ let entries: KernelLogEntry[] = [];
177
+ const r = reporter(() => entries);
178
+ await r.check(sample(0), 0, 1);
179
+
180
+ entries = [entry(70, LEGACY_KILL)];
181
+ expect(await r.check(sample(0), 61_000, 1)).toBeNull();
182
+ expect(recorded).toHaveLength(0);
183
+
184
+ // A later local kill is still found, and the neighbour's is not swept in.
185
+ entries = [entry(70, LEGACY_KILL), entry(80, TOOL_KILL)];
186
+ const report = await r.check(sample(1), 122_000, 1);
187
+ expect(report?.victims.map((v) => v.pid)).toEqual([4242]);
188
+ });
189
+ test("reports every new kill when there is no counter at all", async () => {
190
+ const r = reporter(() => [entry(5, TOOL_KILL)]);
191
+ await r.check(sample(null, false), 0, 1);
192
+ const r2 = reporter(() => [entry(5, TOOL_KILL), entry(9, LEGACY_KILL)]);
193
+ const report = await r2.check(sample(null, false), 61_000, 1);
194
+ expect(report?.attribution).toBe("kernel_log");
195
+ expect(report?.victims.map((v) => v.pid)).toEqual([77]);
196
+ });
197
+
198
+ test("a counter move with no readable log still produces an event", async () => {
199
+ const report = await reporter(() => null).check(sample(3), 0, 1);
200
+ expect(report).toEqual({
201
+ attribution: "cgroup_counter",
202
+ victims: [],
203
+ unnamed: 3,
204
+ });
205
+ expect(recorded[0]).toMatchObject({ value: 3 });
206
+ expect(recorded[0].detail).toMatchObject({
207
+ unnamed_victims: 3,
208
+ killed_daemon: false,
209
+ });
210
+ });
211
+
212
+ test("a rebooted kernel resets the cursor even when the boot id is unreadable", async () => {
213
+ await reporter(() => [entry(500, TOOL_KILL)], null).check(sample(0), 0, 1);
214
+ const report = await reporter(() => [entry(5, TOOL_KILL)], null).check(
215
+ sample(0),
216
+ 0,
217
+ 1,
218
+ );
219
+ expect(report?.victims).toHaveLength(1);
220
+ expect(recorded).toHaveLength(2);
221
+ });
222
+
223
+ test("keeps the cursor and retries when the telemetry store refuses the event", async () => {
224
+ recordOk = false;
225
+ const r = reporter(() => [entry(5, TOOL_KILL)]);
226
+ expect(await r.check(sample(1), 0, 1)).not.toBeNull();
227
+ expect(recorded).toHaveLength(0);
228
+
229
+ recordOk = true;
230
+ await r.check(sample(0), 1_000, 1);
231
+ expect(recorded).toHaveLength(1);
232
+
233
+ // Now durably acknowledged: a new incarnation does not repeat it.
234
+ expect(
235
+ await reporter(() => [entry(5, TOOL_KILL)]).check(sample(0), 0, 1),
236
+ ).toBeNull();
237
+ expect(recorded).toHaveLength(1);
238
+ });
239
+
240
+ test("an analytics opt-out advances the cursor without recording", async () => {
241
+ consent = false;
242
+ await reporter(() => [entry(5, TOOL_KILL)]).check(sample(1), 0, 1);
243
+ consent = true;
244
+ expect(
245
+ await reporter(() => [entry(5, TOOL_KILL)]).check(sample(0), 0, 1),
246
+ ).toBeNull();
247
+ expect(recorded).toHaveLength(0);
248
+ });
249
+
250
+ test("scans on the first tick, on a counter move, and on the fallback cadence", async () => {
251
+ const r = reporter(() => []);
252
+ await r.check(sample(0), 0, 1);
253
+ await r.check(sample(0), 1_000, 1);
254
+ expect(reads).toBe(1);
255
+ await r.check(sample(1), 2_000, 1);
256
+ expect(reads).toBe(2);
257
+ await r.check(sample(0), 61_999, 1);
258
+ expect(reads).toBe(2);
259
+ await r.check(sample(0), 62_000, 1);
260
+ expect(reads).toBe(3);
261
+ });
262
+ });
@@ -0,0 +1,345 @@
1
+ /**
2
+ * Reports kernel OOM kills in this container as `oom_kill` watchdog events.
3
+ *
4
+ * The log comes from `dmesg`, not `/dev/kmsg`: the device is not mounted in
5
+ * the container, while `dmesg` uses the syslog syscall the container is
6
+ * allowed. The log is per kernel, not per container, and `/proc/self/cgroup`
7
+ * reads `/` inside the container, so kills are attributed by the cgroup
8
+ * `memory.events` counter rather than by cgroup path.
9
+ */
10
+
11
+ import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
12
+ import { dirname, join } from "node:path";
13
+
14
+ import { getRawShareAnalytics } from "../platform/consent-cache.js";
15
+ import { recordWatchdogEvent } from "../telemetry/watchdog-events-store.js";
16
+ import { getLogger } from "../util/logger.js";
17
+ import { DAEMON_OOM_SCORE_ADJ } from "../util/oom-priority.js";
18
+ import { getMonitoringDataDir } from "../util/platform.js";
19
+ import type { ResourceSample } from "./resource-sample-types.js";
20
+
21
+ const log = getLogger("oom-kill-reporter");
22
+
23
+ export const OOM_KILL_CHECK_NAME = "oom_kill";
24
+ const CURSOR_FILENAME = "oom-kill-cursor.json";
25
+ /** Cadence for hosts without `memory.events`. */
26
+ const FALLBACK_SCAN_INTERVAL_MS = 60_000;
27
+ /** The detail bag is capped at 4 KiB server-side. */
28
+ const MAX_VICTIMS_IN_DETAIL = 10;
29
+ const DMESG_TIMEOUT_MS = 5_000;
30
+
31
+ export interface KernelLogEntry {
32
+ /** Seconds since boot, monotonic for the life of the kernel. */
33
+ time: number;
34
+ msg: string;
35
+ }
36
+
37
+ export interface OomKill {
38
+ time: number;
39
+ pid: number;
40
+ comm: string;
41
+ oomScoreAdj: number | null;
42
+ anonRssKb: number | null;
43
+ totalVmKb: number | null;
44
+ }
45
+
46
+ type Attribution = "cgroup_counter" | "kernel_log";
47
+
48
+ /** `dmesg --json`: `{"dmesg":[{"pri":3,"time":44170.542,"msg":"..."}]}`. */
49
+ export function parseDmesgJson(text: string): KernelLogEntry[] | null {
50
+ try {
51
+ const parsed: unknown = JSON.parse(text);
52
+ const rows = (parsed as { dmesg?: unknown }).dmesg;
53
+ if (!Array.isArray(rows)) {
54
+ return null;
55
+ }
56
+ const entries: KernelLogEntry[] = [];
57
+ for (const row of rows) {
58
+ const { time, msg } = row as { time?: unknown; msg?: unknown };
59
+ if (typeof time === "number" && typeof msg === "string") {
60
+ entries.push({ time, msg });
61
+ }
62
+ }
63
+ return entries;
64
+ } catch {
65
+ return null;
66
+ }
67
+ }
68
+
69
+ /** Plain `dmesg`: `[ 44170.542000] message` per line. */
70
+ export function parseDmesgText(text: string): KernelLogEntry[] {
71
+ const entries: KernelLogEntry[] = [];
72
+ for (const line of text.split("\n")) {
73
+ const match = /^\[\s*(\d+\.\d+)\]\s?(.*)$/.exec(line);
74
+ if (match) {
75
+ entries.push({ time: parseFloat(match[1]), msg: match[2] });
76
+ }
77
+ }
78
+ return entries;
79
+ }
80
+
81
+ const OOM_KILL_RE = /[Oo]ut of memory: Killed process (\d+) \(([^)]*)\)(.*)$/;
82
+
83
+ function kb(fields: string, key: string): number | null {
84
+ const match = new RegExp(`${key}:(\\d+)kB`).exec(fields);
85
+ return match ? parseInt(match[1], 10) : null;
86
+ }
87
+
88
+ export function oomKillsFromEntries(entries: KernelLogEntry[]): OomKill[] {
89
+ const kills: OomKill[] = [];
90
+ for (const entry of entries) {
91
+ const kill = OOM_KILL_RE.exec(entry.msg);
92
+ if (!kill) {
93
+ continue;
94
+ }
95
+ const fields = kill[3];
96
+ const adj = /oom_score_adj:(-?\d+)/.exec(fields);
97
+ kills.push({
98
+ time: entry.time,
99
+ pid: parseInt(kill[1], 10),
100
+ comm: kill[2],
101
+ oomScoreAdj: adj ? parseInt(adj[1], 10) : null,
102
+ anonRssKb: kb(fields, "anon-rss"),
103
+ totalVmKb: kb(fields, "total-vm"),
104
+ });
105
+ }
106
+ return kills.sort((a, b) => a.time - b.time);
107
+ }
108
+
109
+ /** Null when `dmesg` is absent or lacks CAP_SYSLOG. */
110
+ export async function readKernelLog(): Promise<KernelLogEntry[] | null> {
111
+ if (process.platform !== "linux") {
112
+ return null;
113
+ }
114
+ const run = async (args: string[]): Promise<string | null> => {
115
+ try {
116
+ const proc = Bun.spawn(["dmesg", ...args], {
117
+ stdout: "pipe",
118
+ stderr: "ignore",
119
+ windowsHide: true,
120
+ timeout: DMESG_TIMEOUT_MS,
121
+ });
122
+ const text = await new Response(proc.stdout).text();
123
+ return (await proc.exited) === 0 ? text : null;
124
+ } catch {
125
+ return null;
126
+ }
127
+ };
128
+ const json = await run(["--json"]);
129
+ const fromJson = json != null ? parseDmesgJson(json) : null;
130
+ if (fromJson) {
131
+ return fromJson;
132
+ }
133
+ const text = await run([]);
134
+ return text != null ? parseDmesgText(text) : null;
135
+ }
136
+
137
+ function readBootId(): string | null {
138
+ try {
139
+ return readFileSync("/proc/sys/kernel/random/boot_id", "utf-8").trim();
140
+ } catch {
141
+ return null;
142
+ }
143
+ }
144
+
145
+ interface Cursor {
146
+ bootId: string | null;
147
+ lastTime: number;
148
+ }
149
+
150
+ function readCursor(path: string): Cursor | null {
151
+ try {
152
+ const parsed = JSON.parse(readFileSync(path, "utf-8")) as Partial<Cursor>;
153
+ if (typeof parsed.lastTime === "number") {
154
+ return { bootId: parsed.bootId ?? null, lastTime: parsed.lastTime };
155
+ }
156
+ } catch {
157
+ // Missing or unreadable.
158
+ }
159
+ return null;
160
+ }
161
+
162
+ function writeCursor(path: string, cursor: Cursor): void {
163
+ mkdirSync(dirname(path), { recursive: true });
164
+ writeFileSync(path, JSON.stringify(cursor));
165
+ }
166
+
167
+ export interface OomKillReporterOptions {
168
+ readKernelLog?: () => Promise<KernelLogEntry[] | null>;
169
+ readBootId?: () => string | null;
170
+ shareAnalytics?: () => boolean | null;
171
+ cursorPath?: string;
172
+ record?: typeof recordWatchdogEvent;
173
+ }
174
+
175
+ export interface OomKillReport {
176
+ attribution: Attribution;
177
+ victims: OomKill[];
178
+ /** Kills the counter saw that the log could not name. */
179
+ unnamed: number;
180
+ }
181
+
182
+ export interface OomKillReporter {
183
+ /** Returns what was queued, for tests. */
184
+ check(
185
+ sample: ResourceSample,
186
+ now: number,
187
+ daemonPid: number | null,
188
+ ): Promise<OomKillReport | null>;
189
+ }
190
+
191
+ export function createOomKillReporter(
192
+ options: OomKillReporterOptions = {},
193
+ ): OomKillReporter {
194
+ const read = options.readKernelLog ?? readKernelLog;
195
+ const bootId = options.readBootId ?? readBootId;
196
+ const shareAnalytics = options.shareAnalytics ?? getRawShareAnalytics;
197
+ const cursorPath =
198
+ options.cursorPath ?? join(getMonitoringDataDir(), CURSOR_FILENAME);
199
+ const record = options.record ?? recordWatchdogEvent;
200
+
201
+ let firstScan = true;
202
+ let lastScanAt = 0;
203
+ /** A report the telemetry store refused; retried before the next scan. */
204
+ let pending: {
205
+ report: OomKillReport;
206
+ sample: ResourceSample;
207
+ daemonPid: number | null;
208
+ cursor: Cursor;
209
+ } | null = null;
210
+
211
+ const queue = (
212
+ report: OomKillReport,
213
+ sample: ResourceSample,
214
+ daemonPid: number | null,
215
+ cursor: Cursor,
216
+ ): boolean => {
217
+ if (shareAnalytics() === false) {
218
+ writeCursor(cursorPath, cursor);
219
+ return true;
220
+ }
221
+ const killedDaemon = report.victims.some(
222
+ (kill) =>
223
+ kill.oomScoreAdj === DAEMON_OOM_SCORE_ADJ ||
224
+ (daemonPid != null && kill.pid === daemonPid),
225
+ );
226
+ const detail = {
227
+ attribution: report.attribution,
228
+ victim_count: report.victims.length + report.unnamed,
229
+ unnamed_victims: report.unnamed,
230
+ victims: report.victims.slice(-MAX_VICTIMS_IN_DETAIL).map((kill) => ({
231
+ pid: kill.pid,
232
+ comm: kill.comm,
233
+ oom_score_adj: kill.oomScoreAdj,
234
+ anon_rss_kb: kill.anonRssKb,
235
+ total_vm_kb: kill.totalVmKb,
236
+ })),
237
+ daemon_pid: daemonPid,
238
+ killed_daemon: killedDaemon,
239
+ memory_current_bytes: sample.memory?.currentBytes ?? null,
240
+ memory_limit_bytes: sample.memory?.limitBytes ?? null,
241
+ memory_peak_bytes: sample.memory?.peakBytes ?? null,
242
+ };
243
+ const queued = record({
244
+ checkName: OOM_KILL_CHECK_NAME,
245
+ value: detail.victim_count,
246
+ detail,
247
+ });
248
+ if (!queued) {
249
+ return false;
250
+ }
251
+ writeCursor(cursorPath, cursor);
252
+ log.warn(detail, "Kernel OOM kill in the assistant container");
253
+ return true;
254
+ };
255
+
256
+ return {
257
+ async check(sample, now, daemonPid) {
258
+ if (pending) {
259
+ if (
260
+ !queue(
261
+ pending.report,
262
+ pending.sample,
263
+ pending.daemonPid,
264
+ pending.cursor,
265
+ )
266
+ ) {
267
+ return null;
268
+ }
269
+ pending = null;
270
+ }
271
+
272
+ const counterDelta = sample.deltas?.events?.oomKill ?? 0;
273
+ const countersAvailable = sample.events != null;
274
+ if (
275
+ !firstScan &&
276
+ counterDelta === 0 &&
277
+ now - lastScanAt < FALLBACK_SCAN_INTERVAL_MS
278
+ ) {
279
+ return null;
280
+ }
281
+ const isFirstScan = firstScan;
282
+ firstScan = false;
283
+ lastScanAt = now;
284
+
285
+ const entries = await read();
286
+ const currentBootId = bootId();
287
+ if (entries == null) {
288
+ if (counterDelta === 0) {
289
+ return null;
290
+ }
291
+ const report: OomKillReport = {
292
+ attribution: "cgroup_counter",
293
+ victims: [],
294
+ unnamed: counterDelta,
295
+ };
296
+ const cursor = { bootId: currentBootId, lastTime: 0 };
297
+ if (!queue(report, sample, daemonPid, cursor)) {
298
+ pending = { report, sample, daemonPid, cursor };
299
+ }
300
+ return report;
301
+ }
302
+
303
+ const all = oomKillsFromEntries(entries);
304
+ const maxTime = all.length > 0 ? all[all.length - 1].time : 0;
305
+ const stored = readCursor(cursorPath);
306
+ // A different boot id, or a log that has not yet reached the stored
307
+ // timestamp, means the kernel restarted and the cursor is stale.
308
+ const sameBoot =
309
+ stored != null &&
310
+ stored.bootId === currentBootId &&
311
+ maxTime >= stored.lastTime;
312
+ const sinceTime = sameBoot ? stored.lastTime : -1;
313
+ const fresh = all.filter((kill) => kill.time > sinceTime);
314
+ const cursor: Cursor = {
315
+ bootId: currentBootId,
316
+ lastTime: Math.max(sinceTime, maxTime),
317
+ };
318
+
319
+ let report: OomKillReport | null = null;
320
+ // The counter says how many were ours; older new kills are neighbours'.
321
+ if (counterDelta > 0) {
322
+ report = {
323
+ attribution: "cgroup_counter",
324
+ victims: fresh.slice(-counterDelta),
325
+ unnamed: Math.max(0, counterDelta - fresh.length),
326
+ };
327
+ } else if ((isFirstScan || !countersAvailable) && fresh.length > 0) {
328
+ // No counter to lean on: a kill that took the daemon down restarted
329
+ // this monitor with it, and the new container's counter starts at 0.
330
+ report = { attribution: "kernel_log", victims: fresh, unnamed: 0 };
331
+ }
332
+
333
+ if (report == null) {
334
+ if (!sameBoot || cursor.lastTime !== stored?.lastTime) {
335
+ writeCursor(cursorPath, cursor);
336
+ }
337
+ return null;
338
+ }
339
+ if (!queue(report, sample, daemonPid, cursor)) {
340
+ pending = { report, sample, daemonPid, cursor };
341
+ }
342
+ return report;
343
+ },
344
+ };
345
+ }
@@ -47,6 +47,7 @@ import { readActiveConversations } from "./active-conversations.js";
47
47
  import { readDaemonHeartbeat } from "./daemon-heartbeat.js";
48
48
  import { topProcessesByFd } from "./file-descriptors.js";
49
49
  import { sweepInheritedOomProtection } from "./oom-inheritance-sweep.js";
50
+ import { createOomKillReporter } from "./oom-kill-reporter.js";
50
51
  import { getTrackedDataFiles, readFileResidency } from "./page-cache.js";
51
52
  import { topProcessesByMemory } from "./process-memory.js";
52
53
  import {
@@ -267,6 +268,7 @@ export function startResourceSampler(
267
268
  // thread's kernel state mid-stall when the heartbeat goes stale.
268
269
  const stallCapture = createStallCaptureMonitor(dataDir);
269
270
  const processUsage = createProcessUsageTracker();
271
+ const oomKills = createOomKillReporter();
270
272
  let lastOomSweepAt = 0;
271
273
 
272
274
  // Skip ticks while a sample is in flight: the disk measurement can take
@@ -306,6 +308,12 @@ export function startResourceSampler(
306
308
  log.warn({ err }, "Daemon stall check failed");
307
309
  }
308
310
 
311
+ try {
312
+ await oomKills.check(sample, now, readDaemonHeartbeat(now)?.pid ?? null);
313
+ } catch (err) {
314
+ log.warn({ err }, "OOM kill report failed");
315
+ }
316
+
309
317
  if (now - lastOomSweepAt >= OOM_SWEEP_INTERVAL_MS) {
310
318
  lastOomSweepAt = now;
311
319
  const daemonPid = readDaemonHeartbeat(now)?.pid;
@@ -94,8 +94,7 @@ async function handleEmitSignal({ body = {} }: RouteHandlerArgs) {
94
94
  parsed.error.issues[0]?.message ?? "Invalid signal parameters",
95
95
  );
96
96
  }
97
- const validated = parsed.data;
98
- const result = await emitNotificationSignal(validated);
97
+ const result = await emitNotificationSignal(parsed.data);
99
98
  return {
100
99
  signalId: result.signalId,
101
100
  dispatched: result.dispatched,
@@ -174,8 +174,7 @@ export function composeVerificationText(
174
174
  templateKey: TextVerifyTemplateKey,
175
175
  vars: GuardianVerifyTemplateVars,
176
176
  ): string {
177
- const composer = templates[templateKey];
178
- return composer(vars);
177
+ return templates[templateKey](vars);
179
178
  }
180
179
 
181
180
  // ---------------------------------------------------------------------------
@@ -287,8 +287,7 @@ export const EXTRACT_LINKS_EXPRESSION = `
287
287
  export function parseBrowserMode(
288
288
  input: Record<string, unknown>,
289
289
  ): { ok: true; mode: BrowserMode } | { ok: false; error: string } {
290
- const raw = input.browser_mode;
291
- const result = normalizeBrowserMode(raw);
290
+ const result = normalizeBrowserMode(input.browser_mode);
292
291
  if ("error" in result) {
293
292
  return { ok: false, error: `Error: ${result.error}` };
294
293
  }
@@ -2672,9 +2671,11 @@ async function checkExtensionModeStatus(
2672
2671
  context: ToolContext,
2673
2672
  autoCandidate: boolean,
2674
2673
  ): Promise<BrowserStatusModeResult> {
2675
- const proxy = HostBrowserProxy.instance;
2676
-
2677
- if (!proxy.hasExtensionClient(context.sourceActorPrincipalId)) {
2674
+ if (
2675
+ !HostBrowserProxy.instance.hasExtensionClient(
2676
+ context.sourceActorPrincipalId,
2677
+ )
2678
+ ) {
2678
2679
  return {
2679
2680
  mode: BROWSER_STATUS_MODE.EXTENSION,
2680
2681
  available: false,
@@ -482,8 +482,7 @@ export function computePerToolTimeoutMs(
482
482
  const { questionResponseTimeoutSec } = getConfig().timeouts;
483
483
  return (questionResponseTimeoutSec + 5) * 1000;
484
484
  }
485
- const rawTimeoutSec = getConfig().timeouts.toolExecutionTimeoutSec;
486
- return safeTimeoutMs(rawTimeoutSec);
485
+ return safeTimeoutMs(getConfig().timeouts.toolExecutionTimeoutSec);
487
486
  }
488
487
 
489
488
  /**
@@ -1530,9 +1530,8 @@ export async function executeFastcrwScrape(
1530
1530
  input: Record<string, unknown>,
1531
1531
  options: { apiKey: string; signal?: AbortSignal },
1532
1532
  ): Promise<ToolExecutionResult> {
1533
- const apiBase = getConfig().services["web-fetch"]?.apiBase;
1534
1533
  const endpoint = resolveProviderApiUrl(
1535
- apiBase,
1534
+ getConfig().services["web-fetch"]?.apiBase,
1536
1535
  FASTCRW_SCRAPE_PATH,
1537
1536
  FASTCRW_DEFAULT_API_BASE,
1538
1537
  );
@@ -1587,9 +1586,8 @@ export async function executeTinyfishFetch(
1587
1586
  1,
1588
1587
  MAX_TIMEOUT_SECONDS,
1589
1588
  );
1590
- const apiBase = getConfig().services["web-fetch"]?.apiBase;
1591
1589
  const endpoint = resolveProviderApiUrl(
1592
- apiBase,
1590
+ getConfig().services["web-fetch"]?.apiBase,
1593
1591
  "/",
1594
1592
  TINYFISH_DEFAULT_FETCH_API_BASE,
1595
1593
  );
@@ -236,8 +236,11 @@ function getWebSearchProvider(): WebSearchProvider {
236
236
  async function getApiKey(
237
237
  provider: WebSearchProvider,
238
238
  ): Promise<string | undefined> {
239
- const adapter = WEB_SEARCH_ADAPTERS[provider];
240
- return (await getProviderKeyAsync(adapter.providerKeyName)) ?? undefined;
239
+ return (
240
+ (await getProviderKeyAsync(
241
+ WEB_SEARCH_ADAPTERS[provider].providerKeyName,
242
+ )) ?? undefined
243
+ );
241
244
  }
242
245
 
243
246
  function fallbackProvidersFor(
@@ -1517,9 +1520,8 @@ function executeFastcrwSearch(
1517
1520
  apiKey: string,
1518
1521
  signal?: AbortSignal,
1519
1522
  ): Promise<ToolExecutionResult> {
1520
- const apiBase = getConfig().services["web-search"]?.apiBase;
1521
1523
  const endpoint = resolveProviderApiUrl(
1522
- apiBase,
1524
+ getConfig().services["web-search"]?.apiBase,
1523
1525
  FASTCRW_SEARCH_PATH,
1524
1526
  FASTCRW_DEFAULT_API_BASE,
1525
1527
  );
@@ -1779,9 +1781,8 @@ async function executeTinyfishSearch(
1779
1781
  signal?: AbortSignal,
1780
1782
  ): Promise<ToolExecutionResult> {
1781
1783
  const startedAt = Date.now();
1782
- const apiBase = getConfig().services["web-search"]?.apiBase;
1783
1784
  const endpoint = resolveProviderApiUrl(
1784
- apiBase,
1785
+ getConfig().services["web-search"]?.apiBase,
1785
1786
  "/",
1786
1787
  TINYFISH_DEFAULT_SEARCH_API_BASE,
1787
1788
  );
@@ -2274,8 +2275,10 @@ export const webSearchTool = {
2274
2275
  // `mode: "managed"` did for installs that never configured a search
2275
2276
  // key. Read the configured provider off config — the local `provider`
2276
2277
  // has already been coerced to "perplexity".
2277
- const configured = getConfig().services["web-search"].provider;
2278
- if (configured === "inference-provider-native") {
2278
+ if (
2279
+ getConfig().services["web-search"].provider ===
2280
+ "inference-provider-native"
2281
+ ) {
2279
2282
  const { managedSearchAvailable } =
2280
2283
  await import("./managed-search-proxy.js");
2281
2284
  if (await managedSearchAvailable()) {
@@ -23,6 +23,5 @@ export function formatWriteSummary(
23
23
  if (isNewFile) {
24
24
  return `(new file, ${newLineCount} line${newLineCount !== 1 ? "s" : ""})`;
25
25
  }
26
- const oldLineCount = oldContent.split("\n").length;
27
- return `(${oldLineCount} → ${newLineCount} lines)`;
26
+ return `(${oldContent.split("\n").length} → ${newLineCount} lines)`;
28
27
  }
@@ -302,8 +302,7 @@ function collectSurfaceFillOnTextProblems(html: string): string[] {
302
302
  const offenders: string[] = [];
303
303
  for (const rule of html.matchAll(/([^{}<>]{1,120})\{([^{}]*)\}/g)) {
304
304
  const selector = rule[1].trim();
305
- const body = rule[2];
306
- if (!/fill\s*:\s*var\(--surface-/.test(body)) {
305
+ if (!/fill\s*:\s*var\(--surface-/.test(rule[2])) {
307
306
  continue;
308
307
  }
309
308
  if (SHAPE_SCOPED_SELECTOR_PATTERN.test(selector)) {