@vellumai/assistant 0.12.5-dev.202609281913.10969f6 → 0.12.5-dev.202609282115.bb92e6d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/ARCHITECTURE.md
CHANGED
|
@@ -49,7 +49,7 @@ Resource pressure monitoring reports sustained CPU/memory pressure on platform-h
|
|
|
49
49
|
|
|
50
50
|
When the container hits its memory limit, the kernel kills the process with the highest `oom_score` (memory share plus `oom_score_adj`). `src/util/oom-priority.ts` steers that choice so the assistant is never the victim of a runaway tool: the daemon sets itself to -700 as the first thing in `runDaemon`, before any spawn. Children inherit the parent's value at fork, so every spawn path resets its child: shell children get +1000 from the Linux prefix in `buildShellInvocation` (`packages/environments`), desktop children and ACP agents and terminals get +1000 from `withOomScoreAdj`, an `sh -c ... exec` wrapper that keeps pid and process group, and workers, the embedding worker, and Qdrant get 0 from a parent-side write right after spawn (the resource monitor gets -500 so it outlives everything but the daemon and records the kill). The monitor also sweeps the daemon's descendants every few seconds (`src/monitoring/oom-inheritance-sweep.ts`) and writes 0 over any negative value it finds, logging the comm, so a spawn path that forgets the reset degrades to a few seconds of inherited protection instead of a protected runaway. Bundled plugins (the memory plugin's rerank worker, for one) rely on the sweep rather than importing the helper, since the plugin import boundary forbids reaching outside the plugin. Raising never needs a capability; lowering needs `CAP_SYS_RESOURCE`, which the platform grants and self-hosted Docker does not, where the daemon simply stays at 0. Nothing here caps a single process or acts early: it only decides who dies once the kernel has to kill.
|
|
51
51
|
|
|
52
|
-
**Verification telemetry:** the resource monitor reports kernel OOM kills in the container as a `watchdog` event with `check_name` `oom_kill` (`src/monitoring/oom-kill-reporter.ts`). It reads the kernel log through `dmesg` (the `/dev/kmsg` device is not mounted in the container; `dmesg` uses the syslog syscall, which the Kata profile permits) when the cgroup `memory.events` kill counter moves, on its first scan after start (the kill that took the daemon down restarts the monitor with it), and once a minute as a fallback. The
|
|
52
|
+
**Verification telemetry:** the resource monitor reports kernel OOM kills in the container as a `watchdog` event with `check_name` `oom_kill` (`src/monitoring/oom-kill-reporter.ts`). It reads the kernel log through `dmesg` (the `/dev/kmsg` device is not mounted in the container; `dmesg` uses the syslog syscall, which the Kata profile permits) when the cgroup `memory.events` kill counter moves, on its first scan after start (the kill that took the daemon down restarts the monitor with it), and once a minute as a fallback. The log is shared by every container on that kernel, so a kill is attributed by the cgroup path in the kernel's `oom-kill:` header, matched against the container's own id, which it reads from the runtime-named mounts in `/proc/self/mountinfo` because `/proc/self/cgroup` reads `/` inside a cgroup namespace (`attribution: "cgroup"`). The kernel rate-limits that header in a burst, so header-less kills are attributed by the counter, re-read after the log: credit left after the header-attributed kills names header-less kills oldest first, header-less kills beyond the credit are another container's, and credit for a kill counted but not yet logged is spent on the next scan or reported unnamed (`attribution: "cgroup_counter"`). The first scan and hosts without the counter claim every new kill not assigned elsewhere by a header (`attribution: "kernel_log"`). The `detail` bag lists each victim's comm, `oom_score_adj` and resident size, the cgroup memory figures at the time, and `killed_daemon`, detected by the daemon's -700 score rather than a pid the restart has since replaced; where the log is unreadable a counter move still yields a victimless event. A cursor (kernel boot id plus last log timestamp) on the monitor data dir advances only after the event is queued, so a telemetry store outage delays a report rather than losing one, and resets when the kernel reboots.
|
|
53
53
|
|
|
54
54
|
### Single-Header JWT Auth Model
|
|
55
55
|
|
package/package.json
CHANGED
|
@@ -135,6 +135,19 @@
|
|
|
135
135
|
"homepage": "https://github.com/topoteretes/cognee-integrations/tree/main/integrations/vellum-assistant",
|
|
136
136
|
"icon": "\ud83d\udd78\ufe0f"
|
|
137
137
|
},
|
|
138
|
+
{
|
|
139
|
+
"name": "contractor-missed-call-dispatcher",
|
|
140
|
+
"description": "Captures missed trade contractor inbound calls, fires an instant 5-second SMS qualification text-back, triages emergencies, and stages draft jobs in Jobber or ServiceTitan.",
|
|
141
|
+
"category": "commerce",
|
|
142
|
+
"icon": "📲",
|
|
143
|
+
"license": "MIT",
|
|
144
|
+
"homepage": "https://github.com/ZeebBoyBlue/contractor-missed-call-dispatcher",
|
|
145
|
+
"source": {
|
|
146
|
+
"source": "github",
|
|
147
|
+
"repo": "ZeebBoyBlue/contractor-missed-call-dispatcher",
|
|
148
|
+
"ref": "2d293cdcb68af03d446b61a9915a0ee3dc842d4e"
|
|
149
|
+
}
|
|
150
|
+
},
|
|
138
151
|
{
|
|
139
152
|
"name": "contractor-voice-invoicer",
|
|
140
153
|
"description": "Converts 20-second truck voice memos and job photos into itemized client invoices with custom labor rates, material markups, and one-click accounting staging.",
|
|
@@ -9,21 +9,41 @@ import {
|
|
|
9
9
|
oomKillsFromEntries,
|
|
10
10
|
parseDmesgJson,
|
|
11
11
|
parseDmesgText,
|
|
12
|
+
parseOwnContainerIds,
|
|
12
13
|
} from "../oom-kill-reporter.js";
|
|
13
14
|
import type { ResourceSample } from "../resource-sample-types.js";
|
|
14
15
|
|
|
15
|
-
const
|
|
16
|
-
"
|
|
17
|
-
const
|
|
18
|
-
"
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
16
|
+
const OUR_ID =
|
|
17
|
+
"de939a251b36f11a400aab40a43b756ca376056239227aec2795953ca219e7d5";
|
|
18
|
+
const OTHER_ID =
|
|
19
|
+
"0f4c3b2a1908f7e6d5c4b3a29180f7e6d5c4b3a29180f7e6d5c4b3a29180f7e6";
|
|
20
|
+
const memcg = (id: string) =>
|
|
21
|
+
`/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-poda5d2033b.slice/cri-containerd-${id}.scope`;
|
|
22
|
+
|
|
23
|
+
const header = (pid: number, comm: string, id: string) =>
|
|
24
|
+
`oom-kill:constraint=CONSTRAINT_MEMCG,nodemask=(null),cpuset=cri-containerd-${id}.scope,mems_allowed=0,oom_memcg=${memcg(id)},task_memcg=${memcg(id)},task=${comm},pid=${pid},uid=0`;
|
|
25
|
+
const killed = (
|
|
26
|
+
pid: number,
|
|
27
|
+
comm: string,
|
|
28
|
+
adj: number | null,
|
|
29
|
+
anonKb = 409600,
|
|
30
|
+
) =>
|
|
31
|
+
`Memory cgroup out of memory: Killed process ${pid} (${comm}) total-vm:1263184kB, anon-rss:${anonKb}kB, file-rss:1024kB, shmem-rss:0kB, UID:0 pgtables:900kB${adj == null ? "" : ` oom_score_adj:${adj}`}`;
|
|
25
32
|
|
|
26
33
|
const entry = (time: number, msg: string): KernelLogEntry => ({ time, msg });
|
|
34
|
+
/** A kill with its header, as the kernel logs it when not rate-limited. */
|
|
35
|
+
const ourKill = (time: number, pid: number, adj = 1000): KernelLogEntry[] => [
|
|
36
|
+
entry(time - 0.00001, header(pid, "python3", OUR_ID)),
|
|
37
|
+
entry(time, killed(pid, "python3", adj)),
|
|
38
|
+
];
|
|
39
|
+
const theirKill = (time: number, pid: number): KernelLogEntry[] => [
|
|
40
|
+
entry(time - 0.00001, header(pid, "node", OTHER_ID)),
|
|
41
|
+
entry(time, killed(pid, "node", 0)),
|
|
42
|
+
];
|
|
43
|
+
/** A kill whose header the kernel rate-limited away. */
|
|
44
|
+
const bareKill = (time: number, pid: number): KernelLogEntry[] => [
|
|
45
|
+
entry(time, killed(pid, "python3", 1000)),
|
|
46
|
+
];
|
|
27
47
|
|
|
28
48
|
function sample(
|
|
29
49
|
oomKillDelta: number | null,
|
|
@@ -57,18 +77,22 @@ function sample(
|
|
|
57
77
|
|
|
58
78
|
describe("kernel log parsing", () => {
|
|
59
79
|
test("parses dmesg --json and plain dmesg into the same entries", () => {
|
|
60
|
-
const
|
|
61
|
-
const
|
|
80
|
+
const msg = killed(4242, "python3", 1000);
|
|
81
|
+
const json = `{"dmesg":[{"pri":3,"time": 44170.542000,"msg":${JSON.stringify(msg)}},{"pri":6,"time":44171.0,"msg":"x"}]}`;
|
|
82
|
+
const text = `[44170.542000] ${msg}\n[ 44171.000000] x\n`;
|
|
62
83
|
expect(parseDmesgJson(json)).toEqual(parseDmesgText(text));
|
|
63
84
|
expect(parseDmesgJson("not json")).toBeNull();
|
|
64
85
|
});
|
|
65
86
|
|
|
66
|
-
test("
|
|
87
|
+
test("pairs each kill with its header and tolerates missing headers and noise", () => {
|
|
67
88
|
const kills = oomKillsFromEntries([
|
|
68
|
-
entry(
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
89
|
+
entry(
|
|
90
|
+
30,
|
|
91
|
+
"bun invoked oom-killer: gfp_mask=0xcc0(GFP_KERNEL), order=0, oom_score_adj=-700",
|
|
92
|
+
),
|
|
93
|
+
...ourKill(31, 4242),
|
|
94
|
+
entry(32, "oom_reaper: reaped process 4242 (python3), now anon-rss:0kB"),
|
|
95
|
+
entry(50, killed(77, "node", null, 300000)),
|
|
72
96
|
]);
|
|
73
97
|
expect(kills).toEqual([
|
|
74
98
|
{
|
|
@@ -78,6 +102,7 @@ describe("kernel log parsing", () => {
|
|
|
78
102
|
oomScoreAdj: 1000,
|
|
79
103
|
anonRssKb: 409600,
|
|
80
104
|
totalVmKb: 1263184,
|
|
105
|
+
memcg: memcg(OUR_ID),
|
|
81
106
|
},
|
|
82
107
|
{
|
|
83
108
|
time: 50,
|
|
@@ -85,10 +110,22 @@ describe("kernel log parsing", () => {
|
|
|
85
110
|
comm: "node",
|
|
86
111
|
oomScoreAdj: null,
|
|
87
112
|
anonRssKb: 300000,
|
|
88
|
-
totalVmKb:
|
|
113
|
+
totalVmKb: 1263184,
|
|
114
|
+
memcg: null,
|
|
89
115
|
},
|
|
90
116
|
]);
|
|
91
117
|
});
|
|
118
|
+
|
|
119
|
+
test("finds the container id in the runtime-named mounts only", () => {
|
|
120
|
+
const layer =
|
|
121
|
+
"aaaa000000000000000000000000000000000000000000000000000000000000";
|
|
122
|
+
const mountinfo = [
|
|
123
|
+
`151 137 0:34 /${OUR_ID}-1f3ead8083566ffb-hosts /etc/hosts rw,relatime - virtiofs kataShared rw`,
|
|
124
|
+
`152 139 0:34 /${OUR_ID}-175b29da4eda0e77-termination-log /dev/termination-log rw,relatime - virtiofs kataShared rw`,
|
|
125
|
+
`140 120 0:30 / / rw,relatime - overlay overlay rw,lowerdir=/var/lib/x/${layer}/fs`,
|
|
126
|
+
].join("\n");
|
|
127
|
+
expect(parseOwnContainerIds(mountinfo)).toEqual(new Set([OUR_ID]));
|
|
128
|
+
});
|
|
92
129
|
});
|
|
93
130
|
|
|
94
131
|
describe("createOomKillReporter", () => {
|
|
@@ -97,6 +134,8 @@ describe("createOomKillReporter", () => {
|
|
|
97
134
|
let reads: number;
|
|
98
135
|
let recordOk: boolean;
|
|
99
136
|
let consent: boolean | null;
|
|
137
|
+
let killCounter: number | null;
|
|
138
|
+
let entries: KernelLogEntry[] | null;
|
|
100
139
|
|
|
101
140
|
beforeEach(() => {
|
|
102
141
|
dir = mkdtempSync(join(tmpdir(), "oom-kill-reporter-"));
|
|
@@ -104,21 +143,22 @@ describe("createOomKillReporter", () => {
|
|
|
104
143
|
reads = 0;
|
|
105
144
|
recordOk = true;
|
|
106
145
|
consent = true;
|
|
146
|
+
killCounter = 0;
|
|
147
|
+
entries = [];
|
|
107
148
|
});
|
|
108
149
|
|
|
109
150
|
afterEach(() => {
|
|
110
151
|
rmSync(dir, { recursive: true, force: true });
|
|
111
152
|
});
|
|
112
153
|
|
|
113
|
-
function reporter(
|
|
114
|
-
log: () => KernelLogEntry[] | null,
|
|
115
|
-
bootId: string | null = "boot-a",
|
|
116
|
-
) {
|
|
154
|
+
function reporter(bootId: string | null = "boot-a") {
|
|
117
155
|
return createOomKillReporter({
|
|
118
156
|
readKernelLog: async () => {
|
|
119
157
|
reads++;
|
|
120
|
-
return
|
|
158
|
+
return entries;
|
|
121
159
|
},
|
|
160
|
+
readKillCounter: () => killCounter,
|
|
161
|
+
readOwnContainerIds: () => new Set([OUR_ID]),
|
|
122
162
|
readBootId: () => bootId,
|
|
123
163
|
shareAnalytics: () => consent,
|
|
124
164
|
cursorPath: join(dir, "cursor.json"),
|
|
@@ -132,32 +172,93 @@ describe("createOomKillReporter", () => {
|
|
|
132
172
|
});
|
|
133
173
|
}
|
|
134
174
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
|
|
175
|
+
const pids = (report: { victims: { pid: number }[] } | null) =>
|
|
176
|
+
report?.victims.map((v) => v.pid) ?? [];
|
|
177
|
+
|
|
178
|
+
test("attributes by the header's cgroup regardless of order or count", async () => {
|
|
179
|
+
const r = reporter();
|
|
180
|
+
await r.check(sample(0), 0, 1);
|
|
181
|
+
|
|
182
|
+
entries = [...theirKill(70, 900), ...ourKill(80, 4242)];
|
|
183
|
+
killCounter = 1;
|
|
184
|
+
const report = await r.check(sample(1), 1_000, 1);
|
|
185
|
+
expect(report?.attribution).toBe("cgroup");
|
|
186
|
+
expect(pids(report)).toEqual([4242]);
|
|
187
|
+
expect(recorded[0].detail).toMatchObject({ killed_daemon: false });
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test("uses the counter for header-less kills, oldest first; the rest are neighbours'", async () => {
|
|
191
|
+
const r = reporter();
|
|
192
|
+
await r.check(sample(0), 0, 1);
|
|
193
|
+
|
|
194
|
+
entries = [...bareKill(20, 1), ...bareKill(21, 2), ...bareKill(22, 3)];
|
|
195
|
+
killCounter = 2;
|
|
196
|
+
const first = await r.check(sample(2), 1_000, 1);
|
|
197
|
+
expect(first).toMatchObject({ attribution: "cgroup_counter", unnamed: 0 });
|
|
198
|
+
expect(pids(first)).toEqual([1, 2]);
|
|
199
|
+
|
|
200
|
+
// The third was logged but not counted, so it was another container's.
|
|
201
|
+
expect(await r.check(sample(0), 61_000, 1)).toBeNull();
|
|
154
202
|
expect(recorded).toHaveLength(1);
|
|
155
|
-
|
|
203
|
+
});
|
|
204
|
+
test("spends counter credit on the next scan and reports it unnamed only if unspent", async () => {
|
|
205
|
+
const r = reporter();
|
|
206
|
+
await r.check(sample(0), 0, 1);
|
|
207
|
+
|
|
208
|
+
// The kill is counted but its line is not yet visible: nothing to say yet.
|
|
209
|
+
killCounter = 1;
|
|
210
|
+
expect(await r.check(sample(1), 1_000, 1)).toBeNull();
|
|
211
|
+
|
|
212
|
+
entries = [...bareKill(5, 1)];
|
|
213
|
+
const spent = await r.check(sample(0), 61_000, 1);
|
|
214
|
+
expect(spent).toMatchObject({ attribution: "cgroup_counter", unnamed: 0 });
|
|
215
|
+
expect(pids(spent)).toEqual([1]);
|
|
216
|
+
|
|
217
|
+
// Credit that never finds a line is reported unnamed after one scan.
|
|
218
|
+
killCounter = 2;
|
|
219
|
+
expect(await r.check(sample(1), 62_000, 1)).toBeNull();
|
|
220
|
+
const expired = await r.check(sample(0), 123_000, 1);
|
|
221
|
+
expect(expired).toEqual({
|
|
222
|
+
attribution: "cgroup_counter",
|
|
223
|
+
victims: [],
|
|
224
|
+
unnamed: 1,
|
|
225
|
+
});
|
|
226
|
+
});
|
|
227
|
+
test("names every victim of a burst where the kernel rate-limits headers", async () => {
|
|
228
|
+
const r = reporter();
|
|
229
|
+
await r.check(sample(0), 0, 1);
|
|
230
|
+
|
|
231
|
+
const victims = [468, 472, 474, 477, 476, 467, 480, 471, 478, 473, 481];
|
|
232
|
+
const headerless = new Set([474, 468]);
|
|
233
|
+
const perKill = victims.map((pid, i) =>
|
|
234
|
+
headerless.has(pid)
|
|
235
|
+
? bareKill(216386 + i * 0.15, pid)
|
|
236
|
+
: ourKill(216386 + i * 0.15, pid),
|
|
237
|
+
);
|
|
238
|
+
// Kills visible in the log, then the counter read after them, per tick.
|
|
239
|
+
const ticks: Array<[number, number]> = [
|
|
240
|
+
[2, 2],
|
|
241
|
+
[5, 6],
|
|
242
|
+
[8, 8],
|
|
243
|
+
[11, 11],
|
|
244
|
+
];
|
|
245
|
+
const named: number[] = [];
|
|
246
|
+
let t = 1_000;
|
|
247
|
+
for (const [visible, counter] of ticks) {
|
|
248
|
+
entries = perKill.slice(0, visible).flat();
|
|
249
|
+
killCounter = counter;
|
|
250
|
+
const report = await r.check(sample(1), (t += 250), 1);
|
|
251
|
+
named.push(...pids(report));
|
|
252
|
+
}
|
|
253
|
+
expect([...named].sort((a, b) => a - b)).toEqual(
|
|
254
|
+
[...victims].sort((a, b) => a - b),
|
|
255
|
+
);
|
|
256
|
+
expect(recorded.reduce((n, e) => n + (e.value ?? 0), 0)).toBe(11);
|
|
156
257
|
});
|
|
157
258
|
|
|
158
259
|
test("first scan reports the kill that preceded the monitor and flags the daemon by score", async () => {
|
|
159
|
-
|
|
160
|
-
const report = await
|
|
260
|
+
entries = [...ourKill(5, 1, -700)];
|
|
261
|
+
const report = await reporter().check(sample(0), 0, 999);
|
|
161
262
|
expect(report?.attribution).toBe("kernel_log");
|
|
162
263
|
expect(recorded[0].detail).toMatchObject({
|
|
163
264
|
killed_daemon: true,
|
|
@@ -165,90 +266,101 @@ describe("createOomKillReporter", () => {
|
|
|
165
266
|
memory_limit_bytes: 3_221_225_472,
|
|
166
267
|
});
|
|
167
268
|
|
|
168
|
-
// A second
|
|
169
|
-
expect(
|
|
170
|
-
await reporter(() => [entry(5, DAEMON_KILL)]).check(sample(0), 0, 999),
|
|
171
|
-
).toBeNull();
|
|
269
|
+
// A second incarnation on the same kernel sees the same log: nothing new.
|
|
270
|
+
expect(await reporter().check(sample(0), 0, 999)).toBeNull();
|
|
172
271
|
expect(recorded).toHaveLength(1);
|
|
173
272
|
});
|
|
174
273
|
|
|
175
|
-
test("
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
274
|
+
test("first scan still excludes kills the header assigns elsewhere", async () => {
|
|
275
|
+
entries = [...theirKill(4, 900), ...ourKill(5, 1)];
|
|
276
|
+
expect(pids(await reporter().check(sample(0), 0, 1))).toEqual([1]);
|
|
277
|
+
});
|
|
179
278
|
|
|
180
|
-
|
|
279
|
+
test("ignores header-less neighbours' kills when the local counter did not move", async () => {
|
|
280
|
+
const r = reporter();
|
|
281
|
+
await r.check(sample(0), 0, 1);
|
|
282
|
+
entries = [...bareKill(70, 900)];
|
|
181
283
|
expect(await r.check(sample(0), 61_000, 1)).toBeNull();
|
|
182
284
|
expect(recorded).toHaveLength(0);
|
|
183
|
-
|
|
184
|
-
// A later local kill is still found, and the neighbour's is not swept in.
|
|
185
|
-
entries = [entry(70, LEGACY_KILL), entry(80, TOOL_KILL)];
|
|
186
|
-
const report = await r.check(sample(1), 122_000, 1);
|
|
187
|
-
expect(report?.victims.map((v) => v.pid)).toEqual([4242]);
|
|
188
285
|
});
|
|
286
|
+
|
|
189
287
|
test("reports every new kill when there is no counter at all", async () => {
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
288
|
+
killCounter = null;
|
|
289
|
+
entries = [...bareKill(5, 1)];
|
|
290
|
+
await reporter().check(sample(null, false), 0, 1);
|
|
291
|
+
entries = [...bareKill(5, 1), ...bareKill(9, 2)];
|
|
292
|
+
const report = await reporter().check(sample(null, false), 61_000, 1);
|
|
194
293
|
expect(report?.attribution).toBe("kernel_log");
|
|
195
|
-
expect(report
|
|
294
|
+
expect(pids(report)).toEqual([2]);
|
|
196
295
|
});
|
|
197
296
|
|
|
198
|
-
test("a
|
|
199
|
-
const
|
|
297
|
+
test("skips a scan when a counter that exists fails to read", async () => {
|
|
298
|
+
const r = reporter();
|
|
299
|
+
await r.check(sample(0), 0, 1);
|
|
300
|
+
entries = [...bareKill(70, 900)];
|
|
301
|
+
killCounter = null;
|
|
302
|
+
expect(await r.check(sample(1), 1_000, 1)).toBeNull();
|
|
303
|
+
expect(recorded).toHaveLength(0);
|
|
304
|
+
// The scan was not consumed: the kill is still claimable once the counter reads.
|
|
305
|
+
killCounter = 1;
|
|
306
|
+
expect(pids(await r.check(sample(0), 1_250, 1))).toEqual([900]);
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
test("a counter move with no readable log still produces an event, and does not double-count later", async () => {
|
|
310
|
+
entries = null;
|
|
311
|
+
killCounter = 3;
|
|
312
|
+
const r = reporter();
|
|
313
|
+
const report = await r.check(sample(3), 0, 1);
|
|
200
314
|
expect(report).toEqual({
|
|
201
315
|
attribution: "cgroup_counter",
|
|
202
316
|
victims: [],
|
|
203
317
|
unnamed: 3,
|
|
204
318
|
});
|
|
205
319
|
expect(recorded[0]).toMatchObject({ value: 3 });
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
320
|
+
|
|
321
|
+
entries = [...bareKill(1, 1), ...bareKill(2, 2), ...bareKill(3, 3)];
|
|
322
|
+
expect(await r.check(sample(0), 61_000, 1)).toBeNull();
|
|
323
|
+
expect(recorded).toHaveLength(1);
|
|
210
324
|
});
|
|
211
325
|
|
|
212
326
|
test("a rebooted kernel resets the cursor even when the boot id is unreadable", async () => {
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
1,
|
|
218
|
-
);
|
|
327
|
+
entries = [...bareKill(500, 1)];
|
|
328
|
+
await reporter(null).check(sample(0), 0, 1);
|
|
329
|
+
entries = [...bareKill(5, 1)];
|
|
330
|
+
const report = await reporter(null).check(sample(0), 0, 1);
|
|
219
331
|
expect(report?.victims).toHaveLength(1);
|
|
220
332
|
expect(recorded).toHaveLength(2);
|
|
221
333
|
});
|
|
222
334
|
|
|
223
335
|
test("keeps the cursor and retries when the telemetry store refuses the event", async () => {
|
|
336
|
+
const r = reporter();
|
|
337
|
+
await r.check(sample(0), 0, 1);
|
|
224
338
|
recordOk = false;
|
|
225
|
-
|
|
226
|
-
|
|
339
|
+
entries = [...ourKill(5, 1)];
|
|
340
|
+
killCounter = 1;
|
|
341
|
+
expect(await r.check(sample(1), 1_000, 1)).not.toBeNull();
|
|
227
342
|
expect(recorded).toHaveLength(0);
|
|
228
343
|
|
|
229
344
|
recordOk = true;
|
|
230
|
-
await r.check(sample(0),
|
|
345
|
+
await r.check(sample(0), 2_000, 1);
|
|
231
346
|
expect(recorded).toHaveLength(1);
|
|
232
347
|
|
|
233
|
-
//
|
|
234
|
-
expect(
|
|
235
|
-
await reporter(() => [entry(5, TOOL_KILL)]).check(sample(0), 0, 1),
|
|
236
|
-
).toBeNull();
|
|
348
|
+
// Durably acknowledged: a new incarnation does not repeat it.
|
|
349
|
+
expect(await reporter().check(sample(0), 0, 1)).toBeNull();
|
|
237
350
|
expect(recorded).toHaveLength(1);
|
|
238
351
|
});
|
|
239
352
|
|
|
240
353
|
test("an analytics opt-out advances the cursor without recording", async () => {
|
|
241
354
|
consent = false;
|
|
242
|
-
|
|
355
|
+
entries = [...ourKill(5, 1)];
|
|
356
|
+
await reporter().check(sample(0), 0, 1);
|
|
243
357
|
consent = true;
|
|
244
|
-
expect(
|
|
245
|
-
await reporter(() => [entry(5, TOOL_KILL)]).check(sample(0), 0, 1),
|
|
246
|
-
).toBeNull();
|
|
358
|
+
expect(await reporter().check(sample(0), 0, 1)).toBeNull();
|
|
247
359
|
expect(recorded).toHaveLength(0);
|
|
248
360
|
});
|
|
249
361
|
|
|
250
362
|
test("scans on the first tick, on a counter move, and on the fallback cadence", async () => {
|
|
251
|
-
const r = reporter(
|
|
363
|
+
const r = reporter();
|
|
252
364
|
await r.check(sample(0), 0, 1);
|
|
253
365
|
await r.check(sample(0), 1_000, 1);
|
|
254
366
|
expect(reads).toBe(1);
|
|
@@ -3,9 +3,12 @@
|
|
|
3
3
|
*
|
|
4
4
|
* The log comes from `dmesg`, not `/dev/kmsg`: the device is not mounted in
|
|
5
5
|
* the container, while `dmesg` uses the syslog syscall the container is
|
|
6
|
-
* allowed. The log is per kernel, not per container
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* allowed. The log is per kernel, not per container. A kill is this
|
|
7
|
+
* container's when the kernel's `oom-kill:` header names a cgroup carrying
|
|
8
|
+
* this container's id, which the container learns from its own mount table
|
|
9
|
+
* since `/proc/self/cgroup` reads `/` inside a cgroup namespace. The kernel
|
|
10
|
+
* rate-limits that header in a burst, so header-less kills fall back to the
|
|
11
|
+
* cgroup `memory.events` counter.
|
|
9
12
|
*/
|
|
10
13
|
|
|
11
14
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
@@ -13,6 +16,7 @@ import { dirname, join } from "node:path";
|
|
|
13
16
|
|
|
14
17
|
import { getRawShareAnalytics } from "../platform/consent-cache.js";
|
|
15
18
|
import { recordWatchdogEvent } from "../telemetry/watchdog-events-store.js";
|
|
19
|
+
import { getContainerMemoryEvents } from "../util/cgroup-memory.js";
|
|
16
20
|
import { getLogger } from "../util/logger.js";
|
|
17
21
|
import { DAEMON_OOM_SCORE_ADJ } from "../util/oom-priority.js";
|
|
18
22
|
import { getMonitoringDataDir } from "../util/platform.js";
|
|
@@ -27,6 +31,13 @@ const FALLBACK_SCAN_INTERVAL_MS = 60_000;
|
|
|
27
31
|
/** The detail bag is capped at 4 KiB server-side. */
|
|
28
32
|
const MAX_VICTIMS_IN_DETAIL = 10;
|
|
29
33
|
const DMESG_TIMEOUT_MS = 5_000;
|
|
34
|
+
/** Mounts whose source path the container runtime names after the container. */
|
|
35
|
+
const CONTAINER_ID_MOUNTPOINTS = new Set([
|
|
36
|
+
"/etc/hosts",
|
|
37
|
+
"/etc/hostname",
|
|
38
|
+
"/etc/resolv.conf",
|
|
39
|
+
"/dev/termination-log",
|
|
40
|
+
]);
|
|
30
41
|
|
|
31
42
|
export interface KernelLogEntry {
|
|
32
43
|
/** Seconds since boot, monotonic for the life of the kernel. */
|
|
@@ -41,9 +52,11 @@ export interface OomKill {
|
|
|
41
52
|
oomScoreAdj: number | null;
|
|
42
53
|
anonRssKb: number | null;
|
|
43
54
|
totalVmKb: number | null;
|
|
55
|
+
/** Victim's cgroup from the `oom-kill:` header; null when the header was rate-limited away. */
|
|
56
|
+
memcg: string | null;
|
|
44
57
|
}
|
|
45
58
|
|
|
46
|
-
type Attribution = "cgroup_counter" | "kernel_log";
|
|
59
|
+
type Attribution = "cgroup" | "cgroup_counter" | "kernel_log";
|
|
47
60
|
|
|
48
61
|
/** `dmesg --json`: `{"dmesg":[{"pri":3,"time":44170.542,"msg":"..."}]}`. */
|
|
49
62
|
export function parseDmesgJson(text: string): KernelLogEntry[] | null {
|
|
@@ -79,6 +92,7 @@ export function parseDmesgText(text: string): KernelLogEntry[] {
|
|
|
79
92
|
}
|
|
80
93
|
|
|
81
94
|
const OOM_KILL_RE = /[Oo]ut of memory: Killed process (\d+) \(([^)]*)\)(.*)$/;
|
|
95
|
+
const OOM_HEADER_RE = /^oom-kill:.*task_memcg=([^,]*),task=[^,]*,pid=(\d+)/;
|
|
82
96
|
|
|
83
97
|
function kb(fields: string, key: string): number | null {
|
|
84
98
|
const match = new RegExp(`${key}:(\\d+)kB`).exec(fields);
|
|
@@ -86,24 +100,35 @@ function kb(fields: string, key: string): number | null {
|
|
|
86
100
|
}
|
|
87
101
|
|
|
88
102
|
export function oomKillsFromEntries(entries: KernelLogEntry[]): OomKill[] {
|
|
103
|
+
const sorted = [...entries].sort((a, b) => a.time - b.time);
|
|
104
|
+
// The header precedes its "Killed process" line and names the same pid.
|
|
105
|
+
const memcgByPid = new Map<number, string>();
|
|
89
106
|
const kills: OomKill[] = [];
|
|
90
|
-
for (const entry of
|
|
107
|
+
for (const entry of sorted) {
|
|
108
|
+
const header = OOM_HEADER_RE.exec(entry.msg);
|
|
109
|
+
if (header) {
|
|
110
|
+
memcgByPid.set(parseInt(header[2], 10), header[1]);
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
91
113
|
const kill = OOM_KILL_RE.exec(entry.msg);
|
|
92
114
|
if (!kill) {
|
|
93
115
|
continue;
|
|
94
116
|
}
|
|
117
|
+
const pid = parseInt(kill[1], 10);
|
|
95
118
|
const fields = kill[3];
|
|
96
119
|
const adj = /oom_score_adj:(-?\d+)/.exec(fields);
|
|
97
120
|
kills.push({
|
|
98
121
|
time: entry.time,
|
|
99
|
-
pid
|
|
122
|
+
pid,
|
|
100
123
|
comm: kill[2],
|
|
101
124
|
oomScoreAdj: adj ? parseInt(adj[1], 10) : null,
|
|
102
125
|
anonRssKb: kb(fields, "anon-rss"),
|
|
103
126
|
totalVmKb: kb(fields, "total-vm"),
|
|
127
|
+
memcg: memcgByPid.get(pid) ?? null,
|
|
104
128
|
});
|
|
129
|
+
memcgByPid.delete(pid);
|
|
105
130
|
}
|
|
106
|
-
return kills
|
|
131
|
+
return kills;
|
|
107
132
|
}
|
|
108
133
|
|
|
109
134
|
/** Null when `dmesg` is absent or lacks CAP_SYSLOG. */
|
|
@@ -134,6 +159,33 @@ export async function readKernelLog(): Promise<KernelLogEntry[] | null> {
|
|
|
134
159
|
return text != null ? parseDmesgText(text) : null;
|
|
135
160
|
}
|
|
136
161
|
|
|
162
|
+
/**
|
|
163
|
+
* Container ids from `/proc/self/mountinfo`: the runtime names the source of
|
|
164
|
+
* the hosts, hostname, resolv.conf and termination-log mounts after the
|
|
165
|
+
* container (Kata: `<id>-<hash>-hosts`; Docker: `.../containers/<id>/hosts`).
|
|
166
|
+
*/
|
|
167
|
+
export function parseOwnContainerIds(mountinfo: string): Set<string> {
|
|
168
|
+
const ids = new Set<string>();
|
|
169
|
+
for (const line of mountinfo.split("\n")) {
|
|
170
|
+
const fields = line.split(" ");
|
|
171
|
+
if (fields.length < 5 || !CONTAINER_ID_MOUNTPOINTS.has(fields[4])) {
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
for (const id of line.match(/[0-9a-f]{64}/g) ?? []) {
|
|
175
|
+
ids.add(id);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return ids;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function readOwnContainerIds(): Set<string> {
|
|
182
|
+
try {
|
|
183
|
+
return parseOwnContainerIds(readFileSync("/proc/self/mountinfo", "utf-8"));
|
|
184
|
+
} catch {
|
|
185
|
+
return new Set();
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
137
189
|
function readBootId(): string | null {
|
|
138
190
|
try {
|
|
139
191
|
return readFileSync("/proc/sys/kernel/random/boot_id", "utf-8").trim();
|
|
@@ -164,8 +216,20 @@ function writeCursor(path: string, cursor: Cursor): void {
|
|
|
164
216
|
writeFileSync(path, JSON.stringify(cursor));
|
|
165
217
|
}
|
|
166
218
|
|
|
219
|
+
function memcgIsOurs(memcg: string, ids: ReadonlySet<string>): boolean {
|
|
220
|
+
for (const id of ids) {
|
|
221
|
+
if (memcg.includes(id)) {
|
|
222
|
+
return true;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return false;
|
|
226
|
+
}
|
|
227
|
+
|
|
167
228
|
export interface OomKillReporterOptions {
|
|
168
229
|
readKernelLog?: () => Promise<KernelLogEntry[] | null>;
|
|
230
|
+
/** Cumulative cgroup oom_kill count, read after the log so it covers every logged kill. */
|
|
231
|
+
readKillCounter?: () => number | null;
|
|
232
|
+
readOwnContainerIds?: () => ReadonlySet<string>;
|
|
169
233
|
readBootId?: () => string | null;
|
|
170
234
|
shareAnalytics?: () => boolean | null;
|
|
171
235
|
cursorPath?: string;
|
|
@@ -192,6 +256,10 @@ export function createOomKillReporter(
|
|
|
192
256
|
options: OomKillReporterOptions = {},
|
|
193
257
|
): OomKillReporter {
|
|
194
258
|
const read = options.readKernelLog ?? readKernelLog;
|
|
259
|
+
const readKillCounter =
|
|
260
|
+
options.readKillCounter ??
|
|
261
|
+
(() => getContainerMemoryEvents()?.oomKill ?? null);
|
|
262
|
+
const ownIds = options.readOwnContainerIds ?? readOwnContainerIds;
|
|
195
263
|
const bootId = options.readBootId ?? readBootId;
|
|
196
264
|
const shareAnalytics = options.shareAnalytics ?? getRawShareAnalytics;
|
|
197
265
|
const cursorPath =
|
|
@@ -200,6 +268,17 @@ export function createOomKillReporter(
|
|
|
200
268
|
|
|
201
269
|
let firstScan = true;
|
|
202
270
|
let lastScanAt = 0;
|
|
271
|
+
let lastKillCounter: number | null = null;
|
|
272
|
+
/**
|
|
273
|
+
* Counter credit with no log line yet: the kernel prints the kill line
|
|
274
|
+
* before it increments the counter, so a kill between the log read and the
|
|
275
|
+
* counter read is counted but not yet visible. Credit is spent on the next
|
|
276
|
+
* scan and reported as unnamed if still unspent, so a kill that never logs
|
|
277
|
+
* a line cannot claim a neighbour's later.
|
|
278
|
+
*/
|
|
279
|
+
let owed = 0;
|
|
280
|
+
/** A scan was skipped mid-way; the next tick scans regardless of the sampler. */
|
|
281
|
+
let scanDue = false;
|
|
203
282
|
/** A report the telemetry store refused; retried before the next scan. */
|
|
204
283
|
let pending: {
|
|
205
284
|
report: OomKillReport;
|
|
@@ -269,29 +348,30 @@ export function createOomKillReporter(
|
|
|
269
348
|
pending = null;
|
|
270
349
|
}
|
|
271
350
|
|
|
272
|
-
const
|
|
273
|
-
const countersAvailable = sample.events != null;
|
|
351
|
+
const sampleDelta = sample.deltas?.events?.oomKill ?? 0;
|
|
274
352
|
if (
|
|
275
353
|
!firstScan &&
|
|
276
|
-
|
|
354
|
+
!scanDue &&
|
|
355
|
+
sampleDelta === 0 &&
|
|
277
356
|
now - lastScanAt < FALLBACK_SCAN_INTERVAL_MS
|
|
278
357
|
) {
|
|
279
358
|
return null;
|
|
280
359
|
}
|
|
281
|
-
|
|
282
|
-
firstScan = false;
|
|
283
|
-
lastScanAt = now;
|
|
360
|
+
scanDue = false;
|
|
284
361
|
|
|
285
362
|
const entries = await read();
|
|
286
363
|
const currentBootId = bootId();
|
|
287
364
|
if (entries == null) {
|
|
288
|
-
|
|
365
|
+
firstScan = false;
|
|
366
|
+
lastScanAt = now;
|
|
367
|
+
lastKillCounter = readKillCounter() ?? lastKillCounter;
|
|
368
|
+
if (sampleDelta === 0) {
|
|
289
369
|
return null;
|
|
290
370
|
}
|
|
291
371
|
const report: OomKillReport = {
|
|
292
372
|
attribution: "cgroup_counter",
|
|
293
373
|
victims: [],
|
|
294
|
-
unnamed:
|
|
374
|
+
unnamed: sampleDelta,
|
|
295
375
|
};
|
|
296
376
|
const cursor = { bootId: currentBootId, lastTime: 0 };
|
|
297
377
|
if (!queue(report, sample, daemonPid, cursor)) {
|
|
@@ -300,6 +380,23 @@ export function createOomKillReporter(
|
|
|
300
380
|
return report;
|
|
301
381
|
}
|
|
302
382
|
|
|
383
|
+
// Read after the log so the count covers every line the log holds.
|
|
384
|
+
const killCounter = readKillCounter();
|
|
385
|
+
if (killCounter == null && sample.events != null) {
|
|
386
|
+
// One failed read of a counter that exists: try again next tick
|
|
387
|
+
// rather than treating every kill on the kernel as ours.
|
|
388
|
+
scanDue = true;
|
|
389
|
+
return null;
|
|
390
|
+
}
|
|
391
|
+
const isFirstScan = firstScan;
|
|
392
|
+
firstScan = false;
|
|
393
|
+
lastScanAt = now;
|
|
394
|
+
const counterDelta =
|
|
395
|
+
killCounter != null && lastKillCounter != null
|
|
396
|
+
? Math.max(0, killCounter - lastKillCounter)
|
|
397
|
+
: null;
|
|
398
|
+
lastKillCounter = killCounter ?? lastKillCounter;
|
|
399
|
+
|
|
303
400
|
const all = oomKillsFromEntries(entries);
|
|
304
401
|
const maxTime = all.length > 0 ? all[all.length - 1].time : 0;
|
|
305
402
|
const stored = readCursor(cursorPath);
|
|
@@ -315,19 +412,46 @@ export function createOomKillReporter(
|
|
|
315
412
|
bootId: currentBootId,
|
|
316
413
|
lastTime: Math.max(sinceTime, maxTime),
|
|
317
414
|
};
|
|
415
|
+
const ids = ownIds();
|
|
318
416
|
|
|
319
417
|
let report: OomKillReport | null = null;
|
|
320
|
-
|
|
321
|
-
if (counterDelta > 0) {
|
|
322
|
-
report = {
|
|
323
|
-
attribution: "cgroup_counter",
|
|
324
|
-
victims: fresh.slice(-counterDelta),
|
|
325
|
-
unnamed: Math.max(0, counterDelta - fresh.length),
|
|
326
|
-
};
|
|
327
|
-
} else if ((isFirstScan || !countersAvailable) && fresh.length > 0) {
|
|
418
|
+
if (counterDelta == null || isFirstScan) {
|
|
328
419
|
// No counter to lean on: a kill that took the daemon down restarted
|
|
329
420
|
// this monitor with it, and the new container's counter starts at 0.
|
|
330
|
-
|
|
421
|
+
const ours = fresh.filter(
|
|
422
|
+
(kill) => kill.memcg == null || memcgIsOurs(kill.memcg, ids),
|
|
423
|
+
);
|
|
424
|
+
if (ours.length > 0) {
|
|
425
|
+
report = { attribution: "kernel_log", victims: ours, unnamed: 0 };
|
|
426
|
+
}
|
|
427
|
+
owed = 0;
|
|
428
|
+
} else {
|
|
429
|
+
const byHeader = fresh.filter(
|
|
430
|
+
(kill) => kill.memcg != null && memcgIsOurs(kill.memcg, ids),
|
|
431
|
+
);
|
|
432
|
+
const unknown = fresh.filter((kill) => kill.memcg == null);
|
|
433
|
+
// Header-attributed kills spend counter credit first; what is left
|
|
434
|
+
// names header-less kills oldest first. Header-less kills beyond the
|
|
435
|
+
// credit are neighbours', since our counter would have covered ours.
|
|
436
|
+
const carry = owed;
|
|
437
|
+
let credit = counterDelta + carry - byHeader.length;
|
|
438
|
+
const byCounter = unknown.slice(0, Math.max(0, credit));
|
|
439
|
+
credit = Math.max(0, credit - byCounter.length);
|
|
440
|
+
owed = Math.min(credit, counterDelta);
|
|
441
|
+
const unnamed = credit - owed;
|
|
442
|
+
const victims = [...byHeader, ...byCounter].sort(
|
|
443
|
+
(a, b) => a.time - b.time,
|
|
444
|
+
);
|
|
445
|
+
if (victims.length > 0 || unnamed > 0) {
|
|
446
|
+
report = {
|
|
447
|
+
attribution:
|
|
448
|
+
byCounter.length === 0 && unnamed === 0
|
|
449
|
+
? "cgroup"
|
|
450
|
+
: "cgroup_counter",
|
|
451
|
+
victims,
|
|
452
|
+
unnamed,
|
|
453
|
+
};
|
|
454
|
+
}
|
|
331
455
|
}
|
|
332
456
|
|
|
333
457
|
if (report == null) {
|