@bli-cockpit/cli 0.2.281 → 0.2.283
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/autostart-contract.js +10 -0
- package/dist/autostart-memory-guard.js +275 -0
- package/dist/autostart-self-heal.js +15 -1
- package/dist/autostart.js +9 -4
- package/dist/commands/autostart-command.js +15 -0
- package/dist/commands/autostart-heal.js +36 -1
- package/dist/commands/autostart-reading.js +17 -3
- package/dist/commands/local-args-tower-notes.js +25 -1
- package/dist/commands/local-help-notes.js +2 -1
- package/dist/commands/notes-review.js +204 -0
- package/dist/commands/notes.js +4 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sync-followups-memory-guard.js +13 -0
- package/dist/commands/sync.js +3 -0
- package/dist/memory-guard-contract.js +121 -0
- package/dist/memory-guard-forward.js +82 -0
- package/dist/memory-guard-script-darwin.js +128 -0
- package/dist/memory-guard-script-windows.js +144 -0
- package/package.json +2 -2
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `cockpit notes review [<note-id>] [--submit|--accept-all] [--json]` (BLI-22350): the attendee's meeting review in
|
|
3
|
+
* the terminal. Sibling of `notes.ts`, named in its header. Owner doc: `docs/runbooks/meeting-review-screen.md`.
|
|
4
|
+
*
|
|
5
|
+
* review the reviews waiting for me (GET /api/notes/review)
|
|
6
|
+
* review <id> one review, in the screen's order (GET /api/notes/review/<id>); --json is the raw read
|
|
7
|
+
* review <id> --submit `{ raw_hash, answers }` JSON from stdin (POST /api/notes/review/<id>)
|
|
8
|
+
* review <id> --accept-all read, then submit every item's default answer, and print what it accepted
|
|
9
|
+
*
|
|
10
|
+
* The same doors the web screen and the `notes_review_*` MCP tools call; the database decides who is a reviewer. This
|
|
11
|
+
* file holds no rule about answers: the door checks them and its refusal (reason and sentence) is relayed verbatim.
|
|
12
|
+
*/
|
|
13
|
+
import { isInteractiveStdin, readPipedText, writeLine } from "./cli-io.js";
|
|
14
|
+
import { ask, emit, errorText, fail, READ_DEADLINE_MS } from "./notes-door.js";
|
|
15
|
+
const WRITE_DEADLINE_MS = 60_000;
|
|
16
|
+
/** A review's answers are a few dozen short objects; this is far above any real one. */
|
|
17
|
+
const SUBMIT_MAX_CHARS = 512_000;
|
|
18
|
+
/** The dashboard's `review-words.ts` sentence, repeated here: this package does not import the dashboard. */
|
|
19
|
+
const AS_TRANSCRIBED_LINE = "This meeting is kept as transcribed; only the speakers need confirming.";
|
|
20
|
+
/** The web screen's plain label per meaning-check issue, repeated here for the same reason. */
|
|
21
|
+
const MEANING_ISSUE_WORDS = {
|
|
22
|
+
number: "A number may have changed",
|
|
23
|
+
negation_or_pronoun: "A 'not' or a 'we/you' may have flipped",
|
|
24
|
+
hedge_or_correction: "A 'maybe' or a correction may have been dropped",
|
|
25
|
+
added_words: "Words may have been added",
|
|
26
|
+
unclear_made_sensible: "An unclear bit may have been made to sound sensible",
|
|
27
|
+
speaker_moved: "A line may have been moved to another speaker",
|
|
28
|
+
passage_left_out: "A passage may have been left out",
|
|
29
|
+
other: "The meaning may have changed",
|
|
30
|
+
};
|
|
31
|
+
const STATE_WORDS = {
|
|
32
|
+
queued: "waiting for Tower to prepare it",
|
|
33
|
+
preparing: "being prepared (transcript cleaned and checked)",
|
|
34
|
+
awaiting_attendee: "waiting for your answers",
|
|
35
|
+
applying: "answered; Tower is applying the answers",
|
|
36
|
+
cleared: "cleared",
|
|
37
|
+
failed: "stuck; an engineer has been told",
|
|
38
|
+
};
|
|
39
|
+
const reviewPath = (noteId) => `/api/notes/review/${encodeURIComponent(noteId)}`;
|
|
40
|
+
export async function runNotesReview(command, door) {
|
|
41
|
+
if (!command.noteId)
|
|
42
|
+
return listReviews(door);
|
|
43
|
+
if (command.submit)
|
|
44
|
+
return submitFromStdin(command.noteId, door);
|
|
45
|
+
if (command.acceptAll)
|
|
46
|
+
return acceptAll(command.noteId, door);
|
|
47
|
+
return showReview(command.noteId, door);
|
|
48
|
+
}
|
|
49
|
+
async function listReviews(door) {
|
|
50
|
+
const answer = await ask(door, { path: "/api/notes/review", method: "GET", label: "notes review", timeoutMs: READ_DEADLINE_MS });
|
|
51
|
+
if (!answer.ok)
|
|
52
|
+
return fail(door, answer.reason, answer.detail);
|
|
53
|
+
const body = answer.body;
|
|
54
|
+
if (door.json)
|
|
55
|
+
return emit(door, { ok: true, ...body });
|
|
56
|
+
const out = door.io.stdout;
|
|
57
|
+
if (body.reviews.length === 0) {
|
|
58
|
+
writeLine(out, "No meeting reviews for you: nothing waits, and nothing was cleared in the last 7 days.");
|
|
59
|
+
return 0;
|
|
60
|
+
}
|
|
61
|
+
writeLine(out, `${body.waiting} meeting(s) wait for your review.`);
|
|
62
|
+
for (const row of body.reviews) {
|
|
63
|
+
writeLine(out, `${row.note_id} ${row.meeting_date} ${row.item_count} item(s) ${STATE_WORDS[row.state] ?? row.state} ${row.note_title ?? "(untitled meeting)"}`);
|
|
64
|
+
}
|
|
65
|
+
writeLine(out, `Open one: cockpit notes review <note-id>, or ${door.dashboardUrl}/meeting-notes/<note-id>/review in a browser.`);
|
|
66
|
+
return 0;
|
|
67
|
+
}
|
|
68
|
+
async function readOne(noteId, door) {
|
|
69
|
+
return ask(door, { path: reviewPath(noteId), method: "GET", label: "notes review", timeoutMs: READ_DEADLINE_MS });
|
|
70
|
+
}
|
|
71
|
+
function lines(value) {
|
|
72
|
+
if (value === null || value === undefined)
|
|
73
|
+
return [];
|
|
74
|
+
if (typeof value === "string")
|
|
75
|
+
return value.trim() ? [value.trim()] : [];
|
|
76
|
+
if (Array.isArray(value))
|
|
77
|
+
return value.flatMap((entry) => (typeof entry === "string" ? [entry] : [JSON.stringify(entry)]));
|
|
78
|
+
return [JSON.stringify(value)];
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* The review in the screen's order: who's who, lines under the wrong speaker, unclear words, meaning checks,
|
|
82
|
+
* sensitive moments.
|
|
83
|
+
*/
|
|
84
|
+
/** A meaning check's stretch is one or more transcript lines: the label on the first, the rest indented under it. */
|
|
85
|
+
function stretch(label, text) {
|
|
86
|
+
const [first = "", ...rest] = text.split(/\r?\n/);
|
|
87
|
+
const pad = " ".repeat(label.length + 5);
|
|
88
|
+
return [` ${label} ${first}`, ...rest.map((line) => `${pad}${line}`)];
|
|
89
|
+
}
|
|
90
|
+
export function renderReview(body, dashboardUrl) {
|
|
91
|
+
const out = [];
|
|
92
|
+
out.push(`${body.note_title ?? "Meeting"}${body.meeting_date ? `, ${body.meeting_date}` : ""}: ${STATE_WORDS[body.state] ?? body.state}.`);
|
|
93
|
+
const of = (kind) => body.items.filter((item) => item.kind === kind);
|
|
94
|
+
const speakers = of("speaker");
|
|
95
|
+
if (speakers.length)
|
|
96
|
+
out.push("", "Who's who (the name you confirm or correct):");
|
|
97
|
+
for (const item of speakers) {
|
|
98
|
+
out.push(` ${item.label}: ${item.guess || "(no guess; leave empty to keep the label)"} [${item.id}]`, ` why: ${item.reasoning}`);
|
|
99
|
+
for (const quote of item.quotes ?? [])
|
|
100
|
+
out.push(` ${quote.timestamp} "${quote.text}"`);
|
|
101
|
+
}
|
|
102
|
+
const moved = of("speaker_line");
|
|
103
|
+
if (moved.length)
|
|
104
|
+
out.push("", "Lines that may be under the wrong speaker (default: keep):");
|
|
105
|
+
for (const item of moved)
|
|
106
|
+
out.push(` ${item.timestamp} under ${item.guess || "(no speaker)"}: "${item.quote.text}" [${item.id}]`);
|
|
107
|
+
const unclear = of("unclear");
|
|
108
|
+
if (unclear.length)
|
|
109
|
+
out.push("", "Unclear words (default: the best guess):");
|
|
110
|
+
for (const item of unclear) {
|
|
111
|
+
out.push(` ${item.timestamp} "${item.words}" -> ${item.guess || "(keep marked unclear)"}${item.changesMeaning ? " CHANGES THE MEANING" : ""} [${item.id}]`, ` context: ${item.context}`);
|
|
112
|
+
}
|
|
113
|
+
const meaning = of("meaning_check");
|
|
114
|
+
if (meaning.length)
|
|
115
|
+
out.push("", "Did the clean-up change the meaning? (default: keep the words as said; answer \"raw\" or \"cleaned\"):");
|
|
116
|
+
for (const item of meaning) {
|
|
117
|
+
out.push(` ${item.timestamp} ${MEANING_ISSUE_WORDS[item.issue] ?? MEANING_ISSUE_WORDS.other} [${item.id}]`);
|
|
118
|
+
if (item.said || item.written)
|
|
119
|
+
out.push(` "${item.said}" -> "${item.written}"`);
|
|
120
|
+
out.push(...stretch("as said: ", item.raw), ...stretch("as cleaned up:", item.cleaned));
|
|
121
|
+
}
|
|
122
|
+
const sensitive = of("sensitive");
|
|
123
|
+
if (sensitive.length)
|
|
124
|
+
out.push("", "Sensitive moments (default: attendees only):");
|
|
125
|
+
for (const item of sensitive)
|
|
126
|
+
out.push(` ${item.timestamp} ${item.what} [${item.id}]`);
|
|
127
|
+
if (body.as_transcribed)
|
|
128
|
+
out.push("", AS_TRANSCRIBED_LINE);
|
|
129
|
+
else {
|
|
130
|
+
for (const [heading, value] of [["Summary", body.summary], ["Decisions", body.decisions], ["Action items", body.action_items]]) {
|
|
131
|
+
const said = lines(value);
|
|
132
|
+
if (said.length)
|
|
133
|
+
out.push("", `${heading}:`, ...said.map((line) => ` ${line}`));
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
if (body.state === "cleared" && body.attendee_notes)
|
|
137
|
+
out.push("", "Kept for the attendees only:", ...body.attendee_notes.split("\n").map((line) => ` ${line}`));
|
|
138
|
+
if (body.state === "awaiting_attendee") {
|
|
139
|
+
out.push("", `Answer: cockpit notes review ${body.note_id} --accept-all (every default above), or edit the defaults from`, `cockpit notes review ${body.note_id} --json and pipe { "raw_hash", "answers" } to cockpit notes review ${body.note_id} --submit.`, `Or open ${dashboardUrl}/meeting-notes/${body.note_id}/review.`);
|
|
140
|
+
}
|
|
141
|
+
return out;
|
|
142
|
+
}
|
|
143
|
+
async function showReview(noteId, door) {
|
|
144
|
+
const answer = await readOne(noteId, door);
|
|
145
|
+
if (!answer.ok)
|
|
146
|
+
return fail(door, answer.reason, answer.detail);
|
|
147
|
+
if (door.json)
|
|
148
|
+
return emit(door, answer.body);
|
|
149
|
+
for (const line of renderReview(answer.body, door.dashboardUrl))
|
|
150
|
+
writeLine(door.io.stdout, line);
|
|
151
|
+
return 0;
|
|
152
|
+
}
|
|
153
|
+
async function post(noteId, door, body, accepted) {
|
|
154
|
+
const answer = await ask(door, { path: reviewPath(noteId), method: "POST", label: "notes review submit", timeoutMs: WRITE_DEADLINE_MS, body });
|
|
155
|
+
if (!answer.ok)
|
|
156
|
+
return fail(door, answer.reason, answer.detail);
|
|
157
|
+
const done = answer.body;
|
|
158
|
+
if (door.json)
|
|
159
|
+
return emit(door, { ok: true, ...done, ...(accepted ? { accepted } : {}) });
|
|
160
|
+
writeLine(door.io.stdout, `Submitted ${done.answers} answer(s). Tower is applying them; the meeting is released to the team once its privacy check passes.`);
|
|
161
|
+
return 0;
|
|
162
|
+
}
|
|
163
|
+
async function submitFromStdin(noteId, door) {
|
|
164
|
+
if (isInteractiveStdin(door.io)) {
|
|
165
|
+
return fail(door, "nothing_piped", `Pipe { "raw_hash": ..., "answers": [...] } in: cockpit notes review ${noteId} --json, edit the defaults, then pipe them to cockpit notes review ${noteId} --submit.`);
|
|
166
|
+
}
|
|
167
|
+
let parsed;
|
|
168
|
+
try {
|
|
169
|
+
parsed = JSON.parse(await readPipedText(door.io.stdin, { maxChars: SUBMIT_MAX_CHARS, overflowMessage: "The answers piped in are far larger than any review." }));
|
|
170
|
+
}
|
|
171
|
+
catch (error) {
|
|
172
|
+
return fail(door, "invalid_body", `The answers piped in are not JSON (${errorText(error)}). Send { "raw_hash": ..., "answers": [...] }.`);
|
|
173
|
+
}
|
|
174
|
+
return post(noteId, door, parsed);
|
|
175
|
+
}
|
|
176
|
+
function acceptedWords(answer) {
|
|
177
|
+
const value = answer.answer;
|
|
178
|
+
if (answer.kind === "speaker")
|
|
179
|
+
return `speaker ${answer.guess || "(no guess)"} -> ${value === "" ? "keep the label" : String(value)}`;
|
|
180
|
+
if (answer.kind === "speaker_line")
|
|
181
|
+
return "line stays where it is";
|
|
182
|
+
if (answer.kind === "unclear") {
|
|
183
|
+
const reading = value.reading;
|
|
184
|
+
return reading === null ? "unclear word stays marked" : `unclear word -> ${reading}`;
|
|
185
|
+
}
|
|
186
|
+
if (answer.kind === "meaning_check")
|
|
187
|
+
return value === "cleaned" ? "meaning check -> keep the clean-up's words" : "meaning check -> keep the words as said";
|
|
188
|
+
return `sensitive moment -> ${String(value) === "keep_in" ? "kept in" : "attendees only"}`;
|
|
189
|
+
}
|
|
190
|
+
async function acceptAll(noteId, door) {
|
|
191
|
+
const read = await readOne(noteId, door);
|
|
192
|
+
if (!read.ok)
|
|
193
|
+
return fail(door, read.reason, read.detail);
|
|
194
|
+
const review = read.body;
|
|
195
|
+
if (review.state !== "awaiting_attendee") {
|
|
196
|
+
return fail(door, "review_not_awaiting_attendee", `This review is ${STATE_WORDS[review.state] ?? review.state}, so there is nothing to answer.`);
|
|
197
|
+
}
|
|
198
|
+
if (!door.json) {
|
|
199
|
+
writeLine(door.io.stdout, `Accepting ${review.defaults.length} default answer(s):`);
|
|
200
|
+
for (const answer of review.defaults)
|
|
201
|
+
writeLine(door.io.stdout, ` ${answer.item_id}: ${acceptedWords(answer)}`);
|
|
202
|
+
}
|
|
203
|
+
return post(noteId, door, { raw_hash: review.raw_hash, answers: review.defaults }, review.defaults);
|
|
204
|
+
}
|
package/dist/commands/notes.js
CHANGED
|
@@ -40,6 +40,8 @@
|
|
|
40
40
|
* Notetaker app on this Mac or Windows PC (Mac LaunchAgent: `notes-app-plist.ts`; Windows: `notes-app-windows.ts`).
|
|
41
41
|
* - `notes-transcription-stats.ts` — BLI-18003: what meeting transcription
|
|
42
42
|
* cost and did per provider for a month, and the spend against the cap.
|
|
43
|
+
* - `notes-review.ts` — BLI-22350: the meeting reviews waiting for me, one
|
|
44
|
+
* review, and answering it (`--submit` from stdin, `--accept-all`).
|
|
43
45
|
*
|
|
44
46
|
* Every public name is still importable from this file.
|
|
45
47
|
*/
|
|
@@ -52,6 +54,7 @@ import { armNotesWebhook, connectNotetaker, detachNotetaker, listNotesAccounts,
|
|
|
52
54
|
import { showBotUpcoming } from "./notes-bot-upcoming.js";
|
|
53
55
|
import { showGranolaStatus } from "./notes-granola-status.js";
|
|
54
56
|
import { showTranscriptionStats } from "./notes-transcription-stats.js";
|
|
57
|
+
import { runNotesReview } from "./notes-review.js";
|
|
55
58
|
import { listRecordings, showRecording } from "./notes-recordings.js";
|
|
56
59
|
import { defaultRecordDeps, runNotesRecord } from "./notes-record.js";
|
|
57
60
|
import { runNotesApp } from "./notes-app.js";
|
|
@@ -94,6 +97,7 @@ const NOTES_VERBS = {
|
|
|
94
97
|
"bot-upcoming": showBotUpcoming,
|
|
95
98
|
"granola-status": showGranolaStatus,
|
|
96
99
|
"transcription-stats": showTranscriptionStats,
|
|
100
|
+
review: runNotesReview,
|
|
97
101
|
};
|
|
98
102
|
export async function runNotes(command, io) {
|
|
99
103
|
// BLI-22979: recording is local until `stop` uploads, so only the upload needs the paired session.
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.283");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { reportInstallEventsBestEffort } from "./install-receipts.js";
|
|
2
|
+
import { forwardMemoryGuardReceipts } from "../memory-guard-forward.js";
|
|
3
|
+
export async function runMemoryGuardForwardAfterSync(command, io, dashboardUrl) {
|
|
4
|
+
try {
|
|
5
|
+
await forwardMemoryGuardReceipts({
|
|
6
|
+
homeDir: command.homeDir,
|
|
7
|
+
report: (events) => reportInstallEventsBestEffort({ homeDir: command.homeDir, dashboardUrl, command: "sync", events, json: command.json, io }),
|
|
8
|
+
});
|
|
9
|
+
}
|
|
10
|
+
catch (error) {
|
|
11
|
+
console.error("[memory-guard-forward] forwarding threw; collection is unaffected", JSON.stringify({ reason: "forward_threw", error_name: error instanceof Error ? error.name : typeof error }));
|
|
12
|
+
}
|
|
13
|
+
}
|
package/dist/commands/sync.js
CHANGED
|
@@ -41,6 +41,7 @@ import { envWithNodeRuntimeOnPath } from "../scheduled-self-update.js";
|
|
|
41
41
|
import { runMemoryExperienceAfterSync } from "./memory-log.js";
|
|
42
42
|
import { classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
|
|
43
43
|
import { runMachineHealAfterSync } from "./sync-heal.js";
|
|
44
|
+
import { runMemoryGuardForwardAfterSync } from "./sync-followups-memory-guard.js";
|
|
44
45
|
import { sendSyncHeartbeat } from "./sync-heartbeat.js";
|
|
45
46
|
import { runSyncWithHealthReceipt } from "./sync-receipt.js";
|
|
46
47
|
import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
|
|
@@ -211,6 +212,8 @@ async function reportTickThrew(command, io, dashboardUrl, error, errorCode) {
|
|
|
211
212
|
async function convergeAfterTick(command, io, dashboardUrl, facts) {
|
|
212
213
|
await runAutostartSelfHealAfterSync(command, io, dashboardUrl);
|
|
213
214
|
await runMemoryInstallAfterSync(command, io, dashboardUrl);
|
|
215
|
+
// BLI-23571: carry the resident memory guard's kills to the fleet.
|
|
216
|
+
await runMemoryGuardForwardAfterSync(command, io, dashboardUrl);
|
|
214
217
|
// BLI-12828: the repairs `cockpit doctor` also makes (agent rules, a lapsed
|
|
215
218
|
// sign-in), so nobody has to be told to run it.
|
|
216
219
|
await runMachineHealAfterSync(command, io, dashboardUrl);
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { getCollectorRuntimePaths } from "./local-state.js";
|
|
5
|
+
/**
|
|
6
|
+
* The resident memory guard's vocabulary (BLI-23571): limits, the log line, the process kinds.
|
|
7
|
+
*
|
|
8
|
+
* Why it exists: on 2026-10-28 a `node (vitest 2)` worker reached 166 GB of macOS footprint
|
|
9
|
+
* (compressed + swap, which `ps` RSS does not show) and froze a 16 GB Mac. `command-guard.ts` only caps
|
|
10
|
+
* the collector's own commands; nothing protected the machine from any OTHER runaway node (test workers,
|
|
11
|
+
* dev servers, agent tools). The guard is a SHELL script on purpose: it keeps running while node is
|
|
12
|
+
* wedged, costs about 2 MB, and spawns no node per tick.
|
|
13
|
+
*
|
|
14
|
+
* What it kills: only a process whose executable is `node`, never itself, never anything else.
|
|
15
|
+
*/
|
|
16
|
+
export const MEMORY_GUARD_LOG = "memory-guard.log";
|
|
17
|
+
export const MEMORY_GUARD_CURSOR = "memory-guard.cursor.json";
|
|
18
|
+
export const MEMORY_GUARD_DISABLED_MARKER = "memory-guard.disabled";
|
|
19
|
+
export const MEMORY_GUARD_LOG_LIMIT_BYTES = 1024 * 1024;
|
|
20
|
+
/** Lines kept when the log is trimmed past its cap. */
|
|
21
|
+
export const MEMORY_GUARD_LOG_KEEP_LINES = 200;
|
|
22
|
+
export const MEMORY_GUARD_ORPHAN_MIN_AGE_SECONDS = 300;
|
|
23
|
+
export const MEMORY_GUARD_STEP = "memory_guard";
|
|
24
|
+
export const MEMORY_GUARD_REASONS = [
|
|
25
|
+
"footprint_over_limit",
|
|
26
|
+
"rss_over_limit",
|
|
27
|
+
"pressure_warning",
|
|
28
|
+
"pressure_critical",
|
|
29
|
+
"orphan_vitest_worker",
|
|
30
|
+
];
|
|
31
|
+
export const MEMORY_GUARD_EVENTS = ["killed", "kill_failed", "orphan_reaped", "started"];
|
|
32
|
+
const MB = 1024 * 1024;
|
|
33
|
+
/**
|
|
34
|
+
* hard = min(6144 MB, 40% of RAM); warning tier = min(4096, 25%); critical tier = min(2048, 12.5%).
|
|
35
|
+
* `COCKPIT_MEMORY_GUARD_LIMIT_MB` replaces the hard limit and can only pull the other tiers down with it.
|
|
36
|
+
*/
|
|
37
|
+
export function resolveMemoryGuardLimits(totalMemoryBytes, env = {}) {
|
|
38
|
+
const totalMb = Math.max(1, Math.floor(totalMemoryBytes / MB));
|
|
39
|
+
const base = {
|
|
40
|
+
hard: Math.min(6144, Math.floor(totalMb * 0.4)),
|
|
41
|
+
warning: Math.min(4096, Math.floor(totalMb * 0.25)),
|
|
42
|
+
critical: Math.min(2048, Math.floor(totalMb * 0.125)),
|
|
43
|
+
};
|
|
44
|
+
const override = Number(env["COCKPIT_MEMORY_GUARD_LIMIT_MB"]);
|
|
45
|
+
if (!Number.isFinite(override) || override <= 0)
|
|
46
|
+
return base;
|
|
47
|
+
const hard = Math.floor(override);
|
|
48
|
+
return { hard, warning: Math.min(base.warning, hard), critical: Math.min(base.critical, hard) };
|
|
49
|
+
}
|
|
50
|
+
export function memoryGuardLimitsFor(options) {
|
|
51
|
+
return resolveMemoryGuardLimits(options.totalMemoryBytes ?? os.totalmem(), options.env ?? process.env);
|
|
52
|
+
}
|
|
53
|
+
/** `COCKPIT_MEMORY_GUARD=off` (or 0/false/no) at install time turns only the guard off. */
|
|
54
|
+
export function memoryGuardTurnedOff(env) {
|
|
55
|
+
return /^(off|0|false|no)$/iu.test((env?.["COCKPIT_MEMORY_GUARD"] ?? "").trim());
|
|
56
|
+
}
|
|
57
|
+
export function memoryGuardTurnedOn(env) {
|
|
58
|
+
return /^(on|1|true|yes)$/iu.test((env?.["COCKPIT_MEMORY_GUARD"] ?? "").trim());
|
|
59
|
+
}
|
|
60
|
+
export function memoryGuardStateFile(homeDir, name) {
|
|
61
|
+
return path.join(getCollectorRuntimePaths(homeDir ?? os.homedir()).state_dir, name);
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* The coarse label a log line carries instead of the command line. ONE table; the bash and PowerShell
|
|
65
|
+
* guards embed these exact patterns (POSIX ERE, JS and .NET read the same text), so a kind means the
|
|
66
|
+
* same thing on every host. Order matters: first match wins, "other" is the floor.
|
|
67
|
+
*/
|
|
68
|
+
export const MEMORY_GUARD_KIND_PATTERNS = [
|
|
69
|
+
["vitest", "vitest"],
|
|
70
|
+
["next", "next[/ \\\\](dev|build|start|dist|server)|next-server"],
|
|
71
|
+
["tsc", "(^|[/ \\\\])(tsc|tsserver)(\\.js)?( |$)|typescript[/\\\\]lib[/\\\\]tsc"],
|
|
72
|
+
["eslint", "eslint"],
|
|
73
|
+
["cockpit", "@bli-cockpit[/\\\\]cli|cockpit-cli[/\\\\]dist|[/\\\\]bin[/\\\\]cockpit( |$)"],
|
|
74
|
+
];
|
|
75
|
+
export function classifyProcessKind(commandLine) {
|
|
76
|
+
for (const [kind, pattern] of MEMORY_GUARD_KIND_PATTERNS)
|
|
77
|
+
if (new RegExp(pattern, "u").test(commandLine))
|
|
78
|
+
return kind;
|
|
79
|
+
return "other";
|
|
80
|
+
}
|
|
81
|
+
/** First 12 hex of the command line's sha256: enough to group identical runaways, useless to read back. */
|
|
82
|
+
export function commandSha12(commandLine) {
|
|
83
|
+
return crypto.createHash("sha256").update(commandLine).digest("hex").slice(0, 12);
|
|
84
|
+
}
|
|
85
|
+
export function parseMemoryGuardLogLine(raw) {
|
|
86
|
+
let value;
|
|
87
|
+
try {
|
|
88
|
+
value = JSON.parse(raw);
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
if (!value || typeof value !== "object")
|
|
94
|
+
return null;
|
|
95
|
+
const line = value;
|
|
96
|
+
const event = line["event"];
|
|
97
|
+
if (typeof line["ts"] !== "string" || !MEMORY_GUARD_EVENTS.includes(event))
|
|
98
|
+
return null;
|
|
99
|
+
const num = (key) => (typeof line[key] === "number" && Number.isFinite(line[key]) ? line[key] : 0);
|
|
100
|
+
const str = (key) => (typeof line[key] === "string" ? line[key].slice(0, 64) : "");
|
|
101
|
+
return {
|
|
102
|
+
ts: line["ts"],
|
|
103
|
+
event: event,
|
|
104
|
+
pid: num("pid"),
|
|
105
|
+
footprint_mb: num("footprint_mb"),
|
|
106
|
+
limit_mb: num("limit_mb"),
|
|
107
|
+
reason: str("reason"),
|
|
108
|
+
kind: str("kind"),
|
|
109
|
+
cmd_sha256: str("cmd_sha256"),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/** The last kill (any event but "started") in a log's text, newest wins. */
|
|
113
|
+
export function lastKillIn(logText) {
|
|
114
|
+
let found;
|
|
115
|
+
for (const raw of logText.split("\n")) {
|
|
116
|
+
const line = raw.trim() ? parseMemoryGuardLogLine(raw) : null;
|
|
117
|
+
if (line && line.event !== "started")
|
|
118
|
+
found = { ts: line.ts, kind: line.kind, reason: line.reason, footprint_mb: line.footprint_mb };
|
|
119
|
+
}
|
|
120
|
+
return found;
|
|
121
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { MEMORY_GUARD_CURSOR, MEMORY_GUARD_LOG, MEMORY_GUARD_STEP, memoryGuardStateFile, parseMemoryGuardLogLine, } from "./memory-guard-contract.js";
|
|
4
|
+
/**
|
|
5
|
+
* Turns the memory guard's kills into fleet receipts (BLI-23571).
|
|
6
|
+
*
|
|
7
|
+
* The guard is a shell loop with no network; the sync tick is the courier. Each tick reads the log lines
|
|
8
|
+
* added since a cursor and queues one install-event receipt per kill (`step: memory_guard`,
|
|
9
|
+
* `status: fail`, `error_code: <reason>`, `error_detail: <the JSON metadata>`), the same outbox
|
|
10
|
+
* `command-guard.ts` uses for its own ceiling trips. The cursor is a timestamp plus how many lines at that
|
|
11
|
+
* exact second were already sent, so the log can be trimmed without re-sending and two kills in one second
|
|
12
|
+
* are both counted. It is written LAST: a crash between queueing and the write re-sends, never loses.
|
|
13
|
+
*/
|
|
14
|
+
const TAG = "[memory-guard-forward]";
|
|
15
|
+
const MAX_LINES_PER_TICK = 40;
|
|
16
|
+
const FORWARDED_EVENTS = new Set(["killed", "kill_failed", "orphan_reaped"]);
|
|
17
|
+
function parseCursor(raw) {
|
|
18
|
+
if (!raw)
|
|
19
|
+
return null;
|
|
20
|
+
try {
|
|
21
|
+
const value = JSON.parse(raw);
|
|
22
|
+
return typeof value["ts"] === "string" && typeof value["ts_lines"] === "number" ? { ts: value["ts"], ts_lines: value["ts_lines"] } : null;
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/** The receipt for one log line. Metadata only: the line never held a command line to begin with. */
|
|
29
|
+
export function memoryGuardReceipt(line) {
|
|
30
|
+
return {
|
|
31
|
+
step: MEMORY_GUARD_STEP,
|
|
32
|
+
status: "fail",
|
|
33
|
+
error_code: line.reason || line.event,
|
|
34
|
+
error_detail: JSON.stringify({ ts: line.ts, event: line.event, pid: line.pid, footprint_mb: line.footprint_mb, limit_mb: line.limit_mb, reason: line.reason, kind: line.kind, cmd_sha256: line.cmd_sha256 }),
|
|
35
|
+
at: line.ts,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/** Reads new log lines, hands their receipts to `report`, then moves the cursor. Never throws. */
|
|
39
|
+
export async function forwardMemoryGuardReceipts(options) {
|
|
40
|
+
const logPath = memoryGuardStateFile(options.homeDir, MEMORY_GUARD_LOG);
|
|
41
|
+
const cursorPath = memoryGuardStateFile(options.homeDir, MEMORY_GUARD_CURSOR);
|
|
42
|
+
const text = await readFile(logPath, "utf8").catch(() => null);
|
|
43
|
+
if (text === null)
|
|
44
|
+
return { forwarded: 0, reason: "no_log" };
|
|
45
|
+
const lines = text.split("\n").flatMap((raw) => (raw.trim() ? [parseMemoryGuardLogLine(raw)] : [])).filter((line) => line !== null);
|
|
46
|
+
const cursor = parseCursor(await readFile(cursorPath, "utf8").catch(() => null));
|
|
47
|
+
let skippedAtCursor = 0;
|
|
48
|
+
const fresh = [];
|
|
49
|
+
lines.forEach((line, index) => {
|
|
50
|
+
if (cursor && line.ts < cursor.ts)
|
|
51
|
+
return;
|
|
52
|
+
if (cursor && line.ts === cursor.ts && skippedAtCursor < cursor.ts_lines) {
|
|
53
|
+
skippedAtCursor += 1;
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
fresh.push({ line, index });
|
|
57
|
+
});
|
|
58
|
+
if (fresh.length === 0)
|
|
59
|
+
return { forwarded: 0, reason: "nothing_new" };
|
|
60
|
+
const taken = fresh.slice(0, MAX_LINES_PER_TICK);
|
|
61
|
+
const events = taken.filter(({ line }) => FORWARDED_EVENTS.has(line.event)).map(({ line }) => memoryGuardReceipt(line));
|
|
62
|
+
try {
|
|
63
|
+
if (events.length > 0)
|
|
64
|
+
await options.report(events);
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
console.error(`${TAG} receipts not queued; the cursor stays and the next tick retries`, JSON.stringify({ reason: "report_threw", count: events.length, error_name: error instanceof Error ? error.name : typeof error }));
|
|
68
|
+
return { forwarded: 0, reason: "report_threw" };
|
|
69
|
+
}
|
|
70
|
+
const last = taken[taken.length - 1];
|
|
71
|
+
const next = { ts: last.line.ts, ts_lines: lines.filter((line, index) => line.ts === last.line.ts && index <= last.index).length };
|
|
72
|
+
try {
|
|
73
|
+
await mkdir(path.dirname(cursorPath), { recursive: true });
|
|
74
|
+
await writeFile(`${cursorPath}.tmp`, JSON.stringify(next), "utf8");
|
|
75
|
+
await rename(`${cursorPath}.tmp`, cursorPath);
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
console.error(`${TAG} cursor not written; the next tick re-sends these receipts`, JSON.stringify({ reason: "cursor_write_failed", error_name: error instanceof Error ? error.name : typeof error }));
|
|
79
|
+
}
|
|
80
|
+
console.error(`${TAG} forwarded`, JSON.stringify({ reason: "forwarded", receipts: events.length, lines: taken.length, remaining: fresh.length - taken.length }));
|
|
81
|
+
return { forwarded: events.length, reason: "forwarded" };
|
|
82
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { MEMORY_GUARD_KIND_PATTERNS, MEMORY_GUARD_LOG_KEEP_LINES, MEMORY_GUARD_LOG_LIMIT_BYTES, MEMORY_GUARD_ORPHAN_MIN_AGE_SECONDS, } from "./memory-guard-contract.js";
|
|
2
|
+
import { shellQuote } from "./autostart-darwin-plist.js";
|
|
3
|
+
/**
|
|
4
|
+
* The macOS memory guard (BLI-23571): a bash loop launchd keeps alive.
|
|
5
|
+
*
|
|
6
|
+
* Every 2 s it reads the kernel's memory-pressure level and a cheap `ps` RSS check. Every 10 s (every 2 s
|
|
7
|
+
* under critical pressure) it scans real memory FOOTPRINT with `top -o mem`, because RSS alone misses
|
|
8
|
+
* runaways: swapped and compressed pages leave RSS, and an 800 MB buffer showed 12 MB RSS. The limit
|
|
9
|
+
* tightens with pressure. Every 60 s it reaps vitest workers orphaned by a dead runner (ppid 1, older than
|
|
10
|
+
* 5 minutes). Only a process whose executable is `node` is ever touched; the guard never kills itself.
|
|
11
|
+
*
|
|
12
|
+
* It logs one JSON line per decision to the state dir: ids, sizes and a sha256 prefix, never a command
|
|
13
|
+
* line (those carry tokens). Test seams, all env: `MEMGUARD_SOURCE_ONLY=1` defines the functions and
|
|
14
|
+
* returns, `MEMGUARD_ONCE=1` runs one tick and exits, `MEMGUARD_NO_NOTIFY=1` skips the notification,
|
|
15
|
+
* `MEMGUARD_ONLY_PID=<pid>` restricts kills to one process so a test cannot shoot its own runner.
|
|
16
|
+
*/
|
|
17
|
+
export function renderDarwinMemoryGuardScript(options) {
|
|
18
|
+
const kindChecks = MEMORY_GUARD_KIND_PATTERNS.map(([kind, pattern]) => ` re=${shellQuote(pattern)}; [[ "$1" =~ $re ]] && { echo ${kind}; return; }`);
|
|
19
|
+
return [
|
|
20
|
+
"#!/bin/bash",
|
|
21
|
+
"# BLI Cockpit memory guard (BLI-23571). Rendered by `cockpit autostart install`; do not edit, it is rewritten.",
|
|
22
|
+
"# Kills a runaway node process by memory footprint. Log: memory-guard.log in the collector state dir.",
|
|
23
|
+
`LIMIT_MB=${options.limits.hard}`,
|
|
24
|
+
`WARN_LIMIT_MB=${options.limits.warning}`,
|
|
25
|
+
`CRITICAL_LIMIT_MB=${options.limits.critical}`,
|
|
26
|
+
`ORPHAN_MIN_AGE_S=${MEMORY_GUARD_ORPHAN_MIN_AGE_SECONDS}`,
|
|
27
|
+
`LOG=${shellQuote(options.logPath)}`,
|
|
28
|
+
`LOG_LIMIT_BYTES=${MEMORY_GUARD_LOG_LIMIT_BYTES}`,
|
|
29
|
+
`LOG_KEEP_LINES=${MEMORY_GUARD_LOG_KEEP_LINES}`,
|
|
30
|
+
'mkdir -p "$(dirname "$LOG")"',
|
|
31
|
+
"",
|
|
32
|
+
"now_iso() { date -u +%Y-%m-%dT%H:%M:%SZ; }",
|
|
33
|
+
"",
|
|
34
|
+
"# event pid footprint_mb limit_mb reason kind cmd_sha256. Every value is a number or a fixed token.",
|
|
35
|
+
"log_json() {",
|
|
36
|
+
` printf '{"ts":"%s","event":"%s","pid":%s,"footprint_mb":%s,"limit_mb":%s,"reason":"%s","kind":"%s","cmd_sha256":"%s"}\\n' \\`,
|
|
37
|
+
' "$(now_iso)" "$1" "$2" "$3" "$4" "$5" "$6" "$7" >> "$LOG"',
|
|
38
|
+
"}",
|
|
39
|
+
"",
|
|
40
|
+
"notify() { [ \"${MEMGUARD_NO_NOTIFY:-}\" = 1 ] || osascript -e \"display notification \\\"$1\\\" with title \\\"Tower\\\"\" >/dev/null 2>&1 & }",
|
|
41
|
+
"",
|
|
42
|
+
"# The coarse label for a command line. The same table as the Windows guard and the CLI's tests.",
|
|
43
|
+
"classify_kind() {",
|
|
44
|
+
" local re",
|
|
45
|
+
...kindChecks,
|
|
46
|
+
" echo other",
|
|
47
|
+
"}",
|
|
48
|
+
"",
|
|
49
|
+
"cmd_sha() { printf '%s' \"$1\" | shasum -a 256 | cut -c1-12; }",
|
|
50
|
+
"",
|
|
51
|
+
"to_mb() { # top MEM like 470M, 69G, 512K, 1024B, with an optional +/- suffix",
|
|
52
|
+
' local v="${1%[+-]}" n u',
|
|
53
|
+
' n="${v%[BKMGT]}"; u="${v: -1}"',
|
|
54
|
+
' case "$u" in B) echo $((n / 1048576)) ;; K) echo $((n / 1024)) ;; M) echo "$n" ;; G) echo $((n * 1024)) ;; T) echo $((n * 1048576)) ;; *) echo 0 ;; esac',
|
|
55
|
+
"}",
|
|
56
|
+
"",
|
|
57
|
+
"kill_node() { # pid footprint_mb limit_mb reason",
|
|
58
|
+
' local cmd kind sha result',
|
|
59
|
+
' [ "$1" = "$$" ] && return',
|
|
60
|
+
' [ -n "${MEMGUARD_ONLY_PID:-}" ] && [ "$1" != "$MEMGUARD_ONLY_PID" ] && return',
|
|
61
|
+
' cmd="$(ps -o command= -p "$1" 2>/dev/null | cut -c1-2000)"',
|
|
62
|
+
' kind="$(classify_kind "$cmd")"; sha="$(cmd_sha "$cmd")"',
|
|
63
|
+
" kill -9 \"$1\" 2>/dev/null && result=killed || result=kill_failed",
|
|
64
|
+
' log_json "$result" "$1" "$2" "$3" "$4" "$kind" "$sha"',
|
|
65
|
+
' if [ "$result" = killed ]; then',
|
|
66
|
+
' if [ "$2" -ge 1024 ]; then notify "Tower killed a runaway node process ($(( ($2 + 512) / 1024 )) GB, $kind)"; else notify "Tower killed a runaway node process ($2 MB, $kind)"; fi',
|
|
67
|
+
" fi",
|
|
68
|
+
"}",
|
|
69
|
+
"",
|
|
70
|
+
"# 1 normal, 2 warning, 4 critical",
|
|
71
|
+
"pressure_level() { sysctl -n kern.memorystatus_vm_pressure_level 2>/dev/null || echo 1; }",
|
|
72
|
+
"",
|
|
73
|
+
"footprint_scan() { # limit_mb reason",
|
|
74
|
+
" top -l 1 -o mem -n 25 -stats pid,mem,command 2>/dev/null | awk 'NR>1 && $1 ~ /^[0-9]+$/' |",
|
|
75
|
+
" while read -r pid mem comm _; do",
|
|
76
|
+
' case "$comm" in node) ;; *) continue ;; esac',
|
|
77
|
+
' mb="$(to_mb "$mem")"',
|
|
78
|
+
' [ "$mb" -gt "$1" ] 2>/dev/null && kill_node "$pid" "$mb" "$1" "$2"',
|
|
79
|
+
" done",
|
|
80
|
+
"}",
|
|
81
|
+
"",
|
|
82
|
+
"rss_scan() { # limit_mb; the cheap every-tick check on resident memory alone",
|
|
83
|
+
` ps -axo pid=,rss=,comm= | awk -v lim="$(($1 * 1024))" '$3 ~ /(^|\\/)node$/ && $2 > lim {print $1, int($2/1024)}' |`,
|
|
84
|
+
' while read -r pid mb; do kill_node "$pid" "$mb" "$1" rss_over_limit; done',
|
|
85
|
+
"}",
|
|
86
|
+
"",
|
|
87
|
+
"orphan_vitest_scan() { # vitest workers whose runner died (ppid 1) and that outlived the grace period",
|
|
88
|
+
" ps -axo pid=,ppid=,etime=,command= | awk '$2 == 1 && $4 ~ /(^|\\/)node$/ && $0 ~ /vitest\\/dist\\/workers\\/forks|vitest\\/dist\\/workers\\/threads|node \\(vitest/ {print $1, $3}' |",
|
|
89
|
+
" while read -r pid etime; do",
|
|
90
|
+
` secs="$(echo "$etime" | awk -F'[-:]' '{n=NF; s=$n+($(n-1))*60; if(n>=3)s+=$(n-2)*3600; if(n==4)s+=$1*86400; print s}')"`,
|
|
91
|
+
' [ "$secs" -ge "$ORPHAN_MIN_AGE_S" ] 2>/dev/null || continue',
|
|
92
|
+
' [ -n "${MEMGUARD_ONLY_PID:-}" ] && [ "$pid" != "$MEMGUARD_ONLY_PID" ] && continue',
|
|
93
|
+
' cmd="$(ps -o command= -p "$pid" 2>/dev/null | cut -c1-2000)"',
|
|
94
|
+
' kill -9 "$pid" 2>/dev/null && log_json orphan_reaped "$pid" 0 0 orphan_vitest_worker "$(classify_kind "$cmd")" "$(cmd_sha "$cmd")"',
|
|
95
|
+
" done",
|
|
96
|
+
"}",
|
|
97
|
+
"",
|
|
98
|
+
"trim_log() { # keep the log small: past the cap, keep the newest lines",
|
|
99
|
+
' local size; size="$(wc -c < "$LOG" 2>/dev/null | tr -d \' \')"',
|
|
100
|
+
' [ "${size:-0}" -gt "$LOG_LIMIT_BYTES" ] 2>/dev/null || return 0',
|
|
101
|
+
' tail -n "$LOG_KEEP_LINES" "$LOG" > "$LOG.tmp" 2>/dev/null && mv "$LOG.tmp" "$LOG"',
|
|
102
|
+
"}",
|
|
103
|
+
"",
|
|
104
|
+
'[ "${MEMGUARD_SOURCE_ONLY:-}" = 1 ] && return 0 2>/dev/null',
|
|
105
|
+
"",
|
|
106
|
+
'log_json started "$$" 0 "$LIMIT_MB" started guard ""',
|
|
107
|
+
"tick=0",
|
|
108
|
+
"while :; do",
|
|
109
|
+
' level="$(pressure_level)"',
|
|
110
|
+
' if [ "$level" -ge 4 ] 2>/dev/null; then',
|
|
111
|
+
' rss_scan "$CRITICAL_LIMIT_MB"',
|
|
112
|
+
' footprint_scan "$CRITICAL_LIMIT_MB" pressure_critical',
|
|
113
|
+
' elif [ "$level" -ge 2 ] 2>/dev/null; then',
|
|
114
|
+
' rss_scan "$WARN_LIMIT_MB"',
|
|
115
|
+
' [ $((tick % 5)) -eq 0 ] && footprint_scan "$WARN_LIMIT_MB" pressure_warning',
|
|
116
|
+
" else",
|
|
117
|
+
' rss_scan "$LIMIT_MB"',
|
|
118
|
+
' [ $((tick % 5)) -eq 0 ] && footprint_scan "$LIMIT_MB" footprint_over_limit',
|
|
119
|
+
" fi",
|
|
120
|
+
' [ $((tick % 30)) -eq 0 ] && orphan_vitest_scan',
|
|
121
|
+
' [ $((tick % 30)) -eq 15 ] && trim_log',
|
|
122
|
+
' [ "${MEMGUARD_ONCE:-}" = 1 ] && exit 0',
|
|
123
|
+
" tick=$((tick + 1))",
|
|
124
|
+
" sleep 2",
|
|
125
|
+
"done",
|
|
126
|
+
"",
|
|
127
|
+
].join("\n");
|
|
128
|
+
}
|