@teamscala/service-boot 0.23.369 → 0.23.373
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -17
- package/src/handlers/platform-fault-escalation.test.ts +15 -15
- package/src/handlers/platform-fault-escalation.ts +7 -8
- package/src/pg-boss-mount.ts +3 -2
- package/src/handlers/lane-attention-sweep.ts +0 -81
- package/src/handlers/sweep-host-cache.test.ts +0 -133
- package/src/handlers/sweep-host-cache.ts +0 -328
package/package.json
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
},
|
|
7
7
|
"name": "@teamscala/service-boot",
|
|
8
8
|
"type": "module",
|
|
9
|
-
"version": "0.23.
|
|
9
|
+
"version": "0.23.373",
|
|
10
10
|
"sideEffects": false,
|
|
11
11
|
"imports": {
|
|
12
12
|
"#service-boot/*": "./src/*",
|
|
@@ -410,14 +410,6 @@
|
|
|
410
410
|
"types": "./src/handlers/bun-stable-watch.d.ts",
|
|
411
411
|
"default": "./src/handlers/bun-stable-watch.ts"
|
|
412
412
|
},
|
|
413
|
-
"./handlers/lane-attention-sweep": {
|
|
414
|
-
"types": "./src/handlers/lane-attention-sweep.d.ts",
|
|
415
|
-
"default": "./src/handlers/lane-attention-sweep.ts"
|
|
416
|
-
},
|
|
417
|
-
"./handlers/lane-attention-sweep.ts": {
|
|
418
|
-
"types": "./src/handlers/lane-attention-sweep.d.ts",
|
|
419
|
-
"default": "./src/handlers/lane-attention-sweep.ts"
|
|
420
|
-
},
|
|
421
413
|
"./handlers/pane-outbox-drain": {
|
|
422
414
|
"types": "./src/handlers/pane-outbox-drain.d.ts",
|
|
423
415
|
"default": "./src/handlers/pane-outbox-drain.ts"
|
|
@@ -490,14 +482,6 @@
|
|
|
490
482
|
"types": "./src/handlers/sweep-event-log.d.ts",
|
|
491
483
|
"default": "./src/handlers/sweep-event-log.ts"
|
|
492
484
|
},
|
|
493
|
-
"./handlers/sweep-host-cache": {
|
|
494
|
-
"types": "./src/handlers/sweep-host-cache.d.ts",
|
|
495
|
-
"default": "./src/handlers/sweep-host-cache.ts"
|
|
496
|
-
},
|
|
497
|
-
"./handlers/sweep-host-cache.ts": {
|
|
498
|
-
"types": "./src/handlers/sweep-host-cache.d.ts",
|
|
499
|
-
"default": "./src/handlers/sweep-host-cache.ts"
|
|
500
|
-
},
|
|
501
485
|
"./handlers/upstream-issue-sync": {
|
|
502
486
|
"types": "./src/handlers/upstream-issue-sync.d.ts",
|
|
503
487
|
"default": "./src/handlers/upstream-issue-sync.ts"
|
|
@@ -65,10 +65,10 @@ describe("decideSweep — thresholds", () => {
|
|
|
65
65
|
});
|
|
66
66
|
|
|
67
67
|
test("the 490-over-12h shape escalates (duration + recurrence)", () => {
|
|
68
|
-
const sustained = agg("host-command-bus.
|
|
68
|
+
const sustained = agg("host-command-bus.command.failed", 490, 720, 0);
|
|
69
69
|
const decision = decideSweep([sustained], [], POLICY, NOW);
|
|
70
70
|
expect(decision.escalate).toHaveLength(1);
|
|
71
|
-
expect(decision.escalate[0]!.aggregate.kind).toBe("host-command-bus.
|
|
71
|
+
expect(decision.escalate[0]!.aggregate.kind).toBe("host-command-bus.command.failed");
|
|
72
72
|
expect(decision.escalate[0]!.episode).toBeNull();
|
|
73
73
|
});
|
|
74
74
|
|
|
@@ -85,9 +85,9 @@ describe("decideSweep — thresholds", () => {
|
|
|
85
85
|
});
|
|
86
86
|
|
|
87
87
|
test("override raises a floor too — bursty kinds can be tightened", () => {
|
|
88
|
-
const sustained = agg("llm.provider.
|
|
88
|
+
const sustained = agg("llm.provider.call.failed", 40, 60, 0); // passes default
|
|
89
89
|
const decision = decideSweep([sustained], [], POLICY, NOW, {
|
|
90
|
-
"llm.provider.
|
|
90
|
+
"llm.provider.call.failed": { minCount: 100 },
|
|
91
91
|
});
|
|
92
92
|
expect(decision.escalate).toHaveLength(0);
|
|
93
93
|
});
|
|
@@ -95,16 +95,16 @@ describe("decideSweep — thresholds", () => {
|
|
|
95
95
|
|
|
96
96
|
describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
|
|
97
97
|
test("already escalated and unresolved does NOT re-escalate", () => {
|
|
98
|
-
const sustained = agg("host-command-bus.
|
|
99
|
-
const open = episode({ kind: "host-command-bus.
|
|
98
|
+
const sustained = agg("host-command-bus.command.failed", 500, 720, 0);
|
|
99
|
+
const open = episode({ kind: "host-command-bus.command.failed", escalatedAt: new Date(NOW.getTime() - 3_600_000) });
|
|
100
100
|
const decision = decideSweep([sustained], [open], POLICY, NOW);
|
|
101
101
|
expect(decision.escalate).toHaveLength(0);
|
|
102
102
|
});
|
|
103
103
|
|
|
104
104
|
test("open delivered episode past the update window gets an UPDATE, not a message", () => {
|
|
105
|
-
const sustained = agg("llm.provider.
|
|
105
|
+
const sustained = agg("llm.provider.call.failed", 300, 720, 0);
|
|
106
106
|
const open = episode({
|
|
107
|
-
kind: "llm.provider.
|
|
107
|
+
kind: "llm.provider.call.failed",
|
|
108
108
|
escalatedAt: new Date(NOW.getTime() - 90 * 60_000), // last update long ago
|
|
109
109
|
lastUpdatedAt: new Date(NOW.getTime() - 90 * 60_000),
|
|
110
110
|
});
|
|
@@ -115,9 +115,9 @@ describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
|
|
|
115
115
|
});
|
|
116
116
|
|
|
117
117
|
test("open episode inside the update window gets NOTHING (dedup window)", () => {
|
|
118
|
-
const sustained = agg("llm.provider.
|
|
118
|
+
const sustained = agg("llm.provider.call.failed", 300, 720, 0);
|
|
119
119
|
const open = episode({
|
|
120
|
-
kind: "llm.provider.
|
|
120
|
+
kind: "llm.provider.call.failed",
|
|
121
121
|
escalatedAt: new Date(NOW.getTime() - 10 * 60_000),
|
|
122
122
|
lastUpdatedAt: new Date(NOW.getTime() - 10 * 60_000),
|
|
123
123
|
});
|
|
@@ -150,9 +150,9 @@ describe("decideSweep — resolution + re-arm", () => {
|
|
|
150
150
|
});
|
|
151
151
|
|
|
152
152
|
test("after resolution the same fault escalates again (new episode)", () => {
|
|
153
|
-
const recurring = agg("host-command-bus.
|
|
153
|
+
const recurring = agg("host-command-bus.command.failed", 50, 40, 0);
|
|
154
154
|
const resolved = episode({
|
|
155
|
-
kind: "host-command-bus.
|
|
155
|
+
kind: "host-command-bus.command.failed",
|
|
156
156
|
resolvedAt: new Date(NOW.getTime() - 35 * 60_000),
|
|
157
157
|
});
|
|
158
158
|
const decision = decideSweep([recurring], [resolved], POLICY, NOW);
|
|
@@ -190,12 +190,12 @@ describe("extractGowaMessageId — delivery verification", () => {
|
|
|
190
190
|
describe("composeEscalationMessage", () => {
|
|
191
191
|
test("carries the three escalation-routing fields and stays phone-sized", () => {
|
|
192
192
|
const message = composeEscalationMessage({
|
|
193
|
-
kind: "host-command-bus.
|
|
194
|
-
aggregate: agg("host-command-bus.
|
|
193
|
+
kind: "host-command-bus.command.failed",
|
|
194
|
+
aggregate: agg("host-command-bus.command.failed", 490, 720, 0),
|
|
195
195
|
impact: "scala-tools verbs are failing host-wide",
|
|
196
196
|
recommendation: "check oxmgr logs for the bus consumer",
|
|
197
197
|
});
|
|
198
|
-
expect(message).toContain("Platform fault: host-command-bus.
|
|
198
|
+
expect(message).toContain("Platform fault: host-command-bus.command.failed");
|
|
199
199
|
expect(message).toContain("Observed: 490 errors over 12 h");
|
|
200
200
|
expect(message).toContain("Impact: scala-tools verbs are failing host-wide");
|
|
201
201
|
expect(message).toContain("Recommendation: check oxmgr logs for the bus consumer");
|
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
*
|
|
6
6
|
* The `platform-fault-escalation` pg-boss handler — closes the "receiving is
|
|
7
7
|
* decoupled from consequence" gap measured 2026-08-15 (760 error events over
|
|
8
|
-
* 12 hours, zero escalations: host-command-bus.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* 12 hours, zero escalations: host-command-bus.command.failed (then spelled
|
|
9
|
+
* host-command-bus.error) 490, llm.provider.call.failed (then spelled
|
|
10
|
+
* llm.provider.error) 231, guard.fail.open 16, scale-to-zero.wake_failed 12,
|
|
11
|
+
* listen.connect_failed 11). Emission works and observability-is-passive is satisfied; nothing
|
|
11
12
|
* acted. This sweep makes RECEIVING have CONSEQUENCE: sustained platform
|
|
12
13
|
* faults escalate to the OPERATOR over the existing PA path (WhatsApp, the
|
|
13
14
|
* scala PA operator-relay bot — escalation-routing's channel; no second
|
|
@@ -38,8 +39,7 @@
|
|
|
38
39
|
* not a state transition any single event can announce. The sweep reads one
|
|
39
40
|
* server-side aggregate (GROUP BY kind) per run against indexed event_log;
|
|
40
41
|
* while the platform is healthy it returns zero rows and costs nothing
|
|
41
|
-
* beyond the tick (idle-costs-nothing).
|
|
42
|
-
* lane-attention-sweep carries on its row as `poll_justification`.
|
|
42
|
+
* beyond the tick (idle-costs-nothing).
|
|
43
43
|
*
|
|
44
44
|
* Registered under slug `@teamscala/service-boot:platform-fault-escalation`.
|
|
45
45
|
*/
|
|
@@ -130,9 +130,8 @@ async function getSystemIdentity(): Promise<{ userId: string | undefined; organi
|
|
|
130
130
|
* TypeError this replaced). Per `no-direct-prisma-outside-orpc`, every WRITE
|
|
131
131
|
* in this handler routes through this client; the single raw read (the
|
|
132
132
|
* event_log recurrence aggregate, a GROUP BY no generated CRUD expresses) uses
|
|
133
|
-
* the assigned-client form with $queryRawUnsafe
|
|
134
|
-
*
|
|
135
|
-
* instances (the generated valibot schemas require it).
|
|
133
|
+
* the assigned-client form with $queryRawUnsafe. Timestamps are ISO strings,
|
|
134
|
+
* never Date instances (the generated valibot schemas require it).
|
|
136
135
|
*/
|
|
137
136
|
let cachedOrpc: OrpcSurface | null = null;
|
|
138
137
|
async function orpc(): Promise<OrpcSurface> {
|
package/src/pg-boss-mount.ts
CHANGED
|
@@ -26,7 +26,9 @@ import { pgBossHandlerRegistry } from "./pg-boss-handler-registry.ts";
|
|
|
26
26
|
import "./handlers/prune-stale-data.ts";
|
|
27
27
|
import "./handlers/projects-digest.ts";
|
|
28
28
|
import "./handlers/sweep-event-log.ts";
|
|
29
|
-
|
|
29
|
+
// `sweep-host-cache` is Rust-native: loga (the job's service_slug) registers it
|
|
30
|
+
// via scala_os::host_cache_retention with the config/cleanup-allowed-roots
|
|
31
|
+
// bound. No TypeScript copy may exist beside it.
|
|
30
32
|
import "./handlers/bun-stable-watch.ts";
|
|
31
33
|
import "./handlers/whatsapp-liveness.ts";
|
|
32
34
|
import "./handlers/upstream-issue-sync.ts";
|
|
@@ -34,7 +36,6 @@ import "./handlers/scala-tools-currency-watch.ts";
|
|
|
34
36
|
import "./handlers/singleton-tools-currency-watch.ts";
|
|
35
37
|
import "./handlers/registry-stale-inactive-sweep.ts";
|
|
36
38
|
import "./handlers/pane-outbox-drain.ts";
|
|
37
|
-
import "./handlers/lane-attention-sweep.ts";
|
|
38
39
|
import "./handlers/platform-fault-escalation.ts";
|
|
39
40
|
|
|
40
41
|
/**
|
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* The `lane-attention-sweep` pg-boss handler. Calls the database function that
|
|
7
|
-
* reminds an orchestrator about lanes waiting on it, and records the outcome.
|
|
8
|
-
*
|
|
9
|
-
* This handler holds NO POLICY. Which lanes are waiting, how long counts as
|
|
10
|
-
* waiting, who is told, what the message says, and how often it may repeat are
|
|
11
|
-
* all in `lane_needs_attention` and `sweep_lanes_needing_attention` — so
|
|
12
|
-
* changing any of them is a migration, not a rebuild and redeploy of this
|
|
13
|
-
* service. All this file supplies is the tick and the telemetry.
|
|
14
|
-
*
|
|
15
|
-
* WHY A TICK EXISTS AT ALL, given `observability-is-passive` and
|
|
16
|
-
* `no-polling-for-pushable-state`. Both forbid polling for state a source CAN
|
|
17
|
-
* push. Here the trigger is the PASSAGE OF TIME against a threshold: a lane
|
|
18
|
-
* going silent emits nothing, because emitting is exactly what it stopped
|
|
19
|
-
* doing. There is no state transition to react to, so nothing could push it.
|
|
20
|
-
* The same justification `registry-stale-inactive-sweep` carries, recorded on
|
|
21
|
-
* the row as `poll_justification`.
|
|
22
|
-
*
|
|
23
|
-
* Registered under slug `@teamscala/service-boot:lane-attention-sweep`.
|
|
24
|
-
*/
|
|
25
|
-
import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
|
|
26
|
-
import { getPrisma } from "@teamscala/db/prisma-registry";
|
|
27
|
-
import { recordEvent } from "@teamscala/event-log/record-event";
|
|
28
|
-
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
29
|
-
import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
|
|
30
|
-
|
|
31
|
-
interface RemindedLane {
|
|
32
|
-
task_id: string;
|
|
33
|
-
window_name: string | null;
|
|
34
|
-
reason: string;
|
|
35
|
-
orchestrator_project_id: string;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
async function laneAttentionSweep(ctx: { config: ScheduledJobConfig }): Promise<void> {
|
|
39
|
-
const log = getAppLogger();
|
|
40
|
-
const prisma = getPrisma();
|
|
41
|
-
|
|
42
|
-
// The repeat window is the row's, so silencing a chatty reminder is a
|
|
43
|
-
// registry edit. The default lives in the SQL function's signature, which is
|
|
44
|
-
// the one place it is defined.
|
|
45
|
-
const repeatAfter = (ctx.config as { repeat_after?: string }).repeat_after;
|
|
46
|
-
|
|
47
|
-
const reminded = repeatAfter
|
|
48
|
-
? await prisma.$queryRawUnsafe<RemindedLane[]>(
|
|
49
|
-
"SELECT * FROM sweep_lanes_needing_attention($1::interval)",
|
|
50
|
-
repeatAfter,
|
|
51
|
-
)
|
|
52
|
-
: await prisma.$queryRawUnsafe<RemindedLane[]>("SELECT * FROM sweep_lanes_needing_attention()");
|
|
53
|
-
|
|
54
|
-
// Emitted even when nothing was due. A sweep that only reports on the runs
|
|
55
|
-
// where it found something is indistinguishable from a sweep that stopped
|
|
56
|
-
// running, and this exists precisely to notice silence.
|
|
57
|
-
recordEvent({
|
|
58
|
-
kind: "fleet.lane-attention.swept",
|
|
59
|
-
payload: {
|
|
60
|
-
remindedCount: reminded.length,
|
|
61
|
-
lanes: reminded.map((lane) => ({
|
|
62
|
-
task: lane.task_id,
|
|
63
|
-
window: lane.window_name,
|
|
64
|
-
reason: lane.reason,
|
|
65
|
-
toldOrchestrator: lane.orchestrator_project_id,
|
|
66
|
-
})),
|
|
67
|
-
},
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
if (reminded.length > 0) {
|
|
71
|
-
log.info(
|
|
72
|
-
`[lane-attention-sweep] reminded ${reminded.length} orchestrator(s) about waiting lanes`,
|
|
73
|
-
{ lanes: reminded.map((lane) => `${lane.window_name}: ${lane.reason}`) },
|
|
74
|
-
);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
pgBossHandlerRegistry.register(
|
|
79
|
-
"@teamscala/service-boot:lane-attention-sweep",
|
|
80
|
-
laneAttentionSweep as (ctx: { config: ScheduledJobConfig }) => Promise<void>,
|
|
81
|
-
);
|
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* Unit tests for the `retention_dirs` helpers (`expandGlob`, `checkDirStale`) —
|
|
7
|
-
* the whole-directory prune that bounds Rust `target/` waste. The file-by-mtime
|
|
8
|
-
* `retention` path is unchanged; these cover ONLY the new build-cache model:
|
|
9
|
-
* (1) glob expansion reaches every repo's target dir; (2) an actively-built dir
|
|
10
|
-
* is kept (short-circuits on a fresh file); (3) an abandoned dir is reported
|
|
11
|
-
* stale with its full size. Lives next to the handler per co-located-test rule.
|
|
12
|
-
*/
|
|
13
|
-
import { describe, expect, test, beforeEach, afterEach } from "bun:test";
|
|
14
|
-
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, utimesSync } from "node:fs";
|
|
15
|
-
import { join } from "node:path";
|
|
16
|
-
import { tmpdir } from "node:os";
|
|
17
|
-
import { expandGlob, checkDirStale, parseRootDiskPct } from "./sweep-host-cache.ts";
|
|
18
|
-
|
|
19
|
-
const DAY = 86_400_000;
|
|
20
|
-
const now = Date.now();
|
|
21
|
-
const daysAgo = (n: number) => new Date(now - n * DAY);
|
|
22
|
-
const touch = (p: string, t: Date) => utimesSync(p, t, t);
|
|
23
|
-
|
|
24
|
-
let root: string;
|
|
25
|
-
beforeEach(() => {
|
|
26
|
-
root = mkdtempSync(join(tmpdir(), "sweep-"));
|
|
27
|
-
});
|
|
28
|
-
afterEach(() => {
|
|
29
|
-
rmSync(root, { recursive: true, force: true });
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
describe("expandGlob", () => {
|
|
33
|
-
test("expands a single-segment wildcard to concrete dirs", () => {
|
|
34
|
-
mkdirSync(join(root, "scala-rust", "target"), { recursive: true });
|
|
35
|
-
mkdirSync(join(root, "scala-os", "target"), { recursive: true });
|
|
36
|
-
mkdirSync(join(root, "scala-os", "src"), { recursive: true });
|
|
37
|
-
const matches = expandGlob(`${root}/*/target`).sort();
|
|
38
|
-
expect(matches).toEqual(
|
|
39
|
-
[join(root, "scala-os", "target"), join(root, "scala-rust", "target")].sort(),
|
|
40
|
-
);
|
|
41
|
-
});
|
|
42
|
-
test("literal segment matches only itself", () => {
|
|
43
|
-
mkdirSync(join(root, "scala-rust", "target"), { recursive: true });
|
|
44
|
-
mkdirSync(join(root, "other", "target"), { recursive: true });
|
|
45
|
-
const matches = expandGlob(`${root}/scala-rust/target`);
|
|
46
|
-
expect(matches).toEqual([join(root, "scala-rust", "target")]);
|
|
47
|
-
});
|
|
48
|
-
test("no matches when the wildcard segment has no children", () => {
|
|
49
|
-
const matches = expandGlob(`${root}/does-not-exist/*/target`);
|
|
50
|
-
expect(matches).toEqual([]);
|
|
51
|
-
});
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
describe("checkDirStale", () => {
|
|
55
|
-
test("stale dir: all files older than cutoff → stale + full size", async () => {
|
|
56
|
-
const dir = join(root, "target");
|
|
57
|
-
mkdirSync(join(dir, "debug", "deps"), { recursive: true });
|
|
58
|
-
writeFileSync(join(dir, "debug", "deps", "a.o"), "x".repeat(40));
|
|
59
|
-
writeFileSync(join(dir, "debug", "deps", "b.o"), "y".repeat(60));
|
|
60
|
-
touch(join(dir, "debug", "deps", "a.o"), daysAgo(40));
|
|
61
|
-
touch(join(dir, "debug", "deps", "b.o"), daysAgo(35));
|
|
62
|
-
const res = await checkDirStale(dir, now - 30 * DAY);
|
|
63
|
-
expect(res.stale).toBe(true);
|
|
64
|
-
expect(res.fileCount).toBe(2);
|
|
65
|
-
expect(res.bytes).toBe(100);
|
|
66
|
-
});
|
|
67
|
-
test("active dir: a file newer than cutoff → not stale (short-circuit)", async () => {
|
|
68
|
-
const dir = join(root, "target");
|
|
69
|
-
mkdirSync(join(dir, "debug"), { recursive: true });
|
|
70
|
-
writeFileSync(join(dir, "debug", "old.o"), "x".repeat(40));
|
|
71
|
-
touch(join(dir, "debug", "old.o"), daysAgo(40));
|
|
72
|
-
writeFileSync(join(dir, "debug", "fresh.o"), "y".repeat(60));
|
|
73
|
-
touch(join(dir, "debug", "fresh.o"), daysAgo(1));
|
|
74
|
-
const res = await checkDirStale(dir, now - 30 * DAY);
|
|
75
|
-
expect(res.stale).toBe(false);
|
|
76
|
-
});
|
|
77
|
-
test("empty dir → stale with zero bytes", async () => {
|
|
78
|
-
const dir = join(root, "target");
|
|
79
|
-
mkdirSync(dir, { recursive: true });
|
|
80
|
-
const res = await checkDirStale(dir, now - 30 * DAY);
|
|
81
|
-
expect(res.stale).toBe(true);
|
|
82
|
-
expect(res.bytes).toBe(0);
|
|
83
|
-
expect(res.fileCount).toBe(0);
|
|
84
|
-
});
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
describe("parseRootDiskPct", () => {
|
|
88
|
-
test("extracts root-fs Use% from df --output=pcent,target output", () => {
|
|
89
|
-
const df =
|
|
90
|
-
"Use% Mounted on\n" +
|
|
91
|
-
" 97% /\n" +
|
|
92
|
-
" 45% /home\n";
|
|
93
|
-
expect(parseRootDiskPct(df)).toBe(97);
|
|
94
|
-
});
|
|
95
|
-
test("picks the root mount (/) over nested mounts", () => {
|
|
96
|
-
const df =
|
|
97
|
-
"Use% Mounted on\n" +
|
|
98
|
-
" 10% /proc/x\n" +
|
|
99
|
-
" 76% /\n";
|
|
100
|
-
expect(parseRootDiskPct(df)).toBe(76);
|
|
101
|
-
});
|
|
102
|
-
test("returns null when no root-mount line present", () => {
|
|
103
|
-
const df = "Use% Mounted on\n 40% /mnt/other\n";
|
|
104
|
-
expect(parseRootDiskPct(df)).toBeNull();
|
|
105
|
-
});
|
|
106
|
-
test("returns null on empty / malformed output", () => {
|
|
107
|
-
expect(parseRootDiskPct("")).toBeNull();
|
|
108
|
-
expect(parseRootDiskPct("Use% Mounted on")).toBeNull();
|
|
109
|
-
});
|
|
110
|
-
});
|
|
111
|
-
|
|
112
|
-
describe("disk-threshold early-warning regression lock", () => {
|
|
113
|
-
const { readFileSync } = require("node:fs");
|
|
114
|
-
// Source-shape lock (same pattern as singleton-tools-currency-watch.test.ts):
|
|
115
|
-
// the df-check that rides this periodic sweep was once removed in favour of an
|
|
116
|
-
// in-process interval watchdog — the wrong architecture (interval watchdogs
|
|
117
|
-
// are forbidden; observability is event-driven). This asserts the corrected
|
|
118
|
-
// shape survives: the sweep handler MUST carry the event-driven host.disk.high
|
|
119
|
-
// emission (logger.warn -> GlitchTip) + the pure parseRootDiskPct helper that
|
|
120
|
-
// backs it. Removing either fails this test. The asserted surface is the
|
|
121
|
-
// disk early-warning cell of host-resources-have-complete-monitoring.
|
|
122
|
-
const src = readFileSync(import.meta.dir + "/sweep-host-cache.ts", "utf-8");
|
|
123
|
-
test("handler emits the host.disk.high event (event-driven, not an interval)", () => {
|
|
124
|
-
expect(src).toContain('log.warn("host.disk.high"');
|
|
125
|
-
expect(src).toContain('kind: "host.disk.high"');
|
|
126
|
-
});
|
|
127
|
-
test("handler reads df via the sanctioned spawn (no in-process poller)", () => {
|
|
128
|
-
expect(src).toContain('command: ["df", "--output=pcent,target"]');
|
|
129
|
-
});
|
|
130
|
-
test("threshold parse is the exported pure helper", () => {
|
|
131
|
-
expect(src).toContain("export function parseRootDiskPct");
|
|
132
|
-
});
|
|
133
|
-
});
|
|
@@ -1,328 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* The `host-cache-sweep` pg-boss handler. Reads the `retention` map from the
|
|
7
|
-
* `scheduled_job` row config — where each KEY is a host cache path (with
|
|
8
|
-
* `${HOME}`/`${WORKSPACE}` placeholders) and each VALUE is a day-granularity
|
|
9
|
-
* duration — and prunes files older than the cutoff. This is the mechanical
|
|
10
|
-
* automation that makes "host disk bounded by construction" real: it extends
|
|
11
|
-
* `no-uncontrolled-repetition-or-cascade` / `shared-resource-bounds-are-host-wide`
|
|
12
|
-
* (which bound install CONCURRENCY) to cache RETENTION, so the dev-host caches
|
|
13
|
-
* that accumulate every published version forever — canonical:
|
|
14
|
-
* `~/.bun/install/cache`, the @teamscala/* install cache — cannot grow unbounded.
|
|
15
|
-
*
|
|
16
|
-
* Path convention (config.retention keys, substituted at runtime via `userHome()`
|
|
17
|
-
* / `workspaceRoot()` per `host-literals-via-substitution` — source never
|
|
18
|
-
* hardcodes host paths):
|
|
19
|
-
* "${HOME}/.bun/install/cache" → "30 days"
|
|
20
|
-
* "${HOME}/.config/*.retired-*" → "14 days" (glob leaf, optional)
|
|
21
|
-
*
|
|
22
|
-
* config.retention_dirs — whole-DIRECTORY prune for build caches (Rust
|
|
23
|
-
* `target/`) whose unit is the dir, not the file (file-by-mtime would corrupt
|
|
24
|
-
* the cache). Keys support `*` globs; a matched dir is deleted ENTIRELY only
|
|
25
|
-
* when its newest file is older than the cutoff:
|
|
26
|
-
* "${WORKSPACE}/<repo>/target" → "30 days" (glob wildcard, whole-dir delete)
|
|
27
|
-
*
|
|
28
|
-
* Deletion runs through Bun fs APIs — pure filesystem deletion needs no shell,
|
|
29
|
-
* so `spawn-is-the-only-spawn` does not apply (that governs shell commands).
|
|
30
|
-
* Hardlinked node_modules copies SURVIVE cache pruning (a hardlink shares an
|
|
31
|
-
* inode; removing the cache's link leaves the installed copy's link intact), so
|
|
32
|
-
* installed packages stay whole — a pruned version simply re-downloads on the
|
|
33
|
-
* next install that needs it. Empty parent directories are left in place
|
|
34
|
-
* (negligible — directory entries are a few KB total); only file data is reclaimed.
|
|
35
|
-
*
|
|
36
|
-
* Liveness: records a `host.cache.swept` event after a run (distinct from
|
|
37
|
-
* event-log-sweep's `sweep.completed` so the event-log sweep-liveness watchdog
|
|
38
|
-
* is not confused). A silently-stopped sweep is detectable from the event stream.
|
|
39
|
-
*
|
|
40
|
-
* Registered under slug `@teamscala/service-boot:sweep-host-cache`.
|
|
41
|
-
*/
|
|
42
|
-
import { rm, readdir, stat } from "node:fs/promises";
|
|
43
|
-
import { readdirSync, type Dirent } from "node:fs";
|
|
44
|
-
import { join } from "node:path";
|
|
45
|
-
import { spawnSync } from "@teamscala/os/spawn/spawn";
|
|
46
|
-
import { userHome, workspaceRoot } from "@teamscala/os/host-paths";
|
|
47
|
-
import { formatBytes } from "@teamscala/formatters/format-bytes";
|
|
48
|
-
import { recordEvent } from "@teamscala/event-log/record-event";
|
|
49
|
-
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
50
|
-
import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
|
|
51
|
-
|
|
52
|
-
const DURATION_RE = /^(\d+)\s+(day|week|month)s?$/;
|
|
53
|
-
const UNIT_TO_DAYS: Record<string, number> = {
|
|
54
|
-
day: 1,
|
|
55
|
-
week: 7,
|
|
56
|
-
month: 30,
|
|
57
|
-
};
|
|
58
|
-
const MS_PER_DAY = 86_400_000;
|
|
59
|
-
|
|
60
|
-
function parseDays(raw: string): number | null {
|
|
61
|
-
const match = DURATION_RE.exec(raw.trim());
|
|
62
|
-
if (!match) return null;
|
|
63
|
-
return Number(match[1]) * (UNIT_TO_DAYS[match[2]] ?? 0);
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
/** Resolve `${HOME}` / `${WORKSPACE}` placeholders against the live host. */
|
|
67
|
-
function resolveHostPath(raw: string): string {
|
|
68
|
-
return raw
|
|
69
|
-
.replaceAll("${HOME}", userHome())
|
|
70
|
-
.replaceAll("${WORKSPACE}", workspaceRoot());
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
interface SweepResult {
|
|
74
|
-
path: string;
|
|
75
|
-
scanned: number;
|
|
76
|
-
deleted: number;
|
|
77
|
-
bytesFreed: number;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Walk `rootDir` (stack-based, bounded memory) and delete regular files whose
|
|
82
|
-
* mtime is older than `cutoffMs`. Directories are traversed but not removed.
|
|
83
|
-
*/
|
|
84
|
-
async function sweepPath(
|
|
85
|
-
rootDir: string,
|
|
86
|
-
cutoffMs: number,
|
|
87
|
-
): Promise<SweepResult> {
|
|
88
|
-
const log = getAppLogger();
|
|
89
|
-
let scanned = 0;
|
|
90
|
-
let deleted = 0;
|
|
91
|
-
let bytesFreed = 0;
|
|
92
|
-
const stack: string[] = [rootDir];
|
|
93
|
-
|
|
94
|
-
while (stack.length > 0) {
|
|
95
|
-
const dir = stack.pop();
|
|
96
|
-
if (dir === undefined) break;
|
|
97
|
-
let entries: Dirent[];
|
|
98
|
-
try {
|
|
99
|
-
entries = await readdir(dir, { withFileTypes: true });
|
|
100
|
-
} catch {
|
|
101
|
-
continue; // unreadable / vanished — skip
|
|
102
|
-
}
|
|
103
|
-
for (const entry of entries) {
|
|
104
|
-
const full = join(dir, entry.name);
|
|
105
|
-
if (entry.isDirectory()) {
|
|
106
|
-
stack.push(full);
|
|
107
|
-
} else if (entry.isFile()) {
|
|
108
|
-
scanned++;
|
|
109
|
-
try {
|
|
110
|
-
const st = await stat(full);
|
|
111
|
-
if (st.mtimeMs < cutoffMs) {
|
|
112
|
-
bytesFreed += st.size;
|
|
113
|
-
await rm(full, { force: true });
|
|
114
|
-
deleted++;
|
|
115
|
-
}
|
|
116
|
-
} catch (error) {
|
|
117
|
-
log.warn(`[host-cache-sweep] could not process ${full}: ${String(error)}`);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
return { path: rootDir, scanned, deleted, bytesFreed };
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
/**
|
|
127
|
-
* Expand a path pattern containing single-segment `*` wildcards into concrete
|
|
128
|
-
* paths (files or dirs). Used by `retention_dirs` so a single config key like
|
|
129
|
-
* a pattern like `${WORKSPACE}/<star>/target` reaches every repo's target dir.
|
|
130
|
-
* Sync + bounded: the
|
|
131
|
-
* pattern is shallow (3 segments), and this runs once
|
|
132
|
-
* nightly. Non-existent segments yield no matches (the caller stats each result).
|
|
133
|
-
*/
|
|
134
|
-
export function expandGlob(pattern: string): string[] {
|
|
135
|
-
const absolute = pattern.startsWith("/");
|
|
136
|
-
const segments = (absolute ? pattern.slice(1) : pattern)
|
|
137
|
-
.split("/")
|
|
138
|
-
.filter((s) => s.length > 0);
|
|
139
|
-
let current: string[] = [absolute ? "/" : "."];
|
|
140
|
-
for (const seg of segments) {
|
|
141
|
-
if (current.length === 0) break;
|
|
142
|
-
const next: string[] = [];
|
|
143
|
-
for (const dir of current) {
|
|
144
|
-
let names: string[];
|
|
145
|
-
try {
|
|
146
|
-
names = readdirSync(dir);
|
|
147
|
-
} catch {
|
|
148
|
-
continue;
|
|
149
|
-
}
|
|
150
|
-
for (const name of names) {
|
|
151
|
-
if (seg === "*" || seg === name) next.push(join(dir, name));
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
current = next;
|
|
155
|
-
}
|
|
156
|
-
return current;
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
/**
|
|
160
|
-
* Is `dir` stale — i.e. NO file under it is newer than `cutoffMs`? Walks
|
|
161
|
-
* recursively and SHORT-CIRCUITS the instant it finds a fresh file (so an
|
|
162
|
-
* actively-built target dir returns fast without walking its full tree). On a
|
|
163
|
-
* stale dir the walk is complete and returns the total size + file count, so
|
|
164
|
-
* the caller can report bytes-freed before deleting the dir whole.
|
|
165
|
-
*/
|
|
166
|
-
export async function checkDirStale(
|
|
167
|
-
dir: string,
|
|
168
|
-
cutoffMs: number,
|
|
169
|
-
): Promise<{ stale: boolean; bytes: number; fileCount: number }> {
|
|
170
|
-
const stack: string[] = [dir];
|
|
171
|
-
let bytes = 0;
|
|
172
|
-
let fileCount = 0;
|
|
173
|
-
while (stack.length > 0) {
|
|
174
|
-
const d = stack.pop();
|
|
175
|
-
if (d === undefined) break;
|
|
176
|
-
let entries: Dirent[];
|
|
177
|
-
try {
|
|
178
|
-
entries = await readdir(d, { withFileTypes: true });
|
|
179
|
-
} catch {
|
|
180
|
-
continue;
|
|
181
|
-
}
|
|
182
|
-
for (const entry of entries) {
|
|
183
|
-
const full = join(d, entry.name);
|
|
184
|
-
if (entry.isDirectory()) {
|
|
185
|
-
stack.push(full);
|
|
186
|
-
} else if (entry.isFile()) {
|
|
187
|
-
try {
|
|
188
|
-
const st = await stat(full);
|
|
189
|
-
if (st.mtimeMs >= cutoffMs) {
|
|
190
|
-
return { stale: false, bytes: 0, fileCount: 0 };
|
|
191
|
-
}
|
|
192
|
-
bytes += st.size;
|
|
193
|
-
fileCount++;
|
|
194
|
-
} catch {
|
|
195
|
-
// vanished/unreadable — skip
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
return { stale: true, bytes, fileCount };
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
/** Parse the root-fs Use% from `df --output=pcent,target` output (pure, for tests).
|
|
204
|
-
* Returns null if no root-mount line is found. */
|
|
205
|
-
export function parseRootDiskPct(out: string): number | null {
|
|
206
|
-
const rootLine = out.split("\n").find((l) => l.trim().endsWith("/"));
|
|
207
|
-
const m = rootLine?.match(/(\d+)%/);
|
|
208
|
-
return m ? Number(m[1]) : null;
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
async function sweepHostCache(ctx: {
|
|
212
|
-
config: { retention?: Record<string, string>; retention_dirs?: Record<string, string> };
|
|
213
|
-
}): Promise<void> {
|
|
214
|
-
const log = getAppLogger();
|
|
215
|
-
const retention = ctx.config.retention ?? {};
|
|
216
|
-
const retentionDirs = ctx.config.retention_dirs ?? {};
|
|
217
|
-
const now = Date.now();
|
|
218
|
-
const results: SweepResult[] = [];
|
|
219
|
-
|
|
220
|
-
for (const [rawPath, rawDuration] of Object.entries(retention)) {
|
|
221
|
-
const days = parseDays(rawDuration);
|
|
222
|
-
if (days === null || days <= 0) {
|
|
223
|
-
log.warn(
|
|
224
|
-
`[host-cache-sweep] skipping "${rawPath}" — value "${rawDuration}" is not a day-granularity duration (expected e.g. "30 days")`,
|
|
225
|
-
);
|
|
226
|
-
continue;
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
const dir = resolveHostPath(rawPath);
|
|
230
|
-
try {
|
|
231
|
-
const st = await stat(dir);
|
|
232
|
-
if (!st.isDirectory()) {
|
|
233
|
-
log.warn(`[host-cache-sweep] skipping "${rawPath}" — resolved path is not a directory`);
|
|
234
|
-
continue;
|
|
235
|
-
}
|
|
236
|
-
} catch {
|
|
237
|
-
log.warn(`[host-cache-sweep] skipping "${rawPath}" — path does not exist or is unreadable`);
|
|
238
|
-
continue;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
const cutoffMs = now - days * MS_PER_DAY;
|
|
242
|
-
const result = await sweepPath(dir, cutoffMs);
|
|
243
|
-
results.push(result);
|
|
244
|
-
log.info(
|
|
245
|
-
`[host-cache-sweep] ${rawPath}: scanned ${result.scanned}, deleted ${result.deleted} file(s) (${formatBytes(result.bytesFreed)}) older than ${rawDuration}`,
|
|
246
|
-
);
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
// retention_dirs — whole-directory prune for build caches (Rust target/).
|
|
250
|
-
// A matched dir is deleted ENTIRELY only when its newest file is older than
|
|
251
|
-
// the cutoff, so an actively-built repo keeps its cache and an abandoned one
|
|
252
|
-
// is reclaimed whole (never the partial slice that would corrupt the cache).
|
|
253
|
-
for (const [rawPattern, rawDuration] of Object.entries(retentionDirs)) {
|
|
254
|
-
const days = parseDays(rawDuration);
|
|
255
|
-
if (days === null || days <= 0) {
|
|
256
|
-
log.warn(
|
|
257
|
-
`[host-cache-sweep] skipping retention_dirs "${rawPattern}" — value "${rawDuration}" is not a day-granularity duration (expected e.g. "30 days")`,
|
|
258
|
-
);
|
|
259
|
-
continue;
|
|
260
|
-
}
|
|
261
|
-
const cutoffMs = now - days * MS_PER_DAY;
|
|
262
|
-
for (const dir of expandGlob(resolveHostPath(rawPattern))) {
|
|
263
|
-
try {
|
|
264
|
-
const st = await stat(dir);
|
|
265
|
-
if (!st.isDirectory()) continue;
|
|
266
|
-
} catch {
|
|
267
|
-
continue; // glob match that is not a dir (or vanished) — skip
|
|
268
|
-
}
|
|
269
|
-
const { stale, bytes, fileCount } = await checkDirStale(dir, cutoffMs);
|
|
270
|
-
if (!stale) continue;
|
|
271
|
-
await rm(dir, { recursive: true, force: true });
|
|
272
|
-
results.push({ path: dir, scanned: fileCount, deleted: fileCount, bytesFreed: bytes });
|
|
273
|
-
log.info(
|
|
274
|
-
`[host-cache-sweep] ${dir}: no build in ${rawDuration}; reclaimed whole dir — ${fileCount} file(s), ${formatBytes(bytes)}`,
|
|
275
|
-
);
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
const totalDeleted = results.reduce((sum, r) => sum + r.deleted, 0);
|
|
280
|
-
const totalBytes = results.reduce((sum, r) => sum + r.bytesFreed, 0);
|
|
281
|
-
log.info(
|
|
282
|
-
`[host-cache-sweep] complete: ${results.length} target(s), ${totalDeleted} file(s) deleted, ${formatBytes(totalBytes)} reclaimed`,
|
|
283
|
-
);
|
|
284
|
-
|
|
285
|
-
// Liveness signal — distinct kind so the event-log sweep watchdog is unaffected.
|
|
286
|
-
recordEvent({
|
|
287
|
-
kind: "host.cache.swept",
|
|
288
|
-
payload: {
|
|
289
|
-
targets: results.map((r) => ({
|
|
290
|
-
path: r.path,
|
|
291
|
-
scanned: r.scanned,
|
|
292
|
-
deleted: r.deleted,
|
|
293
|
-
bytesFreed: r.bytesFreed,
|
|
294
|
-
})),
|
|
295
|
-
totalDeleted,
|
|
296
|
-
totalBytesFreed: totalBytes,
|
|
297
|
-
completedAt: new Date().toISOString(),
|
|
298
|
-
},
|
|
299
|
-
});
|
|
300
|
-
|
|
301
|
-
// Disk-threshold observability — rides this periodic sweep (NOT an interval:
|
|
302
|
-
// this job is genuinely periodic application-DATA work — deletion — and the
|
|
303
|
-
// df read piggybacks once per run). Emits through the event-driven surface:
|
|
304
|
-
// logger.warn -> GlitchTip (the paging path, same as host.pressure) +
|
|
305
|
-
// recordEvent for the durable event_log record. Catches a fill faster than
|
|
306
|
-
// the nightly prune can absorb (a cache with no retention entry, an external
|
|
307
|
-
// dump, a runaway log) so the operator learns it within a day, not at 100%.
|
|
308
|
-
const diskHighPct = 90;
|
|
309
|
-
try {
|
|
310
|
-
const dfRes = spawnSync({ name: "df:/", command: ["df", "--output=pcent,target"], timeoutMs: 5_000 });
|
|
311
|
-
const dfOut = (dfRes.stdout ?? "").trim();
|
|
312
|
-
const usePct = parseRootDiskPct(dfOut);
|
|
313
|
-
if (usePct !== null && usePct >= diskHighPct) {
|
|
314
|
-
log.warn("host.disk.high", { usePct, threshold: diskHighPct, df: dfOut });
|
|
315
|
-
recordEvent({
|
|
316
|
-
kind: "host.disk.high",
|
|
317
|
-
payload: { usePct, threshold: diskHighPct, df: dfOut, completedAt: new Date().toISOString() },
|
|
318
|
-
});
|
|
319
|
-
}
|
|
320
|
-
} catch (error) {
|
|
321
|
-
log.warn("[host-cache-sweep] disk-threshold read failed: " + String(error));
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
pgBossHandlerRegistry.register(
|
|
326
|
-
"@teamscala/service-boot:sweep-host-cache",
|
|
327
|
-
sweepHostCache,
|
|
328
|
-
);
|