@teamscala/service-boot 0.23.369 → 0.23.372
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
},
|
|
7
7
|
"name": "@teamscala/service-boot",
|
|
8
8
|
"type": "module",
|
|
9
|
-
"version": "0.23.
|
|
9
|
+
"version": "0.23.372",
|
|
10
10
|
"sideEffects": false,
|
|
11
11
|
"imports": {
|
|
12
12
|
"#service-boot/*": "./src/*",
|
|
@@ -410,14 +410,6 @@
|
|
|
410
410
|
"types": "./src/handlers/bun-stable-watch.d.ts",
|
|
411
411
|
"default": "./src/handlers/bun-stable-watch.ts"
|
|
412
412
|
},
|
|
413
|
-
"./handlers/lane-attention-sweep": {
|
|
414
|
-
"types": "./src/handlers/lane-attention-sweep.d.ts",
|
|
415
|
-
"default": "./src/handlers/lane-attention-sweep.ts"
|
|
416
|
-
},
|
|
417
|
-
"./handlers/lane-attention-sweep.ts": {
|
|
418
|
-
"types": "./src/handlers/lane-attention-sweep.d.ts",
|
|
419
|
-
"default": "./src/handlers/lane-attention-sweep.ts"
|
|
420
|
-
},
|
|
421
413
|
"./handlers/pane-outbox-drain": {
|
|
422
414
|
"types": "./src/handlers/pane-outbox-drain.d.ts",
|
|
423
415
|
"default": "./src/handlers/pane-outbox-drain.ts"
|
|
@@ -490,14 +482,6 @@
|
|
|
490
482
|
"types": "./src/handlers/sweep-event-log.d.ts",
|
|
491
483
|
"default": "./src/handlers/sweep-event-log.ts"
|
|
492
484
|
},
|
|
493
|
-
"./handlers/sweep-host-cache": {
|
|
494
|
-
"types": "./src/handlers/sweep-host-cache.d.ts",
|
|
495
|
-
"default": "./src/handlers/sweep-host-cache.ts"
|
|
496
|
-
},
|
|
497
|
-
"./handlers/sweep-host-cache.ts": {
|
|
498
|
-
"types": "./src/handlers/sweep-host-cache.d.ts",
|
|
499
|
-
"default": "./src/handlers/sweep-host-cache.ts"
|
|
500
|
-
},
|
|
501
485
|
"./handlers/upstream-issue-sync": {
|
|
502
486
|
"types": "./src/handlers/upstream-issue-sync.d.ts",
|
|
503
487
|
"default": "./src/handlers/upstream-issue-sync.ts"
|
|
@@ -38,8 +38,7 @@
|
|
|
38
38
|
* not a state transition any single event can announce. The sweep reads one
|
|
39
39
|
* server-side aggregate (GROUP BY kind) per run against indexed event_log;
|
|
40
40
|
* while the platform is healthy it returns zero rows and costs nothing
|
|
41
|
-
* beyond the tick (idle-costs-nothing).
|
|
42
|
-
* lane-attention-sweep carries on its row as `poll_justification`.
|
|
41
|
+
* beyond the tick (idle-costs-nothing).
|
|
43
42
|
*
|
|
44
43
|
* Registered under slug `@teamscala/service-boot:platform-fault-escalation`.
|
|
45
44
|
*/
|
|
@@ -130,9 +129,8 @@ async function getSystemIdentity(): Promise<{ userId: string | undefined; organi
|
|
|
130
129
|
* TypeError this replaced). Per `no-direct-prisma-outside-orpc`, every WRITE
|
|
131
130
|
* in this handler routes through this client; the single raw read (the
|
|
132
131
|
* event_log recurrence aggregate, a GROUP BY no generated CRUD expresses) uses
|
|
133
|
-
* the assigned-client form with $queryRawUnsafe
|
|
134
|
-
*
|
|
135
|
-
* instances (the generated valibot schemas require it).
|
|
132
|
+
* the assigned-client form with $queryRawUnsafe. Timestamps are ISO strings,
|
|
133
|
+
* never Date instances (the generated valibot schemas require it).
|
|
136
134
|
*/
|
|
137
135
|
let cachedOrpc: OrpcSurface | null = null;
|
|
138
136
|
async function orpc(): Promise<OrpcSurface> {
|
package/src/pg-boss-mount.ts
CHANGED
|
@@ -26,7 +26,9 @@ import { pgBossHandlerRegistry } from "./pg-boss-handler-registry.ts";
|
|
|
26
26
|
import "./handlers/prune-stale-data.ts";
|
|
27
27
|
import "./handlers/projects-digest.ts";
|
|
28
28
|
import "./handlers/sweep-event-log.ts";
|
|
29
|
-
|
|
29
|
+
// `sweep-host-cache` is Rust-native: loga (the job's service_slug) registers it
|
|
30
|
+
// via scala_os::host_cache_retention with the config/cleanup-allowed-roots
|
|
31
|
+
// bound. No TypeScript copy may exist beside it.
|
|
30
32
|
import "./handlers/bun-stable-watch.ts";
|
|
31
33
|
import "./handlers/whatsapp-liveness.ts";
|
|
32
34
|
import "./handlers/upstream-issue-sync.ts";
|
|
@@ -34,7 +36,6 @@ import "./handlers/scala-tools-currency-watch.ts";
|
|
|
34
36
|
import "./handlers/singleton-tools-currency-watch.ts";
|
|
35
37
|
import "./handlers/registry-stale-inactive-sweep.ts";
|
|
36
38
|
import "./handlers/pane-outbox-drain.ts";
|
|
37
|
-
import "./handlers/lane-attention-sweep.ts";
|
|
38
39
|
import "./handlers/platform-fault-escalation.ts";
|
|
39
40
|
|
|
40
41
|
/**
|
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* The `lane-attention-sweep` pg-boss handler. Calls the database function that
|
|
7
|
-
* reminds an orchestrator about lanes waiting on it, and records the outcome.
|
|
8
|
-
*
|
|
9
|
-
* This handler holds NO POLICY. Which lanes are waiting, how long counts as
|
|
10
|
-
* waiting, who is told, what the message says, and how often it may repeat are
|
|
11
|
-
* all in `lane_needs_attention` and `sweep_lanes_needing_attention` — so
|
|
12
|
-
* changing any of them is a migration, not a rebuild and redeploy of this
|
|
13
|
-
* service. All this file supplies is the tick and the telemetry.
|
|
14
|
-
*
|
|
15
|
-
* WHY A TICK EXISTS AT ALL, given `observability-is-passive` and
|
|
16
|
-
* `no-polling-for-pushable-state`. Both forbid polling for state a source CAN
|
|
17
|
-
* push. Here the trigger is the PASSAGE OF TIME against a threshold: a lane
|
|
18
|
-
* going silent emits nothing, because emitting is exactly what it stopped
|
|
19
|
-
* doing. There is no state transition to react to, so nothing could push it.
|
|
20
|
-
* The same justification `registry-stale-inactive-sweep` carries, recorded on
|
|
21
|
-
* the row as `poll_justification`.
|
|
22
|
-
*
|
|
23
|
-
* Registered under slug `@teamscala/service-boot:lane-attention-sweep`.
|
|
24
|
-
*/
|
|
25
|
-
import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
|
|
26
|
-
import { getPrisma } from "@teamscala/db/prisma-registry";
|
|
27
|
-
import { recordEvent } from "@teamscala/event-log/record-event";
|
|
28
|
-
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
29
|
-
import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
|
|
30
|
-
|
|
31
|
-
interface RemindedLane {
|
|
32
|
-
task_id: string;
|
|
33
|
-
window_name: string | null;
|
|
34
|
-
reason: string;
|
|
35
|
-
orchestrator_project_id: string;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
async function laneAttentionSweep(ctx: { config: ScheduledJobConfig }): Promise<void> {
|
|
39
|
-
const log = getAppLogger();
|
|
40
|
-
const prisma = getPrisma();
|
|
41
|
-
|
|
42
|
-
// The repeat window is the row's, so silencing a chatty reminder is a
|
|
43
|
-
// registry edit. The default lives in the SQL function's signature, which is
|
|
44
|
-
// the one place it is defined.
|
|
45
|
-
const repeatAfter = (ctx.config as { repeat_after?: string }).repeat_after;
|
|
46
|
-
|
|
47
|
-
const reminded = repeatAfter
|
|
48
|
-
? await prisma.$queryRawUnsafe<RemindedLane[]>(
|
|
49
|
-
"SELECT * FROM sweep_lanes_needing_attention($1::interval)",
|
|
50
|
-
repeatAfter,
|
|
51
|
-
)
|
|
52
|
-
: await prisma.$queryRawUnsafe<RemindedLane[]>("SELECT * FROM sweep_lanes_needing_attention()");
|
|
53
|
-
|
|
54
|
-
// Emitted even when nothing was due. A sweep that only reports on the runs
|
|
55
|
-
// where it found something is indistinguishable from a sweep that stopped
|
|
56
|
-
// running, and this exists precisely to notice silence.
|
|
57
|
-
recordEvent({
|
|
58
|
-
kind: "fleet.lane-attention.swept",
|
|
59
|
-
payload: {
|
|
60
|
-
remindedCount: reminded.length,
|
|
61
|
-
lanes: reminded.map((lane) => ({
|
|
62
|
-
task: lane.task_id,
|
|
63
|
-
window: lane.window_name,
|
|
64
|
-
reason: lane.reason,
|
|
65
|
-
toldOrchestrator: lane.orchestrator_project_id,
|
|
66
|
-
})),
|
|
67
|
-
},
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
if (reminded.length > 0) {
|
|
71
|
-
log.info(
|
|
72
|
-
`[lane-attention-sweep] reminded ${reminded.length} orchestrator(s) about waiting lanes`,
|
|
73
|
-
{ lanes: reminded.map((lane) => `${lane.window_name}: ${lane.reason}`) },
|
|
74
|
-
);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
pgBossHandlerRegistry.register(
|
|
79
|
-
"@teamscala/service-boot:lane-attention-sweep",
|
|
80
|
-
laneAttentionSweep as (ctx: { config: ScheduledJobConfig }) => Promise<void>,
|
|
81
|
-
);
|
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* Unit tests for the `retention_dirs` helpers (`expandGlob`, `checkDirStale`) —
|
|
7
|
-
* the whole-directory prune that bounds Rust `target/` waste. The file-by-mtime
|
|
8
|
-
* `retention` path is unchanged; these cover ONLY the new build-cache model:
|
|
9
|
-
* (1) glob expansion reaches every repo's target dir; (2) an actively-built dir
|
|
10
|
-
* is kept (short-circuits on a fresh file); (3) an abandoned dir is reported
|
|
11
|
-
* stale with its full size. Lives next to the handler per co-located-test rule.
|
|
12
|
-
*/
|
|
13
|
-
import { describe, expect, test, beforeEach, afterEach } from "bun:test";
|
|
14
|
-
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, utimesSync } from "node:fs";
|
|
15
|
-
import { join } from "node:path";
|
|
16
|
-
import { tmpdir } from "node:os";
|
|
17
|
-
import { expandGlob, checkDirStale, parseRootDiskPct } from "./sweep-host-cache.ts";
|
|
18
|
-
|
|
19
|
-
const DAY = 86_400_000;
|
|
20
|
-
const now = Date.now();
|
|
21
|
-
const daysAgo = (n: number) => new Date(now - n * DAY);
|
|
22
|
-
const touch = (p: string, t: Date) => utimesSync(p, t, t);
|
|
23
|
-
|
|
24
|
-
let root: string;
|
|
25
|
-
beforeEach(() => {
|
|
26
|
-
root = mkdtempSync(join(tmpdir(), "sweep-"));
|
|
27
|
-
});
|
|
28
|
-
afterEach(() => {
|
|
29
|
-
rmSync(root, { recursive: true, force: true });
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
describe("expandGlob", () => {
|
|
33
|
-
test("expands a single-segment wildcard to concrete dirs", () => {
|
|
34
|
-
mkdirSync(join(root, "scala-rust", "target"), { recursive: true });
|
|
35
|
-
mkdirSync(join(root, "scala-os", "target"), { recursive: true });
|
|
36
|
-
mkdirSync(join(root, "scala-os", "src"), { recursive: true });
|
|
37
|
-
const matches = expandGlob(`${root}/*/target`).sort();
|
|
38
|
-
expect(matches).toEqual(
|
|
39
|
-
[join(root, "scala-os", "target"), join(root, "scala-rust", "target")].sort(),
|
|
40
|
-
);
|
|
41
|
-
});
|
|
42
|
-
test("literal segment matches only itself", () => {
|
|
43
|
-
mkdirSync(join(root, "scala-rust", "target"), { recursive: true });
|
|
44
|
-
mkdirSync(join(root, "other", "target"), { recursive: true });
|
|
45
|
-
const matches = expandGlob(`${root}/scala-rust/target`);
|
|
46
|
-
expect(matches).toEqual([join(root, "scala-rust", "target")]);
|
|
47
|
-
});
|
|
48
|
-
test("no matches when the wildcard segment has no children", () => {
|
|
49
|
-
const matches = expandGlob(`${root}/does-not-exist/*/target`);
|
|
50
|
-
expect(matches).toEqual([]);
|
|
51
|
-
});
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
describe("checkDirStale", () => {
|
|
55
|
-
test("stale dir: all files older than cutoff → stale + full size", async () => {
|
|
56
|
-
const dir = join(root, "target");
|
|
57
|
-
mkdirSync(join(dir, "debug", "deps"), { recursive: true });
|
|
58
|
-
writeFileSync(join(dir, "debug", "deps", "a.o"), "x".repeat(40));
|
|
59
|
-
writeFileSync(join(dir, "debug", "deps", "b.o"), "y".repeat(60));
|
|
60
|
-
touch(join(dir, "debug", "deps", "a.o"), daysAgo(40));
|
|
61
|
-
touch(join(dir, "debug", "deps", "b.o"), daysAgo(35));
|
|
62
|
-
const res = await checkDirStale(dir, now - 30 * DAY);
|
|
63
|
-
expect(res.stale).toBe(true);
|
|
64
|
-
expect(res.fileCount).toBe(2);
|
|
65
|
-
expect(res.bytes).toBe(100);
|
|
66
|
-
});
|
|
67
|
-
test("active dir: a file newer than cutoff → not stale (short-circuit)", async () => {
|
|
68
|
-
const dir = join(root, "target");
|
|
69
|
-
mkdirSync(join(dir, "debug"), { recursive: true });
|
|
70
|
-
writeFileSync(join(dir, "debug", "old.o"), "x".repeat(40));
|
|
71
|
-
touch(join(dir, "debug", "old.o"), daysAgo(40));
|
|
72
|
-
writeFileSync(join(dir, "debug", "fresh.o"), "y".repeat(60));
|
|
73
|
-
touch(join(dir, "debug", "fresh.o"), daysAgo(1));
|
|
74
|
-
const res = await checkDirStale(dir, now - 30 * DAY);
|
|
75
|
-
expect(res.stale).toBe(false);
|
|
76
|
-
});
|
|
77
|
-
test("empty dir → stale with zero bytes", async () => {
|
|
78
|
-
const dir = join(root, "target");
|
|
79
|
-
mkdirSync(dir, { recursive: true });
|
|
80
|
-
const res = await checkDirStale(dir, now - 30 * DAY);
|
|
81
|
-
expect(res.stale).toBe(true);
|
|
82
|
-
expect(res.bytes).toBe(0);
|
|
83
|
-
expect(res.fileCount).toBe(0);
|
|
84
|
-
});
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
describe("parseRootDiskPct", () => {
|
|
88
|
-
test("extracts root-fs Use% from df --output=pcent,target output", () => {
|
|
89
|
-
const df =
|
|
90
|
-
"Use% Mounted on\n" +
|
|
91
|
-
" 97% /\n" +
|
|
92
|
-
" 45% /home\n";
|
|
93
|
-
expect(parseRootDiskPct(df)).toBe(97);
|
|
94
|
-
});
|
|
95
|
-
test("picks the root mount (/) over nested mounts", () => {
|
|
96
|
-
const df =
|
|
97
|
-
"Use% Mounted on\n" +
|
|
98
|
-
" 10% /proc/x\n" +
|
|
99
|
-
" 76% /\n";
|
|
100
|
-
expect(parseRootDiskPct(df)).toBe(76);
|
|
101
|
-
});
|
|
102
|
-
test("returns null when no root-mount line present", () => {
|
|
103
|
-
const df = "Use% Mounted on\n 40% /mnt/other\n";
|
|
104
|
-
expect(parseRootDiskPct(df)).toBeNull();
|
|
105
|
-
});
|
|
106
|
-
test("returns null on empty / malformed output", () => {
|
|
107
|
-
expect(parseRootDiskPct("")).toBeNull();
|
|
108
|
-
expect(parseRootDiskPct("Use% Mounted on")).toBeNull();
|
|
109
|
-
});
|
|
110
|
-
});
|
|
111
|
-
|
|
112
|
-
describe("disk-threshold early-warning regression lock", () => {
|
|
113
|
-
const { readFileSync } = require("node:fs");
|
|
114
|
-
// Source-shape lock (same pattern as singleton-tools-currency-watch.test.ts):
|
|
115
|
-
// the df-check that rides this periodic sweep was once removed in favour of an
|
|
116
|
-
// in-process interval watchdog — the wrong architecture (interval watchdogs
|
|
117
|
-
// are forbidden; observability is event-driven). This asserts the corrected
|
|
118
|
-
// shape survives: the sweep handler MUST carry the event-driven host.disk.high
|
|
119
|
-
// emission (logger.warn -> GlitchTip) + the pure parseRootDiskPct helper that
|
|
120
|
-
// backs it. Removing either fails this test. The asserted surface is the
|
|
121
|
-
// disk early-warning cell of host-resources-have-complete-monitoring.
|
|
122
|
-
const src = readFileSync(import.meta.dir + "/sweep-host-cache.ts", "utf-8");
|
|
123
|
-
test("handler emits the host.disk.high event (event-driven, not an interval)", () => {
|
|
124
|
-
expect(src).toContain('log.warn("host.disk.high"');
|
|
125
|
-
expect(src).toContain('kind: "host.disk.high"');
|
|
126
|
-
});
|
|
127
|
-
test("handler reads df via the sanctioned spawn (no in-process poller)", () => {
|
|
128
|
-
expect(src).toContain('command: ["df", "--output=pcent,target"]');
|
|
129
|
-
});
|
|
130
|
-
test("threshold parse is the exported pure helper", () => {
|
|
131
|
-
expect(src).toContain("export function parseRootDiskPct");
|
|
132
|
-
});
|
|
133
|
-
});
|
|
@@ -1,328 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* The `host-cache-sweep` pg-boss handler. Reads the `retention` map from the
|
|
7
|
-
* `scheduled_job` row config — where each KEY is a host cache path (with
|
|
8
|
-
* `${HOME}`/`${WORKSPACE}` placeholders) and each VALUE is a day-granularity
|
|
9
|
-
* duration — and prunes files older than the cutoff. This is the mechanical
|
|
10
|
-
* automation that makes "host disk bounded by construction" real: it extends
|
|
11
|
-
* `no-uncontrolled-repetition-or-cascade` / `shared-resource-bounds-are-host-wide`
|
|
12
|
-
* (which bound install CONCURRENCY) to cache RETENTION, so the dev-host caches
|
|
13
|
-
* that accumulate every published version forever — canonical:
|
|
14
|
-
* `~/.bun/install/cache`, the @teamscala/* install cache — cannot grow unbounded.
|
|
15
|
-
*
|
|
16
|
-
* Path convention (config.retention keys, substituted at runtime via `userHome()`
|
|
17
|
-
* / `workspaceRoot()` per `host-literals-via-substitution` — source never
|
|
18
|
-
* hardcodes host paths):
|
|
19
|
-
* "${HOME}/.bun/install/cache" → "30 days"
|
|
20
|
-
* "${HOME}/.config/*.retired-*" → "14 days" (glob leaf, optional)
|
|
21
|
-
*
|
|
22
|
-
* config.retention_dirs — whole-DIRECTORY prune for build caches (Rust
|
|
23
|
-
* `target/`) whose unit is the dir, not the file (file-by-mtime would corrupt
|
|
24
|
-
* the cache). Keys support `*` globs; a matched dir is deleted ENTIRELY only
|
|
25
|
-
* when its newest file is older than the cutoff:
|
|
26
|
-
* "${WORKSPACE}/<repo>/target" → "30 days" (glob wildcard, whole-dir delete)
|
|
27
|
-
*
|
|
28
|
-
* Deletion runs through Bun fs APIs — pure filesystem deletion needs no shell,
|
|
29
|
-
* so `spawn-is-the-only-spawn` does not apply (that governs shell commands).
|
|
30
|
-
* Hardlinked node_modules copies SURVIVE cache pruning (a hardlink shares an
|
|
31
|
-
* inode; removing the cache's link leaves the installed copy's link intact), so
|
|
32
|
-
* installed packages stay whole — a pruned version simply re-downloads on the
|
|
33
|
-
* next install that needs it. Empty parent directories are left in place
|
|
34
|
-
* (negligible — directory entries are a few KB total); only file data is reclaimed.
|
|
35
|
-
*
|
|
36
|
-
* Liveness: records a `host.cache.swept` event after a run (distinct from
|
|
37
|
-
* event-log-sweep's `sweep.completed` so the event-log sweep-liveness watchdog
|
|
38
|
-
* is not confused). A silently-stopped sweep is detectable from the event stream.
|
|
39
|
-
*
|
|
40
|
-
* Registered under slug `@teamscala/service-boot:sweep-host-cache`.
|
|
41
|
-
*/
|
|
42
|
-
import { rm, readdir, stat } from "node:fs/promises";
|
|
43
|
-
import { readdirSync, type Dirent } from "node:fs";
|
|
44
|
-
import { join } from "node:path";
|
|
45
|
-
import { spawnSync } from "@teamscala/os/spawn/spawn";
|
|
46
|
-
import { userHome, workspaceRoot } from "@teamscala/os/host-paths";
|
|
47
|
-
import { formatBytes } from "@teamscala/formatters/format-bytes";
|
|
48
|
-
import { recordEvent } from "@teamscala/event-log/record-event";
|
|
49
|
-
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
50
|
-
import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
|
|
51
|
-
|
|
52
|
-
const DURATION_RE = /^(\d+)\s+(day|week|month)s?$/;
|
|
53
|
-
const UNIT_TO_DAYS: Record<string, number> = {
|
|
54
|
-
day: 1,
|
|
55
|
-
week: 7,
|
|
56
|
-
month: 30,
|
|
57
|
-
};
|
|
58
|
-
const MS_PER_DAY = 86_400_000;
|
|
59
|
-
|
|
60
|
-
function parseDays(raw: string): number | null {
|
|
61
|
-
const match = DURATION_RE.exec(raw.trim());
|
|
62
|
-
if (!match) return null;
|
|
63
|
-
return Number(match[1]) * (UNIT_TO_DAYS[match[2]] ?? 0);
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
/** Resolve `${HOME}` / `${WORKSPACE}` placeholders against the live host. */
|
|
67
|
-
function resolveHostPath(raw: string): string {
|
|
68
|
-
return raw
|
|
69
|
-
.replaceAll("${HOME}", userHome())
|
|
70
|
-
.replaceAll("${WORKSPACE}", workspaceRoot());
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
interface SweepResult {
|
|
74
|
-
path: string;
|
|
75
|
-
scanned: number;
|
|
76
|
-
deleted: number;
|
|
77
|
-
bytesFreed: number;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Walk `rootDir` (stack-based, bounded memory) and delete regular files whose
|
|
82
|
-
* mtime is older than `cutoffMs`. Directories are traversed but not removed.
|
|
83
|
-
*/
|
|
84
|
-
async function sweepPath(
|
|
85
|
-
rootDir: string,
|
|
86
|
-
cutoffMs: number,
|
|
87
|
-
): Promise<SweepResult> {
|
|
88
|
-
const log = getAppLogger();
|
|
89
|
-
let scanned = 0;
|
|
90
|
-
let deleted = 0;
|
|
91
|
-
let bytesFreed = 0;
|
|
92
|
-
const stack: string[] = [rootDir];
|
|
93
|
-
|
|
94
|
-
while (stack.length > 0) {
|
|
95
|
-
const dir = stack.pop();
|
|
96
|
-
if (dir === undefined) break;
|
|
97
|
-
let entries: Dirent[];
|
|
98
|
-
try {
|
|
99
|
-
entries = await readdir(dir, { withFileTypes: true });
|
|
100
|
-
} catch {
|
|
101
|
-
continue; // unreadable / vanished — skip
|
|
102
|
-
}
|
|
103
|
-
for (const entry of entries) {
|
|
104
|
-
const full = join(dir, entry.name);
|
|
105
|
-
if (entry.isDirectory()) {
|
|
106
|
-
stack.push(full);
|
|
107
|
-
} else if (entry.isFile()) {
|
|
108
|
-
scanned++;
|
|
109
|
-
try {
|
|
110
|
-
const st = await stat(full);
|
|
111
|
-
if (st.mtimeMs < cutoffMs) {
|
|
112
|
-
bytesFreed += st.size;
|
|
113
|
-
await rm(full, { force: true });
|
|
114
|
-
deleted++;
|
|
115
|
-
}
|
|
116
|
-
} catch (error) {
|
|
117
|
-
log.warn(`[host-cache-sweep] could not process ${full}: ${String(error)}`);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
return { path: rootDir, scanned, deleted, bytesFreed };
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
/**
|
|
127
|
-
* Expand a path pattern containing single-segment `*` wildcards into concrete
|
|
128
|
-
* paths (files or dirs). Used by `retention_dirs` so a single config key like
|
|
129
|
-
* a pattern like `${WORKSPACE}/<star>/target` reaches every repo's target dir.
|
|
130
|
-
* Sync + bounded: the
|
|
131
|
-
* pattern is shallow (3 segments), and this runs once
|
|
132
|
-
* nightly. Non-existent segments yield no matches (the caller stats each result).
|
|
133
|
-
*/
|
|
134
|
-
export function expandGlob(pattern: string): string[] {
|
|
135
|
-
const absolute = pattern.startsWith("/");
|
|
136
|
-
const segments = (absolute ? pattern.slice(1) : pattern)
|
|
137
|
-
.split("/")
|
|
138
|
-
.filter((s) => s.length > 0);
|
|
139
|
-
let current: string[] = [absolute ? "/" : "."];
|
|
140
|
-
for (const seg of segments) {
|
|
141
|
-
if (current.length === 0) break;
|
|
142
|
-
const next: string[] = [];
|
|
143
|
-
for (const dir of current) {
|
|
144
|
-
let names: string[];
|
|
145
|
-
try {
|
|
146
|
-
names = readdirSync(dir);
|
|
147
|
-
} catch {
|
|
148
|
-
continue;
|
|
149
|
-
}
|
|
150
|
-
for (const name of names) {
|
|
151
|
-
if (seg === "*" || seg === name) next.push(join(dir, name));
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
current = next;
|
|
155
|
-
}
|
|
156
|
-
return current;
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
/**
|
|
160
|
-
* Is `dir` stale — i.e. NO file under it is newer than `cutoffMs`? Walks
|
|
161
|
-
* recursively and SHORT-CIRCUITS the instant it finds a fresh file (so an
|
|
162
|
-
* actively-built target dir returns fast without walking its full tree). On a
|
|
163
|
-
* stale dir the walk is complete and returns the total size + file count, so
|
|
164
|
-
* the caller can report bytes-freed before deleting the dir whole.
|
|
165
|
-
*/
|
|
166
|
-
export async function checkDirStale(
|
|
167
|
-
dir: string,
|
|
168
|
-
cutoffMs: number,
|
|
169
|
-
): Promise<{ stale: boolean; bytes: number; fileCount: number }> {
|
|
170
|
-
const stack: string[] = [dir];
|
|
171
|
-
let bytes = 0;
|
|
172
|
-
let fileCount = 0;
|
|
173
|
-
while (stack.length > 0) {
|
|
174
|
-
const d = stack.pop();
|
|
175
|
-
if (d === undefined) break;
|
|
176
|
-
let entries: Dirent[];
|
|
177
|
-
try {
|
|
178
|
-
entries = await readdir(d, { withFileTypes: true });
|
|
179
|
-
} catch {
|
|
180
|
-
continue;
|
|
181
|
-
}
|
|
182
|
-
for (const entry of entries) {
|
|
183
|
-
const full = join(d, entry.name);
|
|
184
|
-
if (entry.isDirectory()) {
|
|
185
|
-
stack.push(full);
|
|
186
|
-
} else if (entry.isFile()) {
|
|
187
|
-
try {
|
|
188
|
-
const st = await stat(full);
|
|
189
|
-
if (st.mtimeMs >= cutoffMs) {
|
|
190
|
-
return { stale: false, bytes: 0, fileCount: 0 };
|
|
191
|
-
}
|
|
192
|
-
bytes += st.size;
|
|
193
|
-
fileCount++;
|
|
194
|
-
} catch {
|
|
195
|
-
// vanished/unreadable — skip
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
return { stale: true, bytes, fileCount };
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
/** Parse the root-fs Use% from `df --output=pcent,target` output (pure, for tests).
|
|
204
|
-
* Returns null if no root-mount line is found. */
|
|
205
|
-
export function parseRootDiskPct(out: string): number | null {
|
|
206
|
-
const rootLine = out.split("\n").find((l) => l.trim().endsWith("/"));
|
|
207
|
-
const m = rootLine?.match(/(\d+)%/);
|
|
208
|
-
return m ? Number(m[1]) : null;
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
async function sweepHostCache(ctx: {
|
|
212
|
-
config: { retention?: Record<string, string>; retention_dirs?: Record<string, string> };
|
|
213
|
-
}): Promise<void> {
|
|
214
|
-
const log = getAppLogger();
|
|
215
|
-
const retention = ctx.config.retention ?? {};
|
|
216
|
-
const retentionDirs = ctx.config.retention_dirs ?? {};
|
|
217
|
-
const now = Date.now();
|
|
218
|
-
const results: SweepResult[] = [];
|
|
219
|
-
|
|
220
|
-
for (const [rawPath, rawDuration] of Object.entries(retention)) {
|
|
221
|
-
const days = parseDays(rawDuration);
|
|
222
|
-
if (days === null || days <= 0) {
|
|
223
|
-
log.warn(
|
|
224
|
-
`[host-cache-sweep] skipping "${rawPath}" — value "${rawDuration}" is not a day-granularity duration (expected e.g. "30 days")`,
|
|
225
|
-
);
|
|
226
|
-
continue;
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
const dir = resolveHostPath(rawPath);
|
|
230
|
-
try {
|
|
231
|
-
const st = await stat(dir);
|
|
232
|
-
if (!st.isDirectory()) {
|
|
233
|
-
log.warn(`[host-cache-sweep] skipping "${rawPath}" — resolved path is not a directory`);
|
|
234
|
-
continue;
|
|
235
|
-
}
|
|
236
|
-
} catch {
|
|
237
|
-
log.warn(`[host-cache-sweep] skipping "${rawPath}" — path does not exist or is unreadable`);
|
|
238
|
-
continue;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
const cutoffMs = now - days * MS_PER_DAY;
|
|
242
|
-
const result = await sweepPath(dir, cutoffMs);
|
|
243
|
-
results.push(result);
|
|
244
|
-
log.info(
|
|
245
|
-
`[host-cache-sweep] ${rawPath}: scanned ${result.scanned}, deleted ${result.deleted} file(s) (${formatBytes(result.bytesFreed)}) older than ${rawDuration}`,
|
|
246
|
-
);
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
// retention_dirs — whole-directory prune for build caches (Rust target/).
|
|
250
|
-
// A matched dir is deleted ENTIRELY only when its newest file is older than
|
|
251
|
-
// the cutoff, so an actively-built repo keeps its cache and an abandoned one
|
|
252
|
-
// is reclaimed whole (never the partial slice that would corrupt the cache).
|
|
253
|
-
for (const [rawPattern, rawDuration] of Object.entries(retentionDirs)) {
|
|
254
|
-
const days = parseDays(rawDuration);
|
|
255
|
-
if (days === null || days <= 0) {
|
|
256
|
-
log.warn(
|
|
257
|
-
`[host-cache-sweep] skipping retention_dirs "${rawPattern}" — value "${rawDuration}" is not a day-granularity duration (expected e.g. "30 days")`,
|
|
258
|
-
);
|
|
259
|
-
continue;
|
|
260
|
-
}
|
|
261
|
-
const cutoffMs = now - days * MS_PER_DAY;
|
|
262
|
-
for (const dir of expandGlob(resolveHostPath(rawPattern))) {
|
|
263
|
-
try {
|
|
264
|
-
const st = await stat(dir);
|
|
265
|
-
if (!st.isDirectory()) continue;
|
|
266
|
-
} catch {
|
|
267
|
-
continue; // glob match that is not a dir (or vanished) — skip
|
|
268
|
-
}
|
|
269
|
-
const { stale, bytes, fileCount } = await checkDirStale(dir, cutoffMs);
|
|
270
|
-
if (!stale) continue;
|
|
271
|
-
await rm(dir, { recursive: true, force: true });
|
|
272
|
-
results.push({ path: dir, scanned: fileCount, deleted: fileCount, bytesFreed: bytes });
|
|
273
|
-
log.info(
|
|
274
|
-
`[host-cache-sweep] ${dir}: no build in ${rawDuration}; reclaimed whole dir — ${fileCount} file(s), ${formatBytes(bytes)}`,
|
|
275
|
-
);
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
const totalDeleted = results.reduce((sum, r) => sum + r.deleted, 0);
|
|
280
|
-
const totalBytes = results.reduce((sum, r) => sum + r.bytesFreed, 0);
|
|
281
|
-
log.info(
|
|
282
|
-
`[host-cache-sweep] complete: ${results.length} target(s), ${totalDeleted} file(s) deleted, ${formatBytes(totalBytes)} reclaimed`,
|
|
283
|
-
);
|
|
284
|
-
|
|
285
|
-
// Liveness signal — distinct kind so the event-log sweep watchdog is unaffected.
|
|
286
|
-
recordEvent({
|
|
287
|
-
kind: "host.cache.swept",
|
|
288
|
-
payload: {
|
|
289
|
-
targets: results.map((r) => ({
|
|
290
|
-
path: r.path,
|
|
291
|
-
scanned: r.scanned,
|
|
292
|
-
deleted: r.deleted,
|
|
293
|
-
bytesFreed: r.bytesFreed,
|
|
294
|
-
})),
|
|
295
|
-
totalDeleted,
|
|
296
|
-
totalBytesFreed: totalBytes,
|
|
297
|
-
completedAt: new Date().toISOString(),
|
|
298
|
-
},
|
|
299
|
-
});
|
|
300
|
-
|
|
301
|
-
// Disk-threshold observability — rides this periodic sweep (NOT an interval:
|
|
302
|
-
// this job is genuinely periodic application-DATA work — deletion — and the
|
|
303
|
-
// df read piggybacks once per run). Emits through the event-driven surface:
|
|
304
|
-
// logger.warn -> GlitchTip (the paging path, same as host.pressure) +
|
|
305
|
-
// recordEvent for the durable event_log record. Catches a fill faster than
|
|
306
|
-
// the nightly prune can absorb (a cache with no retention entry, an external
|
|
307
|
-
// dump, a runaway log) so the operator learns it within a day, not at 100%.
|
|
308
|
-
const diskHighPct = 90;
|
|
309
|
-
try {
|
|
310
|
-
const dfRes = spawnSync({ name: "df:/", command: ["df", "--output=pcent,target"], timeoutMs: 5_000 });
|
|
311
|
-
const dfOut = (dfRes.stdout ?? "").trim();
|
|
312
|
-
const usePct = parseRootDiskPct(dfOut);
|
|
313
|
-
if (usePct !== null && usePct >= diskHighPct) {
|
|
314
|
-
log.warn("host.disk.high", { usePct, threshold: diskHighPct, df: dfOut });
|
|
315
|
-
recordEvent({
|
|
316
|
-
kind: "host.disk.high",
|
|
317
|
-
payload: { usePct, threshold: diskHighPct, df: dfOut, completedAt: new Date().toISOString() },
|
|
318
|
-
});
|
|
319
|
-
}
|
|
320
|
-
} catch (error) {
|
|
321
|
-
log.warn("[host-cache-sweep] disk-threshold read failed: " + String(error));
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
pgBossHandlerRegistry.register(
|
|
326
|
-
"@teamscala/service-boot:sweep-host-cache",
|
|
327
|
-
sweepHostCache,
|
|
328
|
-
);
|