@specific.dev/spectest 0.89.0 → 0.90.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/android.d.ts +84 -0
- package/dist/android.js +355 -0
- package/dist/components/android-runtime.mjs +257 -0
- package/dist/components/android.d.ts +74 -0
- package/dist/components/android.js +92 -0
- package/dist/components/index.d.ts +1 -0
- package/dist/components/index.js +1 -0
- package/dist/daemon.js +65 -88
- package/dist/harness/main.js +6 -1
- package/dist/harness/sticky-consumer.d.ts +28 -0
- package/dist/harness/sticky-consumer.js +178 -0
- package/dist/harness/sticky-maintenance.d.ts +1 -0
- package/dist/harness/sticky-maintenance.js +11 -0
- package/dist/harness/ws-bridge.d.ts +78 -0
- package/dist/harness/ws-bridge.js +180 -0
- package/dist/index.d.ts +4 -0
- package/dist/recorder.d.ts +1 -1
- package/dist/recorder.js +6 -3
- package/package.json +7 -3
- package/src/android.test.ts +180 -0
- package/src/android.ts +444 -0
- package/src/components/android-runtime.mjs +257 -0
- package/src/components/android.test.ts +68 -0
- package/src/components/android.ts +123 -0
- package/src/components/index.ts +2 -0
- package/src/daemon.ts +72 -89
- package/src/harness/main.ts +6 -1
- package/src/harness/sticky-consumer.test.ts +45 -0
- package/src/harness/sticky-consumer.ts +176 -0
- package/src/harness/sticky-maintenance.ts +7 -0
- package/src/harness/ws-bridge.test.ts +224 -0
- package/src/harness/ws-bridge.ts +210 -0
- package/src/index.ts +4 -0
- package/src/recorder.ts +3 -3
package/src/daemon.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { androidScope, androidRecordingChunk, releaseAndroidRecording, isAndroidApp, type AndroidRecording } from "./android.js";
|
|
1
2
|
// Long-running HTTP daemon. Runs as a systemd unit on the VM host (not in
|
|
2
3
|
// a container). Owns:
|
|
3
4
|
//
|
|
@@ -124,6 +125,13 @@ import {
|
|
|
124
125
|
corsPreflightResponse,
|
|
125
126
|
isCorsPreflight,
|
|
126
127
|
} from "./harness/http-proxy.js";
|
|
128
|
+
import {
|
|
129
|
+
WS_BRIDGE_HANDLERS,
|
|
130
|
+
openUpstreamWebSocket,
|
|
131
|
+
parseSubprotocols,
|
|
132
|
+
upstreamHandshakeHeaders,
|
|
133
|
+
type WsBridgeData,
|
|
134
|
+
} from "./harness/ws-bridge.js";
|
|
127
135
|
import {
|
|
128
136
|
certCovers as hostmatchCertCovers,
|
|
129
137
|
hostWithoutPort,
|
|
@@ -768,6 +776,11 @@ function ensureLocalBuildkitd(): Promise<boolean> {
|
|
|
768
776
|
}
|
|
769
777
|
|
|
770
778
|
async function bringUpLocalBuildkitd(): Promise<boolean> {
|
|
779
|
+
if (existsSync("/run/spectest-sticky/consumer.json")) {
|
|
780
|
+
const ready = await docker(["buildx", "inspect", "--bootstrap", LOCAL_BUILDER_NAME], 30_000);
|
|
781
|
+
if (ready.code !== 0) throw new Error(`sticky BuildKit unavailable: ${ready.stderr}`);
|
|
782
|
+
return true;
|
|
783
|
+
}
|
|
771
784
|
const paths = await imageCachePaths();
|
|
772
785
|
if (!paths) return false;
|
|
773
786
|
const state = paths.root;
|
|
@@ -2112,14 +2125,6 @@ async function bridgeGatewayIp(): Promise<string> {
|
|
|
2112
2125
|
* The shape is `harness/ingress-table.ts`'s, generic over that record. */
|
|
2113
2126
|
type Route = IngressRoute<FakeRuntime>;
|
|
2114
2127
|
|
|
2115
|
-
/** Per-WS-upgrade context: the upstream URL we'll open in `open()` and
|
|
2116
|
-
* keep the bridge in `message()`/`close()`. Stored on `ws.data`. */
|
|
2117
|
-
interface WsBridgeData {
|
|
2118
|
-
upstreamUrl: string;
|
|
2119
|
-
upstream: WebSocket | null;
|
|
2120
|
-
pending: Array<string | ArrayBuffer | Uint8Array>;
|
|
2121
|
-
}
|
|
2122
|
-
|
|
2123
2128
|
/**
|
|
2124
2129
|
* Pristine `fetch` captured at module load, before any test-scoped
|
|
2125
2130
|
* fetch wrapper can monkey-patch `globalThis.fetch`. The reverse-proxy
|
|
@@ -2440,8 +2445,8 @@ async function unbindRuntimeTls(hostname: string): Promise<void> {
|
|
|
2440
2445
|
* Spin up one Bun.serve listener bound to (port, optional TLS) that
|
|
2441
2446
|
* dispatches every request to the matching Route by Host header.
|
|
2442
2447
|
*
|
|
2443
|
-
* Shared by the HTTP and HTTPS branches. Also
|
|
2444
|
-
* handler so reverse-proxy targets can transparently bridge WS
|
|
2448
|
+
* Shared by the HTTP and HTTPS branches. Also carries the `websocket`
|
|
2449
|
+
* handler set so reverse-proxy targets can transparently bridge WS
|
|
2445
2450
|
* upgrades through to their upstream service.
|
|
2446
2451
|
*/
|
|
2447
2452
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -2485,74 +2490,9 @@ function bindIngressServer(
|
|
|
2485
2490
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
2486
2491
|
fetch: (req: Request, server: any): Response | Promise<Response> =>
|
|
2487
2492
|
dispatchIngress(req, server, byHost, listenerLabel, proto),
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
const data = ws.data as WsBridgeData;
|
|
2492
|
-
try {
|
|
2493
|
-
const upstream = new WebSocket(data.upstreamUrl);
|
|
2494
|
-
// ArrayBuffer so binary frames can be ws.send()'d to the
|
|
2495
|
-
// downstream client verbatim — Blob would need an extra
|
|
2496
|
-
// .arrayBuffer() round-trip on every message.
|
|
2497
|
-
upstream.binaryType = "arraybuffer";
|
|
2498
|
-
data.upstream = upstream;
|
|
2499
|
-
upstream.addEventListener("open", () => {
|
|
2500
|
-
for (const m of data.pending) upstream.send(m);
|
|
2501
|
-
data.pending = [];
|
|
2502
|
-
});
|
|
2503
|
-
upstream.addEventListener("message", (ev: MessageEvent) => {
|
|
2504
|
-
try {
|
|
2505
|
-
ws.send(ev.data);
|
|
2506
|
-
} catch {
|
|
2507
|
-
/* client gone */
|
|
2508
|
-
}
|
|
2509
|
-
});
|
|
2510
|
-
upstream.addEventListener("close", (ev: CloseEvent) => {
|
|
2511
|
-
try {
|
|
2512
|
-
ws.close(ev.code, ev.reason);
|
|
2513
|
-
} catch {
|
|
2514
|
-
/* already closed */
|
|
2515
|
-
}
|
|
2516
|
-
});
|
|
2517
|
-
upstream.addEventListener("error", () => {
|
|
2518
|
-
try {
|
|
2519
|
-
ws.close(1011, "upstream error");
|
|
2520
|
-
} catch {
|
|
2521
|
-
/* already closed */
|
|
2522
|
-
}
|
|
2523
|
-
});
|
|
2524
|
-
} catch (err) {
|
|
2525
|
-
// eslint-disable-next-line no-console
|
|
2526
|
-
console.warn(`[ingress] ws upstream open failed for ${data.upstreamUrl}:`, err);
|
|
2527
|
-
try {
|
|
2528
|
-
ws.close(1011, "upstream open failed");
|
|
2529
|
-
} catch {
|
|
2530
|
-
/* ignore */
|
|
2531
|
-
}
|
|
2532
|
-
}
|
|
2533
|
-
},
|
|
2534
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
2535
|
-
message(ws: any, message: string | Buffer) {
|
|
2536
|
-
const data = ws.data as WsBridgeData;
|
|
2537
|
-
const payload =
|
|
2538
|
-
typeof message === "string" ? message : new Uint8Array(message);
|
|
2539
|
-
if (data.upstream && data.upstream.readyState === WebSocket.OPEN) {
|
|
2540
|
-
data.upstream.send(payload);
|
|
2541
|
-
} else {
|
|
2542
|
-
// Buffer until the upstream finishes its handshake.
|
|
2543
|
-
data.pending.push(payload);
|
|
2544
|
-
}
|
|
2545
|
-
},
|
|
2546
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
2547
|
-
close(ws: any, code: number, reason: string) {
|
|
2548
|
-
const data = ws.data as WsBridgeData;
|
|
2549
|
-
try {
|
|
2550
|
-
data.upstream?.close(code, reason);
|
|
2551
|
-
} catch {
|
|
2552
|
-
/* ignore */
|
|
2553
|
-
}
|
|
2554
|
-
},
|
|
2555
|
-
},
|
|
2493
|
+
// The frame relay for bridged WebSocket upgrades; `proxyToService`
|
|
2494
|
+
// opens the upstream before it upgrades the client (harness/ws-bridge.ts).
|
|
2495
|
+
websocket: WS_BRIDGE_HANDLERS,
|
|
2556
2496
|
};
|
|
2557
2497
|
return Bun.serve(opts);
|
|
2558
2498
|
}
|
|
@@ -2657,8 +2597,12 @@ function ingressClientIp(server: any, req: Request, proto: string): string | und
|
|
|
2657
2597
|
* Reverse-proxy a request to `http://<service>:<port>` on
|
|
2658
2598
|
* `spectest-net`. Handles plain HTTP/1.1 + 2 and WebSocket upgrades:
|
|
2659
2599
|
*
|
|
2660
|
-
* - WS upgrade requests
|
|
2661
|
-
*
|
|
2600
|
+
* - WS upgrade requests open the upstream WebSocket first, with the
|
|
2601
|
+
* client's full `Sec-WebSocket-Protocol` list and its forwardable
|
|
2602
|
+
* headers (cookies, authorization, origin, x-forwarded-*). Only after
|
|
2603
|
+
* that handshake succeeds is the client upgraded, with the protocol
|
|
2604
|
+
* the upstream selected; a refused upstream handshake is a 502, not a
|
|
2605
|
+
* 101 followed by a close. The open upstream rides on `ws.data`; the shared `websocket`
|
|
2662
2606
|
* handler opens the upstream and bridges frames both ways.
|
|
2663
2607
|
* - Plain requests pass through via `fetch()` with hop-by-hop
|
|
2664
2608
|
* headers stripped; the response body is a ReadableStream returned
|
|
@@ -2688,16 +2632,40 @@ async function proxyToService(
|
|
|
2688
2632
|
const upgrade = req.headers.get("upgrade")?.toLowerCase() ?? "";
|
|
2689
2633
|
if (upgrade === "websocket") {
|
|
2690
2634
|
const upstreamUrl = `ws://${await proxyUpstreamHost(service)}:${port}${upstreamPath}`;
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2635
|
+
// The upstream handshake carries what the client's did: the whole
|
|
2636
|
+
// subprotocol list (a console ticket often rides as the second entry)
|
|
2637
|
+
// and the forwardable headers. Same provenance stamps as the HTTP path.
|
|
2638
|
+
const protocols = parseSubprotocols(req.headers.get("sec-websocket-protocol"));
|
|
2639
|
+
const headers = upstreamHandshakeHeaders(req.headers, {
|
|
2640
|
+
service,
|
|
2641
|
+
port,
|
|
2642
|
+
proto,
|
|
2643
|
+
clientIp: ingressClientIp(server, req, proto),
|
|
2644
|
+
});
|
|
2645
|
+
const opened = await openUpstreamWebSocket(upstreamUrl, { protocols, headers });
|
|
2646
|
+
if (!opened.ok) {
|
|
2647
|
+
return new Response(
|
|
2648
|
+
`spectest-daemon: ${opened.reason} (${upstreamUrl}) on ${listenerLabel}\n`,
|
|
2649
|
+
{ status: 502, headers: { "content-type": "text/plain" } },
|
|
2650
|
+
);
|
|
2651
|
+
}
|
|
2652
|
+
const wsData: WsBridgeData = { upstreamUrl, upstream: opened.socket };
|
|
2653
|
+
// Echo the protocol the upstream selected; Bun would otherwise pick
|
|
2654
|
+
// the client's first entry by itself, which can differ.
|
|
2655
|
+
const selected = opened.socket.protocol;
|
|
2656
|
+
const ok = server.upgrade(req, {
|
|
2657
|
+
data: wsData,
|
|
2658
|
+
headers: selected ? { "sec-websocket-protocol": selected } : undefined,
|
|
2659
|
+
});
|
|
2697
2660
|
if (ok) {
|
|
2698
2661
|
// Bun has already taken over the response — return a stub.
|
|
2699
2662
|
return new Response(null, { status: 101 });
|
|
2700
2663
|
}
|
|
2664
|
+
try {
|
|
2665
|
+
opened.socket.close(1011, "client upgrade refused");
|
|
2666
|
+
} catch {
|
|
2667
|
+
/* ignore */
|
|
2668
|
+
}
|
|
2701
2669
|
return new Response(
|
|
2702
2670
|
`spectest-daemon: ws upgrade refused on ${listenerLabel}\n`,
|
|
2703
2671
|
{ status: 426, headers: { "content-type": "text/plain" } },
|
|
@@ -3802,6 +3770,7 @@ async function runProjectSetupInner(): Promise<ProjectSetupResult> {
|
|
|
3802
3770
|
// ────────────────────────────────────────────────────────────────────────
|
|
3803
3771
|
|
|
3804
3772
|
interface RunResult {
|
|
3773
|
+
androidRecordings: AndroidRecording[];
|
|
3805
3774
|
status: "passed" | "failed";
|
|
3806
3775
|
durationMs: number;
|
|
3807
3776
|
log: string;
|
|
@@ -5328,6 +5297,8 @@ function captureConsole(chunks: LineSink): () => void {
|
|
|
5328
5297
|
|
|
5329
5298
|
async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
|
|
5330
5299
|
const start = Date.now();
|
|
5300
|
+
const nativeAndroid = androidScope(testCase.id, start);
|
|
5301
|
+
let androidRecordings: AndroidRecording[] = [];
|
|
5331
5302
|
const chunks: string[] = [];
|
|
5332
5303
|
const origStdout = process.stdout.write.bind(process.stdout);
|
|
5333
5304
|
const origStderr = process.stderr.write.bind(process.stderr);
|
|
@@ -5520,7 +5491,8 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
|
|
|
5520
5491
|
return term;
|
|
5521
5492
|
};
|
|
5522
5493
|
|
|
5523
|
-
|
|
5494
|
+
// Share the case clock with native video timestamps and other artifacts.
|
|
5495
|
+
startRecording(start);
|
|
5524
5496
|
const restoreFetch = installFetchWrapper();
|
|
5525
5497
|
const fetchScope = createInstrumentationScope();
|
|
5526
5498
|
|
|
@@ -5635,6 +5607,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
|
|
|
5635
5607
|
openTerminal: recordedOpenTerminal,
|
|
5636
5608
|
browser: trackedOpenBrowser,
|
|
5637
5609
|
mobile: trackedOpenMobile,
|
|
5610
|
+
android: nativeAndroid.open,
|
|
5638
5611
|
desktop: desktopOpener(start, testCase.id, sessions, browserDetaches),
|
|
5639
5612
|
mcp: openMcp,
|
|
5640
5613
|
testName: testCase.name,
|
|
@@ -5662,10 +5635,10 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
|
|
|
5662
5635
|
|
|
5663
5636
|
// Result events are gathered inside finally (after the final browser
|
|
5664
5637
|
// drains) so we hoist these out of the try/catch.
|
|
5665
|
-
let outcome: { status: "passed" | "failed"; error?: RunResult["error"] };
|
|
5638
|
+
let outcome: { status: "passed" | "failed"; error?: RunResult["error"] } = { status: "passed" };
|
|
5666
5639
|
try {
|
|
5667
5640
|
const value = await Promise.race([
|
|
5668
|
-
Promise.resolve(runInstrumented(fetchScope, () => testCase.run(ctx))),
|
|
5641
|
+
Promise.resolve(runInstrumented(fetchScope, async () => { await nativeAndroid.begin(Object.values(base.svc).filter(isAndroidApp)); return testCase.run(ctx); })),
|
|
5669
5642
|
timedOut,
|
|
5670
5643
|
]);
|
|
5671
5644
|
// Stash the return value so child cases — which fork from the snapshot
|
|
@@ -5680,6 +5653,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
|
|
|
5680
5653
|
};
|
|
5681
5654
|
} finally {
|
|
5682
5655
|
if (timer) clearTimeout(timer);
|
|
5656
|
+
nativeAndroid.deactivate();
|
|
5683
5657
|
RECORDING_EXEC = undefined;
|
|
5684
5658
|
INTERCEPTORS.endScope();
|
|
5685
5659
|
fetchScope.active = false;
|
|
@@ -5702,6 +5676,9 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
|
|
|
5702
5676
|
}
|
|
5703
5677
|
}
|
|
5704
5678
|
for (const s of sessions) s.markClosed();
|
|
5679
|
+
androidRecordings = await nativeAndroid.finish();
|
|
5680
|
+
const captureErrors = androidRecordings.flatMap(r => r.error ? [r.error] : []);
|
|
5681
|
+
if (captureErrors.length && outcome.status === "passed") outcome = { status: "failed", error: { message: captureErrors.join("\n") } };
|
|
5705
5682
|
}
|
|
5706
5683
|
const durationMs = Date.now() - start;
|
|
5707
5684
|
// Capture each service's log delta (the lines THIS case added beyond its
|
|
@@ -5757,6 +5734,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
|
|
|
5757
5734
|
}
|
|
5758
5735
|
return {
|
|
5759
5736
|
status: outcome.status,
|
|
5737
|
+
androidRecordings,
|
|
5760
5738
|
durationMs,
|
|
5761
5739
|
log: chunks.join(""),
|
|
5762
5740
|
events,
|
|
@@ -6032,6 +6010,7 @@ function explainEvalExportError(code: string, message: string): string {
|
|
|
6032
6010
|
|
|
6033
6011
|
async function evalCode(code: string): Promise<EvalResult> {
|
|
6034
6012
|
const start = Date.now();
|
|
6013
|
+
const nativeAndroid = androidScope("eval", start, { record: false });
|
|
6035
6014
|
const chunks: string[] = [];
|
|
6036
6015
|
const origStdout = process.stdout.write.bind(process.stdout);
|
|
6037
6016
|
const origStderr = process.stderr.write.bind(process.stderr);
|
|
@@ -6183,6 +6162,7 @@ async function evalCode(code: string): Promise<EvalResult> {
|
|
|
6183
6162
|
openTerminal: evalOpenTerminal,
|
|
6184
6163
|
browser: trackedOpenBrowser,
|
|
6185
6164
|
mobile: trackedOpenMobile,
|
|
6165
|
+
android: nativeAndroid.open,
|
|
6186
6166
|
desktop: desktopOpener(start, "eval", sessions, browserDetaches, artifactCollector),
|
|
6187
6167
|
mcp: openMcp,
|
|
6188
6168
|
testName: "eval",
|
|
@@ -6243,6 +6223,7 @@ async function evalCode(code: string): Promise<EvalResult> {
|
|
|
6243
6223
|
}
|
|
6244
6224
|
}
|
|
6245
6225
|
for (const s of sessions) s.markClosed();
|
|
6226
|
+
await nativeAndroid.finish();
|
|
6246
6227
|
if (filePath) {
|
|
6247
6228
|
fs.unlink(filePath).catch(() => {
|
|
6248
6229
|
/* best-effort cleanup */
|
|
@@ -6833,6 +6814,8 @@ export function harnessMethods(state: RouteState): MethodTable {
|
|
|
6833
6814
|
|
|
6834
6815
|
// One chunk of a parked replay bundle. Case ids are arbitrary user
|
|
6835
6816
|
// strings, so they travel in params rather than a URL path.
|
|
6817
|
+
androidRecordingChunk: async (params) => androidRecordingChunk(String(params.caseId ?? params.id), params.offset, params.limit),
|
|
6818
|
+
releaseAndroidRecording: async (params) => releaseAndroidRecording(String(params.id)),
|
|
6836
6819
|
replayChunk: async (params) => {
|
|
6837
6820
|
const caseId = requireString(params, "caseId", "caseId");
|
|
6838
6821
|
const gz = REPLAY_BUNDLES.get(caseId);
|
package/src/harness/main.ts
CHANGED
|
@@ -34,6 +34,7 @@
|
|
|
34
34
|
*/
|
|
35
35
|
|
|
36
36
|
import net from "node:net";
|
|
37
|
+
import { initializeStickyConsumer, STICKY_CAPABILITY, type StickyConsumerConfig } from "./sticky-consumer";
|
|
37
38
|
|
|
38
39
|
import { METHODS } from "../daemon";
|
|
39
40
|
import { codeOf } from "./methods";
|
|
@@ -50,7 +51,7 @@ import {
|
|
|
50
51
|
|
|
51
52
|
/** Capabilities this build announces. Empty is fine — the slot is what
|
|
52
53
|
* matters, so a later release has something to negotiate against. */
|
|
53
|
-
const CAPABILITIES: string[] = [];
|
|
54
|
+
const CAPABILITIES: string[] = [STICKY_CAPABILITY];
|
|
54
55
|
|
|
55
56
|
type Handler = (params: Record<string, unknown>) => Promise<unknown>;
|
|
56
57
|
|
|
@@ -65,6 +66,10 @@ export function handlers(
|
|
|
65
66
|
return {
|
|
66
67
|
// Everything the environment actually does.
|
|
67
68
|
...methods,
|
|
69
|
+
stickyInitialize: async (params) => {
|
|
70
|
+
await initializeStickyConsumer(params as unknown as StickyConsumerConfig);
|
|
71
|
+
return { initialized: true };
|
|
72
|
+
},
|
|
68
73
|
// Transport-level methods, which the method table has no business
|
|
69
74
|
// knowing about: they concern this connection, not this environment.
|
|
70
75
|
// Declared after the spread so the table can never shadow them — a
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { expect, test } from "bun:test";
|
|
2
|
+
import { STICKY_COMPATIBILITY, validateStickyConfig, collectStickyCache } from "./sticky-consumer";
|
|
3
|
+
|
|
4
|
+
const GiB = 1024 ** 3;
|
|
5
|
+
const config = { path: "/var/lib/spectest-sticky", capacityBytes: 256 * GiB,
|
|
6
|
+
headroomBytes: 32 * GiB, cacheTargetBytes: 64 * GiB, buildkitTargetBytes: 32 * GiB,
|
|
7
|
+
compatibility: STICKY_COMPATIBILITY };
|
|
8
|
+
|
|
9
|
+
test("sticky consumer validates independent retention and working budgets", () => {
|
|
10
|
+
expect(() => validateStickyConfig(config)).not.toThrow();
|
|
11
|
+
// Changing capacity does not require increasing either retention target.
|
|
12
|
+
expect(() => validateStickyConfig({ ...config, capacityBytes: 512 * GiB })).not.toThrow();
|
|
13
|
+
for (const override of [{ path: "/" }, { compatibility: "legacy" }, { headroomBytes: 0 },
|
|
14
|
+
{ headroomBytes: config.capacityBytes }, { capacityBytes: Number.NaN },
|
|
15
|
+
{ cacheTargetBytes: 240 * GiB }, { buildkitTargetBytes: 65 * GiB }, { cacheTargetBytes: Number.NaN }]) {
|
|
16
|
+
expect(() => validateStickyConfig({ ...config, ...override })).toThrow();
|
|
17
|
+
}
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test("filesystem pressure removes oldest Docker images when BuildKit pruning frees nothing", async () => {
|
|
21
|
+
let used = 100 * GiB;
|
|
22
|
+
const calls: string[][] = [];
|
|
23
|
+
const run = async (command: string, args: string[]) => {
|
|
24
|
+
calls.push([command, ...args]);
|
|
25
|
+
if (command === "docker" && args[1] === "ls") return "new\nold\n";
|
|
26
|
+
if (command === "docker" && args[1] === "inspect") return JSON.stringify([
|
|
27
|
+
{ Id: "new", Created: "2026-09-29T10:00:00Z" }, { Id: "old", Created: "2026-09-28T10:00:00Z" }]);
|
|
28
|
+
if (command === "docker" && args[1] === "rm") used -= 40 * GiB;
|
|
29
|
+
return ""; // Model Harmony: BuildKit's own GC reports no reclaimable data.
|
|
30
|
+
};
|
|
31
|
+
await collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: 256 * GiB - used }));
|
|
32
|
+
expect(calls.filter(c => c[2] === "rm")).toEqual([["docker", "image", "rm", "-f", "old"]]);
|
|
33
|
+
const prune = calls.find(c => c.includes("--free-storage"))!;
|
|
34
|
+
expect(prune[prune.indexOf("--keep-storage") + 1]).toBe(String(Math.floor(32 * GiB / 1e6)));
|
|
35
|
+
expect(prune[prune.indexOf("--free-storage") + 1]).toBe(String(Math.ceil(192 * GiB / 1e6)));
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test("healthy cache keeps recent images and a cache that cannot shrink is not published", async () => {
|
|
39
|
+
const calls: string[][] = [];
|
|
40
|
+
const run = async (command: string, args: string[]) => { calls.push([command, ...args]); return ""; };
|
|
41
|
+
await collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: 220 * GiB }));
|
|
42
|
+
expect(calls.some(c => c[2] === "rm" || c[2] === "inspect")).toBe(false);
|
|
43
|
+
await expect(collectStickyCache(config, run, async () => ({ totalBytes: 256 * GiB, freeBytes: 5 * GiB })))
|
|
44
|
+
.rejects.toThrow("cannot meet filesystem retention target");
|
|
45
|
+
});
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/** Docker/BuildKit policy for the generic sticky disk. No host publication API. */
|
|
2
|
+
import { execFile } from "node:child_process";
|
|
3
|
+
import { mkdir, readFile, statfs, writeFile } from "node:fs/promises";
|
|
4
|
+
|
|
5
|
+
export const STICKY_CAPABILITY = "sticky-disks-v1";
|
|
6
|
+
export const STICKY_COMPATIBILITY = "docker29-ctr21-buildkit032-overlayfs-256g-v2";
|
|
7
|
+
export const STICKY_RUNTIME = "/run/spectest-sticky";
|
|
8
|
+
export interface StickyConsumerConfig {
|
|
9
|
+
path: string;
|
|
10
|
+
capacityBytes: number;
|
|
11
|
+
headroomBytes: number;
|
|
12
|
+
cacheTargetBytes: number;
|
|
13
|
+
buildkitTargetBytes: number;
|
|
14
|
+
compatibility: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function exec(program: string, args: string[], timeout = 60_000): Promise<string> {
|
|
18
|
+
return new Promise((resolve, reject) => {
|
|
19
|
+
execFile(program, args, { timeout, maxBuffer: 8 * 1024 * 1024 }, (error, stdout, stderr) => {
|
|
20
|
+
if (error) reject(new Error(`${program} failed: ${stderr || stdout || error.message}`));
|
|
21
|
+
else resolve(stdout.trim());
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
async function ready(program: string, args: string[]): Promise<void> {
|
|
26
|
+
const deadline = Date.now() + 30_000;
|
|
27
|
+
let last: unknown;
|
|
28
|
+
do {
|
|
29
|
+
try { await exec(program, args, 2000); return; } catch (error) { last = error; }
|
|
30
|
+
await Bun.sleep(100);
|
|
31
|
+
} while (Date.now() < deadline);
|
|
32
|
+
throw last;
|
|
33
|
+
}
|
|
34
|
+
export function validateStickyConfig(config: StickyConsumerConfig): void {
|
|
35
|
+
if (config.path !== "/var/lib/spectest-sticky" || config.compatibility !== STICKY_COMPATIBILITY ||
|
|
36
|
+
!Number.isSafeInteger(config.capacityBytes) || config.capacityBytes <= 0 ||
|
|
37
|
+
!Number.isSafeInteger(config.headroomBytes) || config.headroomBytes < 16 * 1024 * 1024 ||
|
|
38
|
+
!Number.isSafeInteger(config.cacheTargetBytes) || config.cacheTargetBytes <= 0 ||
|
|
39
|
+
!Number.isSafeInteger(config.buildkitTargetBytes) || config.buildkitTargetBytes <= 0 ||
|
|
40
|
+
config.buildkitTargetBytes > config.cacheTargetBytes ||
|
|
41
|
+
config.cacheTargetBytes + config.headroomBytes >= config.capacityBytes) {
|
|
42
|
+
throw new Error("unsupported sticky consumer profile or budget");
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export async function initializeStickyConsumer(config: StickyConsumerConfig): Promise<void> {
|
|
47
|
+
validateStickyConfig(config);
|
|
48
|
+
await exec("mountpoint", ["-q", config.path]);
|
|
49
|
+
// The base must have stopped these before its RAM was captured. Refuse an
|
|
50
|
+
// unexpected writer rather than racing it for the standard daemon sockets.
|
|
51
|
+
for (const unit of ["docker.service", "docker.socket", "containerd.service"]) {
|
|
52
|
+
const state = await exec("systemctl", ["show", unit, "-p", "ActiveState", "--value"]);
|
|
53
|
+
if (!["inactive", "failed"].includes(state)) throw new Error(`sticky base has active legacy consumer: ${unit}: ${state}`);
|
|
54
|
+
}
|
|
55
|
+
for (const [program, version] of [["dockerd", "29.7.2"], ["containerd", "v2.1.5"], ["buildkitd", "v0.32.2"]]) {
|
|
56
|
+
if (!(await exec(program!, ["--version"])).includes(version!)) throw new Error(`unsupported ${program} version for sticky profile`);
|
|
57
|
+
}
|
|
58
|
+
await mkdir(STICKY_RUNTIME, { recursive: true });
|
|
59
|
+
for (const child of ["docker", "containerd", "buildkit"]) await mkdir(`${config.path}/${child}`, { recursive: true });
|
|
60
|
+
await writeFile(`${STICKY_RUNTIME}/consumer.json`, JSON.stringify(config));
|
|
61
|
+
await writeFile(`${STICKY_RUNTIME}/containerd.toml`, `version = 2
|
|
62
|
+
root = "${config.path}/containerd"
|
|
63
|
+
state = "/run/containerd"
|
|
64
|
+
disabled_plugins = ["io.containerd.grpc.v1.cri"]
|
|
65
|
+
[grpc]
|
|
66
|
+
address = "/run/containerd/containerd.sock"
|
|
67
|
+
`);
|
|
68
|
+
const dockerConfig = JSON.parse(await readFile("/etc/docker/daemon.json", "utf8"));
|
|
69
|
+
dockerConfig["data-root"] = `${config.path}/docker`;
|
|
70
|
+
dockerConfig.features = { "containerd-snapshotter": true };
|
|
71
|
+
dockerConfig["storage-driver"] = "overlayfs";
|
|
72
|
+
await writeFile(`${STICKY_RUNTIME}/docker.json`, JSON.stringify(dockerConfig));
|
|
73
|
+
await writeFile(`${STICKY_RUNTIME}/buildkit.toml`, `root = "${config.path}/buildkit"
|
|
74
|
+
[grpc]
|
|
75
|
+
address = ["tcp://127.0.0.1:1234"]
|
|
76
|
+
[worker.oci]
|
|
77
|
+
enabled = false
|
|
78
|
+
[worker.containerd]
|
|
79
|
+
enabled = true
|
|
80
|
+
address = "/run/containerd/containerd.sock"
|
|
81
|
+
namespace = "moby"
|
|
82
|
+
snapshotter = "overlayfs"
|
|
83
|
+
gc = true
|
|
84
|
+
reservedSpace = "1GB"
|
|
85
|
+
maxUsedSpace = ${config.buildkitTargetBytes}
|
|
86
|
+
minFreeSpace = ${config.headroomBytes}
|
|
87
|
+
`);
|
|
88
|
+
const start = async (unit: string, program: string, args: string[]) => {
|
|
89
|
+
await exec("systemd-run", ["--quiet", "--unit", unit, "-p", "TimeoutStopSec=30",
|
|
90
|
+
// Pinned BuildKit 0.32.2 returns 1 after its normal first-SIGTERM
|
|
91
|
+
// shutdown (util/appcontext). A timeout/signal still fails the unit.
|
|
92
|
+
...(unit === "spectest-sticky-buildkit" ? ["-p", "SuccessExitStatus=1"] : []),
|
|
93
|
+
program, ...args]);
|
|
94
|
+
};
|
|
95
|
+
await start("spectest-sticky-containerd", "containerd", ["--config", `${STICKY_RUNTIME}/containerd.toml`]);
|
|
96
|
+
await ready("ctr", ["version"]);
|
|
97
|
+
await start("spectest-sticky-docker", "dockerd", ["--config-file", `${STICKY_RUNTIME}/docker.json`,
|
|
98
|
+
"--host", "unix:///run/docker.sock", "--exec-root", "/run/docker", "--pidfile", "/run/docker.pid",
|
|
99
|
+
"--containerd", "/run/containerd/containerd.sock", "--containerd-namespace", "moby"]);
|
|
100
|
+
await ready("docker", ["info"]);
|
|
101
|
+
await start("spectest-sticky-buildkit", "buildkitd", ["--config", `${STICKY_RUNTIME}/buildkit.toml`]);
|
|
102
|
+
await ready("buildctl", ["--addr", "tcp://127.0.0.1:1234", "debug", "workers"]);
|
|
103
|
+
await exec("docker", ["buildx", "create", "--name", "spectest-local", "--driver", "remote", "tcp://127.0.0.1:1234"]);
|
|
104
|
+
await exec("docker", ["buildx", "inspect", "--bootstrap", "spectest-local"]);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Filesystem pressure takes precedence over cache age/BuildKit record sizes.
|
|
108
|
+
* Only called after runtime containers/volumes have been removed on the
|
|
109
|
+
* isolated publication clone. Never removes files behind the daemons' backs.
|
|
110
|
+
*/
|
|
111
|
+
export async function collectStickyCache(
|
|
112
|
+
config: StickyConsumerConfig,
|
|
113
|
+
run: typeof exec = exec,
|
|
114
|
+
usage = async (): Promise<{ freeBytes: number; totalBytes: number }> => {
|
|
115
|
+
const fs = await statfs(config.path);
|
|
116
|
+
return { freeBytes: fs.bavail * fs.bsize, totalBytes: fs.blocks * fs.bsize };
|
|
117
|
+
},
|
|
118
|
+
): Promise<void> {
|
|
119
|
+
validateStickyConfig(config);
|
|
120
|
+
const buildctl = ["--addr", "tcp://127.0.0.1:1234", "prune"];
|
|
121
|
+
await run("buildctl", [...buildctl, "--keep-duration", "168h"], 120_000);
|
|
122
|
+
await run("docker", ["image", "prune", "-af", "--filter", "until=168h"], 120_000);
|
|
123
|
+
const withinTarget = (fs: { freeBytes: number; totalBytes: number }) =>
|
|
124
|
+
fs.freeBytes >= config.headroomBytes && fs.totalBytes - fs.freeBytes <= config.cacheTargetBytes;
|
|
125
|
+
let fs = await usage();
|
|
126
|
+
// Always enforce the independent BuildKit target. Its accounting alone
|
|
127
|
+
// cannot bound shared images/content, so also request real filesystem space.
|
|
128
|
+
const pruneBuildkit = async () => {
|
|
129
|
+
await run("buildctl", [...buildctl, "--all", "--keep-storage",
|
|
130
|
+
String(Math.floor(config.buildkitTargetBytes / 1e6)), "--free-storage",
|
|
131
|
+
String(Math.ceil(Math.max(config.headroomBytes, fs.totalBytes - config.cacheTargetBytes) / 1e6))], 120_000);
|
|
132
|
+
fs = await usage();
|
|
133
|
+
};
|
|
134
|
+
await pruneBuildkit();
|
|
135
|
+
if (withinTarget(fs)) return;
|
|
136
|
+
// Pressure can be entirely in recent Docker images. Drop oldest images
|
|
137
|
+
// first, even within the age window, until the whole filesystem fits.
|
|
138
|
+
const ids = [...new Set((await run("docker", ["image", "ls", "-aq", "--no-trunc"])).split(/\s+/).filter(Boolean))];
|
|
139
|
+
const images: { Id: string; Created: string }[] = ids.length
|
|
140
|
+
? JSON.parse(await run("docker", ["image", "inspect", ...ids])) : [];
|
|
141
|
+
images.sort((a, b) => a.Created.localeCompare(b.Created) || a.Id.localeCompare(b.Id));
|
|
142
|
+
for (const image of images) {
|
|
143
|
+
await run("docker", ["image", "rm", "-f", image.Id], 120_000);
|
|
144
|
+
await pruneBuildkit();
|
|
145
|
+
if (withinTarget(fs)) return;
|
|
146
|
+
}
|
|
147
|
+
// Containerd GC may lag reference removal; allow bounded additional passes.
|
|
148
|
+
for (let pass = 0; pass < 3 && !withinTarget(fs); pass++) await pruneBuildkit();
|
|
149
|
+
if (!withinTarget(fs)) throw new Error("sticky cache cannot meet filesystem retention target and write headroom");
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Only on an isolated maintenance clone; never on the live ready environment. */
|
|
153
|
+
export async function prepareStickyConsumer(artifact = false): Promise<{ prepared: true; freeBytes: number }> {
|
|
154
|
+
const config: StickyConsumerConfig = JSON.parse(await readFile(`${STICKY_RUNTIME}/consumer.json`, "utf8"));
|
|
155
|
+
validateStickyConfig(config);
|
|
156
|
+
if (!artifact) {
|
|
157
|
+
const containers = (await exec("docker", ["ps", "-aq"])).split(/\s+/).filter(Boolean);
|
|
158
|
+
if (containers.length) await exec("docker", ["rm", "-f", ...containers]);
|
|
159
|
+
const volumes = (await exec("docker", ["volume", "ls", "-q"])).split(/\s+/).filter(Boolean);
|
|
160
|
+
if (volumes.length) await exec("docker", ["volume", "rm", ...volumes]);
|
|
161
|
+
await collectStickyCache(config);
|
|
162
|
+
}
|
|
163
|
+
const units = ["spectest-sticky-buildkit", "spectest-sticky-docker", "spectest-sticky-containerd"];
|
|
164
|
+
for (const unit of units) {
|
|
165
|
+
const active = await exec("systemctl", ["show", unit, "-p", "ActiveState", "--value"]);
|
|
166
|
+
if (active !== "active") throw new Error(`consumer exited before sealing: ${unit}: ${active}`);
|
|
167
|
+
await exec("systemctl", ["stop", unit], 45_000);
|
|
168
|
+
const result = await exec("systemctl", ["show", unit, "-p", "Result", "--value"]);
|
|
169
|
+
if (result !== "success") throw new Error(`unclean consumer shutdown: ${unit}: ${result}`);
|
|
170
|
+
}
|
|
171
|
+
const fs = await statfs(config.path);
|
|
172
|
+
const freeBytes = fs.bavail * fs.bsize;
|
|
173
|
+
if (freeBytes < config.headroomBytes) throw new Error("sticky publication has insufficient write headroom");
|
|
174
|
+
// The supervisor verifies and unmounts; only the worker seals/publishes.
|
|
175
|
+
return { prepared: true, freeBytes };
|
|
176
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// Dedicated entry point: no daemon.ts, project modules, bootstrap or tests.
|
|
2
|
+
import { prepareStickyConsumer } from "./sticky-consumer";
|
|
3
|
+
|
|
4
|
+
if (import.meta.main) {
|
|
5
|
+
try { console.log(JSON.stringify(await prepareStickyConsumer(process.argv.includes("--artifact")))); }
|
|
6
|
+
catch (error) { console.error(error); process.exitCode = 1; }
|
|
7
|
+
}
|