@specific.dev/spectest 0.75.0 → 0.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/daemon.js +36 -111
- package/dist/harness/buildkit-progress.d.ts +6 -5
- package/dist/harness/buildkit-progress.js +6 -5
- package/dist/harness/image-cache.d.ts +23 -22
- package/dist/harness/image-cache.js +23 -22
- package/package.json +1 -1
- package/src/daemon.ts +36 -114
- package/src/harness/buildkit-progress.ts +6 -5
- package/src/harness/image-cache.ts +23 -22
package/dist/daemon.js
CHANGED
|
@@ -35,7 +35,7 @@ import { isMobileApp, openPersistentMobile } from "./mobile.js";
|
|
|
35
35
|
import { buildArgFlags, buildContentKey as computeBuildContentKey, imageTag, isGeneratedDockerignore, serviceDockerignore as composeServiceDockerignore, unionDockerignore, } from "./harness/build-context.js";
|
|
36
36
|
import { validateServiceGraph as validateGraph } from "./harness/service-graph.js";
|
|
37
37
|
import { casesMetadata as catalogueCases, groupsMetadata as catalogueGroups, } from "./harness/catalogue.js";
|
|
38
|
-
import {
|
|
38
|
+
import { summarizeBuildKit } from "./harness/buildkit-progress.js";
|
|
39
39
|
import { LOG_DELTA_MAX_BYTES, capMiddle, streamDelta } from "./harness/log-delta.js";
|
|
40
40
|
import { resolveHostPath as resolveVolumeHostPath, sanitizeSegment, } from "./harness/volume-paths.js";
|
|
41
41
|
import { pollUntilReady } from "./harness/ready-poll.js";
|
|
@@ -50,7 +50,7 @@ import { certCovers as hostmatchCertCovers, hostWithoutPort, matchRoute, selectC
|
|
|
50
50
|
import { INGRESS_HTTPS_PORT, INGRESS_HTTP_PORT, bindRoute, clearTables, emptyTables, planBind, registryTarget, routesFor, unbindRoute, } from "./harness/ingress-table.js";
|
|
51
51
|
import { startTlsTerminator } from "./harness/tls-terminator.js";
|
|
52
52
|
import { runContainerArgs } from "./harness/container-run.js";
|
|
53
|
-
import { BUILDKIT_CACHE_DIR, BUILDKIT_CACHE_UNION,
|
|
53
|
+
import { BUILDKIT_CACHE_DIR, BUILDKIT_CACHE_UNION, IMAGE_CACHE_MANIFEST, imageCachePathsSync, isOnImageCache, mergeCacheIndex, } from "./harness/image-cache.js";
|
|
54
54
|
import { assertAbsolute, certificateHostnames, defaultKeyMode, expandServiceToken, isNoopChown, mountFlag, needsIdTables, numericId, resolveChownIds, } from "./harness/file-mounts.js";
|
|
55
55
|
import { conflict, notFound, requireString, } from "./harness/methods.js";
|
|
56
56
|
import { openTerminal } from "./terminal.js";
|
|
@@ -507,23 +507,14 @@ async function ensureLocalBuildkitd() {
|
|
|
507
507
|
}
|
|
508
508
|
// eslint-disable-next-line no-console
|
|
509
509
|
console.log(`[build] building in this VM against the image cache (root ${paths.root}, layers ${paths.layers})`);
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
else {
|
|
519
|
-
// Imports come from the merged cache on the read-only layers disk and
|
|
520
|
-
// from this lineage's own exports on the root; exports go to the root,
|
|
521
|
-
// where the merge picks them up (image_cache/merge.rs). Every cold
|
|
522
|
-
// build re-exports every blob this way, which is the cost the union
|
|
523
|
-
// removes.
|
|
524
|
-
_localCacheDir = `${state}/${BUILDKIT_CACHE_DIR}`;
|
|
525
|
-
_localCacheImports = [`${paths.layers}/${BUILDKIT_CACHE_DIR}`, `${state}/${BUILDKIT_CACHE_DIR}`];
|
|
526
|
-
}
|
|
510
|
+
// The cache directory union is for the project's OWN BuildKit (the
|
|
511
|
+
// `/services` docs page tells it to mount `BUILDKIT_CACHE_UNION`).
|
|
512
|
+
// The harness's builds no longer touch it: their cache is the
|
|
513
|
+
// daemon's own state, which lives on the root disk and rides the
|
|
514
|
+
// base root into the next cold build (CONTAINER_STORE.md §20), so a
|
|
515
|
+
// hit is a snapshot that already exists — no tar blob to unpack
|
|
516
|
+
// before the first uncached step, no export pass after a miss.
|
|
517
|
+
await ensureBuildkitCacheUnion(paths);
|
|
527
518
|
_localBuilder = true;
|
|
528
519
|
return true;
|
|
529
520
|
}
|
|
@@ -589,11 +580,6 @@ async function ensureBuildkitCacheUnion(paths) {
|
|
|
589
580
|
}
|
|
590
581
|
return BUILDKIT_CACHE_UNION;
|
|
591
582
|
}
|
|
592
|
-
/** The exported-cache directory on the cache disk, once the in-VM builder is up. */
|
|
593
|
-
let _localCacheDir = null;
|
|
594
|
-
/** The cache directories a build imports from: the merged one on the
|
|
595
|
-
* layers disk, then this lineage's own exports. */
|
|
596
|
-
let _localCacheImports = [];
|
|
597
583
|
/**
|
|
598
584
|
* A reference as containerd names it. buildx's `-t` on a remote builder
|
|
599
585
|
* stores an unqualified name (`probe:bx`) that dockerd then cannot
|
|
@@ -1092,62 +1078,31 @@ async function runServiceBuild(name, image, tag) {
|
|
|
1092
1078
|
const useBuildKit = useLocal || (await hasBuildx());
|
|
1093
1079
|
const buildEnv = {};
|
|
1094
1080
|
let buildArgs;
|
|
1095
|
-
// The `--cache-to` flag, held back for a second pass that runs only
|
|
1096
|
-
// when the first one executed a step (see below). Null off the in-VM
|
|
1097
|
-
// builder.
|
|
1098
|
-
let cacheExport = null;
|
|
1099
1081
|
// The user's `buildArgs`, as `--build-arg` flags; a plain client flag,
|
|
1100
|
-
// so every builder —
|
|
1082
|
+
// so every builder — in-VM BuildKit, dockerd's own, legacy — takes it.
|
|
1101
1083
|
const argFlags = buildArgFlags(image.buildArgs);
|
|
1102
|
-
if (useLocal
|
|
1084
|
+
if (useLocal) {
|
|
1103
1085
|
// The in-VM builder is BuildKit's containerd worker on this VM's
|
|
1104
1086
|
// own image store (CONTAINER_STORE.md): the output is an image
|
|
1105
1087
|
// record in dockerd's namespace, unpacked, so there is no `--load`
|
|
1106
|
-
// and nothing crosses a socket.
|
|
1107
|
-
//
|
|
1108
|
-
//
|
|
1109
|
-
// and
|
|
1110
|
-
//
|
|
1111
|
-
//
|
|
1112
|
-
//
|
|
1113
|
-
//
|
|
1114
|
-
//
|
|
1115
|
-
// is named. Specific's `nginx-mod` service tagged its own buildctl
|
|
1116
|
-
// cache `nginx-mod` and the two overwrote each other on alternate
|
|
1117
|
-
// boots (reported 2026-09-06). The prefix is reserved in the docs.
|
|
1118
|
-
const cacheTag = buildkitCacheTag(name);
|
|
1119
|
-
cacheExport = [
|
|
1120
|
-
"--cache-to",
|
|
1121
|
-
`type=local,dest=${_localCacheDir},mode=max,compression=uncompressed,force-compression=true,tag=${cacheTag}`,
|
|
1122
|
-
];
|
|
1123
|
-
// LANDMINE: the local cache importer reads the `latest` entry of
|
|
1124
|
-
// the directory's index unless told otherwise, and the exporter
|
|
1125
|
-
// below writes this service's entry under `tag=<service>`. An
|
|
1126
|
-
// import without the same tag misses every time and every
|
|
1127
|
-
// `RUN` re-executes on a seeded cache (seen on the first deploy,
|
|
1128
|
-
// 2026-09-05: the disk carried the blobs, the build used none).
|
|
1088
|
+
// and nothing crosses a socket. No `--cache-from` and no
|
|
1089
|
+
// `--cache-to` either, since 2026-09-07 (§20): the daemon's own
|
|
1090
|
+
// cache database is on the same disk as the snapshots it names
|
|
1091
|
+
// and rides the base root into the next cold build, so a cached
|
|
1092
|
+
// step is a snapshot that already exists. The exported directory
|
|
1093
|
+
// did the same job through tar blobs, at the price of unpacking
|
|
1094
|
+
// every cached parent before the first uncached step (11–79 s for
|
|
1095
|
+
// one `COPY` on Specific's dashboard) and a second full pass to
|
|
1096
|
+
// export after every miss.
|
|
1129
1097
|
buildArgs = [
|
|
1130
1098
|
"buildx", "build",
|
|
1131
1099
|
"--builder", LOCAL_BUILDER_NAME,
|
|
1132
1100
|
"--progress=plain",
|
|
1133
1101
|
"--output", `type=image,name=${qualifyImageRef(tag)},unpack=true`,
|
|
1134
|
-
..._localCacheImports.flatMap((src) => ["--cache-from", `type=local,src=${src},tag=${cacheTag}`]),
|
|
1135
1102
|
...argFlags,
|
|
1136
1103
|
"-f", dfPath, WORKSPACE,
|
|
1137
1104
|
];
|
|
1138
1105
|
}
|
|
1139
|
-
else if (useLocal) {
|
|
1140
|
-
// The daemon came up but reported no cache directory: build on it
|
|
1141
|
-
// and `--load` the result into dockerd.
|
|
1142
|
-
buildArgs = [
|
|
1143
|
-
"buildx", "build",
|
|
1144
|
-
"--builder", LOCAL_BUILDER_NAME,
|
|
1145
|
-
"--load",
|
|
1146
|
-
"--progress=plain",
|
|
1147
|
-
...argFlags,
|
|
1148
|
-
"-t", tag, "-f", dfPath, WORKSPACE,
|
|
1149
|
-
];
|
|
1150
|
-
}
|
|
1151
1106
|
else if (useBuildKit) {
|
|
1152
1107
|
buildArgs = ["build", ...argFlags, "-t", tag, "-f", dfPath, "--progress=plain", WORKSPACE];
|
|
1153
1108
|
buildEnv.DOCKER_BUILDKIT = "1";
|
|
@@ -1182,27 +1137,6 @@ async function runServiceBuild(name, image, tag) {
|
|
|
1182
1137
|
if (build.code !== 0) {
|
|
1183
1138
|
return { ok: false, log };
|
|
1184
1139
|
}
|
|
1185
|
-
// Export the cache only when there is something new to export. A
|
|
1186
|
-
// `mode=max` export costs ~3 s per service even when every blob is
|
|
1187
|
-
// already in the destination (measured 2026-09-06, api: 2.2–3.0 s
|
|
1188
|
-
// for the cached build alone, 4.9–7.5 s with the export), and a cold
|
|
1189
|
-
// start of a cached project is nothing but such builds. When a step
|
|
1190
|
-
// did run, the second pass is that same build fully cached from the
|
|
1191
|
-
// daemon's own state plus the export — seconds on top of a build
|
|
1192
|
-
// that took tens of them. Best-effort: the image is already in the
|
|
1193
|
-
// store, so a failed export is a slower next cold start, not a
|
|
1194
|
-
// failed service.
|
|
1195
|
-
if (cacheExport) {
|
|
1196
|
-
const ran = executedSteps(build.stderr);
|
|
1197
|
-
if (ran.length > 0) {
|
|
1198
|
-
progressService(name, { status: "building", detail: `exporting cache (${ran.length} step(s) ran)` });
|
|
1199
|
-
const exported = await shxStream("docker", [...buildArgs, ...cacheExport], 1_800_000, buildEnv, () => { });
|
|
1200
|
-
if (exported.code !== 0) {
|
|
1201
|
-
// eslint-disable-next-line no-console
|
|
1202
|
-
console.warn(`[build] ${name}: cache export failed; the next cold start rebuilds it:\n${exported.stderr.trim().slice(-2000)}`);
|
|
1203
|
-
}
|
|
1204
|
-
}
|
|
1205
|
-
}
|
|
1206
1140
|
if (useBuildKit) {
|
|
1207
1141
|
// Keep only the slowest dozen steps ≥1s — enough to profile, small
|
|
1208
1142
|
// enough to ride back in the /bootstrap response and the journal.
|
|
@@ -3023,31 +2957,22 @@ async function bootstrapInner() {
|
|
|
3023
2957
|
// image is pulled and whose deps are up starts immediately; it never sits
|
|
3024
2958
|
// at "image ready" waiting for an unrelated slow build elsewhere.
|
|
3025
2959
|
//
|
|
3026
|
-
// Prep concurrency:
|
|
3027
|
-
//
|
|
3028
|
-
//
|
|
3029
|
-
//
|
|
3030
|
-
//
|
|
3031
|
-
//
|
|
2960
|
+
// Prep concurrency: everything at once. Registry pulls are
|
|
2961
|
+
// network-bound and cheap in RAM. Dockerfile builds all go to the one
|
|
2962
|
+
// in-VM buildkitd, whose `max-parallelism` (all but one vCPU,
|
|
2963
|
+
// `base.rs::BUILDKITD_UP_SH`) caps the steps in flight across every
|
|
2964
|
+
// build it holds, and whose steps run under a cgroup with a memory
|
|
2965
|
+
// ceiling — so a fully cached build finishes at once while a miss is
|
|
2966
|
+
// still running, and two installs that fan out together kill a step,
|
|
2967
|
+
// never the VM. Until 2026-09-07 builds queued behind one FIFO chain
|
|
2968
|
+
// here, the harness's own guard against that OOM (each install fans
|
|
2969
|
+
// out to ~16 fetchers + lifecycle workers, ~70 MB/process): a service
|
|
2970
|
+
// whose build was entirely cached then waited the whole length of a
|
|
2971
|
+
// sibling's miss, 276 s for 4.5 s of work on Specific's `cli`.
|
|
3032
2972
|
const tags = new Map();
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
// waits for the previous to settle. Pulls and host-side builds bypass it.
|
|
3037
|
-
let inVmBuildChain = Promise.resolve();
|
|
3038
|
-
const prepImage = (svc) => {
|
|
3039
|
-
// Bootstrap is the only dedup scope: all its builds share one
|
|
3040
|
-
// /workspace generation (see prepareServiceImage).
|
|
3041
|
-
const run = () => prepareServiceImage(svc, { dedup: true });
|
|
3042
|
-
if (svc.image.type === "dockerfile" && !buildsRunHostSide) {
|
|
3043
|
-
const next = inVmBuildChain.then(run, run);
|
|
3044
|
-
// Keep the chain moving even if a build throws; the chain itself never
|
|
3045
|
-
// rejects (the per-service prep promise below is what surfaces errors).
|
|
3046
|
-
inVmBuildChain = next.then(() => undefined, () => undefined);
|
|
3047
|
-
return next;
|
|
3048
|
-
}
|
|
3049
|
-
return run();
|
|
3050
|
-
};
|
|
2973
|
+
// Bootstrap is the only dedup scope: all its builds share one
|
|
2974
|
+
// /workspace generation (see prepareServiceImage).
|
|
2975
|
+
const prepImage = (svc) => prepareServiceImage(svc, { dedup: true });
|
|
3051
2976
|
const prep = new Map();
|
|
3052
2977
|
for (const svc of services) {
|
|
3053
2978
|
const p = (async () => {
|
|
@@ -39,11 +39,12 @@ export declare function summarizeBuildKit(out: string): BuildStep[];
|
|
|
39
39
|
* The Dockerfile steps a build actually ran — the ones a cache export
|
|
40
40
|
* would have something new to record.
|
|
41
41
|
*
|
|
42
|
-
*
|
|
43
|
-
* export
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
42
|
+
* Between SDK 0.75.0 and 0.76.0 this gated the harness's cache export
|
|
43
|
+
* (a `mode=max` export cost ~3 s per service with nothing to write,
|
|
44
|
+
* measured 2026-09-06). The harness exports nothing since its cache is
|
|
45
|
+
* the daemon's own state (CONTAINER_STORE.md §20); the reading stays,
|
|
46
|
+
* since it is the one machine-readable answer to "did this build run
|
|
47
|
+
* anything".
|
|
47
48
|
*
|
|
48
49
|
* What counts: a bracketed stage step (`[builder 3/6] RUN …`) that ended
|
|
49
50
|
* in `DONE` rather than `CACHED`. What does not: BuildKit's own
|
|
@@ -68,11 +68,12 @@ export function summarizeBuildKit(out) {
|
|
|
68
68
|
* The Dockerfile steps a build actually ran — the ones a cache export
|
|
69
69
|
* would have something new to record.
|
|
70
70
|
*
|
|
71
|
-
*
|
|
72
|
-
* export
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
71
|
+
* Between SDK 0.75.0 and 0.76.0 this gated the harness's cache export
|
|
72
|
+
* (a `mode=max` export cost ~3 s per service with nothing to write,
|
|
73
|
+
* measured 2026-09-06). The harness exports nothing since its cache is
|
|
74
|
+
* the daemon's own state (CONTAINER_STORE.md §20); the reading stays,
|
|
75
|
+
* since it is the one machine-readable answer to "did this build run
|
|
76
|
+
* anything".
|
|
76
77
|
*
|
|
77
78
|
* What counts: a bracketed stage step (`[builder 3/6] RUN …`) that ended
|
|
78
79
|
* in `DONE` rather than `CACHED`. What does not: BuildKit's own
|
|
@@ -45,33 +45,34 @@ export declare const BUILDKIT_CACHE_DIR = "spectest-buildkit-cache";
|
|
|
45
45
|
* One directory that is both: an overlayfs with the layers disk's cache
|
|
46
46
|
* directory as the read-only lower and the root disk's as the upper,
|
|
47
47
|
* mounted here by the harness when the in-VM builder comes up
|
|
48
|
-
* (`daemon.ts::ensureBuildkitCacheUnion`).
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
48
|
+
* (`daemon.ts::ensureBuildkitCacheUnion`). It is for the PROJECT'S OWN
|
|
49
|
+
* BuildKit — the `/services` docs page tells it to mount this one path
|
|
50
|
+
* for both of its cache flags. BuildKit's local exporter skips a blob its
|
|
51
|
+
* destination already holds, and through the union every merged blob is
|
|
52
|
+
* already there, so a fully cached build exports its index and manifest
|
|
53
|
+
* and nothing else (a fresh directory on the root cost every
|
|
54
|
+
* uncompressed layer again, 1.1 GB and 20 s for one service, measured
|
|
55
|
+
* 2026-09-06). New layers land in the upper, which the merge reads.
|
|
56
|
+
*
|
|
57
|
+
* The harness's own service builds used it too until SDK 0.76.0; since
|
|
58
|
+
* then their cache is the in-VM daemon's own state, which lives on the
|
|
59
|
+
* root disk and rides the base root into the next cold build
|
|
60
|
+
* (CONTAINER_STORE.md §20).
|
|
57
61
|
*/
|
|
58
62
|
export declare const BUILDKIT_CACHE_UNION = "/var/lib/spectest-buildkit-cache";
|
|
59
63
|
/**
|
|
60
|
-
* The prefix
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
* `nginx-mod` running `buildctl` tagged its cache `nginx-mod`, and it and
|
|
68
|
-
* the harness's image build for that very container overwrote each other
|
|
69
|
-
* on alternate boots.
|
|
64
|
+
* The prefix the docs reserve for spectest in that directory's one flat
|
|
65
|
+
* tag namespace. The harness tagged its service builds under it from
|
|
66
|
+
* SDK 0.75.0 to 0.76.0, after Specific's `nginx-mod` service tagged its
|
|
67
|
+
* own `buildctl` cache `nginx-mod` and the two overwrote each other on
|
|
68
|
+
* alternate boots (2026-09-06); the harness writes no tags now, and the
|
|
69
|
+
* prefix stays reserved so nothing a project names can ever collide
|
|
70
|
+
* with something spectest writes there later.
|
|
70
71
|
*/
|
|
71
72
|
export declare const BUILDKIT_CACHE_TAG_PREFIX = "spectest/service/";
|
|
72
|
-
/** The
|
|
73
|
-
* the service name, lowercased with anything outside `[a-z0-9-]`
|
|
74
|
-
* to `-`. */
|
|
73
|
+
/** The tag a service's Dockerfile build would carry under the reserved
|
|
74
|
+
* prefix: the service name, lowercased with anything outside `[a-z0-9-]`
|
|
75
|
+
* folded to `-`. Unused by the harness since SDK 0.76.0. */
|
|
75
76
|
export declare function buildkitCacheTag(service: string): string;
|
|
76
77
|
/** The cache's paths, or `null` when this VM carries none.
|
|
77
78
|
* `SPECTEST_IMAGE_CACHE_MANIFEST` points a test at another file. */
|
|
@@ -39,33 +39,34 @@ export const BUILDKIT_CACHE_DIR = "spectest-buildkit-cache";
|
|
|
39
39
|
* One directory that is both: an overlayfs with the layers disk's cache
|
|
40
40
|
* directory as the read-only lower and the root disk's as the upper,
|
|
41
41
|
* mounted here by the harness when the in-VM builder comes up
|
|
42
|
-
* (`daemon.ts::ensureBuildkitCacheUnion`).
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
42
|
+
* (`daemon.ts::ensureBuildkitCacheUnion`). It is for the PROJECT'S OWN
|
|
43
|
+
* BuildKit — the `/services` docs page tells it to mount this one path
|
|
44
|
+
* for both of its cache flags. BuildKit's local exporter skips a blob its
|
|
45
|
+
* destination already holds, and through the union every merged blob is
|
|
46
|
+
* already there, so a fully cached build exports its index and manifest
|
|
47
|
+
* and nothing else (a fresh directory on the root cost every
|
|
48
|
+
* uncompressed layer again, 1.1 GB and 20 s for one service, measured
|
|
49
|
+
* 2026-09-06). New layers land in the upper, which the merge reads.
|
|
50
|
+
*
|
|
51
|
+
* The harness's own service builds used it too until SDK 0.76.0; since
|
|
52
|
+
* then their cache is the in-VM daemon's own state, which lives on the
|
|
53
|
+
* root disk and rides the base root into the next cold build
|
|
54
|
+
* (CONTAINER_STORE.md §20).
|
|
51
55
|
*/
|
|
52
56
|
export const BUILDKIT_CACHE_UNION = "/var/lib/spectest-buildkit-cache";
|
|
53
57
|
/**
|
|
54
|
-
* The prefix
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
* `nginx-mod` running `buildctl` tagged its cache `nginx-mod`, and it and
|
|
62
|
-
* the harness's image build for that very container overwrote each other
|
|
63
|
-
* on alternate boots.
|
|
58
|
+
* The prefix the docs reserve for spectest in that directory's one flat
|
|
59
|
+
* tag namespace. The harness tagged its service builds under it from
|
|
60
|
+
* SDK 0.75.0 to 0.76.0, after Specific's `nginx-mod` service tagged its
|
|
61
|
+
* own `buildctl` cache `nginx-mod` and the two overwrote each other on
|
|
62
|
+
* alternate boots (2026-09-06); the harness writes no tags now, and the
|
|
63
|
+
* prefix stays reserved so nothing a project names can ever collide
|
|
64
|
+
* with something spectest writes there later.
|
|
64
65
|
*/
|
|
65
66
|
export const BUILDKIT_CACHE_TAG_PREFIX = "spectest/service/";
|
|
66
|
-
/** The
|
|
67
|
-
* the service name, lowercased with anything outside `[a-z0-9-]`
|
|
68
|
-
* to `-`. */
|
|
67
|
+
/** The tag a service's Dockerfile build would carry under the reserved
|
|
68
|
+
* prefix: the service name, lowercased with anything outside `[a-z0-9-]`
|
|
69
|
+
* folded to `-`. Unused by the harness since SDK 0.76.0. */
|
|
69
70
|
export function buildkitCacheTag(service) {
|
|
70
71
|
return `${BUILDKIT_CACHE_TAG_PREFIX}${service.replace(/[^a-z0-9-]/gi, "-").toLowerCase()}`;
|
|
71
72
|
}
|
package/package.json
CHANGED
package/src/daemon.ts
CHANGED
|
@@ -83,7 +83,7 @@ import {
|
|
|
83
83
|
type CaseMeta,
|
|
84
84
|
type GroupMeta,
|
|
85
85
|
} from "./harness/catalogue.js";
|
|
86
|
-
import {
|
|
86
|
+
import { summarizeBuildKit, type BuildStep } from "./harness/buildkit-progress.js";
|
|
87
87
|
import { LOG_DELTA_MAX_BYTES, capMiddle, streamDelta } from "./harness/log-delta.js";
|
|
88
88
|
import {
|
|
89
89
|
resolveHostPath as resolveVolumeHostPath,
|
|
@@ -138,7 +138,6 @@ import { runContainerArgs } from "./harness/container-run.js";
|
|
|
138
138
|
import {
|
|
139
139
|
BUILDKIT_CACHE_DIR,
|
|
140
140
|
BUILDKIT_CACHE_UNION,
|
|
141
|
-
buildkitCacheTag,
|
|
142
141
|
IMAGE_CACHE_MANIFEST,
|
|
143
142
|
imageCachePathsSync,
|
|
144
143
|
isOnImageCache,
|
|
@@ -788,22 +787,14 @@ async function ensureLocalBuildkitd(): Promise<boolean> {
|
|
|
788
787
|
}
|
|
789
788
|
// eslint-disable-next-line no-console
|
|
790
789
|
console.log(`[build] building in this VM against the image cache (root ${paths.root}, layers ${paths.layers})`);
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
// Imports come from the merged cache on the read-only layers disk and
|
|
800
|
-
// from this lineage's own exports on the root; exports go to the root,
|
|
801
|
-
// where the merge picks them up (image_cache/merge.rs). Every cold
|
|
802
|
-
// build re-exports every blob this way, which is the cost the union
|
|
803
|
-
// removes.
|
|
804
|
-
_localCacheDir = `${state}/${BUILDKIT_CACHE_DIR}`;
|
|
805
|
-
_localCacheImports = [`${paths.layers}/${BUILDKIT_CACHE_DIR}`, `${state}/${BUILDKIT_CACHE_DIR}`];
|
|
806
|
-
}
|
|
790
|
+
// The cache directory union is for the project's OWN BuildKit (the
|
|
791
|
+
// `/services` docs page tells it to mount `BUILDKIT_CACHE_UNION`).
|
|
792
|
+
// The harness's builds no longer touch it: their cache is the
|
|
793
|
+
// daemon's own state, which lives on the root disk and rides the
|
|
794
|
+
// base root into the next cold build (CONTAINER_STORE.md §20), so a
|
|
795
|
+
// hit is a snapshot that already exists — no tar blob to unpack
|
|
796
|
+
// before the first uncached step, no export pass after a miss.
|
|
797
|
+
await ensureBuildkitCacheUnion(paths);
|
|
807
798
|
_localBuilder = true;
|
|
808
799
|
return true;
|
|
809
800
|
}
|
|
@@ -875,11 +866,6 @@ async function ensureBuildkitCacheUnion(paths: ImageCachePaths): Promise<string
|
|
|
875
866
|
}
|
|
876
867
|
return BUILDKIT_CACHE_UNION;
|
|
877
868
|
}
|
|
878
|
-
/** The exported-cache directory on the cache disk, once the in-VM builder is up. */
|
|
879
|
-
let _localCacheDir: string | null = null;
|
|
880
|
-
/** The cache directories a build imports from: the merged one on the
|
|
881
|
-
* layers disk, then this lineage's own exports. */
|
|
882
|
-
let _localCacheImports: string[] = [];
|
|
883
869
|
/**
|
|
884
870
|
* A reference as containerd names it. buildx's `-t` on a remote builder
|
|
885
871
|
* stores an unqualified name (`probe:bx`) that dockerd then cannot
|
|
@@ -1442,60 +1428,30 @@ async function runServiceBuild(
|
|
|
1442
1428
|
const useBuildKit = useLocal || (await hasBuildx());
|
|
1443
1429
|
const buildEnv: Record<string, string> = {};
|
|
1444
1430
|
let buildArgs: string[];
|
|
1445
|
-
// The `--cache-to` flag, held back for a second pass that runs only
|
|
1446
|
-
// when the first one executed a step (see below). Null off the in-VM
|
|
1447
|
-
// builder.
|
|
1448
|
-
let cacheExport: string[] | null = null;
|
|
1449
1431
|
// The user's `buildArgs`, as `--build-arg` flags; a plain client flag,
|
|
1450
|
-
// so every builder —
|
|
1432
|
+
// so every builder — in-VM BuildKit, dockerd's own, legacy — takes it.
|
|
1451
1433
|
const argFlags = buildArgFlags(image.buildArgs);
|
|
1452
|
-
if (useLocal
|
|
1434
|
+
if (useLocal) {
|
|
1453
1435
|
// The in-VM builder is BuildKit's containerd worker on this VM's
|
|
1454
1436
|
// own image store (CONTAINER_STORE.md): the output is an image
|
|
1455
1437
|
// record in dockerd's namespace, unpacked, so there is no `--load`
|
|
1456
|
-
// and nothing crosses a socket.
|
|
1457
|
-
//
|
|
1458
|
-
//
|
|
1459
|
-
// and
|
|
1460
|
-
//
|
|
1461
|
-
//
|
|
1462
|
-
//
|
|
1463
|
-
//
|
|
1464
|
-
//
|
|
1465
|
-
// is named. Specific's `nginx-mod` service tagged its own buildctl
|
|
1466
|
-
// cache `nginx-mod` and the two overwrote each other on alternate
|
|
1467
|
-
// boots (reported 2026-09-06). The prefix is reserved in the docs.
|
|
1468
|
-
const cacheTag = buildkitCacheTag(name);
|
|
1469
|
-
cacheExport = [
|
|
1470
|
-
"--cache-to",
|
|
1471
|
-
`type=local,dest=${_localCacheDir},mode=max,compression=uncompressed,force-compression=true,tag=${cacheTag}`,
|
|
1472
|
-
];
|
|
1473
|
-
// LANDMINE: the local cache importer reads the `latest` entry of
|
|
1474
|
-
// the directory's index unless told otherwise, and the exporter
|
|
1475
|
-
// below writes this service's entry under `tag=<service>`. An
|
|
1476
|
-
// import without the same tag misses every time and every
|
|
1477
|
-
// `RUN` re-executes on a seeded cache (seen on the first deploy,
|
|
1478
|
-
// 2026-09-05: the disk carried the blobs, the build used none).
|
|
1438
|
+
// and nothing crosses a socket. No `--cache-from` and no
|
|
1439
|
+
// `--cache-to` either, since 2026-09-07 (§20): the daemon's own
|
|
1440
|
+
// cache database is on the same disk as the snapshots it names
|
|
1441
|
+
// and rides the base root into the next cold build, so a cached
|
|
1442
|
+
// step is a snapshot that already exists. The exported directory
|
|
1443
|
+
// did the same job through tar blobs, at the price of unpacking
|
|
1444
|
+
// every cached parent before the first uncached step (11–79 s for
|
|
1445
|
+
// one `COPY` on Specific's dashboard) and a second full pass to
|
|
1446
|
+
// export after every miss.
|
|
1479
1447
|
buildArgs = [
|
|
1480
1448
|
"buildx", "build",
|
|
1481
1449
|
"--builder", LOCAL_BUILDER_NAME,
|
|
1482
1450
|
"--progress=plain",
|
|
1483
1451
|
"--output", `type=image,name=${qualifyImageRef(tag)},unpack=true`,
|
|
1484
|
-
..._localCacheImports.flatMap((src) => ["--cache-from", `type=local,src=${src},tag=${cacheTag}`]),
|
|
1485
1452
|
...argFlags,
|
|
1486
1453
|
"-f", dfPath, WORKSPACE,
|
|
1487
1454
|
];
|
|
1488
|
-
} else if (useLocal) {
|
|
1489
|
-
// The daemon came up but reported no cache directory: build on it
|
|
1490
|
-
// and `--load` the result into dockerd.
|
|
1491
|
-
buildArgs = [
|
|
1492
|
-
"buildx", "build",
|
|
1493
|
-
"--builder", LOCAL_BUILDER_NAME,
|
|
1494
|
-
"--load",
|
|
1495
|
-
"--progress=plain",
|
|
1496
|
-
...argFlags,
|
|
1497
|
-
"-t", tag, "-f", dfPath, WORKSPACE,
|
|
1498
|
-
];
|
|
1499
1455
|
} else if (useBuildKit) {
|
|
1500
1456
|
buildArgs = ["build", ...argFlags, "-t", tag, "-f", dfPath, "--progress=plain", WORKSPACE];
|
|
1501
1457
|
buildEnv.DOCKER_BUILDKIT = "1";
|
|
@@ -1529,27 +1485,6 @@ async function runServiceBuild(
|
|
|
1529
1485
|
if (build.code !== 0) {
|
|
1530
1486
|
return { ok: false, log };
|
|
1531
1487
|
}
|
|
1532
|
-
// Export the cache only when there is something new to export. A
|
|
1533
|
-
// `mode=max` export costs ~3 s per service even when every blob is
|
|
1534
|
-
// already in the destination (measured 2026-09-06, api: 2.2–3.0 s
|
|
1535
|
-
// for the cached build alone, 4.9–7.5 s with the export), and a cold
|
|
1536
|
-
// start of a cached project is nothing but such builds. When a step
|
|
1537
|
-
// did run, the second pass is that same build fully cached from the
|
|
1538
|
-
// daemon's own state plus the export — seconds on top of a build
|
|
1539
|
-
// that took tens of them. Best-effort: the image is already in the
|
|
1540
|
-
// store, so a failed export is a slower next cold start, not a
|
|
1541
|
-
// failed service.
|
|
1542
|
-
if (cacheExport) {
|
|
1543
|
-
const ran = executedSteps(build.stderr);
|
|
1544
|
-
if (ran.length > 0) {
|
|
1545
|
-
progressService(name, { status: "building", detail: `exporting cache (${ran.length} step(s) ran)` });
|
|
1546
|
-
const exported = await shxStream("docker", [...buildArgs, ...cacheExport], 1_800_000, buildEnv, () => {});
|
|
1547
|
-
if (exported.code !== 0) {
|
|
1548
|
-
// eslint-disable-next-line no-console
|
|
1549
|
-
console.warn(`[build] ${name}: cache export failed; the next cold start rebuilds it:\n${exported.stderr.trim().slice(-2000)}`);
|
|
1550
|
-
}
|
|
1551
|
-
}
|
|
1552
|
-
}
|
|
1553
1488
|
if (useBuildKit) {
|
|
1554
1489
|
// Keep only the slowest dozen steps ≥1s — enough to profile, small
|
|
1555
1490
|
// enough to ride back in the /bootstrap response and the journal.
|
|
@@ -3586,36 +3521,23 @@ async function bootstrapInner(): Promise<BootstrapTimings> {
|
|
|
3586
3521
|
// image is pulled and whose deps are up starts immediately; it never sits
|
|
3587
3522
|
// at "image ready" waiting for an unrelated slow build elsewhere.
|
|
3588
3523
|
//
|
|
3589
|
-
// Prep concurrency:
|
|
3590
|
-
//
|
|
3591
|
-
//
|
|
3592
|
-
//
|
|
3593
|
-
//
|
|
3594
|
-
//
|
|
3524
|
+
// Prep concurrency: everything at once. Registry pulls are
|
|
3525
|
+
// network-bound and cheap in RAM. Dockerfile builds all go to the one
|
|
3526
|
+
// in-VM buildkitd, whose `max-parallelism` (all but one vCPU,
|
|
3527
|
+
// `base.rs::BUILDKITD_UP_SH`) caps the steps in flight across every
|
|
3528
|
+
// build it holds, and whose steps run under a cgroup with a memory
|
|
3529
|
+
// ceiling — so a fully cached build finishes at once while a miss is
|
|
3530
|
+
// still running, and two installs that fan out together kill a step,
|
|
3531
|
+
// never the VM. Until 2026-09-07 builds queued behind one FIFO chain
|
|
3532
|
+
// here, the harness's own guard against that OOM (each install fans
|
|
3533
|
+
// out to ~16 fetchers + lifecycle workers, ~70 MB/process): a service
|
|
3534
|
+
// whose build was entirely cached then waited the whole length of a
|
|
3535
|
+
// sibling's miss, 276 s for 4.5 s of work on Specific's `cli`.
|
|
3595
3536
|
const tags = new Map<string, string>();
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
3599
|
-
|
|
3600
|
-
let inVmBuildChain: Promise<unknown> = Promise.resolve();
|
|
3601
|
-
const prepImage = (
|
|
3602
|
-
svc: NamedService,
|
|
3603
|
-
): Promise<{ tag: string; buildSteps?: BuildStep[] }> => {
|
|
3604
|
-
// Bootstrap is the only dedup scope: all its builds share one
|
|
3605
|
-
// /workspace generation (see prepareServiceImage).
|
|
3606
|
-
const run = () => prepareServiceImage(svc, { dedup: true });
|
|
3607
|
-
if (svc.image.type === "dockerfile" && !buildsRunHostSide) {
|
|
3608
|
-
const next = inVmBuildChain.then(run, run);
|
|
3609
|
-
// Keep the chain moving even if a build throws; the chain itself never
|
|
3610
|
-
// rejects (the per-service prep promise below is what surfaces errors).
|
|
3611
|
-
inVmBuildChain = next.then(
|
|
3612
|
-
() => undefined,
|
|
3613
|
-
() => undefined,
|
|
3614
|
-
);
|
|
3615
|
-
return next;
|
|
3616
|
-
}
|
|
3617
|
-
return run();
|
|
3618
|
-
};
|
|
3537
|
+
// Bootstrap is the only dedup scope: all its builds share one
|
|
3538
|
+
// /workspace generation (see prepareServiceImage).
|
|
3539
|
+
const prepImage = (svc: NamedService): Promise<{ tag: string; buildSteps?: BuildStep[] }> =>
|
|
3540
|
+
prepareServiceImage(svc, { dedup: true });
|
|
3619
3541
|
const prep = new Map<string, Promise<void>>();
|
|
3620
3542
|
for (const svc of services) {
|
|
3621
3543
|
const p = (async () => {
|
|
@@ -77,11 +77,12 @@ export function summarizeBuildKit(out: string): BuildStep[] {
|
|
|
77
77
|
* The Dockerfile steps a build actually ran — the ones a cache export
|
|
78
78
|
* would have something new to record.
|
|
79
79
|
*
|
|
80
|
-
*
|
|
81
|
-
* export
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
80
|
+
* Between SDK 0.75.0 and 0.76.0 this gated the harness's cache export
|
|
81
|
+
* (a `mode=max` export cost ~3 s per service with nothing to write,
|
|
82
|
+
* measured 2026-09-06). The harness exports nothing since its cache is
|
|
83
|
+
* the daemon's own state (CONTAINER_STORE.md §20); the reading stays,
|
|
84
|
+
* since it is the one machine-readable answer to "did this build run
|
|
85
|
+
* anything".
|
|
85
86
|
*
|
|
86
87
|
* What counts: a bracketed stage step (`[builder 3/6] RUN …`) that ended
|
|
87
88
|
* in `DONE` rather than `CACHED`. What does not: BuildKit's own
|
|
@@ -54,35 +54,36 @@ export const BUILDKIT_CACHE_DIR = "spectest-buildkit-cache";
|
|
|
54
54
|
* One directory that is both: an overlayfs with the layers disk's cache
|
|
55
55
|
* directory as the read-only lower and the root disk's as the upper,
|
|
56
56
|
* mounted here by the harness when the in-VM builder comes up
|
|
57
|
-
* (`daemon.ts::ensureBuildkitCacheUnion`).
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
57
|
+
* (`daemon.ts::ensureBuildkitCacheUnion`). It is for the PROJECT'S OWN
|
|
58
|
+
* BuildKit — the `/services` docs page tells it to mount this one path
|
|
59
|
+
* for both of its cache flags. BuildKit's local exporter skips a blob its
|
|
60
|
+
* destination already holds, and through the union every merged blob is
|
|
61
|
+
* already there, so a fully cached build exports its index and manifest
|
|
62
|
+
* and nothing else (a fresh directory on the root cost every
|
|
63
|
+
* uncompressed layer again, 1.1 GB and 20 s for one service, measured
|
|
64
|
+
* 2026-09-06). New layers land in the upper, which the merge reads.
|
|
65
|
+
*
|
|
66
|
+
* The harness's own service builds used it too until SDK 0.76.0; since
|
|
67
|
+
* then their cache is the in-VM daemon's own state, which lives on the
|
|
68
|
+
* root disk and rides the base root into the next cold build
|
|
69
|
+
* (CONTAINER_STORE.md §20).
|
|
66
70
|
*/
|
|
67
71
|
export const BUILDKIT_CACHE_UNION = "/var/lib/spectest-buildkit-cache";
|
|
68
72
|
|
|
69
73
|
/**
|
|
70
|
-
* The prefix
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
* `nginx-mod` running `buildctl` tagged its cache `nginx-mod`, and it and
|
|
78
|
-
* the harness's image build for that very container overwrote each other
|
|
79
|
-
* on alternate boots.
|
|
74
|
+
* The prefix the docs reserve for spectest in that directory's one flat
|
|
75
|
+
* tag namespace. The harness tagged its service builds under it from
|
|
76
|
+
* SDK 0.75.0 to 0.76.0, after Specific's `nginx-mod` service tagged its
|
|
77
|
+
* own `buildctl` cache `nginx-mod` and the two overwrote each other on
|
|
78
|
+
* alternate boots (2026-09-06); the harness writes no tags now, and the
|
|
79
|
+
* prefix stays reserved so nothing a project names can ever collide
|
|
80
|
+
* with something spectest writes there later.
|
|
80
81
|
*/
|
|
81
82
|
export const BUILDKIT_CACHE_TAG_PREFIX = "spectest/service/";
|
|
82
83
|
|
|
83
|
-
/** The
|
|
84
|
-
* the service name, lowercased with anything outside `[a-z0-9-]`
|
|
85
|
-
* to `-`. */
|
|
84
|
+
/** The tag a service's Dockerfile build would carry under the reserved
|
|
85
|
+
* prefix: the service name, lowercased with anything outside `[a-z0-9-]`
|
|
86
|
+
* folded to `-`. Unused by the harness since SDK 0.76.0. */
|
|
86
87
|
export function buildkitCacheTag(service: string): string {
|
|
87
88
|
return `${BUILDKIT_CACHE_TAG_PREFIX}${service.replace(/[^a-z0-9-]/gi, "-").toLowerCase()}`;
|
|
88
89
|
}
|