@specific.dev/spectest 0.75.0 → 0.76.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/daemon.js CHANGED
@@ -35,7 +35,7 @@ import { isMobileApp, openPersistentMobile } from "./mobile.js";
35
35
  import { buildArgFlags, buildContentKey as computeBuildContentKey, imageTag, isGeneratedDockerignore, serviceDockerignore as composeServiceDockerignore, unionDockerignore, } from "./harness/build-context.js";
36
36
  import { validateServiceGraph as validateGraph } from "./harness/service-graph.js";
37
37
  import { casesMetadata as catalogueCases, groupsMetadata as catalogueGroups, } from "./harness/catalogue.js";
38
- import { executedSteps, summarizeBuildKit } from "./harness/buildkit-progress.js";
38
+ import { summarizeBuildKit } from "./harness/buildkit-progress.js";
39
39
  import { LOG_DELTA_MAX_BYTES, capMiddle, streamDelta } from "./harness/log-delta.js";
40
40
  import { resolveHostPath as resolveVolumeHostPath, sanitizeSegment, } from "./harness/volume-paths.js";
41
41
  import { pollUntilReady } from "./harness/ready-poll.js";
@@ -50,7 +50,7 @@ import { certCovers as hostmatchCertCovers, hostWithoutPort, matchRoute, selectC
50
50
  import { INGRESS_HTTPS_PORT, INGRESS_HTTP_PORT, bindRoute, clearTables, emptyTables, planBind, registryTarget, routesFor, unbindRoute, } from "./harness/ingress-table.js";
51
51
  import { startTlsTerminator } from "./harness/tls-terminator.js";
52
52
  import { runContainerArgs } from "./harness/container-run.js";
53
- import { BUILDKIT_CACHE_DIR, BUILDKIT_CACHE_UNION, buildkitCacheTag, IMAGE_CACHE_MANIFEST, imageCachePathsSync, isOnImageCache, mergeCacheIndex, } from "./harness/image-cache.js";
53
+ import { BUILDKIT_CACHE_DIR, BUILDKIT_CACHE_UNION, IMAGE_CACHE_MANIFEST, imageCachePathsSync, isOnImageCache, mergeCacheIndex, } from "./harness/image-cache.js";
54
54
  import { assertAbsolute, certificateHostnames, defaultKeyMode, expandServiceToken, isNoopChown, mountFlag, needsIdTables, numericId, resolveChownIds, } from "./harness/file-mounts.js";
55
55
  import { conflict, notFound, requireString, } from "./harness/methods.js";
56
56
  import { openTerminal } from "./terminal.js";
@@ -507,23 +507,14 @@ async function ensureLocalBuildkitd() {
507
507
  }
508
508
  // eslint-disable-next-line no-console
509
509
  console.log(`[build] building in this VM against the image cache (root ${paths.root}, layers ${paths.layers})`);
510
- const union = await ensureBuildkitCacheUnion(paths);
511
- if (union) {
512
- // One directory for both flags: the merged cache on the layers disk
513
- // seen through this lineage's own exports on the root. See
514
- // `BUILDKIT_CACHE_UNION` for why the exporter needs the union.
515
- _localCacheDir = union;
516
- _localCacheImports = [union];
517
- }
518
- else {
519
- // Imports come from the merged cache on the read-only layers disk and
520
- // from this lineage's own exports on the root; exports go to the root,
521
- // where the merge picks them up (image_cache/merge.rs). Every cold
522
- // build re-exports every blob this way, which is the cost the union
523
- // removes.
524
- _localCacheDir = `${state}/${BUILDKIT_CACHE_DIR}`;
525
- _localCacheImports = [`${paths.layers}/${BUILDKIT_CACHE_DIR}`, `${state}/${BUILDKIT_CACHE_DIR}`];
526
- }
510
+ // The cache directory union is for the project's OWN BuildKit (the
511
+ // `/services` docs page tells it to mount `BUILDKIT_CACHE_UNION`).
512
+ // The harness's builds no longer touch it: their cache is the
513
+ // daemon's own state, which lives on the root disk and rides the
514
+ // base root into the next cold build (CONTAINER_STORE.md §20), so a
515
+ // hit is a snapshot that already exists — no tar blob to unpack
516
+ // before the first uncached step, no export pass after a miss.
517
+ await ensureBuildkitCacheUnion(paths);
527
518
  _localBuilder = true;
528
519
  return true;
529
520
  }
@@ -589,11 +580,6 @@ async function ensureBuildkitCacheUnion(paths) {
589
580
  }
590
581
  return BUILDKIT_CACHE_UNION;
591
582
  }
592
- /** The exported-cache directory on the cache disk, once the in-VM builder is up. */
593
- let _localCacheDir = null;
594
- /** The cache directories a build imports from: the merged one on the
595
- * layers disk, then this lineage's own exports. */
596
- let _localCacheImports = [];
597
583
  /**
598
584
  * A reference as containerd names it. buildx's `-t` on a remote builder
599
585
  * stores an unqualified name (`probe:bx`) that dockerd then cannot
@@ -1092,62 +1078,31 @@ async function runServiceBuild(name, image, tag) {
1092
1078
  const useBuildKit = useLocal || (await hasBuildx());
1093
1079
  const buildEnv = {};
1094
1080
  let buildArgs;
1095
- // The `--cache-to` flag, held back for a second pass that runs only
1096
- // when the first one executed a step (see below). Null off the in-VM
1097
- // builder.
1098
- let cacheExport = null;
1099
1081
  // The user's `buildArgs`, as `--build-arg` flags; a plain client flag,
1100
- // so every builder — host buildkitd, in-VM BuildKit, legacy — takes it.
1082
+ // so every builder — in-VM BuildKit, dockerd's own, legacy — takes it.
1101
1083
  const argFlags = buildArgFlags(image.buildArgs);
1102
- if (useLocal && _localCacheDir) {
1084
+ if (useLocal) {
1103
1085
  // The in-VM builder is BuildKit's containerd worker on this VM's
1104
1086
  // own image store (CONTAINER_STORE.md): the output is an image
1105
1087
  // record in dockerd's namespace, unpacked, so there is no `--load`
1106
- // and nothing crosses a socket. The cache directory on the image cache
1107
- // disk is what outlives the VM; `mode=max` keeps every
1108
- // intermediate layer, uncompressed so an import never inflates,
1109
- // and one tag per service so exports do not replace each other.
1110
- // The tag carries the `spectest/service/` prefix because the
1111
- // directory is one flat tag namespace the project's own BuildKit
1112
- // writes into too (the docs tell it to mount this path): tags merge
1113
- // newest-wins with no warning anywhere, and a project naturally tags
1114
- // a build after the thing it builds, which is also what its service
1115
- // is named. Specific's `nginx-mod` service tagged its own buildctl
1116
- // cache `nginx-mod` and the two overwrote each other on alternate
1117
- // boots (reported 2026-09-06). The prefix is reserved in the docs.
1118
- const cacheTag = buildkitCacheTag(name);
1119
- cacheExport = [
1120
- "--cache-to",
1121
- `type=local,dest=${_localCacheDir},mode=max,compression=uncompressed,force-compression=true,tag=${cacheTag}`,
1122
- ];
1123
- // LANDMINE: the local cache importer reads the `latest` entry of
1124
- // the directory's index unless told otherwise, and the exporter
1125
- // below writes this service's entry under `tag=<service>`. An
1126
- // import without the same tag misses every time and every
1127
- // `RUN` re-executes on a seeded cache (seen on the first deploy,
1128
- // 2026-09-05: the disk carried the blobs, the build used none).
1088
+ // and nothing crosses a socket. No `--cache-from` and no
1089
+ // `--cache-to` either, since 2026-09-07 (§20): the daemon's own
1090
+ // cache database is on the same disk as the snapshots it names
1091
+ // and rides the base root into the next cold build, so a cached
1092
+ // step is a snapshot that already exists. The exported directory
1093
+ // did the same job through tar blobs, at the price of unpacking
1094
+ // every cached parent before the first uncached step (11–79 s for
1095
+ // one `COPY` on Specific's dashboard) and a second full pass to
1096
+ // export after every miss.
1129
1097
  buildArgs = [
1130
1098
  "buildx", "build",
1131
1099
  "--builder", LOCAL_BUILDER_NAME,
1132
1100
  "--progress=plain",
1133
1101
  "--output", `type=image,name=${qualifyImageRef(tag)},unpack=true`,
1134
- ..._localCacheImports.flatMap((src) => ["--cache-from", `type=local,src=${src},tag=${cacheTag}`]),
1135
1102
  ...argFlags,
1136
1103
  "-f", dfPath, WORKSPACE,
1137
1104
  ];
1138
1105
  }
1139
- else if (useLocal) {
1140
- // The daemon came up but reported no cache directory: build on it
1141
- // and `--load` the result into dockerd.
1142
- buildArgs = [
1143
- "buildx", "build",
1144
- "--builder", LOCAL_BUILDER_NAME,
1145
- "--load",
1146
- "--progress=plain",
1147
- ...argFlags,
1148
- "-t", tag, "-f", dfPath, WORKSPACE,
1149
- ];
1150
- }
1151
1106
  else if (useBuildKit) {
1152
1107
  buildArgs = ["build", ...argFlags, "-t", tag, "-f", dfPath, "--progress=plain", WORKSPACE];
1153
1108
  buildEnv.DOCKER_BUILDKIT = "1";
@@ -1182,27 +1137,6 @@ async function runServiceBuild(name, image, tag) {
1182
1137
  if (build.code !== 0) {
1183
1138
  return { ok: false, log };
1184
1139
  }
1185
- // Export the cache only when there is something new to export. A
1186
- // `mode=max` export costs ~3 s per service even when every blob is
1187
- // already in the destination (measured 2026-09-06, api: 2.2–3.0 s
1188
- // for the cached build alone, 4.9–7.5 s with the export), and a cold
1189
- // start of a cached project is nothing but such builds. When a step
1190
- // did run, the second pass is that same build fully cached from the
1191
- // daemon's own state plus the export — seconds on top of a build
1192
- // that took tens of them. Best-effort: the image is already in the
1193
- // store, so a failed export is a slower next cold start, not a
1194
- // failed service.
1195
- if (cacheExport) {
1196
- const ran = executedSteps(build.stderr);
1197
- if (ran.length > 0) {
1198
- progressService(name, { status: "building", detail: `exporting cache (${ran.length} step(s) ran)` });
1199
- const exported = await shxStream("docker", [...buildArgs, ...cacheExport], 1_800_000, buildEnv, () => { });
1200
- if (exported.code !== 0) {
1201
- // eslint-disable-next-line no-console
1202
- console.warn(`[build] ${name}: cache export failed; the next cold start rebuilds it:\n${exported.stderr.trim().slice(-2000)}`);
1203
- }
1204
- }
1205
- }
1206
1140
  if (useBuildKit) {
1207
1141
  // Keep only the slowest dozen steps ≥1s — enough to profile, small
1208
1142
  // enough to ride back in the /bootstrap response and the journal.
@@ -3023,31 +2957,22 @@ async function bootstrapInner() {
3023
2957
  // image is pulled and whose deps are up starts immediately; it never sits
3024
2958
  // at "image ready" waiting for an unrelated slow build elsewhere.
3025
2959
  //
3026
- // Prep concurrency: registry pulls always run in parallel (network-bound,
3027
- // low VM RAM). Dockerfile builds run inside the VM, and two or more
3028
- // concurrent builds routinely OOM a single VM on monorepos with parallel
3029
- // pnpm/npm installs (each install fans out to ~16 fetchers + lifecycle
3030
- // workers, ~70 MB/process), so builds serialize behind a FIFO chain —
3031
- // but only the builds; pulls and starts run freely alongside them.
2960
+ // Prep concurrency: everything at once. Registry pulls are
2961
+ // network-bound and cheap in RAM. Dockerfile builds all go to the one
2962
+ // in-VM buildkitd, whose `max-parallelism` (all but one vCPU,
2963
+ // `base.rs::BUILDKITD_UP_SH`) caps the steps in flight across every
2964
+ // build it holds, and whose steps run under a cgroup with a memory
2965
+ // ceiling — so a fully cached build finishes at once while a miss is
2966
+ // still running, and two installs that fan out together kill a step,
2967
+ // never the VM. Until 2026-09-07 builds queued behind one FIFO chain
2968
+ // here, the harness's own guard against that OOM (each install fans
2969
+ // out to ~16 fetchers + lifecycle workers, ~70 MB/process): a service
2970
+ // whose build was entirely cached then waited the whole length of a
2971
+ // sibling's miss, 276 s for 4.5 s of work on Specific's `cli`.
3032
2972
  const tags = new Map();
3033
- const builds = services.filter((s) => s.image.type === "dockerfile");
3034
- const buildsRunHostSide = false;
3035
- // A promise chain is a fair FIFO mutex: when builds run in-VM, each build
3036
- // waits for the previous to settle. Pulls and host-side builds bypass it.
3037
- let inVmBuildChain = Promise.resolve();
3038
- const prepImage = (svc) => {
3039
- // Bootstrap is the only dedup scope: all its builds share one
3040
- // /workspace generation (see prepareServiceImage).
3041
- const run = () => prepareServiceImage(svc, { dedup: true });
3042
- if (svc.image.type === "dockerfile" && !buildsRunHostSide) {
3043
- const next = inVmBuildChain.then(run, run);
3044
- // Keep the chain moving even if a build throws; the chain itself never
3045
- // rejects (the per-service prep promise below is what surfaces errors).
3046
- inVmBuildChain = next.then(() => undefined, () => undefined);
3047
- return next;
3048
- }
3049
- return run();
3050
- };
2973
+ // Bootstrap is the only dedup scope: all its builds share one
2974
+ // /workspace generation (see prepareServiceImage).
2975
+ const prepImage = (svc) => prepareServiceImage(svc, { dedup: true });
3051
2976
  const prep = new Map();
3052
2977
  for (const svc of services) {
3053
2978
  const p = (async () => {
@@ -39,11 +39,12 @@ export declare function summarizeBuildKit(out: string): BuildStep[];
39
39
  * The Dockerfile steps a build actually ran — the ones a cache export
40
40
  * would have something new to record.
41
41
  *
42
- * A fully cached build still has to export its cache today, and that
43
- * export costs ~3 s per service in `mode=max` even when every blob is
44
- * already in the destination (measured 2026-09-06: 2.2–3.0 s for the
45
- * build, 4.9–7.5 s with the export). So the harness builds without
46
- * `--cache-to` first and exports only when this says something ran.
42
+ * Between SDK 0.75.0 and 0.76.0 this gated the harness's cache export
43
+ * (a `mode=max` export cost ~3 s per service with nothing to write,
44
+ * measured 2026-09-06). The harness exports nothing since its cache is
45
+ * the daemon's own state (CONTAINER_STORE.md §20); the reading stays,
46
+ * since it is the one machine-readable answer to "did this build run
47
+ * anything".
47
48
  *
48
49
  * What counts: a bracketed stage step (`[builder 3/6] RUN …`) that ended
49
50
  * in `DONE` rather than `CACHED`. What does not: BuildKit's own
@@ -68,11 +68,12 @@ export function summarizeBuildKit(out) {
68
68
  * The Dockerfile steps a build actually ran — the ones a cache export
69
69
  * would have something new to record.
70
70
  *
71
- * A fully cached build still has to export its cache today, and that
72
- * export costs ~3 s per service in `mode=max` even when every blob is
73
- * already in the destination (measured 2026-09-06: 2.2–3.0 s for the
74
- * build, 4.9–7.5 s with the export). So the harness builds without
75
- * `--cache-to` first and exports only when this says something ran.
71
+ * Between SDK 0.75.0 and 0.76.0 this gated the harness's cache export
72
+ * (a `mode=max` export cost ~3 s per service with nothing to write,
73
+ * measured 2026-09-06). The harness exports nothing since its cache is
74
+ * the daemon's own state (CONTAINER_STORE.md §20); the reading stays,
75
+ * since it is the one machine-readable answer to "did this build run
76
+ * anything".
76
77
  *
77
78
  * What counts: a bracketed stage step (`[builder 3/6] RUN …`) that ended
78
79
  * in `DONE` rather than `CACHED`. What does not: BuildKit's own
@@ -45,33 +45,34 @@ export declare const BUILDKIT_CACHE_DIR = "spectest-buildkit-cache";
45
45
  * One directory that is both: an overlayfs with the layers disk's cache
46
46
  * directory as the read-only lower and the root disk's as the upper,
47
47
  * mounted here by the harness when the in-VM builder comes up
48
- * (`daemon.ts::ensureBuildkitCacheUnion`). A build imports from it and
49
- * exports to it. BuildKit's local exporter skips a blob its destination
50
- * already holds, and through the union every merged blob is already
51
- * there — so a fully cached build exports its index and manifest and
52
- * nothing else, where a fresh directory on the root cost it every
53
- * uncompressed layer again (1.1 GB and 20 s for one service, measured
54
- * 2026-09-06). New layers land in the upper, which is exactly the
55
- * directory the merge read before. A user-run BuildKit mounts this one
56
- * path for both of its cache flags.
48
+ * (`daemon.ts::ensureBuildkitCacheUnion`). It is for the PROJECT'S OWN
49
+ * BuildKit — the `/services` docs page tells it to mount this one path
50
+ * for both of its cache flags. BuildKit's local exporter skips a blob its
51
+ * destination already holds, and through the union every merged blob is
52
+ * already there, so a fully cached build exports its index and manifest
53
+ * and nothing else (a fresh directory on the root cost every
54
+ * uncompressed layer again, 1.1 GB and 20 s for one service, measured
55
+ * 2026-09-06). New layers land in the upper, which the merge reads.
56
+ *
57
+ * The harness's own service builds used it too until SDK 0.76.0; since
58
+ * then their cache is the in-VM daemon's own state, which lives on the
59
+ * root disk and rides the base root into the next cold build
60
+ * (CONTAINER_STORE.md §20).
57
61
  */
58
62
  export declare const BUILDKIT_CACHE_UNION = "/var/lib/spectest-buildkit-cache";
59
63
  /**
60
- * The prefix under which the harness tags its own exports in that
61
- * directory. It is one flat tag namespace shared with the project's own
62
- * BuildKit (the `/services` docs page tells it to mount the union), tags
63
- * merge newest-wins with no warning, and a project naturally tags a build
64
- * after the thing it builds — which is what its spectest service is named
65
- * too. So the harness's tags live under a prefix no project would choose,
66
- * and the docs reserve it. Reported by Specific 2026-09-06: a service
67
- * `nginx-mod` running `buildctl` tagged its cache `nginx-mod`, and it and
68
- * the harness's image build for that very container overwrote each other
69
- * on alternate boots.
64
+ * The prefix the docs reserve for spectest in that directory's one flat
65
+ * tag namespace. The harness tagged its service builds under it from
66
+ * SDK 0.75.0 to 0.76.0, after Specific's `nginx-mod` service tagged its
67
+ * own `buildctl` cache `nginx-mod` and the two overwrote each other on
68
+ * alternate boots (2026-09-06); the harness writes no tags now, and the
69
+ * prefix stays reserved so nothing a project names can ever collide
70
+ * with something spectest writes there later.
70
71
  */
71
72
  export declare const BUILDKIT_CACHE_TAG_PREFIX = "spectest/service/";
72
- /** The BuildKit cache tag for a service's Dockerfile build: the prefix plus
73
- * the service name, lowercased with anything outside `[a-z0-9-]` folded
74
- * to `-`. */
73
+ /** The tag a service's Dockerfile build would carry under the reserved
74
+ * prefix: the service name, lowercased with anything outside `[a-z0-9-]`
75
+ * folded to `-`. Unused by the harness since SDK 0.76.0. */
75
76
  export declare function buildkitCacheTag(service: string): string;
76
77
  /** The cache's paths, or `null` when this VM carries none.
77
78
  * `SPECTEST_IMAGE_CACHE_MANIFEST` points a test at another file. */
@@ -39,33 +39,34 @@ export const BUILDKIT_CACHE_DIR = "spectest-buildkit-cache";
39
39
  * One directory that is both: an overlayfs with the layers disk's cache
40
40
  * directory as the read-only lower and the root disk's as the upper,
41
41
  * mounted here by the harness when the in-VM builder comes up
42
- * (`daemon.ts::ensureBuildkitCacheUnion`). A build imports from it and
43
- * exports to it. BuildKit's local exporter skips a blob its destination
44
- * already holds, and through the union every merged blob is already
45
- * there — so a fully cached build exports its index and manifest and
46
- * nothing else, where a fresh directory on the root cost it every
47
- * uncompressed layer again (1.1 GB and 20 s for one service, measured
48
- * 2026-09-06). New layers land in the upper, which is exactly the
49
- * directory the merge read before. A user-run BuildKit mounts this one
50
- * path for both of its cache flags.
42
+ * (`daemon.ts::ensureBuildkitCacheUnion`). It is for the PROJECT'S OWN
43
+ * BuildKit — the `/services` docs page tells it to mount this one path
44
+ * for both of its cache flags. BuildKit's local exporter skips a blob its
45
+ * destination already holds, and through the union every merged blob is
46
+ * already there, so a fully cached build exports its index and manifest
47
+ * and nothing else (a fresh directory on the root cost every
48
+ * uncompressed layer again, 1.1 GB and 20 s for one service, measured
49
+ * 2026-09-06). New layers land in the upper, which the merge reads.
50
+ *
51
+ * The harness's own service builds used it too until SDK 0.76.0; since
52
+ * then their cache is the in-VM daemon's own state, which lives on the
53
+ * root disk and rides the base root into the next cold build
54
+ * (CONTAINER_STORE.md §20).
51
55
  */
52
56
  export const BUILDKIT_CACHE_UNION = "/var/lib/spectest-buildkit-cache";
53
57
  /**
54
- * The prefix under which the harness tags its own exports in that
55
- * directory. It is one flat tag namespace shared with the project's own
56
- * BuildKit (the `/services` docs page tells it to mount the union), tags
57
- * merge newest-wins with no warning, and a project naturally tags a build
58
- * after the thing it builds — which is what its spectest service is named
59
- * too. So the harness's tags live under a prefix no project would choose,
60
- * and the docs reserve it. Reported by Specific 2026-09-06: a service
61
- * `nginx-mod` running `buildctl` tagged its cache `nginx-mod`, and it and
62
- * the harness's image build for that very container overwrote each other
63
- * on alternate boots.
58
+ * The prefix the docs reserve for spectest in that directory's one flat
59
+ * tag namespace. The harness tagged its service builds under it from
60
+ * SDK 0.75.0 to 0.76.0, after Specific's `nginx-mod` service tagged its
61
+ * own `buildctl` cache `nginx-mod` and the two overwrote each other on
62
+ * alternate boots (2026-09-06); the harness writes no tags now, and the
63
+ * prefix stays reserved so nothing a project names can ever collide
64
+ * with something spectest writes there later.
64
65
  */
65
66
  export const BUILDKIT_CACHE_TAG_PREFIX = "spectest/service/";
66
- /** The BuildKit cache tag for a service's Dockerfile build: the prefix plus
67
- * the service name, lowercased with anything outside `[a-z0-9-]` folded
68
- * to `-`. */
67
+ /** The tag a service's Dockerfile build would carry under the reserved
68
+ * prefix: the service name, lowercased with anything outside `[a-z0-9-]`
69
+ * folded to `-`. Unused by the harness since SDK 0.76.0. */
69
70
  export function buildkitCacheTag(service) {
70
71
  return `${BUILDKIT_CACHE_TAG_PREFIX}${service.replace(/[^a-z0-9-]/gi, "-").toLowerCase()}`;
71
72
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@specific.dev/spectest",
3
- "version": "0.75.0",
3
+ "version": "0.76.0",
4
4
  "description": "Spectest SDK for defining test environments in TypeScript.",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
package/src/daemon.ts CHANGED
@@ -83,7 +83,7 @@ import {
83
83
  type CaseMeta,
84
84
  type GroupMeta,
85
85
  } from "./harness/catalogue.js";
86
- import { executedSteps, summarizeBuildKit, type BuildStep } from "./harness/buildkit-progress.js";
86
+ import { summarizeBuildKit, type BuildStep } from "./harness/buildkit-progress.js";
87
87
  import { LOG_DELTA_MAX_BYTES, capMiddle, streamDelta } from "./harness/log-delta.js";
88
88
  import {
89
89
  resolveHostPath as resolveVolumeHostPath,
@@ -138,7 +138,6 @@ import { runContainerArgs } from "./harness/container-run.js";
138
138
  import {
139
139
  BUILDKIT_CACHE_DIR,
140
140
  BUILDKIT_CACHE_UNION,
141
- buildkitCacheTag,
142
141
  IMAGE_CACHE_MANIFEST,
143
142
  imageCachePathsSync,
144
143
  isOnImageCache,
@@ -788,22 +787,14 @@ async function ensureLocalBuildkitd(): Promise<boolean> {
788
787
  }
789
788
  // eslint-disable-next-line no-console
790
789
  console.log(`[build] building in this VM against the image cache (root ${paths.root}, layers ${paths.layers})`);
791
- const union = await ensureBuildkitCacheUnion(paths);
792
- if (union) {
793
- // One directory for both flags: the merged cache on the layers disk
794
- // seen through this lineage's own exports on the root. See
795
- // `BUILDKIT_CACHE_UNION` for why the exporter needs the union.
796
- _localCacheDir = union;
797
- _localCacheImports = [union];
798
- } else {
799
- // Imports come from the merged cache on the read-only layers disk and
800
- // from this lineage's own exports on the root; exports go to the root,
801
- // where the merge picks them up (image_cache/merge.rs). Every cold
802
- // build re-exports every blob this way, which is the cost the union
803
- // removes.
804
- _localCacheDir = `${state}/${BUILDKIT_CACHE_DIR}`;
805
- _localCacheImports = [`${paths.layers}/${BUILDKIT_CACHE_DIR}`, `${state}/${BUILDKIT_CACHE_DIR}`];
806
- }
790
+ // The cache directory union is for the project's OWN BuildKit (the
791
+ // `/services` docs page tells it to mount `BUILDKIT_CACHE_UNION`).
792
+ // The harness's builds no longer touch it: their cache is the
793
+ // daemon's own state, which lives on the root disk and rides the
794
+ // base root into the next cold build (CONTAINER_STORE.md §20), so a
795
+ // hit is a snapshot that already exists — no tar blob to unpack
796
+ // before the first uncached step, no export pass after a miss.
797
+ await ensureBuildkitCacheUnion(paths);
807
798
  _localBuilder = true;
808
799
  return true;
809
800
  }
@@ -875,11 +866,6 @@ async function ensureBuildkitCacheUnion(paths: ImageCachePaths): Promise<string
875
866
  }
876
867
  return BUILDKIT_CACHE_UNION;
877
868
  }
878
- /** The exported-cache directory on the cache disk, once the in-VM builder is up. */
879
- let _localCacheDir: string | null = null;
880
- /** The cache directories a build imports from: the merged one on the
881
- * layers disk, then this lineage's own exports. */
882
- let _localCacheImports: string[] = [];
883
869
  /**
884
870
  * A reference as containerd names it. buildx's `-t` on a remote builder
885
871
  * stores an unqualified name (`probe:bx`) that dockerd then cannot
@@ -1442,60 +1428,30 @@ async function runServiceBuild(
1442
1428
  const useBuildKit = useLocal || (await hasBuildx());
1443
1429
  const buildEnv: Record<string, string> = {};
1444
1430
  let buildArgs: string[];
1445
- // The `--cache-to` flag, held back for a second pass that runs only
1446
- // when the first one executed a step (see below). Null off the in-VM
1447
- // builder.
1448
- let cacheExport: string[] | null = null;
1449
1431
  // The user's `buildArgs`, as `--build-arg` flags; a plain client flag,
1450
- // so every builder — host buildkitd, in-VM BuildKit, legacy — takes it.
1432
+ // so every builder — in-VM BuildKit, dockerd's own, legacy — takes it.
1451
1433
  const argFlags = buildArgFlags(image.buildArgs);
1452
- if (useLocal && _localCacheDir) {
1434
+ if (useLocal) {
1453
1435
  // The in-VM builder is BuildKit's containerd worker on this VM's
1454
1436
  // own image store (CONTAINER_STORE.md): the output is an image
1455
1437
  // record in dockerd's namespace, unpacked, so there is no `--load`
1456
- // and nothing crosses a socket. The cache directory on the image cache
1457
- // disk is what outlives the VM; `mode=max` keeps every
1458
- // intermediate layer, uncompressed so an import never inflates,
1459
- // and one tag per service so exports do not replace each other.
1460
- // The tag carries the `spectest/service/` prefix because the
1461
- // directory is one flat tag namespace the project's own BuildKit
1462
- // writes into too (the docs tell it to mount this path): tags merge
1463
- // newest-wins with no warning anywhere, and a project naturally tags
1464
- // a build after the thing it builds, which is also what its service
1465
- // is named. Specific's `nginx-mod` service tagged its own buildctl
1466
- // cache `nginx-mod` and the two overwrote each other on alternate
1467
- // boots (reported 2026-09-06). The prefix is reserved in the docs.
1468
- const cacheTag = buildkitCacheTag(name);
1469
- cacheExport = [
1470
- "--cache-to",
1471
- `type=local,dest=${_localCacheDir},mode=max,compression=uncompressed,force-compression=true,tag=${cacheTag}`,
1472
- ];
1473
- // LANDMINE: the local cache importer reads the `latest` entry of
1474
- // the directory's index unless told otherwise, and the exporter
1475
- // below writes this service's entry under `tag=<service>`. An
1476
- // import without the same tag misses every time and every
1477
- // `RUN` re-executes on a seeded cache (seen on the first deploy,
1478
- // 2026-09-05: the disk carried the blobs, the build used none).
1438
+ // and nothing crosses a socket. No `--cache-from` and no
1439
+ // `--cache-to` either, since 2026-09-07 (§20): the daemon's own
1440
+ // cache database is on the same disk as the snapshots it names
1441
+ // and rides the base root into the next cold build, so a cached
1442
+ // step is a snapshot that already exists. The exported directory
1443
+ // did the same job through tar blobs, at the price of unpacking
1444
+ // every cached parent before the first uncached step (11–79 s for
1445
+ // one `COPY` on Specific's dashboard) and a second full pass to
1446
+ // export after every miss.
1479
1447
  buildArgs = [
1480
1448
  "buildx", "build",
1481
1449
  "--builder", LOCAL_BUILDER_NAME,
1482
1450
  "--progress=plain",
1483
1451
  "--output", `type=image,name=${qualifyImageRef(tag)},unpack=true`,
1484
- ..._localCacheImports.flatMap((src) => ["--cache-from", `type=local,src=${src},tag=${cacheTag}`]),
1485
1452
  ...argFlags,
1486
1453
  "-f", dfPath, WORKSPACE,
1487
1454
  ];
1488
- } else if (useLocal) {
1489
- // The daemon came up but reported no cache directory: build on it
1490
- // and `--load` the result into dockerd.
1491
- buildArgs = [
1492
- "buildx", "build",
1493
- "--builder", LOCAL_BUILDER_NAME,
1494
- "--load",
1495
- "--progress=plain",
1496
- ...argFlags,
1497
- "-t", tag, "-f", dfPath, WORKSPACE,
1498
- ];
1499
1455
  } else if (useBuildKit) {
1500
1456
  buildArgs = ["build", ...argFlags, "-t", tag, "-f", dfPath, "--progress=plain", WORKSPACE];
1501
1457
  buildEnv.DOCKER_BUILDKIT = "1";
@@ -1529,27 +1485,6 @@ async function runServiceBuild(
1529
1485
  if (build.code !== 0) {
1530
1486
  return { ok: false, log };
1531
1487
  }
1532
- // Export the cache only when there is something new to export. A
1533
- // `mode=max` export costs ~3 s per service even when every blob is
1534
- // already in the destination (measured 2026-09-06, api: 2.2–3.0 s
1535
- // for the cached build alone, 4.9–7.5 s with the export), and a cold
1536
- // start of a cached project is nothing but such builds. When a step
1537
- // did run, the second pass is that same build fully cached from the
1538
- // daemon's own state plus the export — seconds on top of a build
1539
- // that took tens of them. Best-effort: the image is already in the
1540
- // store, so a failed export is a slower next cold start, not a
1541
- // failed service.
1542
- if (cacheExport) {
1543
- const ran = executedSteps(build.stderr);
1544
- if (ran.length > 0) {
1545
- progressService(name, { status: "building", detail: `exporting cache (${ran.length} step(s) ran)` });
1546
- const exported = await shxStream("docker", [...buildArgs, ...cacheExport], 1_800_000, buildEnv, () => {});
1547
- if (exported.code !== 0) {
1548
- // eslint-disable-next-line no-console
1549
- console.warn(`[build] ${name}: cache export failed; the next cold start rebuilds it:\n${exported.stderr.trim().slice(-2000)}`);
1550
- }
1551
- }
1552
- }
1553
1488
  if (useBuildKit) {
1554
1489
  // Keep only the slowest dozen steps ≥1s — enough to profile, small
1555
1490
  // enough to ride back in the /bootstrap response and the journal.
@@ -3586,36 +3521,23 @@ async function bootstrapInner(): Promise<BootstrapTimings> {
3586
3521
  // image is pulled and whose deps are up starts immediately; it never sits
3587
3522
  // at "image ready" waiting for an unrelated slow build elsewhere.
3588
3523
  //
3589
- // Prep concurrency: registry pulls always run in parallel (network-bound,
3590
- // low VM RAM). Dockerfile builds run inside the VM, and two or more
3591
- // concurrent builds routinely OOM a single VM on monorepos with parallel
3592
- // pnpm/npm installs (each install fans out to ~16 fetchers + lifecycle
3593
- // workers, ~70 MB/process), so builds serialize behind a FIFO chain —
3594
- // but only the builds; pulls and starts run freely alongside them.
3524
+ // Prep concurrency: everything at once. Registry pulls are
3525
+ // network-bound and cheap in RAM. Dockerfile builds all go to the one
3526
+ // in-VM buildkitd, whose `max-parallelism` (all but one vCPU,
3527
+ // `base.rs::BUILDKITD_UP_SH`) caps the steps in flight across every
3528
+ // build it holds, and whose steps run under a cgroup with a memory
3529
+ // ceiling — so a fully cached build finishes at once while a miss is
3530
+ // still running, and two installs that fan out together kill a step,
3531
+ // never the VM. Until 2026-09-07 builds queued behind one FIFO chain
3532
+ // here, the harness's own guard against that OOM (each install fans
3533
+ // out to ~16 fetchers + lifecycle workers, ~70 MB/process): a service
3534
+ // whose build was entirely cached then waited the whole length of a
3535
+ // sibling's miss, 276 s for 4.5 s of work on Specific's `cli`.
3595
3536
  const tags = new Map<string, string>();
3596
- const builds = services.filter((s) => s.image.type === "dockerfile");
3597
- const buildsRunHostSide = false;
3598
- // A promise chain is a fair FIFO mutex: when builds run in-VM, each build
3599
- // waits for the previous to settle. Pulls and host-side builds bypass it.
3600
- let inVmBuildChain: Promise<unknown> = Promise.resolve();
3601
- const prepImage = (
3602
- svc: NamedService,
3603
- ): Promise<{ tag: string; buildSteps?: BuildStep[] }> => {
3604
- // Bootstrap is the only dedup scope: all its builds share one
3605
- // /workspace generation (see prepareServiceImage).
3606
- const run = () => prepareServiceImage(svc, { dedup: true });
3607
- if (svc.image.type === "dockerfile" && !buildsRunHostSide) {
3608
- const next = inVmBuildChain.then(run, run);
3609
- // Keep the chain moving even if a build throws; the chain itself never
3610
- // rejects (the per-service prep promise below is what surfaces errors).
3611
- inVmBuildChain = next.then(
3612
- () => undefined,
3613
- () => undefined,
3614
- );
3615
- return next;
3616
- }
3617
- return run();
3618
- };
3537
+ // Bootstrap is the only dedup scope: all its builds share one
3538
+ // /workspace generation (see prepareServiceImage).
3539
+ const prepImage = (svc: NamedService): Promise<{ tag: string; buildSteps?: BuildStep[] }> =>
3540
+ prepareServiceImage(svc, { dedup: true });
3619
3541
  const prep = new Map<string, Promise<void>>();
3620
3542
  for (const svc of services) {
3621
3543
  const p = (async () => {
@@ -77,11 +77,12 @@ export function summarizeBuildKit(out: string): BuildStep[] {
77
77
  * The Dockerfile steps a build actually ran — the ones a cache export
78
78
  * would have something new to record.
79
79
  *
80
- * A fully cached build still has to export its cache today, and that
81
- * export costs ~3 s per service in `mode=max` even when every blob is
82
- * already in the destination (measured 2026-09-06: 2.2–3.0 s for the
83
- * build, 4.9–7.5 s with the export). So the harness builds without
84
- * `--cache-to` first and exports only when this says something ran.
80
+ * Between SDK 0.75.0 and 0.76.0 this gated the harness's cache export
81
+ * (a `mode=max` export cost ~3 s per service with nothing to write,
82
+ * measured 2026-09-06). The harness exports nothing since its cache is
83
+ * the daemon's own state (CONTAINER_STORE.md §20); the reading stays,
84
+ * since it is the one machine-readable answer to "did this build run
85
+ * anything".
85
86
  *
86
87
  * What counts: a bracketed stage step (`[builder 3/6] RUN …`) that ended
87
88
  * in `DONE` rather than `CACHED`. What does not: BuildKit's own
@@ -54,35 +54,36 @@ export const BUILDKIT_CACHE_DIR = "spectest-buildkit-cache";
54
54
  * One directory that is both: an overlayfs with the layers disk's cache
55
55
  * directory as the read-only lower and the root disk's as the upper,
56
56
  * mounted here by the harness when the in-VM builder comes up
57
- * (`daemon.ts::ensureBuildkitCacheUnion`). A build imports from it and
58
- * exports to it. BuildKit's local exporter skips a blob its destination
59
- * already holds, and through the union every merged blob is already
60
- * there — so a fully cached build exports its index and manifest and
61
- * nothing else, where a fresh directory on the root cost it every
62
- * uncompressed layer again (1.1 GB and 20 s for one service, measured
63
- * 2026-09-06). New layers land in the upper, which is exactly the
64
- * directory the merge read before. A user-run BuildKit mounts this one
65
- * path for both of its cache flags.
57
+ * (`daemon.ts::ensureBuildkitCacheUnion`). It is for the PROJECT'S OWN
58
+ * BuildKit — the `/services` docs page tells it to mount this one path
59
+ * for both of its cache flags. BuildKit's local exporter skips a blob its
60
+ * destination already holds, and through the union every merged blob is
61
+ * already there, so a fully cached build exports its index and manifest
62
+ * and nothing else (a fresh directory on the root cost every
63
+ * uncompressed layer again, 1.1 GB and 20 s for one service, measured
64
+ * 2026-09-06). New layers land in the upper, which the merge reads.
65
+ *
66
+ * The harness's own service builds used it too until SDK 0.76.0; since
67
+ * then their cache is the in-VM daemon's own state, which lives on the
68
+ * root disk and rides the base root into the next cold build
69
+ * (CONTAINER_STORE.md §20).
66
70
  */
67
71
  export const BUILDKIT_CACHE_UNION = "/var/lib/spectest-buildkit-cache";
68
72
 
69
73
  /**
70
- * The prefix under which the harness tags its own exports in that
71
- * directory. It is one flat tag namespace shared with the project's own
72
- * BuildKit (the `/services` docs page tells it to mount the union), tags
73
- * merge newest-wins with no warning, and a project naturally tags a build
74
- * after the thing it builds — which is what its spectest service is named
75
- * too. So the harness's tags live under a prefix no project would choose,
76
- * and the docs reserve it. Reported by Specific 2026-09-06: a service
77
- * `nginx-mod` running `buildctl` tagged its cache `nginx-mod`, and it and
78
- * the harness's image build for that very container overwrote each other
79
- * on alternate boots.
74
+ * The prefix the docs reserve for spectest in that directory's one flat
75
+ * tag namespace. The harness tagged its service builds under it from
76
+ * SDK 0.75.0 to 0.76.0, after Specific's `nginx-mod` service tagged its
77
+ * own `buildctl` cache `nginx-mod` and the two overwrote each other on
78
+ * alternate boots (2026-09-06); the harness writes no tags now, and the
79
+ * prefix stays reserved so nothing a project names can ever collide
80
+ * with something spectest writes there later.
80
81
  */
81
82
  export const BUILDKIT_CACHE_TAG_PREFIX = "spectest/service/";
82
83
 
83
- /** The BuildKit cache tag for a service's Dockerfile build: the prefix plus
84
- * the service name, lowercased with anything outside `[a-z0-9-]` folded
85
- * to `-`. */
84
+ /** The tag a service's Dockerfile build would carry under the reserved
85
+ * prefix: the service name, lowercased with anything outside `[a-z0-9-]`
86
+ * folded to `-`. Unused by the harness since SDK 0.76.0. */
86
87
  export function buildkitCacheTag(service: string): string {
87
88
  return `${BUILDKIT_CACHE_TAG_PREFIX}${service.replace(/[^a-z0-9-]/gi, "-").toLowerCase()}`;
88
89
  }