@specific.dev/spectest 0.73.1 → 0.75.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/daemon.js CHANGED
@@ -35,7 +35,7 @@ import { isMobileApp, openPersistentMobile } from "./mobile.js";
35
35
  import { buildArgFlags, buildContentKey as computeBuildContentKey, imageTag, isGeneratedDockerignore, serviceDockerignore as composeServiceDockerignore, unionDockerignore, } from "./harness/build-context.js";
36
36
  import { validateServiceGraph as validateGraph } from "./harness/service-graph.js";
37
37
  import { casesMetadata as catalogueCases, groupsMetadata as catalogueGroups, } from "./harness/catalogue.js";
38
- import { summarizeBuildKit } from "./harness/buildkit-progress.js";
38
+ import { executedSteps, summarizeBuildKit } from "./harness/buildkit-progress.js";
39
39
  import { LOG_DELTA_MAX_BYTES, capMiddle, streamDelta } from "./harness/log-delta.js";
40
40
  import { resolveHostPath as resolveVolumeHostPath, sanitizeSegment, } from "./harness/volume-paths.js";
41
41
  import { pollUntilReady } from "./harness/ready-poll.js";
@@ -50,7 +50,7 @@ import { certCovers as hostmatchCertCovers, hostWithoutPort, matchRoute, selectC
50
50
  import { INGRESS_HTTPS_PORT, INGRESS_HTTP_PORT, bindRoute, clearTables, emptyTables, planBind, registryTarget, routesFor, unbindRoute, } from "./harness/ingress-table.js";
51
51
  import { startTlsTerminator } from "./harness/tls-terminator.js";
52
52
  import { runContainerArgs } from "./harness/container-run.js";
53
- import { IMAGE_CACHE_MANIFEST, imageCachePathsSync, isOnImageCache } from "./harness/image-cache.js";
53
+ import { BUILDKIT_CACHE_DIR, BUILDKIT_CACHE_UNION, buildkitCacheTag, IMAGE_CACHE_MANIFEST, imageCachePathsSync, isOnImageCache, mergeCacheIndex, } from "./harness/image-cache.js";
54
54
  import { assertAbsolute, certificateHostnames, defaultKeyMode, expandServiceToken, isNoopChown, mountFlag, needsIdTables, numericId, resolveChownIds, } from "./harness/file-mounts.js";
55
55
  import { conflict, notFound, requireString, } from "./harness/methods.js";
56
56
  import { openTerminal } from "./terminal.js";
@@ -507,14 +507,88 @@ async function ensureLocalBuildkitd() {
507
507
  }
508
508
  // eslint-disable-next-line no-console
509
509
  console.log(`[build] building in this VM against the image cache (root ${paths.root}, layers ${paths.layers})`);
510
- // Imports come from the merged cache on the read-only layers disk and
511
- // from this lineage's own exports on the root; exports go to the root,
512
- // where the merge picks them up (image_cache/merge.rs).
513
- _localCacheDir = `${state}/spectest-buildkit-cache`;
514
- _localCacheImports = [`${paths.layers}/spectest-buildkit-cache`, `${state}/spectest-buildkit-cache`];
510
+ const union = await ensureBuildkitCacheUnion(paths);
511
+ if (union) {
512
+ // One directory for both flags: the merged cache on the layers disk
513
+ // seen through this lineage's own exports on the root. See
514
+ // `BUILDKIT_CACHE_UNION` for why the exporter needs the union.
515
+ _localCacheDir = union;
516
+ _localCacheImports = [union];
517
+ }
518
+ else {
519
+ // Imports come from the merged cache on the read-only layers disk and
520
+ // from this lineage's own exports on the root; exports go to the root,
521
+ // where the merge picks them up (image_cache/merge.rs). Every cold
522
+ // build re-exports every blob this way, which is the cost the union
523
+ // removes.
524
+ _localCacheDir = `${state}/${BUILDKIT_CACHE_DIR}`;
525
+ _localCacheImports = [`${paths.layers}/${BUILDKIT_CACHE_DIR}`, `${state}/${BUILDKIT_CACHE_DIR}`];
526
+ }
515
527
  _localBuilder = true;
516
528
  return true;
517
529
  }
530
+ /**
531
+ * Mount the BuildKit cache union (`BUILDKIT_CACHE_UNION`): overlayfs with
532
+ * the layers disk's cache directory as the lower and the root disk's as
533
+ * the upper, so one path serves `--cache-from` and `--cache-to`.
534
+ *
535
+ * Idempotent — a delta restore restarts the harness on a guest that
536
+ * still has the mount. Returns null, and the caller keeps the two-directory
537
+ * scheme, when the lower does not exist yet (a volume that has never
538
+ * been merged has no cache directory on its layers disk) or the mount is
539
+ * refused; the build is then slower, never failed. The work directory
540
+ * has to share a filesystem with the upper, so it lives on the root disk
541
+ * beside it; the merge reads neither it nor anything but the upper.
542
+ */
543
+ async function ensureBuildkitCacheUnion(paths) {
544
+ const lower = `${paths.layers}/${BUILDKIT_CACHE_DIR}`;
545
+ const upper = `${paths.root}/${BUILDKIT_CACHE_DIR}`;
546
+ const work = `${paths.root}/spectest-buildkit-work`;
547
+ if (!existsSync(lower)) {
548
+ // eslint-disable-next-line no-console
549
+ console.log(`[build] no merged BuildKit cache on the layers disk yet; exporting to the root directly`);
550
+ return null;
551
+ }
552
+ // overlayfs shows the upper's `index.json` INSTEAD of the lower's, so
553
+ // an index on the root (a lineage's exports, or one left behind with
554
+ // its blobs gone) would hide every merged tag and, naming manifests
555
+ // that are not there, make BuildKit skip the import altogether. Write
556
+ // the union of the two into the upper first (`mergeCacheIndex`), and
557
+ // say what each side held so a miss can be read off the boot log.
558
+ try {
559
+ const readIndex = (dir) => existsSync(`${dir}/index.json`) ? JSON.parse(readFileSync(`${dir}/index.json`, "utf8")) : null;
560
+ const lowerIdx = readIndex(lower);
561
+ const upperIdx = readIndex(upper);
562
+ if (upperIdx && lowerIdx) {
563
+ const merged = mergeCacheIndex(lowerIdx, upperIdx, (hex) => existsSync(`${upper}/blobs/sha256/${hex}`));
564
+ await fs.mkdir(upper, { recursive: true });
565
+ await fs.writeFile(`${upper}/index.json`, JSON.stringify(merged));
566
+ await fs.writeFile(`${upper}/oci-layout`, JSON.stringify({ imageLayoutVersion: "1.0.0" }));
567
+ // eslint-disable-next-line no-console
568
+ console.log(`[build] BuildKit cache union: ${lowerIdx.manifests.length} tag(s) from the merged cache, ${upperIdx.manifests.length} on this root, ${merged.manifests.length} after the merge`);
569
+ }
570
+ else {
571
+ // eslint-disable-next-line no-console
572
+ console.log(`[build] BuildKit cache union: ${lowerIdx?.manifests.length ?? 0} tag(s) from the merged cache, none on this root`);
573
+ }
574
+ }
575
+ catch (err) {
576
+ // eslint-disable-next-line no-console
577
+ console.warn(`[build] could not reconcile the BuildKit cache indexes (${err}); the union shows the root's`);
578
+ }
579
+ const r = await shx("/bin/sh", [
580
+ "-c",
581
+ `mkdir -p "${BUILDKIT_CACHE_UNION}" "${upper}" "${work}" && ` +
582
+ `{ mountpoint -q "${BUILDKIT_CACHE_UNION}" || ` +
583
+ `mount -t overlay overlay -o "lowerdir=${lower},upperdir=${upper},workdir=${work}" "${BUILDKIT_CACHE_UNION}"; }`,
584
+ ], 60_000);
585
+ if (r.code !== 0) {
586
+ // eslint-disable-next-line no-console
587
+ console.warn(`[build] could not mount the BuildKit cache union; exporting to the root directly:\n${(r.stderr || r.stdout).trim()}`);
588
+ return null;
589
+ }
590
+ return BUILDKIT_CACHE_UNION;
591
+ }
518
592
  /** The exported-cache directory on the cache disk, once the in-VM builder is up. */
519
593
  let _localCacheDir = null;
520
594
  /** The cache directories a build imports from: the merged one on the
@@ -1018,6 +1092,10 @@ async function runServiceBuild(name, image, tag) {
1018
1092
  const useBuildKit = useLocal || (await hasBuildx());
1019
1093
  const buildEnv = {};
1020
1094
  let buildArgs;
1095
+ // The `--cache-to` flag, held back for a second pass that runs only
1096
+ // when the first one executed a step (see below). Null off the in-VM
1097
+ // builder.
1098
+ let cacheExport = null;
1021
1099
  // The user's `buildArgs`, as `--build-arg` flags; a plain client flag,
1022
1100
  // so every builder — host buildkitd, in-VM BuildKit, legacy — takes it.
1023
1101
  const argFlags = buildArgFlags(image.buildArgs);
@@ -1029,7 +1107,19 @@ async function runServiceBuild(name, image, tag) {
1029
1107
  // disk is what outlives the VM; `mode=max` keeps every
1030
1108
  // intermediate layer, uncompressed so an import never inflates,
1031
1109
  // and one tag per service so exports do not replace each other.
1032
- const cacheTag = name.replace(/[^a-z0-9-]/gi, "-").toLowerCase();
1110
+ // The tag carries the `spectest/service/` prefix because the
1111
+ // directory is one flat tag namespace the project's own BuildKit
1112
+ // writes into too (the docs tell it to mount this path): tags merge
1113
+ // newest-wins with no warning anywhere, and a project naturally tags
1114
+ // a build after the thing it builds, which is also what its service
1115
+ // is named. Specific's `nginx-mod` service tagged its own buildctl
1116
+ // cache `nginx-mod` and the two overwrote each other on alternate
1117
+ // boots (reported 2026-09-06). The prefix is reserved in the docs.
1118
+ const cacheTag = buildkitCacheTag(name);
1119
+ cacheExport = [
1120
+ "--cache-to",
1121
+ `type=local,dest=${_localCacheDir},mode=max,compression=uncompressed,force-compression=true,tag=${cacheTag}`,
1122
+ ];
1033
1123
  // LANDMINE: the local cache importer reads the `latest` entry of
1034
1124
  // the directory's index unless told otherwise, and the exporter
1035
1125
  // below writes this service's entry under `tag=<service>`. An
@@ -1042,7 +1132,6 @@ async function runServiceBuild(name, image, tag) {
1042
1132
  "--progress=plain",
1043
1133
  "--output", `type=image,name=${qualifyImageRef(tag)},unpack=true`,
1044
1134
  ..._localCacheImports.flatMap((src) => ["--cache-from", `type=local,src=${src},tag=${cacheTag}`]),
1045
- "--cache-to", `type=local,dest=${_localCacheDir},mode=max,compression=uncompressed,force-compression=true,tag=${cacheTag}`,
1046
1135
  ...argFlags,
1047
1136
  "-f", dfPath, WORKSPACE,
1048
1137
  ];
@@ -1093,6 +1182,27 @@ async function runServiceBuild(name, image, tag) {
1093
1182
  if (build.code !== 0) {
1094
1183
  return { ok: false, log };
1095
1184
  }
1185
+ // Export the cache only when there is something new to export. A
1186
+ // `mode=max` export costs ~3 s per service even when every blob is
1187
+ // already in the destination (measured 2026-09-06, api: 2.2–3.0 s
1188
+ // for the cached build alone, 4.9–7.5 s with the export), and a cold
1189
+ // start of a cached project is nothing but such builds. When a step
1190
+ // did run, the second pass is that same build fully cached from the
1191
+ // daemon's own state plus the export — seconds on top of a build
1192
+ // that took tens of them. Best-effort: the image is already in the
1193
+ // store, so a failed export is a slower next cold start, not a
1194
+ // failed service.
1195
+ if (cacheExport) {
1196
+ const ran = executedSteps(build.stderr);
1197
+ if (ran.length > 0) {
1198
+ progressService(name, { status: "building", detail: `exporting cache (${ran.length} step(s) ran)` });
1199
+ const exported = await shxStream("docker", [...buildArgs, ...cacheExport], 1_800_000, buildEnv, () => { });
1200
+ if (exported.code !== 0) {
1201
+ // eslint-disable-next-line no-console
1202
+ console.warn(`[build] ${name}: cache export failed; the next cold start rebuilds it:\n${exported.stderr.trim().slice(-2000)}`);
1203
+ }
1204
+ }
1205
+ }
1096
1206
  if (useBuildKit) {
1097
1207
  // Keep only the slowest dozen steps ≥1s — enough to profile, small
1098
1208
  // enough to ride back in the /bootstrap response and the journal.
@@ -35,3 +35,21 @@ export interface BuildStep {
35
35
  * omitting it makes a fully-cached build look like it did nothing.
36
36
  */
37
37
  export declare function summarizeBuildKit(out: string): BuildStep[];
38
+ /**
39
+ * The Dockerfile steps a build actually ran — the ones a cache export
40
+ * would have something new to record.
41
+ *
42
+ * A fully cached build still has to export its cache today, and that
43
+ * export costs ~3 s per service in `mode=max` even when every blob is
44
+ * already in the destination (measured 2026-09-06: 2.2–3.0 s for the
45
+ * build, 4.9–7.5 s with the export). So the harness builds without
46
+ * `--cache-to` first and exports only when this says something ran.
47
+ *
48
+ * What counts: a bracketed stage step (`[builder 3/6] RUN …`) that ended
49
+ * in `DONE` rather than `CACHED`. What does not: BuildKit's own
50
+ * `[internal]`/`[auth]` bookkeeping, which is never cached and never a
51
+ * layer, and a `FROM` step that finished in the blink it takes to find
52
+ * the base image locally — a pulled base takes seconds and does count,
53
+ * since the export records its layers too.
54
+ */
55
+ export declare function executedSteps(out: string): string[];
@@ -64,3 +64,57 @@ export function summarizeBuildKit(out) {
64
64
  }
65
65
  return steps.sort((a, b) => b.secs - a.secs);
66
66
  }
67
+ /**
68
+ * The Dockerfile steps a build actually ran — the ones a cache export
69
+ * would have something new to record.
70
+ *
71
+ * A fully cached build still has to export its cache today, and that
72
+ * export costs ~3 s per service in `mode=max` even when every blob is
73
+ * already in the destination (measured 2026-09-06: 2.2–3.0 s for the
74
+ * build, 4.9–7.5 s with the export). So the harness builds without
75
+ * `--cache-to` first and exports only when this says something ran.
76
+ *
77
+ * What counts: a bracketed stage step (`[builder 3/6] RUN …`) that ended
78
+ * in `DONE` rather than `CACHED`. What does not: BuildKit's own
79
+ * `[internal]`/`[auth]` bookkeeping, which is never cached and never a
80
+ * layer, and a `FROM` step that finished in the blink it takes to find
81
+ * the base image locally — a pulled base takes seconds and does count,
82
+ * since the export records its layers too.
83
+ */
84
+ export function executedSteps(out) {
85
+ const names = new Map();
86
+ const stages = new Map();
87
+ const secs = new Map();
88
+ const cached = new Set();
89
+ for (const line of out.split("\n")) {
90
+ let m = line.match(/^#(\d+)\s+\[([^\]]*)\]\s+(.+)$/);
91
+ if (m) {
92
+ const id = `#${m[1]}`;
93
+ if (!names.has(id)) {
94
+ names.set(id, m[3].trim().slice(0, MAX_NAME));
95
+ stages.set(id, m[2].trim());
96
+ }
97
+ continue;
98
+ }
99
+ m = line.match(/^#(\d+)\s+DONE\s+([\d.]+)s/);
100
+ if (m) {
101
+ secs.set(`#${m[1]}`, parseFloat(m[2]));
102
+ continue;
103
+ }
104
+ m = line.match(/^#(\d+)\s+CACHED/);
105
+ if (m)
106
+ cached.add(`#${m[1]}`);
107
+ }
108
+ const ran = [];
109
+ for (const [id, name] of names) {
110
+ if (cached.has(id))
111
+ continue;
112
+ const stage = stages.get(id) ?? "";
113
+ if (stage === "internal" || stage === "auth")
114
+ continue;
115
+ if (/^FROM\s/.test(name) && (secs.get(id) ?? 0) < 0.5)
116
+ continue;
117
+ ran.push(name);
118
+ }
119
+ return ran;
120
+ }
@@ -35,6 +35,44 @@ export declare const MKFS_EROFS_PATH = "/usr/local/bin/mkfs.erofs";
35
35
  /** The guest's adopt helper (`base.rs::STORE_ADOPT_SH`), POSIX sh so a
36
36
  * nested runtime's busybox can run the same file. */
37
37
  export declare const STORE_ADOPT_PATH = "/usr/local/bin/spectest-store-adopt";
38
+ /**
39
+ * The BuildKit cache directory under each cache disk (`merge.rs::CACHE_DIR`):
40
+ * on the layers disk the merged export of every earlier build, read-only;
41
+ * on the root disk this lineage's own exports, which the merge harvests.
42
+ */
43
+ export declare const BUILDKIT_CACHE_DIR = "spectest-buildkit-cache";
44
+ /**
45
+ * One directory that is both: an overlayfs with the layers disk's cache
46
+ * directory as the read-only lower and the root disk's as the upper,
47
+ * mounted here by the harness when the in-VM builder comes up
48
+ * (`daemon.ts::ensureBuildkitCacheUnion`). A build imports from it and
49
+ * exports to it. BuildKit's local exporter skips a blob its destination
50
+ * already holds, and through the union every merged blob is already
51
+ * there — so a fully cached build exports its index and manifest and
52
+ * nothing else, where a fresh directory on the root cost it every
53
+ * uncompressed layer again (1.1 GB and 20 s for one service, measured
54
+ * 2026-09-06). New layers land in the upper, which is exactly the
55
+ * directory the merge read before. A user-run BuildKit mounts this one
56
+ * path for both of its cache flags.
57
+ */
58
+ export declare const BUILDKIT_CACHE_UNION = "/var/lib/spectest-buildkit-cache";
59
+ /**
60
+ * The prefix under which the harness tags its own exports in that
61
+ * directory. It is one flat tag namespace shared with the project's own
62
+ * BuildKit (the `/services` docs page tells it to mount the union), tags
63
+ * merge newest-wins with no warning, and a project naturally tags a build
64
+ * after the thing it builds — which is what its spectest service is named
65
+ * too. So the harness's tags live under a prefix no project would choose,
66
+ * and the docs reserve it. Reported by Specific 2026-09-06: a service
67
+ * `nginx-mod` running `buildctl` tagged its cache `nginx-mod`, and it and
68
+ * the harness's image build for that very container overwrote each other
69
+ * on alternate boots.
70
+ */
71
+ export declare const BUILDKIT_CACHE_TAG_PREFIX = "spectest/service/";
72
+ /** The BuildKit cache tag for a service's Dockerfile build: the prefix plus
73
+ * the service name, lowercased with anything outside `[a-z0-9-]` folded
74
+ * to `-`. */
75
+ export declare function buildkitCacheTag(service: string): string;
38
76
  /** The cache's paths, or `null` when this VM carries none.
39
77
  * `SPECTEST_IMAGE_CACHE_MANIFEST` points a test at another file. */
40
78
  export declare function imageCachePathsSync(manifest?: string): ImageCachePaths | null;
@@ -46,3 +84,33 @@ export declare function nestedStoreDir(paths: ImageCachePaths, service: string):
46
84
  * delta-restore teardown must not wipe it.
47
85
  */
48
86
  export declare function isOnImageCache(hostPath: string, paths: ImageCachePaths | null): boolean;
87
+ /** The shape of an OCI layout's `index.json` the BuildKit local cache
88
+ * exporter writes: one manifest per tag, the tag in the ref-name
89
+ * annotation. */
90
+ export interface OciIndex {
91
+ schemaVersion?: number;
92
+ mediaType?: string;
93
+ manifests: Array<{
94
+ mediaType?: string;
95
+ digest: string;
96
+ size?: number;
97
+ annotations?: Record<string, string>;
98
+ }>;
99
+ }
100
+ /**
101
+ * The index the BuildKit cache union must show: every tag of the merged
102
+ * cache (lower), overridden by this lineage's own exports (upper) where
103
+ * the upper still holds the manifest.
104
+ *
105
+ * overlayfs shadows whole files, so an `index.json` in the upper hides
106
+ * the lower's completely — and a root disk can carry one: the exports
107
+ * of a lineage that built before the union existed, or an index left
108
+ * behind with its blobs gone. Either way BuildKit then imports from an
109
+ * index naming a few tags whose manifests may not exist, skips the
110
+ * import, and every step runs (seen 2026-09-06: a cold build with
111
+ * every service's cache on the layers disk and 4 stale tags in the
112
+ * upper built everything from scratch). `upperHasBlob` says whether the
113
+ * upper holds a manifest's blob; an upper entry without one is dropped,
114
+ * since it would fail the whole import.
115
+ */
116
+ export declare function mergeCacheIndex(lower: OciIndex | null, upper: OciIndex | null, upperHasBlob: (hex: string) => boolean): OciIndex;
@@ -29,6 +29,46 @@ export const MKFS_EROFS_PATH = "/usr/local/bin/mkfs.erofs";
29
29
  /** The guest's adopt helper (`base.rs::STORE_ADOPT_SH`), POSIX sh so a
30
30
  * nested runtime's busybox can run the same file. */
31
31
  export const STORE_ADOPT_PATH = "/usr/local/bin/spectest-store-adopt";
32
+ /**
33
+ * The BuildKit cache directory under each cache disk (`merge.rs::CACHE_DIR`):
34
+ * on the layers disk the merged export of every earlier build, read-only;
35
+ * on the root disk this lineage's own exports, which the merge harvests.
36
+ */
37
+ export const BUILDKIT_CACHE_DIR = "spectest-buildkit-cache";
38
+ /**
39
+ * One directory that is both: an overlayfs with the layers disk's cache
40
+ * directory as the read-only lower and the root disk's as the upper,
41
+ * mounted here by the harness when the in-VM builder comes up
42
+ * (`daemon.ts::ensureBuildkitCacheUnion`). A build imports from it and
43
+ * exports to it. BuildKit's local exporter skips a blob its destination
44
+ * already holds, and through the union every merged blob is already
45
+ * there — so a fully cached build exports its index and manifest and
46
+ * nothing else, where a fresh directory on the root cost it every
47
+ * uncompressed layer again (1.1 GB and 20 s for one service, measured
48
+ * 2026-09-06). New layers land in the upper, which is exactly the
49
+ * directory the merge read before. A user-run BuildKit mounts this one
50
+ * path for both of its cache flags.
51
+ */
52
+ export const BUILDKIT_CACHE_UNION = "/var/lib/spectest-buildkit-cache";
53
+ /**
54
+ * The prefix under which the harness tags its own exports in that
55
+ * directory. It is one flat tag namespace shared with the project's own
56
+ * BuildKit (the `/services` docs page tells it to mount the union), tags
57
+ * merge newest-wins with no warning, and a project naturally tags a build
58
+ * after the thing it builds — which is what its spectest service is named
59
+ * too. So the harness's tags live under a prefix no project would choose,
60
+ * and the docs reserve it. Reported by Specific 2026-09-06: a service
61
+ * `nginx-mod` running `buildctl` tagged its cache `nginx-mod`, and it and
62
+ * the harness's image build for that very container overwrote each other
63
+ * on alternate boots.
64
+ */
65
+ export const BUILDKIT_CACHE_TAG_PREFIX = "spectest/service/";
66
+ /** The BuildKit cache tag for a service's Dockerfile build: the prefix plus
67
+ * the service name, lowercased with anything outside `[a-z0-9-]` folded
68
+ * to `-`. */
69
+ export function buildkitCacheTag(service) {
70
+ return `${BUILDKIT_CACHE_TAG_PREFIX}${service.replace(/[^a-z0-9-]/gi, "-").toLowerCase()}`;
71
+ }
32
72
  function parse(raw) {
33
73
  const parsed = JSON.parse(raw);
34
74
  const root = (parsed.disks ?? []).find((d) => d.role === "root" && d.path)?.path;
@@ -59,5 +99,51 @@ export function nestedStoreDir(paths, service) {
59
99
  export function isOnImageCache(hostPath, paths) {
60
100
  if (!paths)
61
101
  return false;
102
+ // The union is the root's cache directory seen through the layers
103
+ // disk: wiping it would wipe the upper, i.e. this lineage's exports.
104
+ if (hostPath === BUILDKIT_CACHE_UNION || hostPath.startsWith(`${BUILDKIT_CACHE_UNION}/`))
105
+ return true;
62
106
  return hostPath === paths.root || hostPath.startsWith(`${paths.root}/`) || hostPath === paths.layers || hostPath.startsWith(`${paths.layers}/`);
63
107
  }
108
+ const REF_NAME = "org.opencontainers.image.ref.name";
109
+ /**
110
+ * The index the BuildKit cache union must show: every tag of the merged
111
+ * cache (lower), overridden by this lineage's own exports (upper) where
112
+ * the upper still holds the manifest.
113
+ *
114
+ * overlayfs shadows whole files, so an `index.json` in the upper hides
115
+ * the lower's completely — and a root disk can carry one: the exports
116
+ * of a lineage that built before the union existed, or an index left
117
+ * behind with its blobs gone. Either way BuildKit then imports from an
118
+ * index naming a few tags whose manifests may not exist, skips the
119
+ * import, and every step runs (seen 2026-09-06: a cold build with
120
+ * every service's cache on the layers disk and 4 stale tags in the
121
+ * upper built everything from scratch). `upperHasBlob` says whether the
122
+ * upper holds a manifest's blob; an upper entry without one is dropped,
123
+ * since it would fail the whole import.
124
+ */
125
+ export function mergeCacheIndex(lower, upper, upperHasBlob) {
126
+ const byTag = new Map();
127
+ const untagged = [];
128
+ for (const m of lower?.manifests ?? []) {
129
+ const tag = m.annotations?.[REF_NAME];
130
+ if (tag)
131
+ byTag.set(tag, m);
132
+ else
133
+ untagged.push(m);
134
+ }
135
+ for (const m of upper?.manifests ?? []) {
136
+ if (!upperHasBlob(m.digest.replace(/^sha256:/, "")))
137
+ continue;
138
+ const tag = m.annotations?.[REF_NAME];
139
+ if (tag)
140
+ byTag.set(tag, m);
141
+ else
142
+ untagged.push(m);
143
+ }
144
+ return {
145
+ schemaVersion: 2,
146
+ mediaType: lower?.mediaType ?? upper?.mediaType ?? "application/vnd.oci.image.index.v1+json",
147
+ manifests: [...untagged, ...byTag.values()],
148
+ };
149
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@specific.dev/spectest",
3
- "version": "0.73.1",
3
+ "version": "0.75.0",
4
4
  "description": "Spectest SDK for defining test environments in TypeScript.",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
package/src/daemon.ts CHANGED
@@ -83,7 +83,7 @@ import {
83
83
  type CaseMeta,
84
84
  type GroupMeta,
85
85
  } from "./harness/catalogue.js";
86
- import { summarizeBuildKit, type BuildStep } from "./harness/buildkit-progress.js";
86
+ import { executedSteps, summarizeBuildKit, type BuildStep } from "./harness/buildkit-progress.js";
87
87
  import { LOG_DELTA_MAX_BYTES, capMiddle, streamDelta } from "./harness/log-delta.js";
88
88
  import {
89
89
  resolveHostPath as resolveVolumeHostPath,
@@ -135,7 +135,17 @@ import {
135
135
  } from "./harness/ingress-table.js";
136
136
  import { startTlsTerminator, type TlsTerminator } from "./harness/tls-terminator.js";
137
137
  import { runContainerArgs } from "./harness/container-run.js";
138
- import { IMAGE_CACHE_MANIFEST, imageCachePathsSync, isOnImageCache, type ImageCachePaths } from "./harness/image-cache.js";
138
+ import {
139
+ BUILDKIT_CACHE_DIR,
140
+ BUILDKIT_CACHE_UNION,
141
+ buildkitCacheTag,
142
+ IMAGE_CACHE_MANIFEST,
143
+ imageCachePathsSync,
144
+ isOnImageCache,
145
+ mergeCacheIndex,
146
+ type ImageCachePaths,
147
+ type OciIndex,
148
+ } from "./harness/image-cache.js";
139
149
  import {
140
150
  assertAbsolute,
141
151
  certificateHostnames,
@@ -778,14 +788,93 @@ async function ensureLocalBuildkitd(): Promise<boolean> {
778
788
  }
779
789
  // eslint-disable-next-line no-console
780
790
  console.log(`[build] building in this VM against the image cache (root ${paths.root}, layers ${paths.layers})`);
781
- // Imports come from the merged cache on the read-only layers disk and
782
- // from this lineage's own exports on the root; exports go to the root,
783
- // where the merge picks them up (image_cache/merge.rs).
784
- _localCacheDir = `${state}/spectest-buildkit-cache`;
785
- _localCacheImports = [`${paths.layers}/spectest-buildkit-cache`, `${state}/spectest-buildkit-cache`];
791
+ const union = await ensureBuildkitCacheUnion(paths);
792
+ if (union) {
793
+ // One directory for both flags: the merged cache on the layers disk
794
+ // seen through this lineage's own exports on the root. See
795
+ // `BUILDKIT_CACHE_UNION` for why the exporter needs the union.
796
+ _localCacheDir = union;
797
+ _localCacheImports = [union];
798
+ } else {
799
+ // Imports come from the merged cache on the read-only layers disk and
800
+ // from this lineage's own exports on the root; exports go to the root,
801
+ // where the merge picks them up (image_cache/merge.rs). Every cold
802
+ // build re-exports every blob this way, which is the cost the union
803
+ // removes.
804
+ _localCacheDir = `${state}/${BUILDKIT_CACHE_DIR}`;
805
+ _localCacheImports = [`${paths.layers}/${BUILDKIT_CACHE_DIR}`, `${state}/${BUILDKIT_CACHE_DIR}`];
806
+ }
786
807
  _localBuilder = true;
787
808
  return true;
788
809
  }
810
+
811
+ /**
812
+ * Mount the BuildKit cache union (`BUILDKIT_CACHE_UNION`): overlayfs with
813
+ * the layers disk's cache directory as the lower and the root disk's as
814
+ * the upper, so one path serves `--cache-from` and `--cache-to`.
815
+ *
816
+ * Idempotent — a delta restore restarts the harness on a guest that
817
+ * still has the mount. Returns null, and the caller keeps the two-directory
818
+ * scheme, when the lower does not exist yet (a volume that has never
819
+ * been merged has no cache directory on its layers disk) or the mount is
820
+ * refused; the build is then slower, never failed. The work directory
821
+ * has to share a filesystem with the upper, so it lives on the root disk
822
+ * beside it; the merge reads neither it nor anything but the upper.
823
+ */
824
+ async function ensureBuildkitCacheUnion(paths: ImageCachePaths): Promise<string | null> {
825
+ const lower = `${paths.layers}/${BUILDKIT_CACHE_DIR}`;
826
+ const upper = `${paths.root}/${BUILDKIT_CACHE_DIR}`;
827
+ const work = `${paths.root}/spectest-buildkit-work`;
828
+ if (!existsSync(lower)) {
829
+ // eslint-disable-next-line no-console
830
+ console.log(`[build] no merged BuildKit cache on the layers disk yet; exporting to the root directly`);
831
+ return null;
832
+ }
833
+ // overlayfs shows the upper's `index.json` INSTEAD of the lower's, so
834
+ // an index on the root (a lineage's exports, or one left behind with
835
+ // its blobs gone) would hide every merged tag and, naming manifests
836
+ // that are not there, make BuildKit skip the import altogether. Write
837
+ // the union of the two into the upper first (`mergeCacheIndex`), and
838
+ // say what each side held so a miss can be read off the boot log.
839
+ try {
840
+ const readIndex = (dir: string): OciIndex | null =>
841
+ existsSync(`${dir}/index.json`) ? (JSON.parse(readFileSync(`${dir}/index.json`, "utf8")) as OciIndex) : null;
842
+ const lowerIdx = readIndex(lower);
843
+ const upperIdx = readIndex(upper);
844
+ if (upperIdx && lowerIdx) {
845
+ const merged = mergeCacheIndex(lowerIdx, upperIdx, (hex) => existsSync(`${upper}/blobs/sha256/${hex}`));
846
+ await fs.mkdir(upper, { recursive: true });
847
+ await fs.writeFile(`${upper}/index.json`, JSON.stringify(merged));
848
+ await fs.writeFile(`${upper}/oci-layout`, JSON.stringify({ imageLayoutVersion: "1.0.0" }));
849
+ // eslint-disable-next-line no-console
850
+ console.log(
851
+ `[build] BuildKit cache union: ${lowerIdx.manifests.length} tag(s) from the merged cache, ${upperIdx.manifests.length} on this root, ${merged.manifests.length} after the merge`,
852
+ );
853
+ } else {
854
+ // eslint-disable-next-line no-console
855
+ console.log(`[build] BuildKit cache union: ${lowerIdx?.manifests.length ?? 0} tag(s) from the merged cache, none on this root`);
856
+ }
857
+ } catch (err) {
858
+ // eslint-disable-next-line no-console
859
+ console.warn(`[build] could not reconcile the BuildKit cache indexes (${err}); the union shows the root's`);
860
+ }
861
+ const r = await shx(
862
+ "/bin/sh",
863
+ [
864
+ "-c",
865
+ `mkdir -p "${BUILDKIT_CACHE_UNION}" "${upper}" "${work}" && ` +
866
+ `{ mountpoint -q "${BUILDKIT_CACHE_UNION}" || ` +
867
+ `mount -t overlay overlay -o "lowerdir=${lower},upperdir=${upper},workdir=${work}" "${BUILDKIT_CACHE_UNION}"; }`,
868
+ ],
869
+ 60_000,
870
+ );
871
+ if (r.code !== 0) {
872
+ // eslint-disable-next-line no-console
873
+ console.warn(`[build] could not mount the BuildKit cache union; exporting to the root directly:\n${(r.stderr || r.stdout).trim()}`);
874
+ return null;
875
+ }
876
+ return BUILDKIT_CACHE_UNION;
877
+ }
789
878
  /** The exported-cache directory on the cache disk, once the in-VM builder is up. */
790
879
  let _localCacheDir: string | null = null;
791
880
  /** The cache directories a build imports from: the merged one on the
@@ -1353,6 +1442,10 @@ async function runServiceBuild(
1353
1442
  const useBuildKit = useLocal || (await hasBuildx());
1354
1443
  const buildEnv: Record<string, string> = {};
1355
1444
  let buildArgs: string[];
1445
+ // The `--cache-to` flag, held back for a second pass that runs only
1446
+ // when the first one executed a step (see below). Null off the in-VM
1447
+ // builder.
1448
+ let cacheExport: string[] | null = null;
1356
1449
  // The user's `buildArgs`, as `--build-arg` flags; a plain client flag,
1357
1450
  // so every builder — host buildkitd, in-VM BuildKit, legacy — takes it.
1358
1451
  const argFlags = buildArgFlags(image.buildArgs);
@@ -1364,7 +1457,19 @@ async function runServiceBuild(
1364
1457
  // disk is what outlives the VM; `mode=max` keeps every
1365
1458
  // intermediate layer, uncompressed so an import never inflates,
1366
1459
  // and one tag per service so exports do not replace each other.
1367
- const cacheTag = name.replace(/[^a-z0-9-]/gi, "-").toLowerCase();
1460
+ // The tag carries the `spectest/service/` prefix because the
1461
+ // directory is one flat tag namespace the project's own BuildKit
1462
+ // writes into too (the docs tell it to mount this path): tags merge
1463
+ // newest-wins with no warning anywhere, and a project naturally tags
1464
+ // a build after the thing it builds, which is also what its service
1465
+ // is named. Specific's `nginx-mod` service tagged its own buildctl
1466
+ // cache `nginx-mod` and the two overwrote each other on alternate
1467
+ // boots (reported 2026-09-06). The prefix is reserved in the docs.
1468
+ const cacheTag = buildkitCacheTag(name);
1469
+ cacheExport = [
1470
+ "--cache-to",
1471
+ `type=local,dest=${_localCacheDir},mode=max,compression=uncompressed,force-compression=true,tag=${cacheTag}`,
1472
+ ];
1368
1473
  // LANDMINE: the local cache importer reads the `latest` entry of
1369
1474
  // the directory's index unless told otherwise, and the exporter
1370
1475
  // below writes this service's entry under `tag=<service>`. An
@@ -1377,7 +1482,6 @@ async function runServiceBuild(
1377
1482
  "--progress=plain",
1378
1483
  "--output", `type=image,name=${qualifyImageRef(tag)},unpack=true`,
1379
1484
  ..._localCacheImports.flatMap((src) => ["--cache-from", `type=local,src=${src},tag=${cacheTag}`]),
1380
- "--cache-to", `type=local,dest=${_localCacheDir},mode=max,compression=uncompressed,force-compression=true,tag=${cacheTag}`,
1381
1485
  ...argFlags,
1382
1486
  "-f", dfPath, WORKSPACE,
1383
1487
  ];
@@ -1425,6 +1529,27 @@ async function runServiceBuild(
1425
1529
  if (build.code !== 0) {
1426
1530
  return { ok: false, log };
1427
1531
  }
1532
+ // Export the cache only when there is something new to export. A
1533
+ // `mode=max` export costs ~3 s per service even when every blob is
1534
+ // already in the destination (measured 2026-09-06, api: 2.2–3.0 s
1535
+ // for the cached build alone, 4.9–7.5 s with the export), and a cold
1536
+ // start of a cached project is nothing but such builds. When a step
1537
+ // did run, the second pass is that same build fully cached from the
1538
+ // daemon's own state plus the export — seconds on top of a build
1539
+ // that took tens of them. Best-effort: the image is already in the
1540
+ // store, so a failed export is a slower next cold start, not a
1541
+ // failed service.
1542
+ if (cacheExport) {
1543
+ const ran = executedSteps(build.stderr);
1544
+ if (ran.length > 0) {
1545
+ progressService(name, { status: "building", detail: `exporting cache (${ran.length} step(s) ran)` });
1546
+ const exported = await shxStream("docker", [...buildArgs, ...cacheExport], 1_800_000, buildEnv, () => {});
1547
+ if (exported.code !== 0) {
1548
+ // eslint-disable-next-line no-console
1549
+ console.warn(`[build] ${name}: cache export failed; the next cold start rebuilds it:\n${exported.stderr.trim().slice(-2000)}`);
1550
+ }
1551
+ }
1552
+ }
1428
1553
  if (useBuildKit) {
1429
1554
  // Keep only the slowest dozen steps ≥1s — enough to profile, small
1430
1555
  // enough to ride back in the /bootstrap response and the journal.
@@ -1,6 +1,6 @@
1
1
  import { describe, expect, test } from "bun:test";
2
2
 
3
- import { summarizeBuildKit } from "./buildkit-progress";
3
+ import { executedSteps, summarizeBuildKit } from "./buildkit-progress";
4
4
 
5
5
  /** A realistic slice of `docker build --progress=plain` output. */
6
6
  const SAMPLE = `
@@ -96,3 +96,56 @@ describe("summarizeBuildKit", () => {
96
96
  expect(out[0].cached).toBe(true);
97
97
  });
98
98
  });
99
+
100
+ /** Captured from a fully cached in-VM build of a real service
101
+ * (2026-09-06): bookkeeping steps report DONE, the base image resolves
102
+ * in no time, every Dockerfile step is CACHED. */
103
+ const FULLY_CACHED = `
104
+ #1 [internal] load build definition from Dockerfile
105
+ #1 transferring dockerfile: 3.38kB done
106
+ #1 DONE 0.0s
107
+ #2 resolve image config for docker-image://docker.io/docker/dockerfile:1
108
+ #2 DONE 0.1s
109
+ #4 [internal] load metadata for docker.io/library/node:25-slim
110
+ #4 DONE 0.1s
111
+ #5 [stage-0 1/19] FROM docker.io/library/node:25-slim@sha256:81db02c4b671288a03915da9534dbd54f96d0e7c24d80ccc54f5b36b2e684370
112
+ #5 DONE 0.0s
113
+ #6 [internal] load build context
114
+ #6 transferring context: 11.87kB done
115
+ #6 DONE 0.0s
116
+ #7 [stage-0 5/19] RUN --mount=type=cache,target=/pnpm-store,sharing=locked cd config && pnpm install --frozen-lockfile
117
+ #7 CACHED
118
+ #8 [stage-0 10/19] COPY packages/tunnel/client tunnel/client
119
+ #8 CACHED
120
+ #25 exporting to image
121
+ #25 exporting layers done
122
+ #25 DONE 0.3s
123
+ `;
124
+
125
+ describe("executedSteps", () => {
126
+ test("a fully cached build ran nothing", () => {
127
+ expect(executedSteps(FULLY_CACHED)).toEqual([]);
128
+ });
129
+
130
+ test("a step that ended in DONE ran", () => {
131
+ expect(executedSteps(SAMPLE)).toEqual([
132
+ "RUN go build -o /out/api ./cmd/api",
133
+ "COPY --from=builder /out/api /usr/local/bin/api",
134
+ ]);
135
+ });
136
+
137
+ /** A base image that had to be pulled is new content the export
138
+ * records; one found locally in 0.0s is not. */
139
+ test("a pulled base image counts, a local one does not", () => {
140
+ const pulled = FULLY_CACHED.replace("#5 DONE 0.0s", "#5 DONE 4.2s");
141
+ const ran = executedSteps(pulled);
142
+ expect(ran).toHaveLength(1);
143
+ // Names are capped like `summarizeBuildKit`'s, so match the prefix.
144
+ expect(ran[0].startsWith("FROM docker.io/library/node:25-slim@sha256:")).toBe(true);
145
+ });
146
+
147
+ test("bookkeeping steps never count", () => {
148
+ const out = "#1 [internal] load build context\n#1 DONE 2.5s\n#2 [auth] library/node:pull token\n#2 DONE 0.4s\n";
149
+ expect(executedSteps(out)).toEqual([]);
150
+ });
151
+ });
@@ -72,3 +72,54 @@ export function summarizeBuildKit(out: string): BuildStep[] {
72
72
  }
73
73
  return steps.sort((a, b) => b.secs - a.secs);
74
74
  }
75
+
76
+ /**
77
+ * The Dockerfile steps a build actually ran — the ones a cache export
78
+ * would have something new to record.
79
+ *
80
+ * A fully cached build still has to export its cache today, and that
81
+ * export costs ~3 s per service in `mode=max` even when every blob is
82
+ * already in the destination (measured 2026-09-06: 2.2–3.0 s for the
83
+ * build, 4.9–7.5 s with the export). So the harness builds without
84
+ * `--cache-to` first and exports only when this says something ran.
85
+ *
86
+ * What counts: a bracketed stage step (`[builder 3/6] RUN …`) that ended
87
+ * in `DONE` rather than `CACHED`. What does not: BuildKit's own
88
+ * `[internal]`/`[auth]` bookkeeping, which is never cached and never a
89
+ * layer, and a `FROM` step that finished in the blink it takes to find
90
+ * the base image locally — a pulled base takes seconds and does count,
91
+ * since the export records its layers too.
92
+ */
93
+ export function executedSteps(out: string): string[] {
94
+ const names = new Map<string, string>();
95
+ const stages = new Map<string, string>();
96
+ const secs = new Map<string, number>();
97
+ const cached = new Set<string>();
98
+ for (const line of out.split("\n")) {
99
+ let m = line.match(/^#(\d+)\s+\[([^\]]*)\]\s+(.+)$/);
100
+ if (m) {
101
+ const id = `#${m[1]}`;
102
+ if (!names.has(id)) {
103
+ names.set(id, m[3].trim().slice(0, MAX_NAME));
104
+ stages.set(id, m[2].trim());
105
+ }
106
+ continue;
107
+ }
108
+ m = line.match(/^#(\d+)\s+DONE\s+([\d.]+)s/);
109
+ if (m) {
110
+ secs.set(`#${m[1]}`, parseFloat(m[2]));
111
+ continue;
112
+ }
113
+ m = line.match(/^#(\d+)\s+CACHED/);
114
+ if (m) cached.add(`#${m[1]}`);
115
+ }
116
+ const ran: string[] = [];
117
+ for (const [id, name] of names) {
118
+ if (cached.has(id)) continue;
119
+ const stage = stages.get(id) ?? "";
120
+ if (stage === "internal" || stage === "auth") continue;
121
+ if (/^FROM\s/.test(name) && (secs.get(id) ?? 0) < 0.5) continue;
122
+ ran.push(name);
123
+ }
124
+ return ran;
125
+ }
@@ -0,0 +1,81 @@
1
+ import { describe, expect, test } from "bun:test";
2
+
3
+ import { BUILDKIT_CACHE_UNION, isOnImageCache } from "./image-cache";
4
+
5
+ const paths = { root: "/var/lib/containerd", layers: "/var/lib/spectest-image-cache" };
6
+
7
+ describe("isOnImageCache", () => {
8
+ test("the two cache disks and anything under them", () => {
9
+ expect(isOnImageCache("/var/lib/containerd", paths)).toBe(true);
10
+ expect(isOnImageCache("/var/lib/containerd/spectest-nested/k8s", paths)).toBe(true);
11
+ expect(isOnImageCache("/var/lib/spectest-image-cache/spectest-buildkit-cache", paths)).toBe(true);
12
+ });
13
+
14
+ /** The union is the root's export directory seen through the layers
15
+ * disk; wiping it at a delta restore would wipe this lineage's exports. */
16
+ test("the BuildKit cache union counts as cache", () => {
17
+ expect(isOnImageCache(BUILDKIT_CACHE_UNION, paths)).toBe(true);
18
+ expect(isOnImageCache(`${BUILDKIT_CACHE_UNION}/blobs`, paths)).toBe(true);
19
+ });
20
+
21
+ test("an ordinary project volume does not", () => {
22
+ expect(isOnImageCache("/var/cache/spectest/volumes/db", paths)).toBe(false);
23
+ expect(isOnImageCache("/var/lib/containerd-not", paths)).toBe(false);
24
+ });
25
+
26
+ test("no cache, nothing counts", () => {
27
+ expect(isOnImageCache(BUILDKIT_CACHE_UNION, null)).toBe(false);
28
+ });
29
+ });
30
+
31
+ import { BUILDKIT_CACHE_TAG_PREFIX, buildkitCacheTag, mergeCacheIndex, type OciIndex } from "./image-cache";
32
+
33
+ const entry = (tag: string, hex: string) => ({ mediaType: "application/vnd.oci.image.manifest.v1+json", digest: `sha256:${hex}`, size: 1, annotations: { "org.opencontainers.image.ref.name": tag } });
34
+
35
+ describe("mergeCacheIndex", () => {
36
+ const lower: OciIndex = { schemaVersion: 2, manifests: [entry("api", "a1"), entry("cli", "c1")] };
37
+
38
+ test("no upper index: the merged cache's tags, unchanged", () => {
39
+ const out = mergeCacheIndex(lower, null, () => false);
40
+ expect(out.manifests.map((m) => m.annotations?.["org.opencontainers.image.ref.name"])).toEqual(["api", "cli"]);
41
+ });
42
+
43
+ /** The lineage's own export of a service wins over the merged one. */
44
+ test("an upper entry whose manifest exists overrides the lower's tag", () => {
45
+ const upper: OciIndex = { schemaVersion: 2, manifests: [entry("api", "a2")] };
46
+ const out = mergeCacheIndex(lower, upper, (hex) => hex === "a2");
47
+ const api = out.manifests.find((m) => m.annotations?.["org.opencontainers.image.ref.name"] === "api");
48
+ expect(api?.digest).toBe("sha256:a2");
49
+ expect(out.manifests).toHaveLength(2);
50
+ });
51
+
52
+ /** A stale upper index — blobs emptied, index left — would hide every
53
+ * lower tag and fail the import; its entries are dropped instead. */
54
+ test("an upper entry without its manifest blob is dropped", () => {
55
+ const upper: OciIndex = { schemaVersion: 2, manifests: [entry("api", "a2"), entry("worker", "w1")] };
56
+ const out = mergeCacheIndex(lower, upper, () => false);
57
+ expect(out.manifests.map((m) => m.digest)).toEqual(["sha256:a1", "sha256:c1"]);
58
+ });
59
+
60
+ test("a valid upper tag the lower lacks is added", () => {
61
+ const upper: OciIndex = { schemaVersion: 2, manifests: [entry("worker", "w1")] };
62
+ const out = mergeCacheIndex(lower, upper, () => true);
63
+ expect(out.manifests.map((m) => m.annotations?.["org.opencontainers.image.ref.name"])).toEqual(["api", "cli", "worker"]);
64
+ });
65
+
66
+ /** The directory is one tag namespace shared with the project's own
67
+ * BuildKit. The harness's tags are prefixed so a project tag named after
68
+ * the same service sits beside it instead of replacing it. */
69
+ test("a project's tag and the harness's tag for the same service both survive", () => {
70
+ const both: OciIndex = { schemaVersion: 2, manifests: [entry("nginx-mod", "p1"), entry(buildkitCacheTag("nginx-mod"), "h1")] };
71
+ const out = mergeCacheIndex(both, null, () => false);
72
+ expect(out.manifests.map((m) => m.digest)).toEqual(["sha256:p1", "sha256:h1"]);
73
+ });
74
+ });
75
+
76
+ describe("buildkitCacheTag", () => {
77
+ test("the reserved prefix plus the normalised service name", () => {
78
+ expect(buildkitCacheTag("nginx-mod")).toBe(`${BUILDKIT_CACHE_TAG_PREFIX}nginx-mod`);
79
+ expect(buildkitCacheTag("Api_v2")).toBe("spectest/service/api-v2");
80
+ });
81
+ });
@@ -43,6 +43,50 @@ export const MKFS_EROFS_PATH = "/usr/local/bin/mkfs.erofs";
43
43
  * nested runtime's busybox can run the same file. */
44
44
  export const STORE_ADOPT_PATH = "/usr/local/bin/spectest-store-adopt";
45
45
 
46
+ /**
47
+ * The BuildKit cache directory under each cache disk (`merge.rs::CACHE_DIR`):
48
+ * on the layers disk the merged export of every earlier build, read-only;
49
+ * on the root disk this lineage's own exports, which the merge harvests.
50
+ */
51
+ export const BUILDKIT_CACHE_DIR = "spectest-buildkit-cache";
52
+
53
+ /**
54
+ * One directory that is both: an overlayfs with the layers disk's cache
55
+ * directory as the read-only lower and the root disk's as the upper,
56
+ * mounted here by the harness when the in-VM builder comes up
57
+ * (`daemon.ts::ensureBuildkitCacheUnion`). A build imports from it and
58
+ * exports to it. BuildKit's local exporter skips a blob its destination
59
+ * already holds, and through the union every merged blob is already
60
+ * there — so a fully cached build exports its index and manifest and
61
+ * nothing else, where a fresh directory on the root cost it every
62
+ * uncompressed layer again (1.1 GB and 20 s for one service, measured
63
+ * 2026-09-06). New layers land in the upper, which is exactly the
64
+ * directory the merge read before. A user-run BuildKit mounts this one
65
+ * path for both of its cache flags.
66
+ */
67
+ export const BUILDKIT_CACHE_UNION = "/var/lib/spectest-buildkit-cache";
68
+
69
+ /**
70
+ * The prefix under which the harness tags its own exports in that
71
+ * directory. It is one flat tag namespace shared with the project's own
72
+ * BuildKit (the `/services` docs page tells it to mount the union), tags
73
+ * merge newest-wins with no warning, and a project naturally tags a build
74
+ * after the thing it builds — which is what its spectest service is named
75
+ * too. So the harness's tags live under a prefix no project would choose,
76
+ * and the docs reserve it. Reported by Specific 2026-09-06: a service
77
+ * `nginx-mod` running `buildctl` tagged its cache `nginx-mod`, and it and
78
+ * the harness's image build for that very container overwrote each other
79
+ * on alternate boots.
80
+ */
81
+ export const BUILDKIT_CACHE_TAG_PREFIX = "spectest/service/";
82
+
83
+ /** The BuildKit cache tag for a service's Dockerfile build: the prefix plus
84
+ * the service name, lowercased with anything outside `[a-z0-9-]` folded
85
+ * to `-`. */
86
+ export function buildkitCacheTag(service: string): string {
87
+ return `${BUILDKIT_CACHE_TAG_PREFIX}${service.replace(/[^a-z0-9-]/gi, "-").toLowerCase()}`;
88
+ }
89
+
46
90
  function parse(raw: string): ImageCachePaths | null {
47
91
  const parsed = JSON.parse(raw) as { disks?: { role?: string; path?: string }[] };
48
92
  const root = (parsed.disks ?? []).find((d) => d.role === "root" && d.path)?.path;
@@ -75,5 +119,56 @@ export function nestedStoreDir(paths: ImageCachePaths, service: string): string
75
119
  */
76
120
  export function isOnImageCache(hostPath: string, paths: ImageCachePaths | null): boolean {
77
121
  if (!paths) return false;
122
+ // The union is the root's cache directory seen through the layers
123
+ // disk: wiping it would wipe the upper, i.e. this lineage's exports.
124
+ if (hostPath === BUILDKIT_CACHE_UNION || hostPath.startsWith(`${BUILDKIT_CACHE_UNION}/`)) return true;
78
125
  return hostPath === paths.root || hostPath.startsWith(`${paths.root}/`) || hostPath === paths.layers || hostPath.startsWith(`${paths.layers}/`);
79
126
  }
127
+
128
+ /** The shape of an OCI layout's `index.json` the BuildKit local cache
129
+ * exporter writes: one manifest per tag, the tag in the ref-name
130
+ * annotation. */
131
+ export interface OciIndex {
132
+ schemaVersion?: number;
133
+ mediaType?: string;
134
+ manifests: Array<{ mediaType?: string; digest: string; size?: number; annotations?: Record<string, string> }>;
135
+ }
136
+
137
+ const REF_NAME = "org.opencontainers.image.ref.name";
138
+
139
+ /**
140
+ * The index the BuildKit cache union must show: every tag of the merged
141
+ * cache (lower), overridden by this lineage's own exports (upper) where
142
+ * the upper still holds the manifest.
143
+ *
144
+ * overlayfs shadows whole files, so an `index.json` in the upper hides
145
+ * the lower's completely — and a root disk can carry one: the exports
146
+ * of a lineage that built before the union existed, or an index left
147
+ * behind with its blobs gone. Either way BuildKit then imports from an
148
+ * index naming a few tags whose manifests may not exist, skips the
149
+ * import, and every step runs (seen 2026-09-06: a cold build with
150
+ * every service's cache on the layers disk and 4 stale tags in the
151
+ * upper built everything from scratch). `upperHasBlob` says whether the
152
+ * upper holds a manifest's blob; an upper entry without one is dropped,
153
+ * since it would fail the whole import.
154
+ */
155
+ export function mergeCacheIndex(lower: OciIndex | null, upper: OciIndex | null, upperHasBlob: (hex: string) => boolean): OciIndex {
156
+ const byTag = new Map<string, OciIndex["manifests"][number]>();
157
+ const untagged: OciIndex["manifests"] = [];
158
+ for (const m of lower?.manifests ?? []) {
159
+ const tag = m.annotations?.[REF_NAME];
160
+ if (tag) byTag.set(tag, m);
161
+ else untagged.push(m);
162
+ }
163
+ for (const m of upper?.manifests ?? []) {
164
+ if (!upperHasBlob(m.digest.replace(/^sha256:/, ""))) continue;
165
+ const tag = m.annotations?.[REF_NAME];
166
+ if (tag) byTag.set(tag, m);
167
+ else untagged.push(m);
168
+ }
169
+ return {
170
+ schemaVersion: 2,
171
+ mediaType: lower?.mediaType ?? upper?.mediaType ?? "application/vnd.oci.image.index.v1+json",
172
+ manifests: [...untagged, ...byTag.values()],
173
+ };
174
+ }