@specific.dev/spectest 0.91.0 → 0.91.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,7 +15,7 @@ export function androidTargetLabel(target) {
15
15
  case "id":
16
16
  return `id ${quoted}`;
17
17
  case "accessibility":
18
- return `accessibility label ${quoted}`;
18
+ return `label ${quoted}`;
19
19
  case "xpath":
20
20
  return `XPath ${quoted}`;
21
21
  case "custom":
@@ -74,5 +74,5 @@ export function androidStepTitle(step) {
74
74
  label = `run command ${JSON.stringify(step.command)}`;
75
75
  break;
76
76
  }
77
- return `${step.device}: ${label}`;
77
+ return label;
78
78
  }
@@ -328,6 +328,8 @@ export interface K3sHelpers {
328
328
  limit?: number;
329
329
  }): Promise<Wrapped<KubernetesListObject<T>>>;
330
330
  }
331
+ /** @internal Exported for regression tests; not part of the components API. */
332
+ export declare function isTransientApplyError(err: unknown): boolean;
331
333
  export declare function k3s(opts?: K3sOptions): {
332
334
  image: {
333
335
  type: "registry";
@@ -1,5 +1,6 @@
1
1
  import { isIP } from "node:net";
2
2
  import { installPodTrust } from "./k3s-trust.js";
3
+ import { waitForAll } from "../harness/concurrency.js";
3
4
  import { loadK3sBuildImages } from "../harness/k3s-image-loader.js";
4
5
  import { AsyncLocalStorage } from "node:async_hooks";
5
6
  import { spawn as nodeSpawn } from "node:child_process";
@@ -909,8 +910,33 @@ const TRANSIENT_APPLY_PATTERNS = [
909
910
  function errorText(err) {
910
911
  return err?.message ?? String(err);
911
912
  }
912
- function isTransientApplyError(err) {
913
- const msg = errorText(err);
913
+ /** ApiException formats its body with JSON.stringify, escaping Status.message.
914
+ * Read the actual response instead of matching that presentation string. */
915
+ function applyErrorStatus(err) {
916
+ let body = err?.body;
917
+ if (typeof body === "string") {
918
+ try {
919
+ body = JSON.parse(body);
920
+ }
921
+ catch {
922
+ return undefined;
923
+ }
924
+ }
925
+ return body && typeof body === "object" && !Array.isArray(body) ? body : undefined;
926
+ }
927
+ /** @internal Exported for regression tests; not part of the components API. */
928
+ export function isTransientApplyError(err) {
929
+ const status = applyErrorStatus(err);
930
+ const code = status?.code ?? err?.code;
931
+ if ([429, 500, 503, 504].includes(code))
932
+ return true;
933
+ // A missing namespace can converge while another apply creates it. An
934
+ // arbitrary resource 404 cannot: don't classify by HTTP status alone.
935
+ if (code === 404 && status?.reason === "NotFound" &&
936
+ status.details?.kind === "namespaces" && !status.details.group &&
937
+ typeof status.details.name === "string" && status.details.name.length > 0)
938
+ return true;
939
+ const msg = typeof status?.message === "string" ? status.message : errorText(err);
914
940
  return TRANSIENT_APPLY_PATTERNS.some((re) => re.test(msg));
915
941
  }
916
942
  function describeDoc(doc) {
@@ -1243,48 +1269,51 @@ async function waitForJob(deps, name, opts) {
1243
1269
  * reachable over HTTPS with a cert the test framework already trusts.
1244
1270
  */
1245
1271
  async function setupK3sCluster(name, helpers, opts) {
1246
- const tlsEnabled = opts.traefik && opts.ingressDomains.length > 0 && caPresent();
1247
- if (tlsEnabled) {
1248
- const { cert, key } = await issueIngressCert(opts.ingressDomains.map((d) => `*.${d}`));
1249
- // Apply the cert Secret + dynamic-config ConfigMap before the
1250
- // Deployment that mounts them. `stringData` lets us hand over plain
1251
- // PEM; the API server base64-encodes it.
1252
- await helpers.client.core.createNamespacedSecret({
1253
- namespace: "kube-system",
1254
- body: {
1255
- metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
1256
- type: "kubernetes.io/tls",
1257
- stringData: { "tls.crt": cert, "tls.key": key },
1258
- },
1259
- });
1260
- await helpers.client.core.createNamespacedConfigMap({
1261
- namespace: "kube-system",
1262
- body: {
1263
- metadata: {
1264
- name: TRAEFIK_DYNAMIC_CONFIGMAP,
1265
- namespace: "kube-system",
1266
- },
1267
- data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
1268
- },
1269
- });
1270
- }
1271
- if (opts.traefik)
1272
- await helpers.apply(buildTraefikManifest(tlsEnabled));
1273
- if (opts.registry)
1274
- await helpers.apply(REGISTRY_MANIFEST);
1275
- // Both rollouts proceed independently inside the cluster — wait on them
1276
- // concurrently (they used to serialize, wasting up to a rollout's tail).
1277
- // 250ms polling: these two sit on the cold start's critical path, where
1278
- // the default 500ms interval wastes real time.
1272
+ // The registry can start while Traefik's certificates are being prepared.
1279
1273
  const wait = (deployment) => helpers.waitForRollout(deployment, {
1280
1274
  namespace: "kube-system",
1281
1275
  timeoutMs: 120_000,
1282
1276
  intervalMs: 250,
1283
1277
  });
1284
- const waits = opts.traefik ? [wait("traefik")] : [];
1285
- if (opts.registry)
1286
- waits.push(wait("spectest-registry"));
1287
- await Promise.all(waits);
1278
+ await waitForAll([
1279
+ (async () => {
1280
+ if (!opts.traefik)
1281
+ return;
1282
+ const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
1283
+ if (tlsEnabled) {
1284
+ const { cert, key } = await issueIngressCert(opts.ingressDomains.map((d) => `*.${d}`));
1285
+ // Apply the cert Secret + dynamic-config ConfigMap before the
1286
+ // Deployment that mounts them. `stringData` lets us hand over plain
1287
+ // PEM; the API server base64-encodes it.
1288
+ await helpers.client.core.createNamespacedSecret({
1289
+ namespace: "kube-system",
1290
+ body: {
1291
+ metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
1292
+ type: "kubernetes.io/tls",
1293
+ stringData: { "tls.crt": cert, "tls.key": key },
1294
+ },
1295
+ });
1296
+ await helpers.client.core.createNamespacedConfigMap({
1297
+ namespace: "kube-system",
1298
+ body: {
1299
+ metadata: {
1300
+ name: TRAEFIK_DYNAMIC_CONFIGMAP,
1301
+ namespace: "kube-system",
1302
+ },
1303
+ data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
1304
+ },
1305
+ });
1306
+ }
1307
+ await helpers.apply(buildTraefikManifest(tlsEnabled));
1308
+ await wait("traefik");
1309
+ })(),
1310
+ (async () => {
1311
+ if (!opts.registry)
1312
+ return;
1313
+ await helpers.apply(REGISTRY_MANIFEST);
1314
+ await wait("spectest-registry");
1315
+ })(),
1316
+ ]);
1288
1317
  }
1289
1318
  /**
1290
1319
  * Snapshot of a namespace's state, dumped whenever a wait times out: pod
@@ -1571,54 +1600,57 @@ export function k3s(opts = {}) {
1571
1600
  const runtime = componentRuntime(ctx);
1572
1601
  if (images.builds.length && !runtime)
1573
1602
  throw new Error(`k3s(${name}): prepared image inputs are missing`);
1574
- if (runtime?.caBundle && caPresent()) {
1575
- const { cert, key } = await issueIngressCert([runtime.gateway]);
1576
- await installPodTrust({ name, gateway: runtime.gateway, cert, key,
1577
- ca: await readFile(CA_PATH, "utf8"), bundle: await readFile(runtime.caBundle, "utf8"),
1578
- }, {
1579
- apply: async (configuration) => { await helpers.apply(JSON.stringify(configuration)); },
1580
- ensureBundle: async (namespace, configMap, bundle) => {
1581
- try {
1582
- await helpers.client.core.createNamespacedConfigMap({ namespace, body: {
1583
- metadata: { name: configMap, namespace }, immutable: true, data: { "ca-bundle.crt": bundle },
1584
- } });
1585
- }
1586
- catch (err) {
1587
- // A simultaneous Pod may have created the same namespace's bundle.
1588
- const existing = (await helpers.client.core.readNamespacedConfigMap({ namespace, name: configMap })).unwrap();
1589
- if (existing.data?.["ca-bundle.crt"] !== bundle || existing.immutable !== true)
1590
- throw err;
1603
+ await waitForAll([
1604
+ runtime ? loadK3sBuildImages(name, runtime.images, runtime.gateway, exec) : Promise.resolve(),
1605
+ prePullK3sImages(name, images.pulls, exec),
1606
+ (async () => {
1607
+ if (runtime?.caBundle && caPresent()) {
1608
+ const { cert, key } = await issueIngressCert([runtime.gateway]);
1609
+ await installPodTrust({ name, gateway: runtime.gateway, cert, key,
1610
+ ca: await readFile(CA_PATH, "utf8"), bundle: await readFile(runtime.caBundle, "utf8"),
1611
+ }, {
1612
+ apply: async (configuration) => { await helpers.apply(JSON.stringify(configuration)); },
1613
+ ensureBundle: async (namespace, configMap, bundle) => {
1614
+ try {
1615
+ await helpers.client.core.createNamespacedConfigMap({ namespace, body: {
1616
+ metadata: { name: configMap, namespace }, immutable: true, data: { "ca-bundle.crt": bundle },
1617
+ } });
1618
+ }
1619
+ catch (err) {
1620
+ // A simultaneous Pod may have created the same namespace's bundle.
1621
+ const existing = (await helpers.client.core.readNamespacedConfigMap({ namespace, name: configMap })).unwrap();
1622
+ if (existing.data?.["ca-bundle.crt"] !== bundle || existing.immutable !== true)
1623
+ throw err;
1624
+ }
1625
+ },
1626
+ });
1627
+ // The API server observes webhook registration asynchronously. A dry-run
1628
+ // Pod proves TLS reachability and admission before any project workload.
1629
+ const deadline = Date.now() + 30_000;
1630
+ for (;;) {
1631
+ try {
1632
+ const pod = (await helpers.client.core.createNamespacedPod({ namespace: "default", dryRun: "All", body: {
1633
+ metadata: { generateName: "spectest-trust-probe-" }, spec: { containers: [{ name: "probe", image: "registry.k8s.io/pause:3.10" }] },
1634
+ } })).unwrap();
1635
+ if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt"))
1636
+ break;
1637
+ if (Date.now() >= deadline)
1638
+ throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
1639
+ }
1640
+ catch (err) {
1641
+ if (Date.now() >= deadline)
1642
+ throw err;
1643
+ }
1644
+ await new Promise(resolve => setTimeout(resolve, 100));
1591
1645
  }
1592
- },
1593
- });
1594
- // The API server observes webhook registration asynchronously. A dry-run
1595
- // Pod proves TLS reachability and admission before any project workload.
1596
- const deadline = Date.now() + 30_000;
1597
- for (;;) {
1598
- try {
1599
- const pod = (await helpers.client.core.createNamespacedPod({ namespace: "default", dryRun: "All", body: {
1600
- metadata: { generateName: "spectest-trust-probe-" }, spec: { containers: [{ name: "probe", image: "registry.k8s.io/pause:3.10" }] },
1601
- } })).unwrap();
1602
- if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt"))
1603
- break;
1604
- if (Date.now() >= deadline)
1605
- throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
1606
- }
1607
- catch (err) {
1608
- if (Date.now() >= deadline)
1609
- throw err;
1610
1646
  }
1611
- await new Promise(resolve => setTimeout(resolve, 100));
1612
- }
1613
- }
1614
- if (runtime)
1615
- await loadK3sBuildImages(name, runtime.images, runtime.gateway, exec);
1616
- await setupK3sCluster(name, helpers, {
1617
- registry: registryEnabled,
1618
- ingressDomains,
1619
- traefik: traefikEnabled,
1620
- });
1621
- await prePullK3sImages(name, images.pulls, exec);
1647
+ await setupK3sCluster(name, helpers, {
1648
+ registry: registryEnabled,
1649
+ ingressDomains,
1650
+ traefik: traefikEnabled,
1651
+ });
1652
+ })(),
1653
+ ]);
1622
1654
  if (sticky)
1623
1655
  await registerStickyKubernetesStore(name, version);
1624
1656
  },
package/dist/daemon.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { componentBuildTasks, COMPONENT_RUNTIME } from "./harness/component-images.js";
2
+ import { waitForAll } from "./harness/concurrency.js";
2
3
  import { androidScope, androidRecordingChunk, releaseAndroidRecording, isAndroidApp } from "./android.js";
3
4
  // Long-running HTTP daemon. Runs as a systemd unit on the VM host (not in
4
5
  // a container). Owns:
@@ -3103,26 +3104,29 @@ async function bootstrapInner(prepared, identity) {
3103
3104
  await waitForReady(svc);
3104
3105
  const tSetup = Date.now();
3105
3106
  const builds = componentTasks.filter(task => task.owner === svc.name);
3106
- await Promise.all(builds.map(task => prep.get(task.name)));
3107
- if (svc.setup) {
3108
- progressService(svc.name, { status: "probing", detail: "running setup" });
3109
- const helpers = await ensureHelpers(svc.name, svc);
3110
- try {
3111
- await svc.setup({
3112
- name: svc.name,
3113
- helpers,
3114
- ...(await spectestContext({ service: svc.name, includeSelf: true })),
3115
- ...{ [COMPONENT_RUNTIME]: {
3116
- images: builds.map(task => ({ source: tags.get(task.name), reference: task.reference })),
3117
- gateway: await bridgeGatewayIp(),
3118
- caBundle: await ensureCaBundle(),
3119
- } },
3120
- });
3121
- }
3122
- catch (err) {
3123
- throw await withContainerPostMortem(svc.name, err);
3124
- }
3125
- }
3107
+ // Setup can overlap pulls/infrastructure with builds, and consume each
3108
+ // image as it becomes ready. Still drain every build if setup fails.
3109
+ await waitForAll([...builds.map(task => prep.get(task.name)), (async () => {
3110
+ if (svc.setup) {
3111
+ progressService(svc.name, { status: "probing", detail: "running setup" });
3112
+ const helpers = await ensureHelpers(svc.name, svc);
3113
+ try {
3114
+ await svc.setup({
3115
+ name: svc.name,
3116
+ helpers,
3117
+ ...(await spectestContext({ service: svc.name, includeSelf: true })),
3118
+ ...{ [COMPONENT_RUNTIME]: {
3119
+ images: builds.map(task => ({ source: imageTag(task.name), reference: task.reference, ready: prep.get(task.name) })),
3120
+ gateway: await bridgeGatewayIp(),
3121
+ caBundle: await ensureCaBundle(),
3122
+ } },
3123
+ });
3124
+ }
3125
+ catch (err) {
3126
+ throw await withContainerPostMortem(svc.name, err);
3127
+ }
3128
+ }
3129
+ })()]);
3126
3130
  for (const task of builds)
3127
3131
  progressService(task.name, { status: "ready", detail: `loaded ${task.reference}` });
3128
3132
  progressService(svc.name, { status: "ready", detail: undefined });
@@ -20,11 +20,14 @@ export declare function componentBuildTasks<T extends ServiceConfig & {
20
20
  /** Private handoff to a component's setup hook. Builds stay generic; the
21
21
  * component decides how its runtime consumes them and establishes trust. */
22
22
  export declare const COMPONENT_RUNTIME: unique symbol;
23
+ export interface PreparedComponentImage {
24
+ source: string;
25
+ reference: string;
26
+ /** A component can load each result as soon as its own build finishes. */
27
+ ready?: Promise<void>;
28
+ }
23
29
  export interface ComponentRuntime {
24
- images: readonly {
25
- source: string;
26
- reference: string;
27
- }[];
30
+ images: readonly PreparedComponentImage[];
28
31
  gateway: string;
29
32
  caBundle: string | null;
30
33
  }
@@ -0,0 +1,2 @@
1
+ /** Wait for every writer to stop before reporting a failure to setup/cleanup. */
2
+ export declare function waitForAll(work: readonly Promise<unknown>[]): Promise<void>;
@@ -0,0 +1,9 @@
1
+ /** Wait for every writer to stop before reporting a failure to setup/cleanup. */
2
+ export async function waitForAll(work) {
3
+ const results = await Promise.allSettled(work);
4
+ const failures = results.flatMap(result => result.status === "rejected" ? [result.reason] : []);
5
+ if (failures.length === 1)
6
+ throw failures[0];
7
+ if (failures.length > 1)
8
+ throw new AggregateError(failures, `Concurrent environment preparation failed: ${failures.map(error => error instanceof Error ? error.message : String(error)).join("; ")}`);
9
+ }
@@ -1,5 +1,3 @@
1
1
  import type { SpectestContext } from "../index.js";
2
- export declare function loadK3sBuildImages(service: string, images: readonly {
3
- source: string;
4
- reference: string;
5
- }[], gateway: string, exec: SpectestContext["exec"]): Promise<void>;
2
+ import { type PreparedComponentImage } from "./component-images.js";
3
+ export declare function loadK3sBuildImages(service: string, images: readonly PreparedComponentImage[], gateway: string, exec: SpectestContext["exec"]): Promise<void>;
@@ -7,6 +7,7 @@ import { createServer } from "node:http";
7
7
  import { promisify } from "node:util";
8
8
  import { canonicalImageReference } from "./component-images.js";
9
9
  import { waitForK3sRuntime } from "./k3s-prepull.js";
10
+ import { waitForAll } from "./concurrency.js";
10
11
  const digest = (bytes) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
11
12
  const descriptor = (bytes, mediaType) => ({ digest: digest(bytes), size: bytes.length, mediaType });
12
13
  const run = promisify(execFile);
@@ -76,7 +77,8 @@ export async function loadK3sBuildImages(service, images, gateway, exec) {
76
77
  const ctr = ["ctr", "--address", "/run/k3s/containerd/containerd.sock", "--namespace", "k8s.io"];
77
78
  const help = await checked([...ctr, "images", "pull", "--help"]);
78
79
  const local = help.includes("--local") ? ["--local"] : [];
79
- for (const image of images) {
80
+ await waitForAll(images.map(async (image) => {
81
+ await image.ready;
80
82
  const reference = canonicalImageReference(image.reference);
81
83
  const { target, metadata: blobs, allowed, configDigest } = await hostImage(image.source);
82
84
  const streams = new Set();
@@ -146,5 +148,5 @@ export async function loadK3sBuildImages(service, images, gateway, exec) {
146
148
  await new Promise(resolve => server.close(() => resolve()));
147
149
  await exec(service, ["rm", "-rf", directory], { timeoutMs: 5_000 }).catch(() => undefined);
148
150
  }
149
- }
151
+ }));
150
152
  }
@@ -1,7 +1,8 @@
1
1
  import type { SpectestContext } from "../index.js";
2
2
  /** CRI PullImage waits for unpacking and uses k3s's registry configuration.
3
3
  * Always resolve tags on a fresh setup. Immutable digests can be used offline.
4
- * Drain both workers on failure so no pulls outlive the setup hook.
4
+ * Submit every image concurrently; containerd manages downloads and unpacking.
5
+ * Drain all pulls on failure so none outlive the setup hook.
5
6
  */
6
7
  export declare function prePullK3sImages(name: string, images: readonly string[], exec: SpectestContext["exec"]): Promise<void>;
7
8
  export declare function waitForK3sRuntime(name: string, exec: SpectestContext["exec"]): Promise<void>;
@@ -1,28 +1,24 @@
1
+ import { waitForAll } from "./concurrency.js";
1
2
  /** CRI PullImage waits for unpacking and uses k3s's registry configuration.
2
3
  * Always resolve tags on a fresh setup. Immutable digests can be used offline.
3
- * Drain both workers on failure so no pulls outlive the setup hook.
4
+ * Submit every image concurrently; containerd manages downloads and unpacking.
5
+ * Drain all pulls on failure so none outlive the setup hook.
4
6
  */
5
7
  export async function prePullK3sImages(name, images, exec) {
6
8
  if (!images.length)
7
9
  return;
8
10
  await waitForK3sRuntime(name, exec);
9
11
  const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
10
- const queue = [...images];
11
- const results = await Promise.allSettled(Array.from({ length: Math.min(2, queue.length) }, async () => {
12
- for (let image = queue.shift(); image !== undefined; image = queue.shift()) {
13
- if (/@sha256:[a-f0-9]{64}$/.test(image)) {
14
- const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
15
- if (existing.exitCode === 0)
16
- continue;
17
- }
18
- const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
19
- if (result.exitCode !== 0)
20
- throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
12
+ await waitForAll(images.map(async (image) => {
13
+ if (/@sha256:[a-f0-9]{64}$/.test(image)) {
14
+ const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
15
+ if (existing.exitCode === 0)
16
+ return;
21
17
  }
18
+ const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
19
+ if (result.exitCode !== 0)
20
+ throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
22
21
  }));
23
- const failed = results.find(result => result.status === "rejected");
24
- if (failed?.status === "rejected")
25
- throw failed.reason;
26
22
  }
27
23
  export async function waitForK3sRuntime(name, exec) {
28
24
  const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@specific.dev/spectest",
3
- "version": "0.91.0",
3
+ "version": "0.91.2",
4
4
  "description": "Spectest SDK for defining test environments in TypeScript.",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -41,7 +41,7 @@ const cases: [Omit<AndroidStepDetails, "device">, string][] = [
41
41
  target: { kind: "accessibility", value: "Save" },
42
42
  state: "hidden",
43
43
  },
44
- 'wait for accessibility label "Save" to be hidden',
44
+ 'wait for label "Save" to be hidden',
45
45
  ],
46
46
  [
47
47
  {
@@ -86,7 +86,7 @@ const cases: [Omit<AndroidStepDetails, "device">, string][] = [
86
86
  for (const [action, title] of cases) {
87
87
  test(title, () =>
88
88
  expect(androidStepTitle({ device: "phone", ...action })).toBe(
89
- `phone: ${title}`,
89
+ title,
90
90
  ),
91
91
  );
92
92
  }
@@ -100,7 +100,7 @@ test("custom selector expressions remain opaque structured data", () => {
100
100
  value: expression,
101
101
  });
102
102
  expect(androidStepTitle({ device: "phone", action: "tap", target })).toBe(
103
- "phone: tap custom selector",
103
+ "tap custom selector",
104
104
  );
105
105
  for (const [using, kind] of [
106
106
  ["id", "id"],
@@ -43,7 +43,7 @@ export function androidTargetLabel(target: AndroidTarget): string {
43
43
  case "id":
44
44
  return `id ${quoted}`;
45
45
  case "accessibility":
46
- return `accessibility label ${quoted}`;
46
+ return `label ${quoted}`;
47
47
  case "xpath":
48
48
  return `XPath ${quoted}`;
49
49
  case "custom":
@@ -103,5 +103,5 @@ export function androidStepTitle(step: AndroidStepDetails): string {
103
103
  label = `run command ${JSON.stringify(step.command)}`;
104
104
  break;
105
105
  }
106
- return `${step.device}: ${label}`;
106
+ return label;
107
107
  }
@@ -156,12 +156,12 @@ describe("Android case lifecycle", () => {
156
156
  const steps = events.filter((e) => e.kind === "android");
157
157
  expect(steps.length).toBe(7);
158
158
  expect(steps[0]).toMatchObject({
159
- title: String.raw`phone: fill text "a \"quoted\" label"`,
159
+ title: String.raw`fill text "a \"quoted\" label"`,
160
160
  android: { device: "phone", action: "fill", target: { kind: "text", value: 'a "quoted" label' } },
161
161
  });
162
162
  expect(steps[1]).toMatchObject({ android: { device: "phone", action: "text", target: { kind: "id", value: "com.example.app:id/draft" } } });
163
- expect(steps[2]).toMatchObject({ title: "phone: take screenshot", android: { device: "phone", action: "screenshot" } });
164
- expect(steps[3]).toMatchObject({ title: 'phone: tap text "Save"', android: { device: "phone", action: "tap", target: { kind: "text", value: "Save" } } });
163
+ expect(steps[2]).toMatchObject({ title: "take screenshot", android: { device: "phone", action: "screenshot" } });
164
+ expect(steps[3]).toMatchObject({ title: 'tap text "Save"', android: { device: "phone", action: "tap", target: { kind: "text", value: "Save" } } });
165
165
  expect(steps[4]).toMatchObject({ android: { device: "phone", action: "tap", point: { x: 270, y: 1800 } } });
166
166
  expect(steps[5]).toMatchObject({ status: "failed", android: { device: "phone", action: "tap", target: { kind: "text", value: "Gone" } } });
167
167
  expect(JSON.stringify(steps)).not.toContain("private draft");
@@ -0,0 +1,90 @@
1
+ import { expect, spyOn, test } from "bun:test";
2
+ import { KubernetesObjectApi } from "@kubernetes/client-node";
3
+ import { ApiException } from "@kubernetes/client-node/dist/gen/apis/exception.js";
4
+ import { isTransientApplyError, k3s } from "./k3s";
5
+
6
+ const missingNamespace = {
7
+ apiVersion: "v1", kind: "Status", status: "Failure", code: 404, reason: "NotFound",
8
+ message: 'namespaces "agent-sandbox-system" not found',
9
+ details: { name: "agent-sandbox-system", kind: "namespaces" },
10
+ };
11
+ const exception = (code: number, body: unknown) => new ApiException(code, "Unknown API Status Code!", body, {});
12
+
13
+ for (const format of ["object", "JSON string"] as const) {
14
+ const encode = (body: unknown) => format === "object" ? body : JSON.stringify(body);
15
+ test(`missing-namespace ApiException with ${format} body is transient`, () => {
16
+ const error = exception(404, encode(missingNamespace));
17
+ // This is the production client's escaped representation, not a hand-made Error.
18
+ expect(/namespaces? "[^"]+" not found/i.test(error.message)).toBe(false);
19
+ expect(isTransientApplyError(error)).toBe(true);
20
+ expect(isTransientApplyError(exception(404, encode({ ...missingNamespace, message: "Not Found" })))).toBe(true);
21
+ // The Status message also works when structured details are absent.
22
+ expect(isTransientApplyError(exception(404, encode({ ...missingNamespace, details: undefined })))).toBe(true);
23
+ });
24
+
25
+ test(`apply converges after a missing namespace with ${format} response body`, async () => {
26
+ const error = exception(404, encode(missingNamespace));
27
+ let attempts = 0;
28
+ const patch = spyOn(KubernetesObjectApi.prototype, "patch").mockImplementation(async doc => {
29
+ if (++attempts === 1) throw error;
30
+ return doc as never;
31
+ });
32
+ try {
33
+ const helpers = await makeHelpers();
34
+ const applied = await helpers.apply(JSON.stringify(policy), { timeoutMs: 2000 });
35
+ expect(attempts).toBe(2);
36
+ expect(applied[0].unwrap()).toEqual(policy);
37
+ } finally { patch.mockRestore(); }
38
+ });
39
+
40
+ test(`unrelated 404, forbidden and invalid Status bodies remain terminal (${format})`, () => {
41
+ for (const body of [
42
+ { ...missingNamespace, message: 'deployments.apps "controller" not found', details: { name: "controller", kind: "deployments", group: "apps" } },
43
+ { ...missingNamespace, message: 'namespaces.example.dev "widget" not found', details: { name: "widget", kind: "namespaces", group: "example.dev" } },
44
+ { code: 404, reason: "NotFound", message: "Not Found" },
45
+ { code: 403, reason: "Forbidden", message: 'namespaces "agent-sandbox-system" is forbidden', details: missingNamespace.details },
46
+ { code: 422, reason: "Invalid", message: 'NetworkPolicy "policy" is invalid', details: { causes: [{ message: missingNamespace.message }] } },
47
+ ]) expect(isTransientApplyError(exception(body.code, encode(body)))).toBe(false);
48
+ });
49
+ }
50
+
51
+ test("plain client errors and transient HTTP statuses retain their retry behavior", () => {
52
+ for (const code of [429, 500, 503, 504]) {
53
+ expect(isTransientApplyError(exception(code, { message: "Try again" }))).toBe(true);
54
+ }
55
+ for (const message of [missingNamespace.message, 'no matches for kind "Widget"', "Failed to fetch resource metadata", "failed calling webhook", "HTTP-Code: 503\nMessage: unavailable"]) {
56
+ expect(isTransientApplyError(new Error(message))).toBe(true);
57
+ }
58
+ for (const body of ["not JSON", "{", null, [], 42]) {
59
+ expect(isTransientApplyError(exception(404, body))).toBe(false);
60
+ }
61
+ expect(isTransientApplyError(null)).toBe(false);
62
+ });
63
+
64
+ const policy = {
65
+ apiVersion: "networking.k8s.io/v1", kind: "NetworkPolicy",
66
+ metadata: { name: "policy", namespace: "agent-sandbox-system" },
67
+ spec: { podSelector: {}, policyTypes: ["Ingress"] },
68
+ };
69
+
70
+ async function makeHelpers() {
71
+ return k3s().helpers({ name: "cluster", exec: async () => ({ exitCode: 0, stderr: "", stdout: JSON.stringify({
72
+ apiVersion: "v1", kind: "Config", clusters: [{ name: "test", cluster: { server: "https://cluster.internal:6443" } }],
73
+ users: [{ name: "test", user: {} }], contexts: [{ name: "test", context: { cluster: "test", user: "test" } }],
74
+ "current-context": "test",
75
+ }) }) } as never);
76
+ }
77
+
78
+ test("apply fails immediately on an unrelated 404 and still times out on a missing namespace", async () => {
79
+ const helpers = await makeHelpers();
80
+ const patch = spyOn(KubernetesObjectApi.prototype, "patch");
81
+ try {
82
+ patch.mockRejectedValue(exception(404, { reason: "NotFound", message: 'deployments.apps "controller" not found' }));
83
+ await expect(helpers.apply(JSON.stringify(policy))).rejects.toThrow("applying NetworkPolicy/policy");
84
+ expect(patch).toHaveBeenCalledTimes(1);
85
+ patch.mockClear();
86
+ patch.mockRejectedValue(exception(404, missingNamespace));
87
+ await expect(helpers.apply(JSON.stringify(policy), { timeoutMs: 0 })).rejects.toThrow("apply did not converge");
88
+ expect(patch).toHaveBeenCalledTimes(1);
89
+ } finally { patch.mockRestore(); }
90
+ });
@@ -0,0 +1,60 @@
1
+ import { expect, test } from "bun:test";
2
+ import { k3s } from "./k3s";
3
+
4
+ function deferred() {
5
+ let resolve!: () => void;
6
+ const promise = new Promise<void>(r => { resolve = r; });
7
+ return { promise, resolve };
8
+ }
9
+
10
+ test("registry, Traefik, and every declared pull start without waiting for one another", async () => {
11
+ const started = new Set<string>();
12
+ const allStarted = deferred(), release = deferred();
13
+ const enter = async (name: string) => {
14
+ started.add(name);
15
+ if (started.size === 5) allStarted.resolve();
16
+ await release.promise;
17
+ };
18
+ const cluster = k3s({ images: ["app:1", "app:2", "app:3"] });
19
+ let finished = false;
20
+ const work = cluster.setup({ name: "cluster", helpers: {
21
+ apply: async (manifest: string) => { await enter(manifest.includes("spectest-registry") ? "registry" : "traefik"); },
22
+ waitForRollout: async () => {},
23
+ }, exec: async (_name: string, args: string[]) => {
24
+ if (args[3] === "pull") await enter(args.at(-1)!);
25
+ return { exitCode: 0, stdout: "", stderr: "" };
26
+ } } as never).then(() => { finished = true; });
27
+ try {
28
+ await Promise.race([allStarted.promise, Bun.sleep(1000).then(() => { throw new Error(`serialized setup: ${[...started]}`); })]);
29
+ expect(finished).toBe(false);
30
+ expect(started.size).toBe(5);
31
+ } finally { release.resolve(); await work; }
32
+ });
33
+
34
+ test("a failed infrastructure rollout waits for other rollouts and image pulls to stop", async () => {
35
+ const started = deferred(), release = deferred();
36
+ let active = 0, finished = false;
37
+ const hold = async () => {
38
+ if (++active === 2) started.resolve();
39
+ await release.promise;
40
+ --active;
41
+ };
42
+ const failure = new Error("Traefik failed");
43
+ const cluster = k3s({ images: ["app:1"] });
44
+ const work = cluster.setup({ name: "cluster", helpers: {
45
+ apply: async () => {},
46
+ waitForRollout: async (deployment: string) => {
47
+ if (deployment === "traefik") throw failure;
48
+ await hold();
49
+ },
50
+ }, exec: async (_name: string, args: string[]) => {
51
+ if (args[3] === "pull") await hold();
52
+ return { exitCode: 0, stdout: "", stderr: "" };
53
+ } } as never).then(() => { finished = true; }, error => { finished = true; return error; });
54
+ try {
55
+ await started.promise;
56
+ expect(finished).toBe(false);
57
+ } finally { release.resolve(); }
58
+ expect(await work).toBe(failure);
59
+ expect(active).toBe(0);
60
+ });
@@ -1,5 +1,6 @@
1
1
  import { isIP } from "node:net";
2
2
  import { installPodTrust } from "./k3s-trust.js";
3
+ import { waitForAll } from "../harness/concurrency.js";
3
4
  import { loadK3sBuildImages } from "../harness/k3s-image-loader.js";
4
5
  import { AsyncLocalStorage } from "node:async_hooks";
5
6
  import { spawn as nodeSpawn } from "node:child_process";
@@ -1420,8 +1421,27 @@ function errorText(err: unknown): string {
1420
1421
  return (err as Error)?.message ?? String(err);
1421
1422
  }
1422
1423
 
1423
- function isTransientApplyError(err: unknown): boolean {
1424
- const msg = errorText(err);
1424
+ /** ApiException formats its body with JSON.stringify, escaping Status.message.
1425
+ * Read the actual response instead of matching that presentation string. */
1426
+ function applyErrorStatus(err: unknown): V1Status | undefined {
1427
+ let body = (err as { body?: unknown } | null)?.body;
1428
+ if (typeof body === "string") {
1429
+ try { body = JSON.parse(body); } catch { return undefined; }
1430
+ }
1431
+ return body && typeof body === "object" && !Array.isArray(body) ? body as V1Status : undefined;
1432
+ }
1433
+
1434
+ /** @internal Exported for regression tests; not part of the components API. */
1435
+ export function isTransientApplyError(err: unknown): boolean {
1436
+ const status = applyErrorStatus(err);
1437
+ const code = status?.code ?? (err as { code?: unknown } | null)?.code;
1438
+ if ([429, 500, 503, 504].includes(code as number)) return true;
1439
+ // A missing namespace can converge while another apply creates it. An
1440
+ // arbitrary resource 404 cannot: don't classify by HTTP status alone.
1441
+ if (code === 404 && status?.reason === "NotFound" &&
1442
+ status.details?.kind === "namespaces" && !status.details.group &&
1443
+ typeof status.details.name === "string" && status.details.name.length > 0) return true;
1444
+ const msg = typeof status?.message === "string" ? status.message : errorText(err);
1425
1445
  return TRANSIENT_APPLY_PATTERNS.some((re) => re.test(msg));
1426
1446
  }
1427
1447
 
@@ -1865,48 +1885,52 @@ async function setupK3sCluster(
1865
1885
  helpers: K3sHelpers,
1866
1886
  opts: { registry: boolean; ingressDomains: string[]; traefik: boolean },
1867
1887
  ): Promise<void> {
1868
- const tlsEnabled = opts.traefik && opts.ingressDomains.length > 0 && caPresent();
1869
- if (tlsEnabled) {
1870
- const { cert, key } = await issueIngressCert(
1871
- opts.ingressDomains.map((d) => `*.${d}`),
1872
- );
1873
- // Apply the cert Secret + dynamic-config ConfigMap before the
1874
- // Deployment that mounts them. `stringData` lets us hand over plain
1875
- // PEM; the API server base64-encodes it.
1876
- await helpers.client.core.createNamespacedSecret({
1877
- namespace: "kube-system",
1878
- body: {
1879
- metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
1880
- type: "kubernetes.io/tls",
1881
- stringData: { "tls.crt": cert, "tls.key": key },
1882
- },
1883
- });
1884
- await helpers.client.core.createNamespacedConfigMap({
1885
- namespace: "kube-system",
1886
- body: {
1887
- metadata: {
1888
- name: TRAEFIK_DYNAMIC_CONFIGMAP,
1889
- namespace: "kube-system",
1890
- },
1891
- data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
1892
- },
1893
- });
1894
- }
1895
- if (opts.traefik) await helpers.apply(buildTraefikManifest(tlsEnabled));
1896
- if (opts.registry) await helpers.apply(REGISTRY_MANIFEST);
1897
- // Both rollouts proceed independently inside the cluster — wait on them
1898
- // concurrently (they used to serialize, wasting up to a rollout's tail).
1899
- // 250ms polling: these two sit on the cold start's critical path, where
1900
- // the default 500ms interval wastes real time.
1888
+ // The registry can start while Traefik's certificates are being prepared.
1901
1889
  const wait = (deployment: string): Promise<void> =>
1902
1890
  helpers.waitForRollout(deployment, {
1903
1891
  namespace: "kube-system",
1904
1892
  timeoutMs: 120_000,
1905
1893
  intervalMs: 250,
1906
1894
  });
1907
- const waits = opts.traefik ? [wait("traefik")] : [];
1908
- if (opts.registry) waits.push(wait("spectest-registry"));
1909
- await Promise.all(waits);
1895
+ await waitForAll([
1896
+ (async () => {
1897
+ if (!opts.traefik) return;
1898
+ const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
1899
+ if (tlsEnabled) {
1900
+ const { cert, key } = await issueIngressCert(
1901
+ opts.ingressDomains.map((d) => `*.${d}`),
1902
+ );
1903
+ // Apply the cert Secret + dynamic-config ConfigMap before the
1904
+ // Deployment that mounts them. `stringData` lets us hand over plain
1905
+ // PEM; the API server base64-encodes it.
1906
+ await helpers.client.core.createNamespacedSecret({
1907
+ namespace: "kube-system",
1908
+ body: {
1909
+ metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
1910
+ type: "kubernetes.io/tls",
1911
+ stringData: { "tls.crt": cert, "tls.key": key },
1912
+ },
1913
+ });
1914
+ await helpers.client.core.createNamespacedConfigMap({
1915
+ namespace: "kube-system",
1916
+ body: {
1917
+ metadata: {
1918
+ name: TRAEFIK_DYNAMIC_CONFIGMAP,
1919
+ namespace: "kube-system",
1920
+ },
1921
+ data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
1922
+ },
1923
+ });
1924
+ }
1925
+ await helpers.apply(buildTraefikManifest(tlsEnabled));
1926
+ await wait("traefik");
1927
+ })(),
1928
+ (async () => {
1929
+ if (!opts.registry) return;
1930
+ await helpers.apply(REGISTRY_MANIFEST);
1931
+ await wait("spectest-registry");
1932
+ })(),
1933
+ ]);
1910
1934
  }
1911
1935
 
1912
1936
  /**
@@ -2199,47 +2223,51 @@ export function k3s(opts: K3sOptions = {}) {
2199
2223
  const { name, helpers, exec } = ctx;
2200
2224
  const runtime = componentRuntime(ctx);
2201
2225
  if (images.builds.length && !runtime) throw new Error(`k3s(${name}): prepared image inputs are missing`);
2202
- if (runtime?.caBundle && caPresent()) {
2203
- const { cert, key } = await issueIngressCert([runtime.gateway]);
2204
- await installPodTrust({ name, gateway: runtime.gateway, cert, key,
2205
- ca: await readFile(CA_PATH, "utf8"), bundle: await readFile(runtime.caBundle, "utf8"),
2206
- }, {
2207
- apply: async configuration => { await helpers.apply(JSON.stringify(configuration)); },
2208
- ensureBundle: async (namespace, configMap, bundle) => {
2209
- try {
2210
- await helpers.client.core.createNamespacedConfigMap({ namespace, body: {
2211
- metadata: { name: configMap, namespace }, immutable: true, data: { "ca-bundle.crt": bundle },
2212
- } });
2213
- } catch (err) {
2214
- // A simultaneous Pod may have created the same namespace's bundle.
2215
- const existing = (await helpers.client.core.readNamespacedConfigMap({ namespace, name: configMap })).unwrap();
2216
- if (existing.data?.["ca-bundle.crt"] !== bundle || existing.immutable !== true) throw err;
2226
+ await waitForAll([
2227
+ runtime ? loadK3sBuildImages(name, runtime.images, runtime.gateway, exec) : Promise.resolve(),
2228
+ prePullK3sImages(name, images.pulls, exec),
2229
+ (async () => {
2230
+ if (runtime?.caBundle && caPresent()) {
2231
+ const { cert, key } = await issueIngressCert([runtime.gateway]);
2232
+ await installPodTrust({ name, gateway: runtime.gateway, cert, key,
2233
+ ca: await readFile(CA_PATH, "utf8"), bundle: await readFile(runtime.caBundle, "utf8"),
2234
+ }, {
2235
+ apply: async configuration => { await helpers.apply(JSON.stringify(configuration)); },
2236
+ ensureBundle: async (namespace, configMap, bundle) => {
2237
+ try {
2238
+ await helpers.client.core.createNamespacedConfigMap({ namespace, body: {
2239
+ metadata: { name: configMap, namespace }, immutable: true, data: { "ca-bundle.crt": bundle },
2240
+ } });
2241
+ } catch (err) {
2242
+ // A simultaneous Pod may have created the same namespace's bundle.
2243
+ const existing = (await helpers.client.core.readNamespacedConfigMap({ namespace, name: configMap })).unwrap();
2244
+ if (existing.data?.["ca-bundle.crt"] !== bundle || existing.immutable !== true) throw err;
2245
+ }
2246
+ },
2247
+ });
2248
+ // The API server observes webhook registration asynchronously. A dry-run
2249
+ // Pod proves TLS reachability and admission before any project workload.
2250
+ const deadline = Date.now() + 30_000;
2251
+ for (;;) {
2252
+ try {
2253
+ const pod = (await helpers.client.core.createNamespacedPod({ namespace: "default", dryRun: "All", body: {
2254
+ metadata: { generateName: "spectest-trust-probe-" }, spec: { containers: [{ name: "probe", image: "registry.k8s.io/pause:3.10" }] },
2255
+ } })).unwrap();
2256
+ if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt")) break;
2257
+ if (Date.now() >= deadline) throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
2258
+ } catch (err) {
2259
+ if (Date.now() >= deadline) throw err;
2260
+ }
2261
+ await new Promise(resolve => setTimeout(resolve, 100));
2217
2262
  }
2218
- },
2219
- });
2220
- // The API server observes webhook registration asynchronously. A dry-run
2221
- // Pod proves TLS reachability and admission before any project workload.
2222
- const deadline = Date.now() + 30_000;
2223
- for (;;) {
2224
- try {
2225
- const pod = (await helpers.client.core.createNamespacedPod({ namespace: "default", dryRun: "All", body: {
2226
- metadata: { generateName: "spectest-trust-probe-" }, spec: { containers: [{ name: "probe", image: "registry.k8s.io/pause:3.10" }] },
2227
- } })).unwrap();
2228
- if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt")) break;
2229
- if (Date.now() >= deadline) throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
2230
- } catch (err) {
2231
- if (Date.now() >= deadline) throw err;
2232
2263
  }
2233
- await new Promise(resolve => setTimeout(resolve, 100));
2234
- }
2235
- }
2236
- if (runtime) await loadK3sBuildImages(name, runtime.images, runtime.gateway, exec);
2237
- await setupK3sCluster(name, helpers, {
2238
- registry: registryEnabled,
2239
- ingressDomains,
2240
- traefik: traefikEnabled,
2241
- });
2242
- await prePullK3sImages(name, images.pulls, exec);
2264
+ await setupK3sCluster(name, helpers, {
2265
+ registry: registryEnabled,
2266
+ ingressDomains,
2267
+ traefik: traefikEnabled,
2268
+ });
2269
+ })(),
2270
+ ]);
2243
2271
  if (sticky) await registerStickyKubernetesStore(name, version);
2244
2272
  },
2245
2273
  helpers: async ({ name, exec, poll }: ServiceHelpersContext): Promise<K3sHelpers> => {
package/src/daemon.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { componentBuildTasks, COMPONENT_RUNTIME } from "./harness/component-images.js";
2
+ import { waitForAll } from "./harness/concurrency.js";
2
3
  import { androidScope, androidRecordingChunk, releaseAndroidRecording, isAndroidApp, type AndroidRecording } from "./android.js";
3
4
  // Long-running HTTP daemon. Runs as a systemd unit on the VM host (not in
4
5
  // a container). Owns:
@@ -3687,25 +3688,28 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
3687
3688
  await waitForReady(svc);
3688
3689
  const tSetup = Date.now();
3689
3690
  const builds = componentTasks.filter(task => task.owner === svc.name);
3690
- await Promise.all(builds.map(task => prep.get(task.name)!));
3691
- if (svc.setup) {
3692
- progressService(svc.name, { status: "probing", detail: "running setup" });
3693
- const helpers = await ensureHelpers(svc.name, svc);
3694
- try {
3695
- await svc.setup({
3696
- name: svc.name,
3697
- helpers,
3698
- ...(await spectestContext({ service: svc.name, includeSelf: true })),
3699
- ...{ [COMPONENT_RUNTIME]: {
3700
- images: builds.map(task => ({ source: tags.get(task.name)!, reference: task.reference })),
3701
- gateway: await bridgeGatewayIp(),
3702
- caBundle: await ensureCaBundle(),
3703
- } },
3704
- });
3705
- } catch (err) {
3706
- throw await withContainerPostMortem(svc.name, err);
3691
+ // Setup can overlap pulls/infrastructure with builds, and consume each
3692
+ // image as it becomes ready. Still drain every build if setup fails.
3693
+ await waitForAll([...builds.map(task => prep.get(task.name)!), (async () => {
3694
+ if (svc.setup) {
3695
+ progressService(svc.name, { status: "probing", detail: "running setup" });
3696
+ const helpers = await ensureHelpers(svc.name, svc);
3697
+ try {
3698
+ await svc.setup({
3699
+ name: svc.name,
3700
+ helpers,
3701
+ ...(await spectestContext({ service: svc.name, includeSelf: true })),
3702
+ ...{ [COMPONENT_RUNTIME]: {
3703
+ images: builds.map(task => ({ source: imageTag(task.name), reference: task.reference, ready: prep.get(task.name)! })),
3704
+ gateway: await bridgeGatewayIp(),
3705
+ caBundle: await ensureCaBundle(),
3706
+ } },
3707
+ });
3708
+ } catch (err) {
3709
+ throw await withContainerPostMortem(svc.name, err);
3710
+ }
3707
3711
  }
3708
- }
3712
+ })()]);
3709
3713
  for (const task of builds) progressService(task.name, { status: "ready", detail: `loaded ${task.reference}` });
3710
3714
  progressService(svc.name, { status: "ready", detail: undefined });
3711
3715
  const ti = timings.get(svc.name);
@@ -63,8 +63,14 @@ export function componentBuildTasks<T extends ServiceConfig & { name: string }>(
63
63
  /** Private handoff to a component's setup hook. Builds stay generic; the
64
64
  * component decides how its runtime consumes them and establishes trust. */
65
65
  export const COMPONENT_RUNTIME = Symbol.for("spectest.component-runtime");
66
+ export interface PreparedComponentImage {
67
+ source: string;
68
+ reference: string;
69
+ /** A component can load each result as soon as its own build finishes. */
70
+ ready?: Promise<void>;
71
+ }
66
72
  export interface ComponentRuntime {
67
- images: readonly { source: string; reference: string }[];
73
+ images: readonly PreparedComponentImage[];
68
74
  gateway: string;
69
75
  caBundle: string | null;
70
76
  }
@@ -0,0 +1,8 @@
1
+ /** Wait for every writer to stop before reporting a failure to setup/cleanup. */
2
+ export async function waitForAll(work: readonly Promise<unknown>[]): Promise<void> {
3
+ const results = await Promise.allSettled(work);
4
+ const failures = results.flatMap(result => result.status === "rejected" ? [result.reason] : []);
5
+ if (failures.length === 1) throw failures[0];
6
+ if (failures.length > 1) throw new AggregateError(failures,
7
+ `Concurrent environment preparation failed: ${failures.map(error => error instanceof Error ? error.message : String(error)).join("; ")}`);
8
+ }
@@ -6,8 +6,9 @@ import { createHash } from "node:crypto";
6
6
  import { createServer } from "node:http";
7
7
  import { promisify } from "node:util";
8
8
  import type { SpectestContext } from "../index.js";
9
- import { canonicalImageReference } from "./component-images.js";
9
+ import { canonicalImageReference, type PreparedComponentImage } from "./component-images.js";
10
10
  import { waitForK3sRuntime } from "./k3s-prepull.js";
11
+ import { waitForAll } from "./concurrency.js";
11
12
 
12
13
  interface Descriptor { digest: string; size: number; mediaType: string; platform?: { os: string; architecture: string } }
13
14
  interface Manifest { schemaVersion: number; mediaType: string; config: Descriptor; layers: Descriptor[]; annotations?: Record<string, string> }
@@ -57,7 +58,7 @@ async function hostImage(source: string) {
57
58
  }
58
59
 
59
60
  export async function loadK3sBuildImages(
60
- service: string, images: readonly { source: string; reference: string }[],
61
+ service: string, images: readonly PreparedComponentImage[],
61
62
  gateway: string, exec: SpectestContext["exec"],
62
63
  ): Promise<void> {
63
64
  if (!images.length) return;
@@ -73,7 +74,8 @@ export async function loadK3sBuildImages(
73
74
  const ctr = ["ctr", "--address", "/run/k3s/containerd/containerd.sock", "--namespace", "k8s.io"];
74
75
  const help = await checked([...ctr, "images", "pull", "--help"]);
75
76
  const local = help.includes("--local") ? ["--local"] : [];
76
- for (const image of images) {
77
+ await waitForAll(images.map(async image => {
78
+ await image.ready;
77
79
  const reference = canonicalImageReference(image.reference);
78
80
  const { target, metadata: blobs, allowed, configDigest } = await hostImage(image.source);
79
81
  const streams = new Set<ReturnType<typeof spawn>>();
@@ -125,5 +127,5 @@ export async function loadK3sBuildImages(
125
127
  await new Promise<void>(resolve => server.close(() => resolve()));
126
128
  await exec(service, ["rm", "-rf", directory], { timeoutMs: 5_000 }).catch(() => undefined);
127
129
  }
128
- }
130
+ }));
129
131
  }
@@ -20,7 +20,7 @@ test("pre-pulls reuse immutable cached images, but resolve tags on every fresh s
20
20
  expect(calls.at(-1)).toEqual([...cri, "pull", digest]);
21
21
  });
22
22
 
23
- test("failed pre-pulls fail setup after outstanding pulls finish, with bounded concurrency", async () => {
23
+ test("failed pre-pulls fail setup after all concurrent pulls finish", async () => {
24
24
  let active = 0, peak = 0, finished = 0;
25
25
  const work = prePullK3sImages("cluster", ["a:1", "b:1", "c:1"], mockExec(async args => {
26
26
  if (args.at(-1) === "info") return { exitCode: 0 };
@@ -31,7 +31,7 @@ test("failed pre-pulls fail setup after outstanding pulls finish, with bounded c
31
31
  }));
32
32
  await expect(work).rejects.toThrow("pre-pull failed for a:1: registry unavailable");
33
33
  expect(active).toBe(0);
34
- expect(peak).toBe(2);
34
+ expect(peak).toBe(3);
35
35
  expect(finished).toBe(3);
36
36
  });
37
37
 
@@ -45,3 +45,13 @@ test("pre-pulls wait for the runtime even when the API server is already ready",
45
45
  }));
46
46
  expect(calls).toEqual(["info", "info", "pull"]);
47
47
  });
48
+
49
+ test("simultaneous pull failures retain every failing image in the setup error", async () => {
50
+ const error = await prePullK3sImages("cluster", ["a:1", "b:1"], mockExec(async args => ({
51
+ exitCode: args[3] === "pull" ? 1 : 0, stderr: "unavailable",
52
+ }))).catch(error => error);
53
+ expect(error).toBeInstanceOf(AggregateError);
54
+ expect(error.errors).toHaveLength(2);
55
+ expect(error.message).toContain("pre-pull failed for a:1");
56
+ expect(error.message).toContain("pre-pull failed for b:1");
57
+ });
@@ -1,26 +1,23 @@
1
1
  import type { SpectestContext } from "../index.js";
2
+ import { waitForAll } from "./concurrency.js";
2
3
 
3
4
  /** CRI PullImage waits for unpacking and uses k3s's registry configuration.
4
5
  * Always resolve tags on a fresh setup. Immutable digests can be used offline.
5
- * Drain both workers on failure so no pulls outlive the setup hook.
6
+ * Submit every image concurrently; containerd manages downloads and unpacking.
7
+ * Drain all pulls on failure so none outlive the setup hook.
6
8
  */
7
9
  export async function prePullK3sImages(name: string, images: readonly string[], exec: SpectestContext["exec"]): Promise<void> {
8
10
  if (!images.length) return;
9
11
  await waitForK3sRuntime(name, exec);
10
12
  const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
11
- const queue = [...images];
12
- const results = await Promise.allSettled(Array.from({ length: Math.min(2, queue.length) }, async () => {
13
- for (let image = queue.shift(); image !== undefined; image = queue.shift()) {
14
- if (/@sha256:[a-f0-9]{64}$/.test(image)) {
15
- const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
16
- if (existing.exitCode === 0) continue;
17
- }
18
- const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
19
- if (result.exitCode !== 0) throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
13
+ await waitForAll(images.map(async image => {
14
+ if (/@sha256:[a-f0-9]{64}$/.test(image)) {
15
+ const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
16
+ if (existing.exitCode === 0) return;
20
17
  }
18
+ const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
19
+ if (result.exitCode !== 0) throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
21
20
  }));
22
- const failed = results.find(result => result.status === "rejected");
23
- if (failed?.status === "rejected") throw failed.reason;
24
21
  }
25
22
 
26
23
  export async function waitForK3sRuntime(name: string, exec: SpectestContext["exec"]): Promise<void> {