@specific.dev/spectest 0.91.0 → 0.91.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/android-step.js +2 -2
- package/dist/components/k3s.d.ts +2 -0
- package/dist/components/k3s.js +117 -85
- package/dist/daemon.js +24 -20
- package/dist/harness/component-images.d.ts +7 -4
- package/dist/harness/concurrency.d.ts +2 -0
- package/dist/harness/concurrency.js +9 -0
- package/dist/harness/k3s-image-loader.d.ts +2 -4
- package/dist/harness/k3s-image-loader.js +4 -2
- package/dist/harness/k3s-prepull.d.ts +2 -1
- package/dist/harness/k3s-prepull.js +11 -15
- package/package.json +1 -1
- package/src/android-step.test.ts +3 -3
- package/src/android-step.ts +2 -2
- package/src/android.test.ts +3 -3
- package/src/components/k3s-apply.test.ts +90 -0
- package/src/components/k3s-concurrency.test.ts +60 -0
- package/src/components/k3s.ts +105 -77
- package/src/daemon.ts +22 -18
- package/src/harness/component-images.ts +7 -1
- package/src/harness/concurrency.ts +8 -0
- package/src/harness/k3s-image-loader.ts +6 -4
- package/src/harness/k3s-prepull.test.ts +12 -2
- package/src/harness/k3s-prepull.ts +9 -12
package/dist/android-step.js
CHANGED
|
@@ -15,7 +15,7 @@ export function androidTargetLabel(target) {
|
|
|
15
15
|
case "id":
|
|
16
16
|
return `id ${quoted}`;
|
|
17
17
|
case "accessibility":
|
|
18
|
-
return `
|
|
18
|
+
return `label ${quoted}`;
|
|
19
19
|
case "xpath":
|
|
20
20
|
return `XPath ${quoted}`;
|
|
21
21
|
case "custom":
|
|
@@ -74,5 +74,5 @@ export function androidStepTitle(step) {
|
|
|
74
74
|
label = `run command ${JSON.stringify(step.command)}`;
|
|
75
75
|
break;
|
|
76
76
|
}
|
|
77
|
-
return
|
|
77
|
+
return label;
|
|
78
78
|
}
|
package/dist/components/k3s.d.ts
CHANGED
|
@@ -328,6 +328,8 @@ export interface K3sHelpers {
|
|
|
328
328
|
limit?: number;
|
|
329
329
|
}): Promise<Wrapped<KubernetesListObject<T>>>;
|
|
330
330
|
}
|
|
331
|
+
/** @internal Exported for regression tests; not part of the components API. */
|
|
332
|
+
export declare function isTransientApplyError(err: unknown): boolean;
|
|
331
333
|
export declare function k3s(opts?: K3sOptions): {
|
|
332
334
|
image: {
|
|
333
335
|
type: "registry";
|
package/dist/components/k3s.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isIP } from "node:net";
|
|
2
2
|
import { installPodTrust } from "./k3s-trust.js";
|
|
3
|
+
import { waitForAll } from "../harness/concurrency.js";
|
|
3
4
|
import { loadK3sBuildImages } from "../harness/k3s-image-loader.js";
|
|
4
5
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
5
6
|
import { spawn as nodeSpawn } from "node:child_process";
|
|
@@ -909,8 +910,33 @@ const TRANSIENT_APPLY_PATTERNS = [
|
|
|
909
910
|
function errorText(err) {
|
|
910
911
|
return err?.message ?? String(err);
|
|
911
912
|
}
|
|
912
|
-
|
|
913
|
-
|
|
913
|
+
/** ApiException formats its body with JSON.stringify, escaping Status.message.
|
|
914
|
+
* Read the actual response instead of matching that presentation string. */
|
|
915
|
+
function applyErrorStatus(err) {
|
|
916
|
+
let body = err?.body;
|
|
917
|
+
if (typeof body === "string") {
|
|
918
|
+
try {
|
|
919
|
+
body = JSON.parse(body);
|
|
920
|
+
}
|
|
921
|
+
catch {
|
|
922
|
+
return undefined;
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
return body && typeof body === "object" && !Array.isArray(body) ? body : undefined;
|
|
926
|
+
}
|
|
927
|
+
/** @internal Exported for regression tests; not part of the components API. */
|
|
928
|
+
export function isTransientApplyError(err) {
|
|
929
|
+
const status = applyErrorStatus(err);
|
|
930
|
+
const code = status?.code ?? err?.code;
|
|
931
|
+
if ([429, 500, 503, 504].includes(code))
|
|
932
|
+
return true;
|
|
933
|
+
// A missing namespace can converge while another apply creates it. An
|
|
934
|
+
// arbitrary resource 404 cannot: don't classify by HTTP status alone.
|
|
935
|
+
if (code === 404 && status?.reason === "NotFound" &&
|
|
936
|
+
status.details?.kind === "namespaces" && !status.details.group &&
|
|
937
|
+
typeof status.details.name === "string" && status.details.name.length > 0)
|
|
938
|
+
return true;
|
|
939
|
+
const msg = typeof status?.message === "string" ? status.message : errorText(err);
|
|
914
940
|
return TRANSIENT_APPLY_PATTERNS.some((re) => re.test(msg));
|
|
915
941
|
}
|
|
916
942
|
function describeDoc(doc) {
|
|
@@ -1243,48 +1269,51 @@ async function waitForJob(deps, name, opts) {
|
|
|
1243
1269
|
* reachable over HTTPS with a cert the test framework already trusts.
|
|
1244
1270
|
*/
|
|
1245
1271
|
async function setupK3sCluster(name, helpers, opts) {
|
|
1246
|
-
|
|
1247
|
-
if (tlsEnabled) {
|
|
1248
|
-
const { cert, key } = await issueIngressCert(opts.ingressDomains.map((d) => `*.${d}`));
|
|
1249
|
-
// Apply the cert Secret + dynamic-config ConfigMap before the
|
|
1250
|
-
// Deployment that mounts them. `stringData` lets us hand over plain
|
|
1251
|
-
// PEM; the API server base64-encodes it.
|
|
1252
|
-
await helpers.client.core.createNamespacedSecret({
|
|
1253
|
-
namespace: "kube-system",
|
|
1254
|
-
body: {
|
|
1255
|
-
metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
|
|
1256
|
-
type: "kubernetes.io/tls",
|
|
1257
|
-
stringData: { "tls.crt": cert, "tls.key": key },
|
|
1258
|
-
},
|
|
1259
|
-
});
|
|
1260
|
-
await helpers.client.core.createNamespacedConfigMap({
|
|
1261
|
-
namespace: "kube-system",
|
|
1262
|
-
body: {
|
|
1263
|
-
metadata: {
|
|
1264
|
-
name: TRAEFIK_DYNAMIC_CONFIGMAP,
|
|
1265
|
-
namespace: "kube-system",
|
|
1266
|
-
},
|
|
1267
|
-
data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
|
|
1268
|
-
},
|
|
1269
|
-
});
|
|
1270
|
-
}
|
|
1271
|
-
if (opts.traefik)
|
|
1272
|
-
await helpers.apply(buildTraefikManifest(tlsEnabled));
|
|
1273
|
-
if (opts.registry)
|
|
1274
|
-
await helpers.apply(REGISTRY_MANIFEST);
|
|
1275
|
-
// Both rollouts proceed independently inside the cluster — wait on them
|
|
1276
|
-
// concurrently (they used to serialize, wasting up to a rollout's tail).
|
|
1277
|
-
// 250ms polling: these two sit on the cold start's critical path, where
|
|
1278
|
-
// the default 500ms interval wastes real time.
|
|
1272
|
+
// The registry can start while Traefik's certificates are being prepared.
|
|
1279
1273
|
const wait = (deployment) => helpers.waitForRollout(deployment, {
|
|
1280
1274
|
namespace: "kube-system",
|
|
1281
1275
|
timeoutMs: 120_000,
|
|
1282
1276
|
intervalMs: 250,
|
|
1283
1277
|
});
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1278
|
+
await waitForAll([
|
|
1279
|
+
(async () => {
|
|
1280
|
+
if (!opts.traefik)
|
|
1281
|
+
return;
|
|
1282
|
+
const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
|
|
1283
|
+
if (tlsEnabled) {
|
|
1284
|
+
const { cert, key } = await issueIngressCert(opts.ingressDomains.map((d) => `*.${d}`));
|
|
1285
|
+
// Apply the cert Secret + dynamic-config ConfigMap before the
|
|
1286
|
+
// Deployment that mounts them. `stringData` lets us hand over plain
|
|
1287
|
+
// PEM; the API server base64-encodes it.
|
|
1288
|
+
await helpers.client.core.createNamespacedSecret({
|
|
1289
|
+
namespace: "kube-system",
|
|
1290
|
+
body: {
|
|
1291
|
+
metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
|
|
1292
|
+
type: "kubernetes.io/tls",
|
|
1293
|
+
stringData: { "tls.crt": cert, "tls.key": key },
|
|
1294
|
+
},
|
|
1295
|
+
});
|
|
1296
|
+
await helpers.client.core.createNamespacedConfigMap({
|
|
1297
|
+
namespace: "kube-system",
|
|
1298
|
+
body: {
|
|
1299
|
+
metadata: {
|
|
1300
|
+
name: TRAEFIK_DYNAMIC_CONFIGMAP,
|
|
1301
|
+
namespace: "kube-system",
|
|
1302
|
+
},
|
|
1303
|
+
data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
|
|
1304
|
+
},
|
|
1305
|
+
});
|
|
1306
|
+
}
|
|
1307
|
+
await helpers.apply(buildTraefikManifest(tlsEnabled));
|
|
1308
|
+
await wait("traefik");
|
|
1309
|
+
})(),
|
|
1310
|
+
(async () => {
|
|
1311
|
+
if (!opts.registry)
|
|
1312
|
+
return;
|
|
1313
|
+
await helpers.apply(REGISTRY_MANIFEST);
|
|
1314
|
+
await wait("spectest-registry");
|
|
1315
|
+
})(),
|
|
1316
|
+
]);
|
|
1288
1317
|
}
|
|
1289
1318
|
/**
|
|
1290
1319
|
* Snapshot of a namespace's state, dumped whenever a wait times out: pod
|
|
@@ -1571,54 +1600,57 @@ export function k3s(opts = {}) {
|
|
|
1571
1600
|
const runtime = componentRuntime(ctx);
|
|
1572
1601
|
if (images.builds.length && !runtime)
|
|
1573
1602
|
throw new Error(`k3s(${name}): prepared image inputs are missing`);
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1603
|
+
await waitForAll([
|
|
1604
|
+
runtime ? loadK3sBuildImages(name, runtime.images, runtime.gateway, exec) : Promise.resolve(),
|
|
1605
|
+
prePullK3sImages(name, images.pulls, exec),
|
|
1606
|
+
(async () => {
|
|
1607
|
+
if (runtime?.caBundle && caPresent()) {
|
|
1608
|
+
const { cert, key } = await issueIngressCert([runtime.gateway]);
|
|
1609
|
+
await installPodTrust({ name, gateway: runtime.gateway, cert, key,
|
|
1610
|
+
ca: await readFile(CA_PATH, "utf8"), bundle: await readFile(runtime.caBundle, "utf8"),
|
|
1611
|
+
}, {
|
|
1612
|
+
apply: async (configuration) => { await helpers.apply(JSON.stringify(configuration)); },
|
|
1613
|
+
ensureBundle: async (namespace, configMap, bundle) => {
|
|
1614
|
+
try {
|
|
1615
|
+
await helpers.client.core.createNamespacedConfigMap({ namespace, body: {
|
|
1616
|
+
metadata: { name: configMap, namespace }, immutable: true, data: { "ca-bundle.crt": bundle },
|
|
1617
|
+
} });
|
|
1618
|
+
}
|
|
1619
|
+
catch (err) {
|
|
1620
|
+
// A simultaneous Pod may have created the same namespace's bundle.
|
|
1621
|
+
const existing = (await helpers.client.core.readNamespacedConfigMap({ namespace, name: configMap })).unwrap();
|
|
1622
|
+
if (existing.data?.["ca-bundle.crt"] !== bundle || existing.immutable !== true)
|
|
1623
|
+
throw err;
|
|
1624
|
+
}
|
|
1625
|
+
},
|
|
1626
|
+
});
|
|
1627
|
+
// The API server observes webhook registration asynchronously. A dry-run
|
|
1628
|
+
// Pod proves TLS reachability and admission before any project workload.
|
|
1629
|
+
const deadline = Date.now() + 30_000;
|
|
1630
|
+
for (;;) {
|
|
1631
|
+
try {
|
|
1632
|
+
const pod = (await helpers.client.core.createNamespacedPod({ namespace: "default", dryRun: "All", body: {
|
|
1633
|
+
metadata: { generateName: "spectest-trust-probe-" }, spec: { containers: [{ name: "probe", image: "registry.k8s.io/pause:3.10" }] },
|
|
1634
|
+
} })).unwrap();
|
|
1635
|
+
if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt"))
|
|
1636
|
+
break;
|
|
1637
|
+
if (Date.now() >= deadline)
|
|
1638
|
+
throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
|
|
1639
|
+
}
|
|
1640
|
+
catch (err) {
|
|
1641
|
+
if (Date.now() >= deadline)
|
|
1642
|
+
throw err;
|
|
1643
|
+
}
|
|
1644
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
1591
1645
|
}
|
|
1592
|
-
},
|
|
1593
|
-
});
|
|
1594
|
-
// The API server observes webhook registration asynchronously. A dry-run
|
|
1595
|
-
// Pod proves TLS reachability and admission before any project workload.
|
|
1596
|
-
const deadline = Date.now() + 30_000;
|
|
1597
|
-
for (;;) {
|
|
1598
|
-
try {
|
|
1599
|
-
const pod = (await helpers.client.core.createNamespacedPod({ namespace: "default", dryRun: "All", body: {
|
|
1600
|
-
metadata: { generateName: "spectest-trust-probe-" }, spec: { containers: [{ name: "probe", image: "registry.k8s.io/pause:3.10" }] },
|
|
1601
|
-
} })).unwrap();
|
|
1602
|
-
if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt"))
|
|
1603
|
-
break;
|
|
1604
|
-
if (Date.now() >= deadline)
|
|
1605
|
-
throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
|
|
1606
|
-
}
|
|
1607
|
-
catch (err) {
|
|
1608
|
-
if (Date.now() >= deadline)
|
|
1609
|
-
throw err;
|
|
1610
1646
|
}
|
|
1611
|
-
await
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
ingressDomains,
|
|
1619
|
-
traefik: traefikEnabled,
|
|
1620
|
-
});
|
|
1621
|
-
await prePullK3sImages(name, images.pulls, exec);
|
|
1647
|
+
await setupK3sCluster(name, helpers, {
|
|
1648
|
+
registry: registryEnabled,
|
|
1649
|
+
ingressDomains,
|
|
1650
|
+
traefik: traefikEnabled,
|
|
1651
|
+
});
|
|
1652
|
+
})(),
|
|
1653
|
+
]);
|
|
1622
1654
|
if (sticky)
|
|
1623
1655
|
await registerStickyKubernetesStore(name, version);
|
|
1624
1656
|
},
|
package/dist/daemon.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { componentBuildTasks, COMPONENT_RUNTIME } from "./harness/component-images.js";
|
|
2
|
+
import { waitForAll } from "./harness/concurrency.js";
|
|
2
3
|
import { androidScope, androidRecordingChunk, releaseAndroidRecording, isAndroidApp } from "./android.js";
|
|
3
4
|
// Long-running HTTP daemon. Runs as a systemd unit on the VM host (not in
|
|
4
5
|
// a container). Owns:
|
|
@@ -3103,26 +3104,29 @@ async function bootstrapInner(prepared, identity) {
|
|
|
3103
3104
|
await waitForReady(svc);
|
|
3104
3105
|
const tSetup = Date.now();
|
|
3105
3106
|
const builds = componentTasks.filter(task => task.owner === svc.name);
|
|
3106
|
-
|
|
3107
|
-
if
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3107
|
+
// Setup can overlap pulls/infrastructure with builds, and consume each
|
|
3108
|
+
// image as it becomes ready. Still drain every build if setup fails.
|
|
3109
|
+
await waitForAll([...builds.map(task => prep.get(task.name)), (async () => {
|
|
3110
|
+
if (svc.setup) {
|
|
3111
|
+
progressService(svc.name, { status: "probing", detail: "running setup" });
|
|
3112
|
+
const helpers = await ensureHelpers(svc.name, svc);
|
|
3113
|
+
try {
|
|
3114
|
+
await svc.setup({
|
|
3115
|
+
name: svc.name,
|
|
3116
|
+
helpers,
|
|
3117
|
+
...(await spectestContext({ service: svc.name, includeSelf: true })),
|
|
3118
|
+
...{ [COMPONENT_RUNTIME]: {
|
|
3119
|
+
images: builds.map(task => ({ source: imageTag(task.name), reference: task.reference, ready: prep.get(task.name) })),
|
|
3120
|
+
gateway: await bridgeGatewayIp(),
|
|
3121
|
+
caBundle: await ensureCaBundle(),
|
|
3122
|
+
} },
|
|
3123
|
+
});
|
|
3124
|
+
}
|
|
3125
|
+
catch (err) {
|
|
3126
|
+
throw await withContainerPostMortem(svc.name, err);
|
|
3127
|
+
}
|
|
3128
|
+
}
|
|
3129
|
+
})()]);
|
|
3126
3130
|
for (const task of builds)
|
|
3127
3131
|
progressService(task.name, { status: "ready", detail: `loaded ${task.reference}` });
|
|
3128
3132
|
progressService(svc.name, { status: "ready", detail: undefined });
|
|
@@ -20,11 +20,14 @@ export declare function componentBuildTasks<T extends ServiceConfig & {
|
|
|
20
20
|
/** Private handoff to a component's setup hook. Builds stay generic; the
|
|
21
21
|
* component decides how its runtime consumes them and establishes trust. */
|
|
22
22
|
export declare const COMPONENT_RUNTIME: unique symbol;
|
|
23
|
+
export interface PreparedComponentImage {
|
|
24
|
+
source: string;
|
|
25
|
+
reference: string;
|
|
26
|
+
/** A component can load each result as soon as its own build finishes. */
|
|
27
|
+
ready?: Promise<void>;
|
|
28
|
+
}
|
|
23
29
|
export interface ComponentRuntime {
|
|
24
|
-
images: readonly
|
|
25
|
-
source: string;
|
|
26
|
-
reference: string;
|
|
27
|
-
}[];
|
|
30
|
+
images: readonly PreparedComponentImage[];
|
|
28
31
|
gateway: string;
|
|
29
32
|
caBundle: string | null;
|
|
30
33
|
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Wait for every writer to stop before reporting a failure to setup/cleanup. */
|
|
2
|
+
export async function waitForAll(work) {
|
|
3
|
+
const results = await Promise.allSettled(work);
|
|
4
|
+
const failures = results.flatMap(result => result.status === "rejected" ? [result.reason] : []);
|
|
5
|
+
if (failures.length === 1)
|
|
6
|
+
throw failures[0];
|
|
7
|
+
if (failures.length > 1)
|
|
8
|
+
throw new AggregateError(failures, `Concurrent environment preparation failed: ${failures.map(error => error instanceof Error ? error.message : String(error)).join("; ")}`);
|
|
9
|
+
}
|
|
@@ -1,5 +1,3 @@
|
|
|
1
1
|
import type { SpectestContext } from "../index.js";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
reference: string;
|
|
5
|
-
}[], gateway: string, exec: SpectestContext["exec"]): Promise<void>;
|
|
2
|
+
import { type PreparedComponentImage } from "./component-images.js";
|
|
3
|
+
export declare function loadK3sBuildImages(service: string, images: readonly PreparedComponentImage[], gateway: string, exec: SpectestContext["exec"]): Promise<void>;
|
|
@@ -7,6 +7,7 @@ import { createServer } from "node:http";
|
|
|
7
7
|
import { promisify } from "node:util";
|
|
8
8
|
import { canonicalImageReference } from "./component-images.js";
|
|
9
9
|
import { waitForK3sRuntime } from "./k3s-prepull.js";
|
|
10
|
+
import { waitForAll } from "./concurrency.js";
|
|
10
11
|
const digest = (bytes) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
11
12
|
const descriptor = (bytes, mediaType) => ({ digest: digest(bytes), size: bytes.length, mediaType });
|
|
12
13
|
const run = promisify(execFile);
|
|
@@ -76,7 +77,8 @@ export async function loadK3sBuildImages(service, images, gateway, exec) {
|
|
|
76
77
|
const ctr = ["ctr", "--address", "/run/k3s/containerd/containerd.sock", "--namespace", "k8s.io"];
|
|
77
78
|
const help = await checked([...ctr, "images", "pull", "--help"]);
|
|
78
79
|
const local = help.includes("--local") ? ["--local"] : [];
|
|
79
|
-
|
|
80
|
+
await waitForAll(images.map(async (image) => {
|
|
81
|
+
await image.ready;
|
|
80
82
|
const reference = canonicalImageReference(image.reference);
|
|
81
83
|
const { target, metadata: blobs, allowed, configDigest } = await hostImage(image.source);
|
|
82
84
|
const streams = new Set();
|
|
@@ -146,5 +148,5 @@ export async function loadK3sBuildImages(service, images, gateway, exec) {
|
|
|
146
148
|
await new Promise(resolve => server.close(() => resolve()));
|
|
147
149
|
await exec(service, ["rm", "-rf", directory], { timeoutMs: 5_000 }).catch(() => undefined);
|
|
148
150
|
}
|
|
149
|
-
}
|
|
151
|
+
}));
|
|
150
152
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { SpectestContext } from "../index.js";
|
|
2
2
|
/** CRI PullImage waits for unpacking and uses k3s's registry configuration.
|
|
3
3
|
* Always resolve tags on a fresh setup. Immutable digests can be used offline.
|
|
4
|
-
*
|
|
4
|
+
* Submit every image concurrently; containerd manages downloads and unpacking.
|
|
5
|
+
* Drain all pulls on failure so none outlive the setup hook.
|
|
5
6
|
*/
|
|
6
7
|
export declare function prePullK3sImages(name: string, images: readonly string[], exec: SpectestContext["exec"]): Promise<void>;
|
|
7
8
|
export declare function waitForK3sRuntime(name: string, exec: SpectestContext["exec"]): Promise<void>;
|
|
@@ -1,28 +1,24 @@
|
|
|
1
|
+
import { waitForAll } from "./concurrency.js";
|
|
1
2
|
/** CRI PullImage waits for unpacking and uses k3s's registry configuration.
|
|
2
3
|
* Always resolve tags on a fresh setup. Immutable digests can be used offline.
|
|
3
|
-
*
|
|
4
|
+
* Submit every image concurrently; containerd manages downloads and unpacking.
|
|
5
|
+
* Drain all pulls on failure so none outlive the setup hook.
|
|
4
6
|
*/
|
|
5
7
|
export async function prePullK3sImages(name, images, exec) {
|
|
6
8
|
if (!images.length)
|
|
7
9
|
return;
|
|
8
10
|
await waitForK3sRuntime(name, exec);
|
|
9
11
|
const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
if (
|
|
14
|
-
|
|
15
|
-
if (existing.exitCode === 0)
|
|
16
|
-
continue;
|
|
17
|
-
}
|
|
18
|
-
const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
|
|
19
|
-
if (result.exitCode !== 0)
|
|
20
|
-
throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
|
|
12
|
+
await waitForAll(images.map(async (image) => {
|
|
13
|
+
if (/@sha256:[a-f0-9]{64}$/.test(image)) {
|
|
14
|
+
const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
|
|
15
|
+
if (existing.exitCode === 0)
|
|
16
|
+
return;
|
|
21
17
|
}
|
|
18
|
+
const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
|
|
19
|
+
if (result.exitCode !== 0)
|
|
20
|
+
throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
|
|
22
21
|
}));
|
|
23
|
-
const failed = results.find(result => result.status === "rejected");
|
|
24
|
-
if (failed?.status === "rejected")
|
|
25
|
-
throw failed.reason;
|
|
26
22
|
}
|
|
27
23
|
export async function waitForK3sRuntime(name, exec) {
|
|
28
24
|
const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
|
package/package.json
CHANGED
package/src/android-step.test.ts
CHANGED
|
@@ -41,7 +41,7 @@ const cases: [Omit<AndroidStepDetails, "device">, string][] = [
|
|
|
41
41
|
target: { kind: "accessibility", value: "Save" },
|
|
42
42
|
state: "hidden",
|
|
43
43
|
},
|
|
44
|
-
'wait for
|
|
44
|
+
'wait for label "Save" to be hidden',
|
|
45
45
|
],
|
|
46
46
|
[
|
|
47
47
|
{
|
|
@@ -86,7 +86,7 @@ const cases: [Omit<AndroidStepDetails, "device">, string][] = [
|
|
|
86
86
|
for (const [action, title] of cases) {
|
|
87
87
|
test(title, () =>
|
|
88
88
|
expect(androidStepTitle({ device: "phone", ...action })).toBe(
|
|
89
|
-
|
|
89
|
+
title,
|
|
90
90
|
),
|
|
91
91
|
);
|
|
92
92
|
}
|
|
@@ -100,7 +100,7 @@ test("custom selector expressions remain opaque structured data", () => {
|
|
|
100
100
|
value: expression,
|
|
101
101
|
});
|
|
102
102
|
expect(androidStepTitle({ device: "phone", action: "tap", target })).toBe(
|
|
103
|
-
"
|
|
103
|
+
"tap custom selector",
|
|
104
104
|
);
|
|
105
105
|
for (const [using, kind] of [
|
|
106
106
|
["id", "id"],
|
package/src/android-step.ts
CHANGED
|
@@ -43,7 +43,7 @@ export function androidTargetLabel(target: AndroidTarget): string {
|
|
|
43
43
|
case "id":
|
|
44
44
|
return `id ${quoted}`;
|
|
45
45
|
case "accessibility":
|
|
46
|
-
return `
|
|
46
|
+
return `label ${quoted}`;
|
|
47
47
|
case "xpath":
|
|
48
48
|
return `XPath ${quoted}`;
|
|
49
49
|
case "custom":
|
|
@@ -103,5 +103,5 @@ export function androidStepTitle(step: AndroidStepDetails): string {
|
|
|
103
103
|
label = `run command ${JSON.stringify(step.command)}`;
|
|
104
104
|
break;
|
|
105
105
|
}
|
|
106
|
-
return
|
|
106
|
+
return label;
|
|
107
107
|
}
|
package/src/android.test.ts
CHANGED
|
@@ -156,12 +156,12 @@ describe("Android case lifecycle", () => {
|
|
|
156
156
|
const steps = events.filter((e) => e.kind === "android");
|
|
157
157
|
expect(steps.length).toBe(7);
|
|
158
158
|
expect(steps[0]).toMatchObject({
|
|
159
|
-
title: String.raw`
|
|
159
|
+
title: String.raw`fill text "a \"quoted\" label"`,
|
|
160
160
|
android: { device: "phone", action: "fill", target: { kind: "text", value: 'a "quoted" label' } },
|
|
161
161
|
});
|
|
162
162
|
expect(steps[1]).toMatchObject({ android: { device: "phone", action: "text", target: { kind: "id", value: "com.example.app:id/draft" } } });
|
|
163
|
-
expect(steps[2]).toMatchObject({ title: "
|
|
164
|
-
expect(steps[3]).toMatchObject({ title: '
|
|
163
|
+
expect(steps[2]).toMatchObject({ title: "take screenshot", android: { device: "phone", action: "screenshot" } });
|
|
164
|
+
expect(steps[3]).toMatchObject({ title: 'tap text "Save"', android: { device: "phone", action: "tap", target: { kind: "text", value: "Save" } } });
|
|
165
165
|
expect(steps[4]).toMatchObject({ android: { device: "phone", action: "tap", point: { x: 270, y: 1800 } } });
|
|
166
166
|
expect(steps[5]).toMatchObject({ status: "failed", android: { device: "phone", action: "tap", target: { kind: "text", value: "Gone" } } });
|
|
167
167
|
expect(JSON.stringify(steps)).not.toContain("private draft");
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { expect, spyOn, test } from "bun:test";
|
|
2
|
+
import { KubernetesObjectApi } from "@kubernetes/client-node";
|
|
3
|
+
import { ApiException } from "@kubernetes/client-node/dist/gen/apis/exception.js";
|
|
4
|
+
import { isTransientApplyError, k3s } from "./k3s";
|
|
5
|
+
|
|
6
|
+
const missingNamespace = {
|
|
7
|
+
apiVersion: "v1", kind: "Status", status: "Failure", code: 404, reason: "NotFound",
|
|
8
|
+
message: 'namespaces "agent-sandbox-system" not found',
|
|
9
|
+
details: { name: "agent-sandbox-system", kind: "namespaces" },
|
|
10
|
+
};
|
|
11
|
+
const exception = (code: number, body: unknown) => new ApiException(code, "Unknown API Status Code!", body, {});
|
|
12
|
+
|
|
13
|
+
for (const format of ["object", "JSON string"] as const) {
|
|
14
|
+
const encode = (body: unknown) => format === "object" ? body : JSON.stringify(body);
|
|
15
|
+
test(`missing-namespace ApiException with ${format} body is transient`, () => {
|
|
16
|
+
const error = exception(404, encode(missingNamespace));
|
|
17
|
+
// This is the production client's escaped representation, not a hand-made Error.
|
|
18
|
+
expect(/namespaces? "[^"]+" not found/i.test(error.message)).toBe(false);
|
|
19
|
+
expect(isTransientApplyError(error)).toBe(true);
|
|
20
|
+
expect(isTransientApplyError(exception(404, encode({ ...missingNamespace, message: "Not Found" })))).toBe(true);
|
|
21
|
+
// The Status message also works when structured details are absent.
|
|
22
|
+
expect(isTransientApplyError(exception(404, encode({ ...missingNamespace, details: undefined })))).toBe(true);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test(`apply converges after a missing namespace with ${format} response body`, async () => {
|
|
26
|
+
const error = exception(404, encode(missingNamespace));
|
|
27
|
+
let attempts = 0;
|
|
28
|
+
const patch = spyOn(KubernetesObjectApi.prototype, "patch").mockImplementation(async doc => {
|
|
29
|
+
if (++attempts === 1) throw error;
|
|
30
|
+
return doc as never;
|
|
31
|
+
});
|
|
32
|
+
try {
|
|
33
|
+
const helpers = await makeHelpers();
|
|
34
|
+
const applied = await helpers.apply(JSON.stringify(policy), { timeoutMs: 2000 });
|
|
35
|
+
expect(attempts).toBe(2);
|
|
36
|
+
expect(applied[0].unwrap()).toEqual(policy);
|
|
37
|
+
} finally { patch.mockRestore(); }
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test(`unrelated 404, forbidden and invalid Status bodies remain terminal (${format})`, () => {
|
|
41
|
+
for (const body of [
|
|
42
|
+
{ ...missingNamespace, message: 'deployments.apps "controller" not found', details: { name: "controller", kind: "deployments", group: "apps" } },
|
|
43
|
+
{ ...missingNamespace, message: 'namespaces.example.dev "widget" not found', details: { name: "widget", kind: "namespaces", group: "example.dev" } },
|
|
44
|
+
{ code: 404, reason: "NotFound", message: "Not Found" },
|
|
45
|
+
{ code: 403, reason: "Forbidden", message: 'namespaces "agent-sandbox-system" is forbidden', details: missingNamespace.details },
|
|
46
|
+
{ code: 422, reason: "Invalid", message: 'NetworkPolicy "policy" is invalid', details: { causes: [{ message: missingNamespace.message }] } },
|
|
47
|
+
]) expect(isTransientApplyError(exception(body.code, encode(body)))).toBe(false);
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
test("plain client errors and transient HTTP statuses retain their retry behavior", () => {
|
|
52
|
+
for (const code of [429, 500, 503, 504]) {
|
|
53
|
+
expect(isTransientApplyError(exception(code, { message: "Try again" }))).toBe(true);
|
|
54
|
+
}
|
|
55
|
+
for (const message of [missingNamespace.message, 'no matches for kind "Widget"', "Failed to fetch resource metadata", "failed calling webhook", "HTTP-Code: 503\nMessage: unavailable"]) {
|
|
56
|
+
expect(isTransientApplyError(new Error(message))).toBe(true);
|
|
57
|
+
}
|
|
58
|
+
for (const body of ["not JSON", "{", null, [], 42]) {
|
|
59
|
+
expect(isTransientApplyError(exception(404, body))).toBe(false);
|
|
60
|
+
}
|
|
61
|
+
expect(isTransientApplyError(null)).toBe(false);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
const policy = {
|
|
65
|
+
apiVersion: "networking.k8s.io/v1", kind: "NetworkPolicy",
|
|
66
|
+
metadata: { name: "policy", namespace: "agent-sandbox-system" },
|
|
67
|
+
spec: { podSelector: {}, policyTypes: ["Ingress"] },
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
async function makeHelpers() {
|
|
71
|
+
return k3s().helpers({ name: "cluster", exec: async () => ({ exitCode: 0, stderr: "", stdout: JSON.stringify({
|
|
72
|
+
apiVersion: "v1", kind: "Config", clusters: [{ name: "test", cluster: { server: "https://cluster.internal:6443" } }],
|
|
73
|
+
users: [{ name: "test", user: {} }], contexts: [{ name: "test", context: { cluster: "test", user: "test" } }],
|
|
74
|
+
"current-context": "test",
|
|
75
|
+
}) }) } as never);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
test("apply fails immediately on an unrelated 404 and still times out on a missing namespace", async () => {
|
|
79
|
+
const helpers = await makeHelpers();
|
|
80
|
+
const patch = spyOn(KubernetesObjectApi.prototype, "patch");
|
|
81
|
+
try {
|
|
82
|
+
patch.mockRejectedValue(exception(404, { reason: "NotFound", message: 'deployments.apps "controller" not found' }));
|
|
83
|
+
await expect(helpers.apply(JSON.stringify(policy))).rejects.toThrow("applying NetworkPolicy/policy");
|
|
84
|
+
expect(patch).toHaveBeenCalledTimes(1);
|
|
85
|
+
patch.mockClear();
|
|
86
|
+
patch.mockRejectedValue(exception(404, missingNamespace));
|
|
87
|
+
await expect(helpers.apply(JSON.stringify(policy), { timeoutMs: 0 })).rejects.toThrow("apply did not converge");
|
|
88
|
+
expect(patch).toHaveBeenCalledTimes(1);
|
|
89
|
+
} finally { patch.mockRestore(); }
|
|
90
|
+
});
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { expect, test } from "bun:test";
|
|
2
|
+
import { k3s } from "./k3s";
|
|
3
|
+
|
|
4
|
+
function deferred() {
|
|
5
|
+
let resolve!: () => void;
|
|
6
|
+
const promise = new Promise<void>(r => { resolve = r; });
|
|
7
|
+
return { promise, resolve };
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
test("registry, Traefik, and every declared pull start without waiting for one another", async () => {
|
|
11
|
+
const started = new Set<string>();
|
|
12
|
+
const allStarted = deferred(), release = deferred();
|
|
13
|
+
const enter = async (name: string) => {
|
|
14
|
+
started.add(name);
|
|
15
|
+
if (started.size === 5) allStarted.resolve();
|
|
16
|
+
await release.promise;
|
|
17
|
+
};
|
|
18
|
+
const cluster = k3s({ images: ["app:1", "app:2", "app:3"] });
|
|
19
|
+
let finished = false;
|
|
20
|
+
const work = cluster.setup({ name: "cluster", helpers: {
|
|
21
|
+
apply: async (manifest: string) => { await enter(manifest.includes("spectest-registry") ? "registry" : "traefik"); },
|
|
22
|
+
waitForRollout: async () => {},
|
|
23
|
+
}, exec: async (_name: string, args: string[]) => {
|
|
24
|
+
if (args[3] === "pull") await enter(args.at(-1)!);
|
|
25
|
+
return { exitCode: 0, stdout: "", stderr: "" };
|
|
26
|
+
} } as never).then(() => { finished = true; });
|
|
27
|
+
try {
|
|
28
|
+
await Promise.race([allStarted.promise, Bun.sleep(1000).then(() => { throw new Error(`serialized setup: ${[...started]}`); })]);
|
|
29
|
+
expect(finished).toBe(false);
|
|
30
|
+
expect(started.size).toBe(5);
|
|
31
|
+
} finally { release.resolve(); await work; }
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("a failed infrastructure rollout waits for other rollouts and image pulls to stop", async () => {
|
|
35
|
+
const started = deferred(), release = deferred();
|
|
36
|
+
let active = 0, finished = false;
|
|
37
|
+
const hold = async () => {
|
|
38
|
+
if (++active === 2) started.resolve();
|
|
39
|
+
await release.promise;
|
|
40
|
+
--active;
|
|
41
|
+
};
|
|
42
|
+
const failure = new Error("Traefik failed");
|
|
43
|
+
const cluster = k3s({ images: ["app:1"] });
|
|
44
|
+
const work = cluster.setup({ name: "cluster", helpers: {
|
|
45
|
+
apply: async () => {},
|
|
46
|
+
waitForRollout: async (deployment: string) => {
|
|
47
|
+
if (deployment === "traefik") throw failure;
|
|
48
|
+
await hold();
|
|
49
|
+
},
|
|
50
|
+
}, exec: async (_name: string, args: string[]) => {
|
|
51
|
+
if (args[3] === "pull") await hold();
|
|
52
|
+
return { exitCode: 0, stdout: "", stderr: "" };
|
|
53
|
+
} } as never).then(() => { finished = true; }, error => { finished = true; return error; });
|
|
54
|
+
try {
|
|
55
|
+
await started.promise;
|
|
56
|
+
expect(finished).toBe(false);
|
|
57
|
+
} finally { release.resolve(); }
|
|
58
|
+
expect(await work).toBe(failure);
|
|
59
|
+
expect(active).toBe(0);
|
|
60
|
+
});
|
package/src/components/k3s.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isIP } from "node:net";
|
|
2
2
|
import { installPodTrust } from "./k3s-trust.js";
|
|
3
|
+
import { waitForAll } from "../harness/concurrency.js";
|
|
3
4
|
import { loadK3sBuildImages } from "../harness/k3s-image-loader.js";
|
|
4
5
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
5
6
|
import { spawn as nodeSpawn } from "node:child_process";
|
|
@@ -1420,8 +1421,27 @@ function errorText(err: unknown): string {
|
|
|
1420
1421
|
return (err as Error)?.message ?? String(err);
|
|
1421
1422
|
}
|
|
1422
1423
|
|
|
1423
|
-
|
|
1424
|
-
|
|
1424
|
+
/** ApiException formats its body with JSON.stringify, escaping Status.message.
|
|
1425
|
+
* Read the actual response instead of matching that presentation string. */
|
|
1426
|
+
function applyErrorStatus(err: unknown): V1Status | undefined {
|
|
1427
|
+
let body = (err as { body?: unknown } | null)?.body;
|
|
1428
|
+
if (typeof body === "string") {
|
|
1429
|
+
try { body = JSON.parse(body); } catch { return undefined; }
|
|
1430
|
+
}
|
|
1431
|
+
return body && typeof body === "object" && !Array.isArray(body) ? body as V1Status : undefined;
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
/** @internal Exported for regression tests; not part of the components API. */
|
|
1435
|
+
export function isTransientApplyError(err: unknown): boolean {
|
|
1436
|
+
const status = applyErrorStatus(err);
|
|
1437
|
+
const code = status?.code ?? (err as { code?: unknown } | null)?.code;
|
|
1438
|
+
if ([429, 500, 503, 504].includes(code as number)) return true;
|
|
1439
|
+
// A missing namespace can converge while another apply creates it. An
|
|
1440
|
+
// arbitrary resource 404 cannot: don't classify by HTTP status alone.
|
|
1441
|
+
if (code === 404 && status?.reason === "NotFound" &&
|
|
1442
|
+
status.details?.kind === "namespaces" && !status.details.group &&
|
|
1443
|
+
typeof status.details.name === "string" && status.details.name.length > 0) return true;
|
|
1444
|
+
const msg = typeof status?.message === "string" ? status.message : errorText(err);
|
|
1425
1445
|
return TRANSIENT_APPLY_PATTERNS.some((re) => re.test(msg));
|
|
1426
1446
|
}
|
|
1427
1447
|
|
|
@@ -1865,48 +1885,52 @@ async function setupK3sCluster(
|
|
|
1865
1885
|
helpers: K3sHelpers,
|
|
1866
1886
|
opts: { registry: boolean; ingressDomains: string[]; traefik: boolean },
|
|
1867
1887
|
): Promise<void> {
|
|
1868
|
-
|
|
1869
|
-
if (tlsEnabled) {
|
|
1870
|
-
const { cert, key } = await issueIngressCert(
|
|
1871
|
-
opts.ingressDomains.map((d) => `*.${d}`),
|
|
1872
|
-
);
|
|
1873
|
-
// Apply the cert Secret + dynamic-config ConfigMap before the
|
|
1874
|
-
// Deployment that mounts them. `stringData` lets us hand over plain
|
|
1875
|
-
// PEM; the API server base64-encodes it.
|
|
1876
|
-
await helpers.client.core.createNamespacedSecret({
|
|
1877
|
-
namespace: "kube-system",
|
|
1878
|
-
body: {
|
|
1879
|
-
metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
|
|
1880
|
-
type: "kubernetes.io/tls",
|
|
1881
|
-
stringData: { "tls.crt": cert, "tls.key": key },
|
|
1882
|
-
},
|
|
1883
|
-
});
|
|
1884
|
-
await helpers.client.core.createNamespacedConfigMap({
|
|
1885
|
-
namespace: "kube-system",
|
|
1886
|
-
body: {
|
|
1887
|
-
metadata: {
|
|
1888
|
-
name: TRAEFIK_DYNAMIC_CONFIGMAP,
|
|
1889
|
-
namespace: "kube-system",
|
|
1890
|
-
},
|
|
1891
|
-
data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
|
|
1892
|
-
},
|
|
1893
|
-
});
|
|
1894
|
-
}
|
|
1895
|
-
if (opts.traefik) await helpers.apply(buildTraefikManifest(tlsEnabled));
|
|
1896
|
-
if (opts.registry) await helpers.apply(REGISTRY_MANIFEST);
|
|
1897
|
-
// Both rollouts proceed independently inside the cluster — wait on them
|
|
1898
|
-
// concurrently (they used to serialize, wasting up to a rollout's tail).
|
|
1899
|
-
// 250ms polling: these two sit on the cold start's critical path, where
|
|
1900
|
-
// the default 500ms interval wastes real time.
|
|
1888
|
+
// The registry can start while Traefik's certificates are being prepared.
|
|
1901
1889
|
const wait = (deployment: string): Promise<void> =>
|
|
1902
1890
|
helpers.waitForRollout(deployment, {
|
|
1903
1891
|
namespace: "kube-system",
|
|
1904
1892
|
timeoutMs: 120_000,
|
|
1905
1893
|
intervalMs: 250,
|
|
1906
1894
|
});
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1895
|
+
await waitForAll([
|
|
1896
|
+
(async () => {
|
|
1897
|
+
if (!opts.traefik) return;
|
|
1898
|
+
const tlsEnabled = opts.ingressDomains.length > 0 && caPresent();
|
|
1899
|
+
if (tlsEnabled) {
|
|
1900
|
+
const { cert, key } = await issueIngressCert(
|
|
1901
|
+
opts.ingressDomains.map((d) => `*.${d}`),
|
|
1902
|
+
);
|
|
1903
|
+
// Apply the cert Secret + dynamic-config ConfigMap before the
|
|
1904
|
+
// Deployment that mounts them. `stringData` lets us hand over plain
|
|
1905
|
+
// PEM; the API server base64-encodes it.
|
|
1906
|
+
await helpers.client.core.createNamespacedSecret({
|
|
1907
|
+
namespace: "kube-system",
|
|
1908
|
+
body: {
|
|
1909
|
+
metadata: { name: TRAEFIK_TLS_SECRET, namespace: "kube-system" },
|
|
1910
|
+
type: "kubernetes.io/tls",
|
|
1911
|
+
stringData: { "tls.crt": cert, "tls.key": key },
|
|
1912
|
+
},
|
|
1913
|
+
});
|
|
1914
|
+
await helpers.client.core.createNamespacedConfigMap({
|
|
1915
|
+
namespace: "kube-system",
|
|
1916
|
+
body: {
|
|
1917
|
+
metadata: {
|
|
1918
|
+
name: TRAEFIK_DYNAMIC_CONFIGMAP,
|
|
1919
|
+
namespace: "kube-system",
|
|
1920
|
+
},
|
|
1921
|
+
data: { "tls.yaml": TRAEFIK_DYNAMIC_TLS },
|
|
1922
|
+
},
|
|
1923
|
+
});
|
|
1924
|
+
}
|
|
1925
|
+
await helpers.apply(buildTraefikManifest(tlsEnabled));
|
|
1926
|
+
await wait("traefik");
|
|
1927
|
+
})(),
|
|
1928
|
+
(async () => {
|
|
1929
|
+
if (!opts.registry) return;
|
|
1930
|
+
await helpers.apply(REGISTRY_MANIFEST);
|
|
1931
|
+
await wait("spectest-registry");
|
|
1932
|
+
})(),
|
|
1933
|
+
]);
|
|
1910
1934
|
}
|
|
1911
1935
|
|
|
1912
1936
|
/**
|
|
@@ -2199,47 +2223,51 @@ export function k3s(opts: K3sOptions = {}) {
|
|
|
2199
2223
|
const { name, helpers, exec } = ctx;
|
|
2200
2224
|
const runtime = componentRuntime(ctx);
|
|
2201
2225
|
if (images.builds.length && !runtime) throw new Error(`k3s(${name}): prepared image inputs are missing`);
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2226
|
+
await waitForAll([
|
|
2227
|
+
runtime ? loadK3sBuildImages(name, runtime.images, runtime.gateway, exec) : Promise.resolve(),
|
|
2228
|
+
prePullK3sImages(name, images.pulls, exec),
|
|
2229
|
+
(async () => {
|
|
2230
|
+
if (runtime?.caBundle && caPresent()) {
|
|
2231
|
+
const { cert, key } = await issueIngressCert([runtime.gateway]);
|
|
2232
|
+
await installPodTrust({ name, gateway: runtime.gateway, cert, key,
|
|
2233
|
+
ca: await readFile(CA_PATH, "utf8"), bundle: await readFile(runtime.caBundle, "utf8"),
|
|
2234
|
+
}, {
|
|
2235
|
+
apply: async configuration => { await helpers.apply(JSON.stringify(configuration)); },
|
|
2236
|
+
ensureBundle: async (namespace, configMap, bundle) => {
|
|
2237
|
+
try {
|
|
2238
|
+
await helpers.client.core.createNamespacedConfigMap({ namespace, body: {
|
|
2239
|
+
metadata: { name: configMap, namespace }, immutable: true, data: { "ca-bundle.crt": bundle },
|
|
2240
|
+
} });
|
|
2241
|
+
} catch (err) {
|
|
2242
|
+
// A simultaneous Pod may have created the same namespace's bundle.
|
|
2243
|
+
const existing = (await helpers.client.core.readNamespacedConfigMap({ namespace, name: configMap })).unwrap();
|
|
2244
|
+
if (existing.data?.["ca-bundle.crt"] !== bundle || existing.immutable !== true) throw err;
|
|
2245
|
+
}
|
|
2246
|
+
},
|
|
2247
|
+
});
|
|
2248
|
+
// The API server observes webhook registration asynchronously. A dry-run
|
|
2249
|
+
// Pod proves TLS reachability and admission before any project workload.
|
|
2250
|
+
const deadline = Date.now() + 30_000;
|
|
2251
|
+
for (;;) {
|
|
2252
|
+
try {
|
|
2253
|
+
const pod = (await helpers.client.core.createNamespacedPod({ namespace: "default", dryRun: "All", body: {
|
|
2254
|
+
metadata: { generateName: "spectest-trust-probe-" }, spec: { containers: [{ name: "probe", image: "registry.k8s.io/pause:3.10" }] },
|
|
2255
|
+
} })).unwrap();
|
|
2256
|
+
if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt")) break;
|
|
2257
|
+
if (Date.now() >= deadline) throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
|
|
2258
|
+
} catch (err) {
|
|
2259
|
+
if (Date.now() >= deadline) throw err;
|
|
2260
|
+
}
|
|
2261
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
2217
2262
|
}
|
|
2218
|
-
},
|
|
2219
|
-
});
|
|
2220
|
-
// The API server observes webhook registration asynchronously. A dry-run
|
|
2221
|
-
// Pod proves TLS reachability and admission before any project workload.
|
|
2222
|
-
const deadline = Date.now() + 30_000;
|
|
2223
|
-
for (;;) {
|
|
2224
|
-
try {
|
|
2225
|
-
const pod = (await helpers.client.core.createNamespacedPod({ namespace: "default", dryRun: "All", body: {
|
|
2226
|
-
metadata: { generateName: "spectest-trust-probe-" }, spec: { containers: [{ name: "probe", image: "registry.k8s.io/pause:3.10" }] },
|
|
2227
|
-
} })).unwrap();
|
|
2228
|
-
if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt")) break;
|
|
2229
|
-
if (Date.now() >= deadline) throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
|
|
2230
|
-
} catch (err) {
|
|
2231
|
-
if (Date.now() >= deadline) throw err;
|
|
2232
2263
|
}
|
|
2233
|
-
await
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
traefik: traefikEnabled,
|
|
2241
|
-
});
|
|
2242
|
-
await prePullK3sImages(name, images.pulls, exec);
|
|
2264
|
+
await setupK3sCluster(name, helpers, {
|
|
2265
|
+
registry: registryEnabled,
|
|
2266
|
+
ingressDomains,
|
|
2267
|
+
traefik: traefikEnabled,
|
|
2268
|
+
});
|
|
2269
|
+
})(),
|
|
2270
|
+
]);
|
|
2243
2271
|
if (sticky) await registerStickyKubernetesStore(name, version);
|
|
2244
2272
|
},
|
|
2245
2273
|
helpers: async ({ name, exec, poll }: ServiceHelpersContext): Promise<K3sHelpers> => {
|
package/src/daemon.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { componentBuildTasks, COMPONENT_RUNTIME } from "./harness/component-images.js";
|
|
2
|
+
import { waitForAll } from "./harness/concurrency.js";
|
|
2
3
|
import { androidScope, androidRecordingChunk, releaseAndroidRecording, isAndroidApp, type AndroidRecording } from "./android.js";
|
|
3
4
|
// Long-running HTTP daemon. Runs as a systemd unit on the VM host (not in
|
|
4
5
|
// a container). Owns:
|
|
@@ -3687,25 +3688,28 @@ async function bootstrapInner(prepared?: PreparedImages, identity?: string): Pro
|
|
|
3687
3688
|
await waitForReady(svc);
|
|
3688
3689
|
const tSetup = Date.now();
|
|
3689
3690
|
const builds = componentTasks.filter(task => task.owner === svc.name);
|
|
3690
|
-
|
|
3691
|
-
if
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
await svc.
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
|
|
3700
|
-
|
|
3701
|
-
|
|
3702
|
-
|
|
3703
|
-
|
|
3704
|
-
|
|
3705
|
-
|
|
3706
|
-
|
|
3691
|
+
// Setup can overlap pulls/infrastructure with builds, and consume each
|
|
3692
|
+
// image as it becomes ready. Still drain every build if setup fails.
|
|
3693
|
+
await waitForAll([...builds.map(task => prep.get(task.name)!), (async () => {
|
|
3694
|
+
if (svc.setup) {
|
|
3695
|
+
progressService(svc.name, { status: "probing", detail: "running setup" });
|
|
3696
|
+
const helpers = await ensureHelpers(svc.name, svc);
|
|
3697
|
+
try {
|
|
3698
|
+
await svc.setup({
|
|
3699
|
+
name: svc.name,
|
|
3700
|
+
helpers,
|
|
3701
|
+
...(await spectestContext({ service: svc.name, includeSelf: true })),
|
|
3702
|
+
...{ [COMPONENT_RUNTIME]: {
|
|
3703
|
+
images: builds.map(task => ({ source: imageTag(task.name), reference: task.reference, ready: prep.get(task.name)! })),
|
|
3704
|
+
gateway: await bridgeGatewayIp(),
|
|
3705
|
+
caBundle: await ensureCaBundle(),
|
|
3706
|
+
} },
|
|
3707
|
+
});
|
|
3708
|
+
} catch (err) {
|
|
3709
|
+
throw await withContainerPostMortem(svc.name, err);
|
|
3710
|
+
}
|
|
3707
3711
|
}
|
|
3708
|
-
}
|
|
3712
|
+
})()]);
|
|
3709
3713
|
for (const task of builds) progressService(task.name, { status: "ready", detail: `loaded ${task.reference}` });
|
|
3710
3714
|
progressService(svc.name, { status: "ready", detail: undefined });
|
|
3711
3715
|
const ti = timings.get(svc.name);
|
|
@@ -63,8 +63,14 @@ export function componentBuildTasks<T extends ServiceConfig & { name: string }>(
|
|
|
63
63
|
/** Private handoff to a component's setup hook. Builds stay generic; the
|
|
64
64
|
* component decides how its runtime consumes them and establishes trust. */
|
|
65
65
|
export const COMPONENT_RUNTIME = Symbol.for("spectest.component-runtime");
|
|
66
|
+
export interface PreparedComponentImage {
|
|
67
|
+
source: string;
|
|
68
|
+
reference: string;
|
|
69
|
+
/** A component can load each result as soon as its own build finishes. */
|
|
70
|
+
ready?: Promise<void>;
|
|
71
|
+
}
|
|
66
72
|
export interface ComponentRuntime {
|
|
67
|
-
images: readonly
|
|
73
|
+
images: readonly PreparedComponentImage[];
|
|
68
74
|
gateway: string;
|
|
69
75
|
caBundle: string | null;
|
|
70
76
|
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** Wait for every writer to stop before reporting a failure to setup/cleanup. */
|
|
2
|
+
export async function waitForAll(work: readonly Promise<unknown>[]): Promise<void> {
|
|
3
|
+
const results = await Promise.allSettled(work);
|
|
4
|
+
const failures = results.flatMap(result => result.status === "rejected" ? [result.reason] : []);
|
|
5
|
+
if (failures.length === 1) throw failures[0];
|
|
6
|
+
if (failures.length > 1) throw new AggregateError(failures,
|
|
7
|
+
`Concurrent environment preparation failed: ${failures.map(error => error instanceof Error ? error.message : String(error)).join("; ")}`);
|
|
8
|
+
}
|
|
@@ -6,8 +6,9 @@ import { createHash } from "node:crypto";
|
|
|
6
6
|
import { createServer } from "node:http";
|
|
7
7
|
import { promisify } from "node:util";
|
|
8
8
|
import type { SpectestContext } from "../index.js";
|
|
9
|
-
import { canonicalImageReference } from "./component-images.js";
|
|
9
|
+
import { canonicalImageReference, type PreparedComponentImage } from "./component-images.js";
|
|
10
10
|
import { waitForK3sRuntime } from "./k3s-prepull.js";
|
|
11
|
+
import { waitForAll } from "./concurrency.js";
|
|
11
12
|
|
|
12
13
|
interface Descriptor { digest: string; size: number; mediaType: string; platform?: { os: string; architecture: string } }
|
|
13
14
|
interface Manifest { schemaVersion: number; mediaType: string; config: Descriptor; layers: Descriptor[]; annotations?: Record<string, string> }
|
|
@@ -57,7 +58,7 @@ async function hostImage(source: string) {
|
|
|
57
58
|
}
|
|
58
59
|
|
|
59
60
|
export async function loadK3sBuildImages(
|
|
60
|
-
service: string, images: readonly
|
|
61
|
+
service: string, images: readonly PreparedComponentImage[],
|
|
61
62
|
gateway: string, exec: SpectestContext["exec"],
|
|
62
63
|
): Promise<void> {
|
|
63
64
|
if (!images.length) return;
|
|
@@ -73,7 +74,8 @@ export async function loadK3sBuildImages(
|
|
|
73
74
|
const ctr = ["ctr", "--address", "/run/k3s/containerd/containerd.sock", "--namespace", "k8s.io"];
|
|
74
75
|
const help = await checked([...ctr, "images", "pull", "--help"]);
|
|
75
76
|
const local = help.includes("--local") ? ["--local"] : [];
|
|
76
|
-
|
|
77
|
+
await waitForAll(images.map(async image => {
|
|
78
|
+
await image.ready;
|
|
77
79
|
const reference = canonicalImageReference(image.reference);
|
|
78
80
|
const { target, metadata: blobs, allowed, configDigest } = await hostImage(image.source);
|
|
79
81
|
const streams = new Set<ReturnType<typeof spawn>>();
|
|
@@ -125,5 +127,5 @@ export async function loadK3sBuildImages(
|
|
|
125
127
|
await new Promise<void>(resolve => server.close(() => resolve()));
|
|
126
128
|
await exec(service, ["rm", "-rf", directory], { timeoutMs: 5_000 }).catch(() => undefined);
|
|
127
129
|
}
|
|
128
|
-
}
|
|
130
|
+
}));
|
|
129
131
|
}
|
|
@@ -20,7 +20,7 @@ test("pre-pulls reuse immutable cached images, but resolve tags on every fresh s
|
|
|
20
20
|
expect(calls.at(-1)).toEqual([...cri, "pull", digest]);
|
|
21
21
|
});
|
|
22
22
|
|
|
23
|
-
test("failed pre-pulls fail setup after
|
|
23
|
+
test("failed pre-pulls fail setup after all concurrent pulls finish", async () => {
|
|
24
24
|
let active = 0, peak = 0, finished = 0;
|
|
25
25
|
const work = prePullK3sImages("cluster", ["a:1", "b:1", "c:1"], mockExec(async args => {
|
|
26
26
|
if (args.at(-1) === "info") return { exitCode: 0 };
|
|
@@ -31,7 +31,7 @@ test("failed pre-pulls fail setup after outstanding pulls finish, with bounded c
|
|
|
31
31
|
}));
|
|
32
32
|
await expect(work).rejects.toThrow("pre-pull failed for a:1: registry unavailable");
|
|
33
33
|
expect(active).toBe(0);
|
|
34
|
-
expect(peak).toBe(
|
|
34
|
+
expect(peak).toBe(3);
|
|
35
35
|
expect(finished).toBe(3);
|
|
36
36
|
});
|
|
37
37
|
|
|
@@ -45,3 +45,13 @@ test("pre-pulls wait for the runtime even when the API server is already ready",
|
|
|
45
45
|
}));
|
|
46
46
|
expect(calls).toEqual(["info", "info", "pull"]);
|
|
47
47
|
});
|
|
48
|
+
|
|
49
|
+
test("simultaneous pull failures retain every failing image in the setup error", async () => {
|
|
50
|
+
const error = await prePullK3sImages("cluster", ["a:1", "b:1"], mockExec(async args => ({
|
|
51
|
+
exitCode: args[3] === "pull" ? 1 : 0, stderr: "unavailable",
|
|
52
|
+
}))).catch(error => error);
|
|
53
|
+
expect(error).toBeInstanceOf(AggregateError);
|
|
54
|
+
expect(error.errors).toHaveLength(2);
|
|
55
|
+
expect(error.message).toContain("pre-pull failed for a:1");
|
|
56
|
+
expect(error.message).toContain("pre-pull failed for b:1");
|
|
57
|
+
});
|
|
@@ -1,26 +1,23 @@
|
|
|
1
1
|
import type { SpectestContext } from "../index.js";
|
|
2
|
+
import { waitForAll } from "./concurrency.js";
|
|
2
3
|
|
|
3
4
|
/** CRI PullImage waits for unpacking and uses k3s's registry configuration.
|
|
4
5
|
* Always resolve tags on a fresh setup. Immutable digests can be used offline.
|
|
5
|
-
*
|
|
6
|
+
* Submit every image concurrently; containerd manages downloads and unpacking.
|
|
7
|
+
* Drain all pulls on failure so none outlive the setup hook.
|
|
6
8
|
*/
|
|
7
9
|
export async function prePullK3sImages(name: string, images: readonly string[], exec: SpectestContext["exec"]): Promise<void> {
|
|
8
10
|
if (!images.length) return;
|
|
9
11
|
await waitForK3sRuntime(name, exec);
|
|
10
12
|
const cri = ["crictl", "--runtime-endpoint", "unix:///run/k3s/containerd/containerd.sock"];
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
if (
|
|
15
|
-
const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
|
|
16
|
-
if (existing.exitCode === 0) continue;
|
|
17
|
-
}
|
|
18
|
-
const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
|
|
19
|
-
if (result.exitCode !== 0) throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
|
|
13
|
+
await waitForAll(images.map(async image => {
|
|
14
|
+
if (/@sha256:[a-f0-9]{64}$/.test(image)) {
|
|
15
|
+
const existing = await exec(name, [...cri, "inspecti", image], { timeoutMs: 30_000 });
|
|
16
|
+
if (existing.exitCode === 0) return;
|
|
20
17
|
}
|
|
18
|
+
const result = await exec(name, [...cri, "pull", image], { timeoutMs: 600_000 });
|
|
19
|
+
if (result.exitCode !== 0) throw new Error(`k3s(${name}): pre-pull failed for ${image}: ${result.stderr || result.stdout}`);
|
|
21
20
|
}));
|
|
22
|
-
const failed = results.find(result => result.status === "rejected");
|
|
23
|
-
if (failed?.status === "rejected") throw failed.reason;
|
|
24
21
|
}
|
|
25
22
|
|
|
26
23
|
export async function waitForK3sRuntime(name: string, exec: SpectestContext["exec"]): Promise<void> {
|