@specific.dev/spectest 0.92.1 → 0.93.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/components/index.d.ts +1 -1
- package/dist/components/k3s.d.ts +26 -4
- package/dist/components/k3s.js +105 -61
- package/dist/daemon.js +3 -0
- package/dist/harness/built-image.d.ts +16 -0
- package/dist/harness/built-image.js +57 -0
- package/dist/harness/k3s-image-loader.js +2 -53
- package/dist/harness/registry-seed.d.ts +12 -0
- package/dist/harness/registry-seed.js +61 -0
- package/dist/index.d.ts +2 -1
- package/package.json +1 -1
- package/src/components/index.ts +1 -0
- package/src/components/k3s-registry.test.ts +50 -0
- package/src/components/k3s.ts +114 -52
- package/src/daemon.ts +3 -0
- package/src/harness/built-image.ts +53 -0
- package/src/harness/k3s-image-loader.ts +2 -47
- package/src/harness/registry-seed.test.ts +60 -0
- package/src/harness/registry-seed.ts +61 -0
- package/src/index.ts +2 -1
package/README.md
CHANGED
|
@@ -106,3 +106,10 @@ renderer DOM from acquisition onward, not native OS surfaces.
|
|
|
106
106
|
|
|
107
107
|
Run `spectest docs /components/electron` for the full configuration reference,
|
|
108
108
|
backend setup, session behavior, and replay scope (CLI 0.2.124 or later).
|
|
109
|
+
|
|
110
|
+
`k3s({ registry: { host: "registry.test" }, images: [...] })` exposes the
|
|
111
|
+
component's writable registry over verified HTTPS through the VM proxy.
|
|
112
|
+
Dockerfile builds in `images` targeting `registry.test/...` are published and
|
|
113
|
+
preloaded from the same cached build, preserving digests without an export/push
|
|
114
|
+
step. Other images keep direct preloading. Registry writes stay isolated across
|
|
115
|
+
test forks; client and containerd CA trust are configured automatically.
|
|
@@ -2,7 +2,7 @@ export { temporal, temporalClient, type TemporalOptions, type TemporalClient, ty
|
|
|
2
2
|
export { postgres, type PostgresOptions, type PostgresHelpers, } from "./postgres.js";
|
|
3
3
|
export { s3, type S3Options, type S3Helpers, } from "./s3.js";
|
|
4
4
|
export { SQL, type SqlClient, type SqlOptions } from "../sql.js";
|
|
5
|
-
export { k3s, type K3sOptions, type KubernetesImage, type K3sHelpers, type K3sClient, type TaggedObjectApi, type KubernetesObjectRef, type ApplyOptions, type RolloutTarget, type WaitOptions, type LogsOptions, type KubernetesObject, type KubernetesListObject, } from "./k3s.js";
|
|
5
|
+
export { k3s, type K3sOptions, type K3sRegistryOptions, type KubernetesImage, type K3sHelpers, type K3sClient, type TaggedObjectApi, type KubernetesObjectRef, type ApplyOptions, type RolloutTarget, type WaitOptions, type LogsOptions, type KubernetesObject, type KubernetesListObject, } from "./k3s.js";
|
|
6
6
|
export { expo, type ExpoOptions, type ExpoHelpers, type ExpoPackageManager, } from "./expo.js";
|
|
7
7
|
export { supabase, type SupabaseOptions, type SupabaseHelpers, type SupabaseStack, } from "./supabase.js";
|
|
8
8
|
export { email, type EmailOptions, type EmailHelpers, type EmailMessage, type EmailSummary, type EmailMatch, type EmailAttachment, } from "./email.js";
|
package/dist/components/k3s.d.ts
CHANGED
|
@@ -4,6 +4,14 @@ import { type KubernetesImage } from "../harness/component-images.js";
|
|
|
4
4
|
export type { KubernetesImage } from "../harness/component-images.js";
|
|
5
5
|
import type { Wrapped } from "../inspect.js";
|
|
6
6
|
export type { KubernetesObject, KubernetesListObject, } from "@kubernetes/client-node";
|
|
7
|
+
/** HTTPS exposure and publication for the component's writable registry. */
|
|
8
|
+
export interface K3sRegistryOptions {
|
|
9
|
+
/** Exact DNS hostname exposed through Spectest's VM HTTPS proxy on port 443.
|
|
10
|
+
* Clients and Kubernetes's containerd trust the environment CA automatically.
|
|
11
|
+
* Builds in `images` whose reference starts with this host are also published
|
|
12
|
+
* here, using their original cached blobs and manifest digests. */
|
|
13
|
+
host: string;
|
|
14
|
+
}
|
|
7
15
|
/** Kubernetes pods receive read-only environment CA mounts during admission.
|
|
8
16
|
* Image bytes and digests stay unchanged; explicit pod trust settings win. */
|
|
9
17
|
export interface K3sOptions {
|
|
@@ -11,6 +19,9 @@ export interface K3sOptions {
|
|
|
11
19
|
* or Dockerfile builds with a full `reference` matching your manifests. Builds
|
|
12
20
|
* support path/content, context, exclude, target and buildArgs, and run in
|
|
13
21
|
* parallel with service builds through the standard BuildKit pipeline.
|
|
22
|
+
* With `registry: { host }`, builds targeting that host are also published
|
|
23
|
+
* to the writable registry before setup completes. Other images remain
|
|
24
|
+
* direct preloads; no external registry receives pushes.
|
|
14
25
|
* Setup waits for every pull and fails if a requested image cannot be pulled.
|
|
15
26
|
* Sticky-enabled projects retain image content and unpacked layers between
|
|
16
27
|
* fresh builds automatically. Digest references can reuse the cache offline;
|
|
@@ -85,10 +96,15 @@ export interface K3sOptions {
|
|
|
85
96
|
* into your platform, e.g. `env: { REGISTRY_URL: "cluster.internal:5000" }`.
|
|
86
97
|
* Plain HTTP, so configure your push client for an insecure registry.
|
|
87
98
|
*
|
|
99
|
+
* Set `{ host: "registry.test" }` for verified HTTPS on port 443 through
|
|
100
|
+
* the built-in VM proxy, including containerd trust and relative upload URLs.
|
|
101
|
+
* This does not require Traefik. Builds targeting that host in `images` are
|
|
102
|
+
* published automatically; tags and test uploads remain isolated per fork.
|
|
103
|
+
*
|
|
88
104
|
* On by default. Set `false` for clusters that only ever run public
|
|
89
105
|
* images — that skips the extra pod.
|
|
90
106
|
*/
|
|
91
|
-
registry?: boolean;
|
|
107
|
+
registry?: boolean | K3sRegistryOptions;
|
|
92
108
|
/**
|
|
93
109
|
* Domains to route into this cluster's ingress via **wildcard DNS**. For
|
|
94
110
|
* each `"example.com"`, spectest-resolver answers any `*.example.com`
|
|
@@ -336,6 +352,10 @@ export declare function k3s(opts?: K3sOptions): {
|
|
|
336
352
|
reference: string;
|
|
337
353
|
};
|
|
338
354
|
imageBuilds: import("../index.js").ComponentImageBuild[];
|
|
355
|
+
tls: {
|
|
356
|
+
hostname: string;
|
|
357
|
+
port: number;
|
|
358
|
+
}[];
|
|
339
359
|
command: string;
|
|
340
360
|
privileged: true;
|
|
341
361
|
tmpfs: string[];
|
|
@@ -348,10 +368,12 @@ export declare function k3s(opts?: K3sOptions): {
|
|
|
348
368
|
} | {
|
|
349
369
|
source: string;
|
|
350
370
|
target: string;
|
|
351
|
-
readOnly:
|
|
352
|
-
}
|
|
371
|
+
readOnly: boolean;
|
|
372
|
+
} | {
|
|
353
373
|
target: string;
|
|
354
|
-
|
|
374
|
+
source?: undefined;
|
|
375
|
+
readOnly?: undefined;
|
|
376
|
+
})[];
|
|
355
377
|
files: {
|
|
356
378
|
path: string;
|
|
357
379
|
content: string;
|
package/dist/components/k3s.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { isIP } from "node:net";
|
|
2
2
|
import { installPodTrust } from "./k3s-trust.js";
|
|
3
3
|
import { waitForAll } from "../harness/concurrency.js";
|
|
4
|
+
import { CONTENT_BLOBS, REGISTRY_CONTENT, REGISTRY_DATA, REGISTRY_IMAGE, registryDirectory, seedRegistry } from "../harness/registry-seed.js";
|
|
4
5
|
import { loadK3sBuildImages } from "../harness/k3s-image-loader.js";
|
|
5
6
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
6
7
|
import { spawn as nodeSpawn } from "node:child_process";
|
|
@@ -792,7 +793,7 @@ const HUB_MIRROR = { registry: "docker.io", mirror: "https://mirror.gcr.io", ups
|
|
|
792
793
|
* shares the node's netns, so this needs no in-container DNS and
|
|
793
794
|
* can't be broken by a clobbered `hostnames`.
|
|
794
795
|
*/
|
|
795
|
-
function buildRegistriesYaml(registryEnabled) {
|
|
796
|
+
function buildRegistriesYaml(registryEnabled, host) {
|
|
796
797
|
const lines = [
|
|
797
798
|
"mirrors:",
|
|
798
799
|
` "${HUB_MIRROR.registry}":`,
|
|
@@ -807,6 +808,11 @@ function buildRegistriesYaml(registryEnabled) {
|
|
|
807
808
|
// makes that explicit and disables any TLS attempt against it.
|
|
808
809
|
` "127.0.0.1:${K3S_REGISTRY_PORT}":`, ` tls:`, ` insecure_skip_verify: true`);
|
|
809
810
|
}
|
|
811
|
+
if (host) {
|
|
812
|
+
// This names the verified HTTPS endpoint itself; do not mirror it to HTTP.
|
|
813
|
+
// container-run mounts the environment CA before k3s/containerd starts.
|
|
814
|
+
lines.push(` "${host}":`, " tls:", ` ca_file: ${JSON.stringify(CA_PATH)}`);
|
|
815
|
+
}
|
|
810
816
|
return lines.join("\n") + "\n";
|
|
811
817
|
}
|
|
812
818
|
/**
|
|
@@ -814,11 +820,13 @@ function buildRegistriesYaml(registryEnabled) {
|
|
|
814
820
|
* binds the cluster container's `:5000` directly — the same trick
|
|
815
821
|
* Traefik uses — so peer services reach it at `<cluster-key>.internal:5000`
|
|
816
822
|
* (the cluster service's own alias) and the node's own containerd reaches
|
|
817
|
-
* it at `127.0.0.1:5000`. Storage is an `emptyDir
|
|
818
|
-
*
|
|
819
|
-
*
|
|
823
|
+
* it at `127.0.0.1:5000`. Storage is an `emptyDir` by default. Declared
|
|
824
|
+
* publications use a rootfs directory plus a read-only content mount; both
|
|
825
|
+
* variants keep writable registry state outside persistent image caches and
|
|
826
|
+
* capture it with the VM snapshot, isolated per test fork.
|
|
820
827
|
*/
|
|
821
|
-
|
|
828
|
+
function registryManifest(seeded) {
|
|
829
|
+
return `apiVersion: apps/v1
|
|
822
830
|
kind: Deployment
|
|
823
831
|
metadata:
|
|
824
832
|
name: spectest-registry
|
|
@@ -841,11 +849,13 @@ spec:
|
|
|
841
849
|
- operator: Exists
|
|
842
850
|
containers:
|
|
843
851
|
- name: registry
|
|
844
|
-
image: registry:2
|
|
852
|
+
image: ${seeded ? REGISTRY_IMAGE : "registry:2"}
|
|
845
853
|
imagePullPolicy: IfNotPresent
|
|
846
854
|
env:
|
|
847
855
|
- name: REGISTRY_HTTP_ADDR
|
|
848
856
|
value: ":${K3S_REGISTRY_PORT}"
|
|
857
|
+
- name: REGISTRY_HTTP_RELATIVEURLS
|
|
858
|
+
value: "true"
|
|
849
859
|
- name: REGISTRY_STORAGE_DELETE_ENABLED
|
|
850
860
|
value: "true"
|
|
851
861
|
ports:
|
|
@@ -854,10 +864,20 @@ spec:
|
|
|
854
864
|
volumeMounts:
|
|
855
865
|
- name: data
|
|
856
866
|
mountPath: /var/lib/registry
|
|
857
|
-
|
|
867
|
+
${seeded ? ` - name: content
|
|
868
|
+
mountPath: ${REGISTRY_CONTENT}
|
|
869
|
+
readOnly: true
|
|
870
|
+
` : ""} volumes:
|
|
858
871
|
- name: data
|
|
859
|
-
|
|
872
|
+
${seeded ? ` hostPath:
|
|
873
|
+
path: ${REGISTRY_DATA}
|
|
874
|
+
type: Directory
|
|
875
|
+
- name: content
|
|
876
|
+
hostPath:
|
|
877
|
+
path: ${REGISTRY_CONTENT}
|
|
878
|
+
type: Directory` : " emptyDir: {}"}
|
|
860
879
|
`;
|
|
880
|
+
}
|
|
861
881
|
// ────────────────────────────────────────────────────────────────────────
|
|
862
882
|
// Apply — server-side, converging
|
|
863
883
|
//
|
|
@@ -1310,7 +1330,8 @@ async function setupK3sCluster(name, helpers, opts) {
|
|
|
1310
1330
|
(async () => {
|
|
1311
1331
|
if (!opts.registry)
|
|
1312
1332
|
return;
|
|
1313
|
-
await
|
|
1333
|
+
await opts.seed?.();
|
|
1334
|
+
await helpers.apply(registryManifest(!!opts.seed));
|
|
1314
1335
|
await wait("spectest-registry");
|
|
1315
1336
|
})(),
|
|
1316
1337
|
]);
|
|
@@ -1438,6 +1459,16 @@ export function k3s(opts = {}) {
|
|
|
1438
1459
|
const extra = opts.extraArgs ?? [];
|
|
1439
1460
|
assertUsableExtraArgs(extra);
|
|
1440
1461
|
const registryEnabled = opts.registry !== false;
|
|
1462
|
+
let registryHost;
|
|
1463
|
+
if (opts.registry !== undefined && typeof opts.registry !== "boolean") {
|
|
1464
|
+
if (!opts.registry || typeof opts.registry.host !== "string" || opts.registry.host.startsWith("*.")) {
|
|
1465
|
+
throw new Error("k3s(): registry must be a boolean or { host: an exact DNS hostname }");
|
|
1466
|
+
}
|
|
1467
|
+
// Reuse ingress validation (no scheme, port, path, or reserved .internal).
|
|
1468
|
+
registryHost = dnsName(opts.registry.host, { ingress: true }).hostname;
|
|
1469
|
+
}
|
|
1470
|
+
const isPublished = (reference) => !!registryHost && reference.startsWith(`${registryHost}/`);
|
|
1471
|
+
const publishBuilds = images.builds.filter(image => isPublished(image.reference));
|
|
1441
1472
|
const traefikEnabled = opts.traefik !== false;
|
|
1442
1473
|
const loadBalancerEnabled = opts.loadBalancer !== false;
|
|
1443
1474
|
// Wildcard ingress domains. Drives both the `provides(... dnsName)`
|
|
@@ -1447,12 +1478,13 @@ export function k3s(opts = {}) {
|
|
|
1447
1478
|
// `/etc/rancher/k3s/registries.yaml` (host-cache mirrors + trust for
|
|
1448
1479
|
// the in-cluster registry). Seeded via `files` because k3s reads it
|
|
1449
1480
|
// only at startup, before any setup hook could run.
|
|
1450
|
-
const registriesYaml = buildRegistriesYaml(registryEnabled);
|
|
1481
|
+
const registriesYaml = buildRegistriesYaml(registryEnabled, registryHost);
|
|
1451
1482
|
// The environment's image cache, when this VM carries one: the
|
|
1452
1483
|
// cluster's store goes on its root disk and its layers come off the
|
|
1453
1484
|
// shared layers disk (see the store section above).
|
|
1454
1485
|
const sticky = stickyDiskPathSync();
|
|
1455
1486
|
const cache = sticky ? null : imageCachePathsSync();
|
|
1487
|
+
const contentDirectory = `${sticky ? `${sticky}/containerd` : cache?.root ?? "/var/lib/containerd"}/${CONTENT_BLOBS}`;
|
|
1456
1488
|
const serverArgs = [
|
|
1457
1489
|
"k3s",
|
|
1458
1490
|
"server",
|
|
@@ -1551,6 +1583,7 @@ export function k3s(opts = {}) {
|
|
|
1551
1583
|
const def = {
|
|
1552
1584
|
image: { type: "registry", reference: `rancher/k3s:${version}` },
|
|
1553
1585
|
imageBuilds: images.builds,
|
|
1586
|
+
tls: registryHost ? [{ hostname: registryHost, port: K3S_REGISTRY_PORT }] : [],
|
|
1554
1587
|
command: cmd,
|
|
1555
1588
|
privileged: true,
|
|
1556
1589
|
tmpfs: ["/run", "/var/run"],
|
|
@@ -1572,15 +1605,20 @@ export function k3s(opts = {}) {
|
|
|
1572
1605
|
// in one environment get separate roots either way (the service
|
|
1573
1606
|
// token names the directory), because two containerd daemons cannot
|
|
1574
1607
|
// share a store: bolt takes an exclusive lock.
|
|
1575
|
-
volumes: sticky
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1608
|
+
volumes: [...(sticky
|
|
1609
|
+
? [{ source: `${stickyKubernetesDirectory(SELF_SERVICE_TOKEN, version)}/containerd`, target: CLUSTER_STORE_TARGET }]
|
|
1610
|
+
: cache ? [
|
|
1611
|
+
{ source: nestedStoreDir(cache, SELF_SERVICE_TOKEN), target: CLUSTER_STORE_TARGET },
|
|
1612
|
+
{ source: cache.layers, target: cache.layers, readOnly: true },
|
|
1613
|
+
{ source: MKFS_EROFS_PATH, target: MKFS_EROFS_PATH, readOnly: true },
|
|
1614
|
+
{ source: STORE_ADOPT_PATH, target: STORE_ADOPT_PATH, readOnly: true },
|
|
1615
|
+
]
|
|
1616
|
+
: [{ target: CLUSTER_STORE_TARGET }]),
|
|
1617
|
+
...(publishBuilds.length ? [
|
|
1618
|
+
{ source: registryDirectory(SELF_SERVICE_TOKEN), target: REGISTRY_DATA },
|
|
1619
|
+
{ source: contentDirectory, target: REGISTRY_CONTENT, readOnly: true },
|
|
1620
|
+
] : []),
|
|
1621
|
+
],
|
|
1584
1622
|
files: [
|
|
1585
1623
|
...(registriesYaml ? [{ path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml }] : []),
|
|
1586
1624
|
...(cache
|
|
@@ -1600,55 +1638,61 @@ export function k3s(opts = {}) {
|
|
|
1600
1638
|
const runtime = componentRuntime(ctx);
|
|
1601
1639
|
if (images.builds.length && !runtime)
|
|
1602
1640
|
throw new Error(`k3s(${name}): prepared image inputs are missing`);
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
}, {
|
|
1612
|
-
apply: async (configuration) => { await helpers.apply(JSON.stringify(configuration)); },
|
|
1613
|
-
ensureBundle: async (namespace, configMap, bundle) => {
|
|
1614
|
-
try {
|
|
1615
|
-
await helpers.client.core.createNamespacedConfigMap({ namespace, body: {
|
|
1616
|
-
metadata: { name: configMap, namespace }, immutable: true, data: { "ca-bundle.crt": bundle },
|
|
1617
|
-
} });
|
|
1618
|
-
}
|
|
1619
|
-
catch (err) {
|
|
1620
|
-
// A simultaneous Pod may have created the same namespace's bundle.
|
|
1621
|
-
const existing = (await helpers.client.core.readNamespacedConfigMap({ namespace, name: configMap })).unwrap();
|
|
1622
|
-
if (existing.data?.["ca-bundle.crt"] !== bundle || existing.immutable !== true)
|
|
1623
|
-
throw err;
|
|
1624
|
-
}
|
|
1625
|
-
},
|
|
1626
|
-
});
|
|
1627
|
-
// The API server observes webhook registration asynchronously. A dry-run
|
|
1628
|
-
// Pod proves TLS reachability and admission before any project workload.
|
|
1629
|
-
const deadline = Date.now() + 30_000;
|
|
1630
|
-
for (;;) {
|
|
1641
|
+
const infrastructure = (async () => {
|
|
1642
|
+
if (runtime?.caBundle && caPresent()) {
|
|
1643
|
+
const { cert, key } = await issueIngressCert([runtime.gateway]);
|
|
1644
|
+
await installPodTrust({ name, gateway: runtime.gateway, cert, key,
|
|
1645
|
+
ca: await readFile(CA_PATH, "utf8"), bundle: await readFile(runtime.caBundle, "utf8"),
|
|
1646
|
+
}, {
|
|
1647
|
+
apply: async (configuration) => { await helpers.apply(JSON.stringify(configuration)); },
|
|
1648
|
+
ensureBundle: async (namespace, configMap, bundle) => {
|
|
1631
1649
|
try {
|
|
1632
|
-
|
|
1633
|
-
metadata: {
|
|
1634
|
-
} })
|
|
1635
|
-
if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt"))
|
|
1636
|
-
break;
|
|
1637
|
-
if (Date.now() >= deadline)
|
|
1638
|
-
throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
|
|
1650
|
+
await helpers.client.core.createNamespacedConfigMap({ namespace, body: {
|
|
1651
|
+
metadata: { name: configMap, namespace }, immutable: true, data: { "ca-bundle.crt": bundle },
|
|
1652
|
+
} });
|
|
1639
1653
|
}
|
|
1640
1654
|
catch (err) {
|
|
1641
|
-
|
|
1655
|
+
// A simultaneous Pod may have created the same namespace's bundle.
|
|
1656
|
+
const existing = (await helpers.client.core.readNamespacedConfigMap({ namespace, name: configMap })).unwrap();
|
|
1657
|
+
if (existing.data?.["ca-bundle.crt"] !== bundle || existing.immutable !== true)
|
|
1642
1658
|
throw err;
|
|
1643
1659
|
}
|
|
1644
|
-
|
|
1660
|
+
},
|
|
1661
|
+
});
|
|
1662
|
+
// The API server observes webhook registration asynchronously. A dry-run
|
|
1663
|
+
// Pod proves TLS reachability and admission before any project workload.
|
|
1664
|
+
const deadline = Date.now() + 30_000;
|
|
1665
|
+
for (;;) {
|
|
1666
|
+
try {
|
|
1667
|
+
const pod = (await helpers.client.core.createNamespacedPod({ namespace: "default", dryRun: "All", body: {
|
|
1668
|
+
metadata: { generateName: "spectest-trust-probe-" }, spec: { containers: [{ name: "probe", image: "registry.k8s.io/pause:3.10" }] },
|
|
1669
|
+
} })).unwrap();
|
|
1670
|
+
if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt"))
|
|
1671
|
+
break;
|
|
1672
|
+
if (Date.now() >= deadline)
|
|
1673
|
+
throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
|
|
1674
|
+
}
|
|
1675
|
+
catch (err) {
|
|
1676
|
+
if (Date.now() >= deadline)
|
|
1677
|
+
throw err;
|
|
1645
1678
|
}
|
|
1679
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
1646
1680
|
}
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1681
|
+
}
|
|
1682
|
+
await setupK3sCluster(name, helpers, {
|
|
1683
|
+
registry: registryEnabled,
|
|
1684
|
+
seed: publishBuilds.length ? () => seedRegistry(name, contentDirectory, runtime.images.filter(image => isPublished(image.reference))) : undefined,
|
|
1685
|
+
ingressDomains,
|
|
1686
|
+
traefik: traefikEnabled,
|
|
1687
|
+
});
|
|
1688
|
+
})();
|
|
1689
|
+
await waitForAll([
|
|
1690
|
+
runtime ? loadK3sBuildImages(name, runtime.images, runtime.gateway, exec) : Promise.resolve(),
|
|
1691
|
+
prePullK3sImages(name, images.pulls.filter(ref => !isPublished(ref)), exec),
|
|
1692
|
+
(async () => {
|
|
1693
|
+
// Registry-local references must not race registry startup/publication.
|
|
1694
|
+
await infrastructure;
|
|
1695
|
+
await prePullK3sImages(name, images.pulls.filter(isPublished), exec);
|
|
1652
1696
|
})(),
|
|
1653
1697
|
]);
|
|
1654
1698
|
if (sticky)
|
package/dist/daemon.js
CHANGED
|
@@ -1958,6 +1958,9 @@ serve = { proto: "http" }) {
|
|
|
1958
1958
|
// {@link stopIngressServers} depends on: a half-alive listener shares
|
|
1959
1959
|
// the port silently instead of colliding loudly.
|
|
1960
1960
|
reusePort: true,
|
|
1961
|
+
// OCI layer uploads can exceed Bun's default 128 MiB request limit.
|
|
1962
|
+
// Large proxy bodies stream through; this limit does not allocate a buffer.
|
|
1963
|
+
maxRequestBodySize: 1024 ** 4,
|
|
1961
1964
|
// Bun.serve defaults to a 10s idleTimeout, which kills any proxied
|
|
1962
1965
|
// request whose upstream takes >10s to produce bytes — under parallel
|
|
1963
1966
|
// test load that surfaced as "fetch failed"/"other side closed" on
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export interface Descriptor {
|
|
2
|
+
digest: string;
|
|
3
|
+
size: number;
|
|
4
|
+
mediaType: string;
|
|
5
|
+
platform?: {
|
|
6
|
+
os: string;
|
|
7
|
+
architecture: string;
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
/** Preserve the original index/manifest bytes and all application descriptors. */
|
|
11
|
+
export declare function hostImage(source: string): Promise<{
|
|
12
|
+
target: Descriptor;
|
|
13
|
+
metadata: Map<string, Buffer<ArrayBufferLike>>;
|
|
14
|
+
allowed: Map<string, Descriptor>;
|
|
15
|
+
configDigest: string;
|
|
16
|
+
}>;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/** Read build results without reserializing manifests or exporting layers. */
|
|
2
|
+
import { execFile } from "node:child_process";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import { promisify } from "node:util";
|
|
5
|
+
import { canonicalImageReference } from "./component-images.js";
|
|
6
|
+
const digest = (bytes) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
7
|
+
const descriptor = (bytes, mediaType) => ({ digest: digest(bytes), size: bytes.length, mediaType });
|
|
8
|
+
const run = promisify(execFile);
|
|
9
|
+
const HOST_CTR = ["--address", "/run/containerd/containerd.sock", "--namespace", "moby"];
|
|
10
|
+
async function hostContent(id) {
|
|
11
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(id))
|
|
12
|
+
throw new Error(`invalid content digest ${id}`);
|
|
13
|
+
const result = await run("ctr", [...HOST_CTR, "content", "get", id], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024, timeout: 30_000 });
|
|
14
|
+
if (digest(result.stdout) !== id)
|
|
15
|
+
throw new Error(`corrupt image metadata ${id}`);
|
|
16
|
+
return result.stdout;
|
|
17
|
+
}
|
|
18
|
+
/** Preserve the original index/manifest bytes and all application descriptors. */
|
|
19
|
+
export async function hostImage(source) {
|
|
20
|
+
const name = canonicalImageReference(source);
|
|
21
|
+
const { stdout } = await run("ctr", [...HOST_CTR, "images", "ls", `name==${name}`], { timeout: 30_000 });
|
|
22
|
+
const found = stdout.split("\n").map(line => line.trim().split(/\s+/)).find(row => row[0] === name)?.[2];
|
|
23
|
+
if (!found)
|
|
24
|
+
throw new Error(`built image ${name} is missing from the VM container store`);
|
|
25
|
+
const metadata = new Map();
|
|
26
|
+
const allowed = new Map();
|
|
27
|
+
let configDigest;
|
|
28
|
+
async function visit(id, selected, depth = 0) {
|
|
29
|
+
if (depth > 4)
|
|
30
|
+
throw new Error(`built image ${name} has a recursive manifest index`);
|
|
31
|
+
const bytes = await hostContent(id);
|
|
32
|
+
const obj = JSON.parse(bytes.toString());
|
|
33
|
+
const desc = descriptor(bytes, obj.mediaType);
|
|
34
|
+
metadata.set(id, bytes);
|
|
35
|
+
allowed.set(id, desc);
|
|
36
|
+
if (obj.config && Array.isArray(obj.layers)) {
|
|
37
|
+
const manifest = obj;
|
|
38
|
+
for (const blob of [manifest.config, ...manifest.layers])
|
|
39
|
+
allowed.set(blob.digest, blob);
|
|
40
|
+
if (selected)
|
|
41
|
+
configDigest = manifest.config.digest;
|
|
42
|
+
}
|
|
43
|
+
else if (Array.isArray(obj.manifests)) {
|
|
44
|
+
for (const entry of obj.manifests) {
|
|
45
|
+
const matches = !entry.platform || (entry.platform.os === "linux" && entry.platform.architecture === (process.arch === "arm64" ? "arm64" : "amd64"));
|
|
46
|
+
await visit(entry.digest, selected && matches, depth + 1);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
else
|
|
50
|
+
throw new Error(`built image ${name} has invalid image metadata`);
|
|
51
|
+
return desc;
|
|
52
|
+
}
|
|
53
|
+
const target = await visit(found, true);
|
|
54
|
+
if (!configDigest)
|
|
55
|
+
throw new Error(`built image ${name} has no image for this VM's platform`);
|
|
56
|
+
return { target, metadata, allowed, configDigest };
|
|
57
|
+
}
|
|
@@ -1,65 +1,14 @@
|
|
|
1
1
|
/** A short-lived, read-only OCI endpoint over the host containerd content store.
|
|
2
2
|
* k3s fetches only missing blobs and unpacks only missing snapshots. No Docker
|
|
3
3
|
* archive, staging registry, or second persistent copy of application blobs. */
|
|
4
|
-
import {
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
5
|
import { createHash } from "node:crypto";
|
|
6
6
|
import { createServer } from "node:http";
|
|
7
|
-
import { promisify } from "node:util";
|
|
8
7
|
import { canonicalImageReference } from "./component-images.js";
|
|
9
8
|
import { waitForK3sRuntime } from "./k3s-prepull.js";
|
|
9
|
+
import { hostImage } from "./built-image.js";
|
|
10
10
|
import { waitForAll } from "./concurrency.js";
|
|
11
|
-
const digest = (bytes) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
12
|
-
const descriptor = (bytes, mediaType) => ({ digest: digest(bytes), size: bytes.length, mediaType });
|
|
13
|
-
const run = promisify(execFile);
|
|
14
11
|
const HOST_CTR = ["--address", "/run/containerd/containerd.sock", "--namespace", "moby"];
|
|
15
|
-
async function hostContent(id) {
|
|
16
|
-
if (!/^sha256:[a-f0-9]{64}$/.test(id))
|
|
17
|
-
throw new Error(`invalid content digest ${id}`);
|
|
18
|
-
const result = await run("ctr", [...HOST_CTR, "content", "get", id], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024, timeout: 30_000 });
|
|
19
|
-
if (digest(result.stdout) !== id)
|
|
20
|
-
throw new Error(`corrupt image metadata ${id}`);
|
|
21
|
-
return result.stdout;
|
|
22
|
-
}
|
|
23
|
-
/** Preserve the original index/manifest bytes and all application descriptors. */
|
|
24
|
-
async function hostImage(source) {
|
|
25
|
-
const name = canonicalImageReference(source);
|
|
26
|
-
const { stdout } = await run("ctr", [...HOST_CTR, "images", "ls", `name==${name}`], { timeout: 30_000 });
|
|
27
|
-
const found = stdout.split("\n").map(line => line.trim().split(/\s+/)).find(row => row[0] === name)?.[2];
|
|
28
|
-
if (!found)
|
|
29
|
-
throw new Error(`built image ${name} is missing from the VM container store`);
|
|
30
|
-
const metadata = new Map();
|
|
31
|
-
const allowed = new Map();
|
|
32
|
-
let configDigest;
|
|
33
|
-
async function visit(id, selected, depth = 0) {
|
|
34
|
-
if (depth > 4)
|
|
35
|
-
throw new Error(`built image ${name} has a recursive manifest index`);
|
|
36
|
-
const bytes = await hostContent(id);
|
|
37
|
-
const obj = JSON.parse(bytes.toString());
|
|
38
|
-
const desc = descriptor(bytes, obj.mediaType);
|
|
39
|
-
metadata.set(id, bytes);
|
|
40
|
-
allowed.set(id, desc);
|
|
41
|
-
if (obj.config && Array.isArray(obj.layers)) {
|
|
42
|
-
const manifest = obj;
|
|
43
|
-
for (const blob of [manifest.config, ...manifest.layers])
|
|
44
|
-
allowed.set(blob.digest, blob);
|
|
45
|
-
if (selected)
|
|
46
|
-
configDigest = manifest.config.digest;
|
|
47
|
-
}
|
|
48
|
-
else if (Array.isArray(obj.manifests)) {
|
|
49
|
-
for (const entry of obj.manifests) {
|
|
50
|
-
const matches = !entry.platform || (entry.platform.os === "linux" && entry.platform.architecture === (process.arch === "arm64" ? "arm64" : "amd64"));
|
|
51
|
-
await visit(entry.digest, selected && matches, depth + 1);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
else
|
|
55
|
-
throw new Error(`built image ${name} has invalid image metadata`);
|
|
56
|
-
return desc;
|
|
57
|
-
}
|
|
58
|
-
const target = await visit(found, true);
|
|
59
|
-
if (!configDigest)
|
|
60
|
-
throw new Error(`built image ${name} has no image for this VM's platform`);
|
|
61
|
-
return { target, metadata, allowed, configDigest };
|
|
62
|
-
}
|
|
63
12
|
export async function loadK3sBuildImages(service, images, gateway, exec) {
|
|
64
13
|
if (!images.length)
|
|
65
14
|
return;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { hostImage } from "./built-image.js";
|
|
2
|
+
import { type PreparedComponentImage } from "./component-images.js";
|
|
3
|
+
export declare const REGISTRY_DATA = "/var/lib/spectest-registry";
|
|
4
|
+
export declare const REGISTRY_CONTENT = "/spectest-registry-content";
|
|
5
|
+
export declare const CONTENT_BLOBS = "io.containerd.content.v1.content/blobs/sha256";
|
|
6
|
+
export declare const REGISTRY_IMAGE = "registry:2.8.3";
|
|
7
|
+
export declare const registryDirectory: (service: string) => string;
|
|
8
|
+
/** Distribution 2.8 registry/storage/paths.go defines this layout. Seed
|
|
9
|
+
* before starting the registry so it cannot cache partial publication.
|
|
10
|
+
* The prepared build's containerd image record retains the referenced blobs. */
|
|
11
|
+
export declare function seedRegistryImage(directory: string, contentDirectory: string, reference: string, image: Awaited<ReturnType<typeof hostImage>>, mountedContent?: string): Promise<void>;
|
|
12
|
+
export declare function seedRegistry(service: string, contentDirectory: string, images: readonly PreparedComponentImage[]): Promise<void>;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/** Distribution v2 publication. Blob data stays in containerd, mounted
|
|
2
|
+
* read-only in the registry. Repository metadata and uploads MUST live on
|
|
3
|
+
* the environment rootfs, outside persistent image/build caches. */
|
|
4
|
+
import { mkdir, stat, symlink, writeFile } from "node:fs/promises";
|
|
5
|
+
import { dirname } from "node:path";
|
|
6
|
+
import { hostImage } from "./built-image.js";
|
|
7
|
+
import { canonicalImageReference } from "./component-images.js";
|
|
8
|
+
export const REGISTRY_DATA = "/var/lib/spectest-registry";
|
|
9
|
+
export const REGISTRY_CONTENT = "/spectest-registry-content";
|
|
10
|
+
export const CONTENT_BLOBS = "io.containerd.content.v1.content/blobs/sha256";
|
|
11
|
+
export const REGISTRY_IMAGE = "registry:2.8.3";
|
|
12
|
+
export const registryDirectory = (service) => `/var/lib/spectest-registries/${service}`;
|
|
13
|
+
/** Distribution 2.8 registry/storage/paths.go defines this layout. Seed
|
|
14
|
+
* before starting the registry so it cannot cache partial publication.
|
|
15
|
+
* The prepared build's containerd image record retains the referenced blobs. */
|
|
16
|
+
export async function seedRegistryImage(directory, contentDirectory, reference, image, mountedContent = REGISTRY_CONTENT) {
|
|
17
|
+
const canonical = canonicalImageReference(reference);
|
|
18
|
+
if (canonical.includes("@"))
|
|
19
|
+
throw new Error("registry publication must name a tag");
|
|
20
|
+
const repository = canonical.slice(canonical.indexOf("/") + 1, canonical.lastIndexOf(":"));
|
|
21
|
+
const tag = canonical.slice(canonical.lastIndexOf(":") + 1);
|
|
22
|
+
const root = `${directory}/docker/registry/v2`;
|
|
23
|
+
const repo = `${root}/repositories/${repository}`;
|
|
24
|
+
const link = async (path, value) => {
|
|
25
|
+
await mkdir(dirname(path), { recursive: true });
|
|
26
|
+
await writeFile(path, value);
|
|
27
|
+
};
|
|
28
|
+
for (const [id, descriptor] of image.allowed) {
|
|
29
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(id))
|
|
30
|
+
throw new Error(`invalid registry seed digest ${id}`);
|
|
31
|
+
const hex = id.slice(7);
|
|
32
|
+
const info = await stat(`${contentDirectory}/${hex}`);
|
|
33
|
+
if (!info.isFile() || info.size !== descriptor.size)
|
|
34
|
+
throw new Error(`missing or incomplete registry seed blob ${id}`);
|
|
35
|
+
const destination = `${root}/blobs/sha256/${hex.slice(0, 2)}/${hex}/data`;
|
|
36
|
+
await mkdir(dirname(destination), { recursive: true });
|
|
37
|
+
try {
|
|
38
|
+
await symlink(`${mountedContent}/${hex}`, destination);
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
if (error.code !== "EEXIST")
|
|
42
|
+
throw error;
|
|
43
|
+
}
|
|
44
|
+
await link(`${repo}/_layers/sha256/${hex}/link`, id);
|
|
45
|
+
if (image.metadata.has(id))
|
|
46
|
+
await link(`${repo}/_manifests/revisions/sha256/${hex}/link`, id);
|
|
47
|
+
}
|
|
48
|
+
const id = image.target.digest;
|
|
49
|
+
await link(`${repo}/_manifests/tags/${tag}/index/sha256/${id.slice(7)}/link`, id);
|
|
50
|
+
await link(`${repo}/_manifests/tags/${tag}/current/link`, id);
|
|
51
|
+
}
|
|
52
|
+
export async function seedRegistry(service, contentDirectory, images) {
|
|
53
|
+
// Sequential metadata updates avoid races when several tags share blobs.
|
|
54
|
+
// Builds themselves still run concurrently with all environment preparation.
|
|
55
|
+
for (const image of images) {
|
|
56
|
+
await image.ready;
|
|
57
|
+
const built = await hostImage(image.source);
|
|
58
|
+
await seedRegistryImage(registryDirectory(service), contentDirectory, image.reference, built);
|
|
59
|
+
console.log(`[k3s-registry] ${service}: ${image.reference} @ ${built.target.digest} (cached blobs, no export)`);
|
|
60
|
+
}
|
|
61
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -831,7 +831,8 @@ export type BuildImage = (Omit<Extract<NormalizedServiceImage, {
|
|
|
831
831
|
}>, "type">) & {
|
|
832
832
|
/** Full image reference, e.g. "k8s.internal:5000/runtime:spectest".
|
|
833
833
|
* Optional for a service; required for a Kubernetes build. This names a
|
|
834
|
-
* local image
|
|
834
|
+
* local image. In k3s, builds targeting `registry.host` are also published
|
|
835
|
+
* to that component's registry; other references are only preloaded. */
|
|
835
836
|
reference?: string;
|
|
836
837
|
};
|
|
837
838
|
/** A registry reference or Dockerfile build. Tagged objects from older SDKs
|
package/package.json
CHANGED
package/src/components/index.ts
CHANGED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { expect, test } from "bun:test";
|
|
2
|
+
import { k3s } from "./k3s";
|
|
3
|
+
import { REGISTRY_CONTENT, REGISTRY_DATA } from "../harness/registry-seed";
|
|
4
|
+
|
|
5
|
+
test("HTTPS registry is component-owned, with explicit containerd trust and no Traefik dependency", () => {
|
|
6
|
+
const cluster = k3s({ registry: { host: "Registry.Test" }, traefik: false });
|
|
7
|
+
expect(cluster.tls).toEqual([{ hostname: "registry.test", port: 5000 }]);
|
|
8
|
+
const registries = cluster.files.find(file => file.path.endsWith("registries.yaml"))!.content;
|
|
9
|
+
expect(registries).toContain('"registry.test":\n tls:\n ca_file:');
|
|
10
|
+
expect(registries).not.toContain('"https://127.0.0.1');
|
|
11
|
+
expect(registries).toContain('"http://127.0.0.1:5000"');
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
test("only builds targeting the configured registry require publication storage", () => {
|
|
15
|
+
const image = { content: "FROM scratch", reference: "registry.test/codenode:latest" };
|
|
16
|
+
const cluster = k3s({ registry: { host: "registry.test" }, images: [image] });
|
|
17
|
+
expect(cluster.imageBuilds).toHaveLength(1);
|
|
18
|
+
expect(cluster.volumes.find(v => v.target === REGISTRY_DATA)?.source).toBe(
|
|
19
|
+
"/var/lib/spectest-registries/{{SPECTEST_SERVICE}}",
|
|
20
|
+
);
|
|
21
|
+
expect(cluster.volumes.find(v => v.target === REGISTRY_CONTENT)?.readOnly).toBe(true);
|
|
22
|
+
for (const registry of [undefined, false, true, { host: "other.test" }]) {
|
|
23
|
+
expect(k3s({ registry, images: [image] }).volumes.some(v => v.target === REGISTRY_DATA)).toBe(false);
|
|
24
|
+
}
|
|
25
|
+
expect(k3s({ registry: { host: "registry.test" }, images: [image.reference] }).volumes.some(v => v.target === REGISTRY_DATA)).toBe(false);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("registry host rejects ports, URLs, wildcard and reserved hostnames", () => {
|
|
29
|
+
for (const host of ["", "https://registry.test", "registry.test:443", "*.registry.test", "k8s.internal", "registry.test/path"]) {
|
|
30
|
+
expect(() => k3s({ registry: { host } })).toThrow();
|
|
31
|
+
}
|
|
32
|
+
for (const registry of [null, "registry.test", {}, { host: 123 }]) {
|
|
33
|
+
expect(() => k3s({ registry } as never)).toThrow();
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test("registry upload URLs are relative and local pre-pulls wait for the registry", async () => {
|
|
38
|
+
const cluster = k3s({ registry: { host: "registry.test" }, traefik: false, images: ["registry.test/app:latest"] });
|
|
39
|
+
let ready = false;
|
|
40
|
+
await cluster.setup({ name: "cluster", helpers: {
|
|
41
|
+
apply: async (manifest: string) => {
|
|
42
|
+
expect(manifest).toContain('REGISTRY_HTTP_RELATIVEURLS\n value: "true"');
|
|
43
|
+
expect(manifest).toContain("emptyDir: {}");
|
|
44
|
+
},
|
|
45
|
+
waitForRollout: async () => { ready = true; },
|
|
46
|
+
}, exec: async (_name: string, args: string[]) => {
|
|
47
|
+
if (args[3] === "pull") expect(ready).toBe(true);
|
|
48
|
+
return { exitCode: 0, stdout: "", stderr: "" };
|
|
49
|
+
} } as never);
|
|
50
|
+
});
|
package/src/components/k3s.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { isIP } from "node:net";
|
|
2
2
|
import { installPodTrust } from "./k3s-trust.js";
|
|
3
3
|
import { waitForAll } from "../harness/concurrency.js";
|
|
4
|
+
import { CONTENT_BLOBS, REGISTRY_CONTENT, REGISTRY_DATA, REGISTRY_IMAGE, registryDirectory, seedRegistry } from "../harness/registry-seed.js";
|
|
4
5
|
import { loadK3sBuildImages } from "../harness/k3s-image-loader.js";
|
|
5
6
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
6
7
|
import { spawn as nodeSpawn } from "node:child_process";
|
|
@@ -60,6 +61,15 @@ export type {
|
|
|
60
61
|
KubernetesListObject,
|
|
61
62
|
} from "@kubernetes/client-node";
|
|
62
63
|
|
|
64
|
+
/** HTTPS exposure and publication for the component's writable registry. */
|
|
65
|
+
export interface K3sRegistryOptions {
|
|
66
|
+
/** Exact DNS hostname exposed through Spectest's VM HTTPS proxy on port 443.
|
|
67
|
+
* Clients and Kubernetes's containerd trust the environment CA automatically.
|
|
68
|
+
* Builds in `images` whose reference starts with this host are also published
|
|
69
|
+
* here, using their original cached blobs and manifest digests. */
|
|
70
|
+
host: string;
|
|
71
|
+
}
|
|
72
|
+
|
|
63
73
|
/** Kubernetes pods receive read-only environment CA mounts during admission.
|
|
64
74
|
* Image bytes and digests stay unchanged; explicit pod trust settings win. */
|
|
65
75
|
export interface K3sOptions {
|
|
@@ -67,6 +77,9 @@ export interface K3sOptions {
|
|
|
67
77
|
* or Dockerfile builds with a full `reference` matching your manifests. Builds
|
|
68
78
|
* support path/content, context, exclude, target and buildArgs, and run in
|
|
69
79
|
* parallel with service builds through the standard BuildKit pipeline.
|
|
80
|
+
* With `registry: { host }`, builds targeting that host are also published
|
|
81
|
+
* to the writable registry before setup completes. Other images remain
|
|
82
|
+
* direct preloads; no external registry receives pushes.
|
|
70
83
|
* Setup waits for every pull and fails if a requested image cannot be pulled.
|
|
71
84
|
* Sticky-enabled projects retain image content and unpacked layers between
|
|
72
85
|
* fresh builds automatically. Digest references can reuse the cache offline;
|
|
@@ -141,10 +154,15 @@ export interface K3sOptions {
|
|
|
141
154
|
* into your platform, e.g. `env: { REGISTRY_URL: "cluster.internal:5000" }`.
|
|
142
155
|
* Plain HTTP, so configure your push client for an insecure registry.
|
|
143
156
|
*
|
|
157
|
+
* Set `{ host: "registry.test" }` for verified HTTPS on port 443 through
|
|
158
|
+
* the built-in VM proxy, including containerd trust and relative upload URLs.
|
|
159
|
+
* This does not require Traefik. Builds targeting that host in `images` are
|
|
160
|
+
* published automatically; tags and test uploads remain isolated per fork.
|
|
161
|
+
*
|
|
144
162
|
* On by default. Set `false` for clusters that only ever run public
|
|
145
163
|
* images — that skips the extra pod.
|
|
146
164
|
*/
|
|
147
|
-
registry?: boolean;
|
|
165
|
+
registry?: boolean | K3sRegistryOptions;
|
|
148
166
|
/**
|
|
149
167
|
* Domains to route into this cluster's ingress via **wildcard DNS**. For
|
|
150
168
|
* each `"example.com"`, spectest-resolver answers any `*.example.com`
|
|
@@ -1290,7 +1308,7 @@ const HUB_MIRROR = { registry: "docker.io", mirror: "https://mirror.gcr.io", ups
|
|
|
1290
1308
|
* shares the node's netns, so this needs no in-container DNS and
|
|
1291
1309
|
* can't be broken by a clobbered `hostnames`.
|
|
1292
1310
|
*/
|
|
1293
|
-
function buildRegistriesYaml(registryEnabled: boolean): string | null {
|
|
1311
|
+
function buildRegistriesYaml(registryEnabled: boolean, host?: string): string | null {
|
|
1294
1312
|
const lines: string[] = [
|
|
1295
1313
|
"mirrors:",
|
|
1296
1314
|
` "${HUB_MIRROR.registry}":`,
|
|
@@ -1312,6 +1330,11 @@ function buildRegistriesYaml(registryEnabled: boolean): string | null {
|
|
|
1312
1330
|
` insecure_skip_verify: true`,
|
|
1313
1331
|
);
|
|
1314
1332
|
}
|
|
1333
|
+
if (host) {
|
|
1334
|
+
// This names the verified HTTPS endpoint itself; do not mirror it to HTTP.
|
|
1335
|
+
// container-run mounts the environment CA before k3s/containerd starts.
|
|
1336
|
+
lines.push(` "${host}":`, " tls:", ` ca_file: ${JSON.stringify(CA_PATH)}`);
|
|
1337
|
+
}
|
|
1315
1338
|
return lines.join("\n") + "\n";
|
|
1316
1339
|
}
|
|
1317
1340
|
|
|
@@ -1320,11 +1343,13 @@ function buildRegistriesYaml(registryEnabled: boolean): string | null {
|
|
|
1320
1343
|
* binds the cluster container's `:5000` directly — the same trick
|
|
1321
1344
|
* Traefik uses — so peer services reach it at `<cluster-key>.internal:5000`
|
|
1322
1345
|
* (the cluster service's own alias) and the node's own containerd reaches
|
|
1323
|
-
* it at `127.0.0.1:5000`. Storage is an `emptyDir
|
|
1324
|
-
*
|
|
1325
|
-
*
|
|
1346
|
+
* it at `127.0.0.1:5000`. Storage is an `emptyDir` by default. Declared
|
|
1347
|
+
* publications use a rootfs directory plus a read-only content mount; both
|
|
1348
|
+
* variants keep writable registry state outside persistent image caches and
|
|
1349
|
+
* capture it with the VM snapshot, isolated per test fork.
|
|
1326
1350
|
*/
|
|
1327
|
-
|
|
1351
|
+
function registryManifest(seeded: boolean): string {
|
|
1352
|
+
return `apiVersion: apps/v1
|
|
1328
1353
|
kind: Deployment
|
|
1329
1354
|
metadata:
|
|
1330
1355
|
name: spectest-registry
|
|
@@ -1347,11 +1372,13 @@ spec:
|
|
|
1347
1372
|
- operator: Exists
|
|
1348
1373
|
containers:
|
|
1349
1374
|
- name: registry
|
|
1350
|
-
image: registry:2
|
|
1375
|
+
image: ${seeded ? REGISTRY_IMAGE : "registry:2"}
|
|
1351
1376
|
imagePullPolicy: IfNotPresent
|
|
1352
1377
|
env:
|
|
1353
1378
|
- name: REGISTRY_HTTP_ADDR
|
|
1354
1379
|
value: ":${K3S_REGISTRY_PORT}"
|
|
1380
|
+
- name: REGISTRY_HTTP_RELATIVEURLS
|
|
1381
|
+
value: "true"
|
|
1355
1382
|
- name: REGISTRY_STORAGE_DELETE_ENABLED
|
|
1356
1383
|
value: "true"
|
|
1357
1384
|
ports:
|
|
@@ -1360,10 +1387,20 @@ spec:
|
|
|
1360
1387
|
volumeMounts:
|
|
1361
1388
|
- name: data
|
|
1362
1389
|
mountPath: /var/lib/registry
|
|
1363
|
-
|
|
1390
|
+
${seeded ? ` - name: content
|
|
1391
|
+
mountPath: ${REGISTRY_CONTENT}
|
|
1392
|
+
readOnly: true
|
|
1393
|
+
` : ""} volumes:
|
|
1364
1394
|
- name: data
|
|
1365
|
-
|
|
1395
|
+
${seeded ? ` hostPath:
|
|
1396
|
+
path: ${REGISTRY_DATA}
|
|
1397
|
+
type: Directory
|
|
1398
|
+
- name: content
|
|
1399
|
+
hostPath:
|
|
1400
|
+
path: ${REGISTRY_CONTENT}
|
|
1401
|
+
type: Directory` : " emptyDir: {}"}
|
|
1366
1402
|
`;
|
|
1403
|
+
}
|
|
1367
1404
|
|
|
1368
1405
|
// ────────────────────────────────────────────────────────────────────────
|
|
1369
1406
|
// Apply — server-side, converging
|
|
@@ -1883,7 +1920,7 @@ async function waitForJob(
|
|
|
1883
1920
|
async function setupK3sCluster(
|
|
1884
1921
|
name: string,
|
|
1885
1922
|
helpers: K3sHelpers,
|
|
1886
|
-
opts: { registry: boolean; ingressDomains: string[]; traefik: boolean },
|
|
1923
|
+
opts: { registry: boolean; ingressDomains: string[]; traefik: boolean; seed?: () => Promise<void> },
|
|
1887
1924
|
): Promise<void> {
|
|
1888
1925
|
// The registry can start while Traefik's certificates are being prepared.
|
|
1889
1926
|
const wait = (deployment: string): Promise<void> =>
|
|
@@ -1927,7 +1964,8 @@ async function setupK3sCluster(
|
|
|
1927
1964
|
})(),
|
|
1928
1965
|
(async () => {
|
|
1929
1966
|
if (!opts.registry) return;
|
|
1930
|
-
await
|
|
1967
|
+
await opts.seed?.();
|
|
1968
|
+
await helpers.apply(registryManifest(!!opts.seed));
|
|
1931
1969
|
await wait("spectest-registry");
|
|
1932
1970
|
})(),
|
|
1933
1971
|
]);
|
|
@@ -2061,6 +2099,16 @@ export function k3s(opts: K3sOptions = {}) {
|
|
|
2061
2099
|
const extra = opts.extraArgs ?? [];
|
|
2062
2100
|
assertUsableExtraArgs(extra);
|
|
2063
2101
|
const registryEnabled = opts.registry !== false;
|
|
2102
|
+
let registryHost: string | undefined;
|
|
2103
|
+
if (opts.registry !== undefined && typeof opts.registry !== "boolean") {
|
|
2104
|
+
if (!opts.registry || typeof opts.registry.host !== "string" || opts.registry.host.startsWith("*.")) {
|
|
2105
|
+
throw new Error("k3s(): registry must be a boolean or { host: an exact DNS hostname }");
|
|
2106
|
+
}
|
|
2107
|
+
// Reuse ingress validation (no scheme, port, path, or reserved .internal).
|
|
2108
|
+
registryHost = dnsName(opts.registry.host, { ingress: true }).hostname;
|
|
2109
|
+
}
|
|
2110
|
+
const isPublished = (reference: string) => !!registryHost && reference.startsWith(`${registryHost}/`);
|
|
2111
|
+
const publishBuilds = images.builds.filter(image => isPublished(image.reference));
|
|
2064
2112
|
const traefikEnabled = opts.traefik !== false;
|
|
2065
2113
|
const loadBalancerEnabled = opts.loadBalancer !== false;
|
|
2066
2114
|
// Wildcard ingress domains. Drives both the `provides(... dnsName)`
|
|
@@ -2070,12 +2118,13 @@ export function k3s(opts: K3sOptions = {}) {
|
|
|
2070
2118
|
// `/etc/rancher/k3s/registries.yaml` (host-cache mirrors + trust for
|
|
2071
2119
|
// the in-cluster registry). Seeded via `files` because k3s reads it
|
|
2072
2120
|
// only at startup, before any setup hook could run.
|
|
2073
|
-
const registriesYaml = buildRegistriesYaml(registryEnabled);
|
|
2121
|
+
const registriesYaml = buildRegistriesYaml(registryEnabled, registryHost);
|
|
2074
2122
|
// The environment's image cache, when this VM carries one: the
|
|
2075
2123
|
// cluster's store goes on its root disk and its layers come off the
|
|
2076
2124
|
// shared layers disk (see the store section above).
|
|
2077
2125
|
const sticky = stickyDiskPathSync();
|
|
2078
2126
|
const cache = sticky ? null : imageCachePathsSync();
|
|
2127
|
+
const contentDirectory = `${sticky ? `${sticky}/containerd` : cache?.root ?? "/var/lib/containerd"}/${CONTENT_BLOBS}`;
|
|
2079
2128
|
const serverArgs = [
|
|
2080
2129
|
"k3s",
|
|
2081
2130
|
"server",
|
|
@@ -2175,6 +2224,7 @@ export function k3s(opts: K3sOptions = {}) {
|
|
|
2175
2224
|
const def = {
|
|
2176
2225
|
image: { type: "registry", reference: `rancher/k3s:${version}` },
|
|
2177
2226
|
imageBuilds: images.builds,
|
|
2227
|
+
tls: registryHost ? [{ hostname: registryHost, port: K3S_REGISTRY_PORT }] : [],
|
|
2178
2228
|
command: cmd,
|
|
2179
2229
|
privileged: true,
|
|
2180
2230
|
tmpfs: ["/run", "/var/run"],
|
|
@@ -2196,7 +2246,7 @@ export function k3s(opts: K3sOptions = {}) {
|
|
|
2196
2246
|
// in one environment get separate roots either way (the service
|
|
2197
2247
|
// token names the directory), because two containerd daemons cannot
|
|
2198
2248
|
// share a store: bolt takes an exclusive lock.
|
|
2199
|
-
volumes: sticky
|
|
2249
|
+
volumes: [...(sticky
|
|
2200
2250
|
? [{ source: `${stickyKubernetesDirectory(SELF_SERVICE_TOKEN, version)}/containerd`, target: CLUSTER_STORE_TARGET }]
|
|
2201
2251
|
: cache ? [
|
|
2202
2252
|
{ source: nestedStoreDir(cache, SELF_SERVICE_TOKEN), target: CLUSTER_STORE_TARGET },
|
|
@@ -2204,7 +2254,12 @@ export function k3s(opts: K3sOptions = {}) {
|
|
|
2204
2254
|
{ source: MKFS_EROFS_PATH, target: MKFS_EROFS_PATH, readOnly: true },
|
|
2205
2255
|
{ source: STORE_ADOPT_PATH, target: STORE_ADOPT_PATH, readOnly: true },
|
|
2206
2256
|
]
|
|
2207
|
-
: [{ target: CLUSTER_STORE_TARGET }],
|
|
2257
|
+
: [{ target: CLUSTER_STORE_TARGET }]),
|
|
2258
|
+
...(publishBuilds.length ? [
|
|
2259
|
+
{ source: registryDirectory(SELF_SERVICE_TOKEN), target: REGISTRY_DATA },
|
|
2260
|
+
{ source: contentDirectory, target: REGISTRY_CONTENT, readOnly: true },
|
|
2261
|
+
] : []),
|
|
2262
|
+
],
|
|
2208
2263
|
files: [
|
|
2209
2264
|
...(registriesYaml ? [{ path: "/etc/rancher/k3s/registries.yaml", content: registriesYaml }] : []),
|
|
2210
2265
|
...(cache
|
|
@@ -2223,49 +2278,56 @@ export function k3s(opts: K3sOptions = {}) {
|
|
|
2223
2278
|
const { name, helpers, exec } = ctx;
|
|
2224
2279
|
const runtime = componentRuntime(ctx);
|
|
2225
2280
|
if (images.builds.length && !runtime) throw new Error(`k3s(${name}): prepared image inputs are missing`);
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
}, {
|
|
2235
|
-
apply: async configuration => { await helpers.apply(JSON.stringify(configuration)); },
|
|
2236
|
-
ensureBundle: async (namespace, configMap, bundle) => {
|
|
2237
|
-
try {
|
|
2238
|
-
await helpers.client.core.createNamespacedConfigMap({ namespace, body: {
|
|
2239
|
-
metadata: { name: configMap, namespace }, immutable: true, data: { "ca-bundle.crt": bundle },
|
|
2240
|
-
} });
|
|
2241
|
-
} catch (err) {
|
|
2242
|
-
// A simultaneous Pod may have created the same namespace's bundle.
|
|
2243
|
-
const existing = (await helpers.client.core.readNamespacedConfigMap({ namespace, name: configMap })).unwrap();
|
|
2244
|
-
if (existing.data?.["ca-bundle.crt"] !== bundle || existing.immutable !== true) throw err;
|
|
2245
|
-
}
|
|
2246
|
-
},
|
|
2247
|
-
});
|
|
2248
|
-
// The API server observes webhook registration asynchronously. A dry-run
|
|
2249
|
-
// Pod proves TLS reachability and admission before any project workload.
|
|
2250
|
-
const deadline = Date.now() + 30_000;
|
|
2251
|
-
for (;;) {
|
|
2281
|
+
const infrastructure = (async () => {
|
|
2282
|
+
if (runtime?.caBundle && caPresent()) {
|
|
2283
|
+
const { cert, key } = await issueIngressCert([runtime.gateway]);
|
|
2284
|
+
await installPodTrust({ name, gateway: runtime.gateway, cert, key,
|
|
2285
|
+
ca: await readFile(CA_PATH, "utf8"), bundle: await readFile(runtime.caBundle, "utf8"),
|
|
2286
|
+
}, {
|
|
2287
|
+
apply: async configuration => { await helpers.apply(JSON.stringify(configuration)); },
|
|
2288
|
+
ensureBundle: async (namespace, configMap, bundle) => {
|
|
2252
2289
|
try {
|
|
2253
|
-
|
|
2254
|
-
metadata: {
|
|
2255
|
-
} })
|
|
2256
|
-
if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt")) break;
|
|
2257
|
-
if (Date.now() >= deadline) throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
|
|
2290
|
+
await helpers.client.core.createNamespacedConfigMap({ namespace, body: {
|
|
2291
|
+
metadata: { name: configMap, namespace }, immutable: true, data: { "ca-bundle.crt": bundle },
|
|
2292
|
+
} });
|
|
2258
2293
|
} catch (err) {
|
|
2259
|
-
|
|
2294
|
+
// A simultaneous Pod may have created the same namespace's bundle.
|
|
2295
|
+
const existing = (await helpers.client.core.readNamespacedConfigMap({ namespace, name: configMap })).unwrap();
|
|
2296
|
+
if (existing.data?.["ca-bundle.crt"] !== bundle || existing.immutable !== true) throw err;
|
|
2260
2297
|
}
|
|
2261
|
-
|
|
2298
|
+
},
|
|
2299
|
+
});
|
|
2300
|
+
// The API server observes webhook registration asynchronously. A dry-run
|
|
2301
|
+
// Pod proves TLS reachability and admission before any project workload.
|
|
2302
|
+
const deadline = Date.now() + 30_000;
|
|
2303
|
+
for (;;) {
|
|
2304
|
+
try {
|
|
2305
|
+
const pod = (await helpers.client.core.createNamespacedPod({ namespace: "default", dryRun: "All", body: {
|
|
2306
|
+
metadata: { generateName: "spectest-trust-probe-" }, spec: { containers: [{ name: "probe", image: "registry.k8s.io/pause:3.10" }] },
|
|
2307
|
+
} })).unwrap();
|
|
2308
|
+
if (pod.spec?.containers[0].env?.some(e => e.name === "SSL_CERT_FILE" && e.value === "/spectest/ca-bundle.crt")) break;
|
|
2309
|
+
if (Date.now() >= deadline) throw new Error(`k3s(${name}): Pod CA mounts did not become ready`);
|
|
2310
|
+
} catch (err) {
|
|
2311
|
+
if (Date.now() >= deadline) throw err;
|
|
2262
2312
|
}
|
|
2313
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
2263
2314
|
}
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2315
|
+
}
|
|
2316
|
+
await setupK3sCluster(name, helpers, {
|
|
2317
|
+
registry: registryEnabled,
|
|
2318
|
+
seed: publishBuilds.length ? () => seedRegistry(name, contentDirectory,
|
|
2319
|
+
runtime!.images.filter(image => isPublished(image.reference))) : undefined,
|
|
2320
|
+
ingressDomains,
|
|
2321
|
+
traefik: traefikEnabled,
|
|
2322
|
+
});
|
|
2323
|
+
})();
|
|
2324
|
+
await waitForAll([
|
|
2325
|
+
runtime ? loadK3sBuildImages(name, runtime.images, runtime.gateway, exec) : Promise.resolve(),
|
|
2326
|
+
prePullK3sImages(name, images.pulls.filter(ref => !isPublished(ref)), exec),
|
|
2327
|
+
(async () => {
|
|
2328
|
+
// Registry-local references must not race registry startup/publication.
|
|
2329
|
+
await infrastructure;
|
|
2330
|
+
await prePullK3sImages(name, images.pulls.filter(isPublished), exec);
|
|
2269
2331
|
})(),
|
|
2270
2332
|
]);
|
|
2271
2333
|
if (sticky) await registerStickyKubernetesStore(name, version);
|
package/src/daemon.ts
CHANGED
|
@@ -2417,6 +2417,9 @@ function bindIngressServer(
|
|
|
2417
2417
|
// {@link stopIngressServers} depends on: a half-alive listener shares
|
|
2418
2418
|
// the port silently instead of colliding loudly.
|
|
2419
2419
|
reusePort: true,
|
|
2420
|
+
// OCI layer uploads can exceed Bun's default 128 MiB request limit.
|
|
2421
|
+
// Large proxy bodies stream through; this limit does not allocate a buffer.
|
|
2422
|
+
maxRequestBodySize: 1024 ** 4,
|
|
2420
2423
|
// Bun.serve defaults to a 10s idleTimeout, which kills any proxied
|
|
2421
2424
|
// request whose upstream takes >10s to produce bytes — under parallel
|
|
2422
2425
|
// test load that surfaced as "fetch failed"/"other side closed" on
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/** Read build results without reserializing manifests or exporting layers. */
|
|
2
|
+
import { execFile } from "node:child_process";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import { promisify } from "node:util";
|
|
5
|
+
import { canonicalImageReference } from "./component-images.js";
|
|
6
|
+
|
|
7
|
+
export interface Descriptor { digest: string; size: number; mediaType: string; platform?: { os: string; architecture: string } }
|
|
8
|
+
interface Manifest { schemaVersion: number; mediaType: string; config: Descriptor; layers: Descriptor[]; annotations?: Record<string, string> }
|
|
9
|
+
const digest = (bytes: Uint8Array) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
10
|
+
const descriptor = (bytes: Uint8Array, mediaType: string): Descriptor => ({ digest: digest(bytes), size: bytes.length, mediaType });
|
|
11
|
+
const run = promisify(execFile);
|
|
12
|
+
const HOST_CTR = ["--address", "/run/containerd/containerd.sock", "--namespace", "moby"];
|
|
13
|
+
|
|
14
|
+
async function hostContent(id: string): Promise<Buffer> {
|
|
15
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(id)) throw new Error(`invalid content digest ${id}`);
|
|
16
|
+
const result = await run("ctr", [...HOST_CTR, "content", "get", id], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024, timeout: 30_000 });
|
|
17
|
+
if (digest(result.stdout) !== id) throw new Error(`corrupt image metadata ${id}`);
|
|
18
|
+
return result.stdout;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Preserve the original index/manifest bytes and all application descriptors. */
|
|
22
|
+
export async function hostImage(source: string) {
|
|
23
|
+
const name = canonicalImageReference(source);
|
|
24
|
+
const { stdout } = await run("ctr", [...HOST_CTR, "images", "ls", `name==${name}`], { timeout: 30_000 });
|
|
25
|
+
const found = stdout.split("\n").map(line => line.trim().split(/\s+/)).find(row => row[0] === name)?.[2];
|
|
26
|
+
if (!found) throw new Error(`built image ${name} is missing from the VM container store`);
|
|
27
|
+
const metadata = new Map<string, Buffer>();
|
|
28
|
+
const allowed = new Map<string, Descriptor>();
|
|
29
|
+
let configDigest: string | undefined;
|
|
30
|
+
async function visit(id: string, selected: boolean, depth = 0): Promise<Descriptor> {
|
|
31
|
+
if (depth > 4) throw new Error(`built image ${name} has a recursive manifest index`);
|
|
32
|
+
const bytes = await hostContent(id);
|
|
33
|
+
const obj = JSON.parse(bytes.toString());
|
|
34
|
+
const desc = descriptor(bytes, obj.mediaType);
|
|
35
|
+
metadata.set(id, bytes);
|
|
36
|
+
allowed.set(id, desc);
|
|
37
|
+
if (obj.config && Array.isArray(obj.layers)) {
|
|
38
|
+
const manifest = obj as Manifest;
|
|
39
|
+
for (const blob of [manifest.config, ...manifest.layers]) allowed.set(blob.digest, blob);
|
|
40
|
+
if (selected) configDigest = manifest.config.digest;
|
|
41
|
+
} else if (Array.isArray(obj.manifests)) {
|
|
42
|
+
for (const entry of obj.manifests as Descriptor[]) {
|
|
43
|
+
const matches = !entry.platform || (entry.platform.os === "linux" && entry.platform.architecture === (process.arch === "arm64" ? "arm64" : "amd64"));
|
|
44
|
+
await visit(entry.digest, selected && matches, depth + 1);
|
|
45
|
+
}
|
|
46
|
+
} else throw new Error(`built image ${name} has invalid image metadata`);
|
|
47
|
+
return desc;
|
|
48
|
+
}
|
|
49
|
+
const target = await visit(found, true);
|
|
50
|
+
if (!configDigest) throw new Error(`built image ${name} has no image for this VM's platform`);
|
|
51
|
+
return { target, metadata, allowed, configDigest };
|
|
52
|
+
}
|
|
53
|
+
|
|
@@ -1,62 +1,17 @@
|
|
|
1
1
|
/** A short-lived, read-only OCI endpoint over the host containerd content store.
|
|
2
2
|
* k3s fetches only missing blobs and unpacks only missing snapshots. No Docker
|
|
3
3
|
* archive, staging registry, or second persistent copy of application blobs. */
|
|
4
|
-
import {
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
5
|
import { createHash } from "node:crypto";
|
|
6
6
|
import { createServer } from "node:http";
|
|
7
|
-
import { promisify } from "node:util";
|
|
8
7
|
import type { SpectestContext } from "../index.js";
|
|
9
8
|
import { canonicalImageReference, type PreparedComponentImage } from "./component-images.js";
|
|
10
9
|
import { waitForK3sRuntime } from "./k3s-prepull.js";
|
|
10
|
+
import { hostImage } from "./built-image.js";
|
|
11
11
|
import { waitForAll } from "./concurrency.js";
|
|
12
12
|
|
|
13
|
-
interface Descriptor { digest: string; size: number; mediaType: string; platform?: { os: string; architecture: string } }
|
|
14
|
-
interface Manifest { schemaVersion: number; mediaType: string; config: Descriptor; layers: Descriptor[]; annotations?: Record<string, string> }
|
|
15
|
-
const digest = (bytes: Uint8Array) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
16
|
-
const descriptor = (bytes: Uint8Array, mediaType: string): Descriptor => ({ digest: digest(bytes), size: bytes.length, mediaType });
|
|
17
|
-
const run = promisify(execFile);
|
|
18
13
|
const HOST_CTR = ["--address", "/run/containerd/containerd.sock", "--namespace", "moby"];
|
|
19
14
|
|
|
20
|
-
async function hostContent(id: string): Promise<Buffer> {
|
|
21
|
-
if (!/^sha256:[a-f0-9]{64}$/.test(id)) throw new Error(`invalid content digest ${id}`);
|
|
22
|
-
const result = await run("ctr", [...HOST_CTR, "content", "get", id], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024, timeout: 30_000 });
|
|
23
|
-
if (digest(result.stdout) !== id) throw new Error(`corrupt image metadata ${id}`);
|
|
24
|
-
return result.stdout;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
/** Preserve the original index/manifest bytes and all application descriptors. */
|
|
28
|
-
async function hostImage(source: string) {
|
|
29
|
-
const name = canonicalImageReference(source);
|
|
30
|
-
const { stdout } = await run("ctr", [...HOST_CTR, "images", "ls", `name==${name}`], { timeout: 30_000 });
|
|
31
|
-
const found = stdout.split("\n").map(line => line.trim().split(/\s+/)).find(row => row[0] === name)?.[2];
|
|
32
|
-
if (!found) throw new Error(`built image ${name} is missing from the VM container store`);
|
|
33
|
-
const metadata = new Map<string, Buffer>();
|
|
34
|
-
const allowed = new Map<string, Descriptor>();
|
|
35
|
-
let configDigest: string | undefined;
|
|
36
|
-
async function visit(id: string, selected: boolean, depth = 0): Promise<Descriptor> {
|
|
37
|
-
if (depth > 4) throw new Error(`built image ${name} has a recursive manifest index`);
|
|
38
|
-
const bytes = await hostContent(id);
|
|
39
|
-
const obj = JSON.parse(bytes.toString());
|
|
40
|
-
const desc = descriptor(bytes, obj.mediaType);
|
|
41
|
-
metadata.set(id, bytes);
|
|
42
|
-
allowed.set(id, desc);
|
|
43
|
-
if (obj.config && Array.isArray(obj.layers)) {
|
|
44
|
-
const manifest = obj as Manifest;
|
|
45
|
-
for (const blob of [manifest.config, ...manifest.layers]) allowed.set(blob.digest, blob);
|
|
46
|
-
if (selected) configDigest = manifest.config.digest;
|
|
47
|
-
} else if (Array.isArray(obj.manifests)) {
|
|
48
|
-
for (const entry of obj.manifests as Descriptor[]) {
|
|
49
|
-
const matches = !entry.platform || (entry.platform.os === "linux" && entry.platform.architecture === (process.arch === "arm64" ? "arm64" : "amd64"));
|
|
50
|
-
await visit(entry.digest, selected && matches, depth + 1);
|
|
51
|
-
}
|
|
52
|
-
} else throw new Error(`built image ${name} has invalid image metadata`);
|
|
53
|
-
return desc;
|
|
54
|
-
}
|
|
55
|
-
const target = await visit(found, true);
|
|
56
|
-
if (!configDigest) throw new Error(`built image ${name} has no image for this VM's platform`);
|
|
57
|
-
return { target, metadata, allowed, configDigest };
|
|
58
|
-
}
|
|
59
|
-
|
|
60
15
|
export async function loadK3sBuildImages(
|
|
61
16
|
service: string, images: readonly PreparedComponentImage[],
|
|
62
17
|
gateway: string, exec: SpectestContext["exec"],
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { afterEach, expect, test } from "bun:test";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { mkdtemp, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { seedRegistryImage } from "./registry-seed";
|
|
7
|
+
import type { Descriptor } from "./built-image";
|
|
8
|
+
|
|
9
|
+
const roots: string[] = [];
|
|
10
|
+
afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
|
|
11
|
+
|
|
12
|
+
async function fixture() {
|
|
13
|
+
const root = await mkdtemp(join(tmpdir(), "registry-seed-")); roots.push(root);
|
|
14
|
+
const content = `${root}/content`; await mkdir(content);
|
|
15
|
+
const metadata = new Map<string, Buffer>(), allowed = new Map<string, Descriptor>();
|
|
16
|
+
async function blob(value: string, mediaType: string, manifest = false) {
|
|
17
|
+
const bytes = Buffer.from(value);
|
|
18
|
+
const digest = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
19
|
+
const descriptor = { digest, size: bytes.length, mediaType };
|
|
20
|
+
await writeFile(`${content}/${digest.slice(7)}`, bytes);
|
|
21
|
+
allowed.set(digest, descriptor);
|
|
22
|
+
if (manifest) metadata.set(digest, bytes);
|
|
23
|
+
return descriptor;
|
|
24
|
+
}
|
|
25
|
+
const config = await blob('{"rootfs":{"type":"layers","diff_ids":[]}}', "application/vnd.oci.image.config.v1+json");
|
|
26
|
+
const layer = await blob("layer bytes", "application/vnd.oci.image.layer.v1.tar");
|
|
27
|
+
const manifestType = "application/vnd.oci.image.manifest.v1+json";
|
|
28
|
+
const manifest = await blob(JSON.stringify({ schemaVersion: 2, mediaType: manifestType, config, layers: [layer] }), manifestType, true);
|
|
29
|
+
const indexType = "application/vnd.oci.image.index.v1+json";
|
|
30
|
+
const target = await blob(JSON.stringify({ schemaVersion: 2, mediaType: indexType, manifests: [manifest] }), indexType, true);
|
|
31
|
+
return { root, content, layer, image: { target, metadata, allowed, configDigest: config.digest } };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
test("seeding preserves index/manifest bytes and shares blobs while repository state stays separate", async () => {
|
|
35
|
+
const { root, content, image } = await fixture();
|
|
36
|
+
for (const name of ["a", "b"]) {
|
|
37
|
+
await seedRegistryImage(`${root}/${name}`, content, "registry.test/team/code:latest", image, content);
|
|
38
|
+
}
|
|
39
|
+
const base = "/docker/registry/v2";
|
|
40
|
+
for (const id of image.allowed.keys()) {
|
|
41
|
+
const hex = id.slice(7), path = `${base}/blobs/sha256/${hex.slice(0, 2)}/${hex}/data`;
|
|
42
|
+
expect(await readFile(`${root}/a${path}`)).toEqual(await readFile(`${content}/${hex}`));
|
|
43
|
+
expect((await stat(`${root}/a${path}`)).ino).toBe((await stat(`${root}/b${path}`)).ino);
|
|
44
|
+
}
|
|
45
|
+
const tag = `${base}/repositories/team/code/_manifests/tags/latest/current/link`;
|
|
46
|
+
expect(await readFile(`${root}/a${tag}`, "utf8")).toBe(image.target.digest);
|
|
47
|
+
await writeFile(`${root}/a${tag}`, `sha256:${"f".repeat(64)}`);
|
|
48
|
+
expect(await readFile(`${root}/b${tag}`, "utf8")).toBe(image.target.digest);
|
|
49
|
+
// A new environment seeded against the same cached blobs has only its declaration.
|
|
50
|
+
await seedRegistryImage(`${root}/fresh`, content, "registry.test/team/code:latest", image, content);
|
|
51
|
+
expect(await readdir(`${root}/fresh${base}/repositories/team/code/_manifests/tags`)).toEqual(["latest"]);
|
|
52
|
+
expect(await readFile(`${root}/fresh${tag}`, "utf8")).toBe(image.target.digest);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("an incomplete cached build cannot publish a usable tag", async () => {
|
|
56
|
+
const { root, content, image, layer } = await fixture();
|
|
57
|
+
await writeFile(`${content}/${layer.digest.slice(7)}`, "truncated");
|
|
58
|
+
await expect(seedRegistryImage(`${root}/data`, content, "registry.test/code:latest", image, content)).rejects.toThrow("incomplete");
|
|
59
|
+
expect(await Bun.file(`${root}/data/docker/registry/v2/repositories/code/_manifests/tags/latest/current/link`).exists()).toBe(false);
|
|
60
|
+
});
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/** Distribution v2 publication. Blob data stays in containerd, mounted
|
|
2
|
+
* read-only in the registry. Repository metadata and uploads MUST live on
|
|
3
|
+
* the environment rootfs, outside persistent image/build caches. */
|
|
4
|
+
import { mkdir, stat, symlink, writeFile } from "node:fs/promises";
|
|
5
|
+
import { dirname } from "node:path";
|
|
6
|
+
import { hostImage } from "./built-image.js";
|
|
7
|
+
import { canonicalImageReference, type PreparedComponentImage } from "./component-images.js";
|
|
8
|
+
|
|
9
|
+
export const REGISTRY_DATA = "/var/lib/spectest-registry";
|
|
10
|
+
export const REGISTRY_CONTENT = "/spectest-registry-content";
|
|
11
|
+
export const CONTENT_BLOBS = "io.containerd.content.v1.content/blobs/sha256";
|
|
12
|
+
export const REGISTRY_IMAGE = "registry:2.8.3";
|
|
13
|
+
export const registryDirectory = (service: string) => `/var/lib/spectest-registries/${service}`;
|
|
14
|
+
|
|
15
|
+
/** Distribution 2.8 registry/storage/paths.go defines this layout. Seed
|
|
16
|
+
* before starting the registry so it cannot cache partial publication.
|
|
17
|
+
* The prepared build's containerd image record retains the referenced blobs. */
|
|
18
|
+
export async function seedRegistryImage(
|
|
19
|
+
directory: string, contentDirectory: string, reference: string,
|
|
20
|
+
image: Awaited<ReturnType<typeof hostImage>>,
|
|
21
|
+
mountedContent = REGISTRY_CONTENT,
|
|
22
|
+
): Promise<void> {
|
|
23
|
+
const canonical = canonicalImageReference(reference);
|
|
24
|
+
if (canonical.includes("@")) throw new Error("registry publication must name a tag");
|
|
25
|
+
const repository = canonical.slice(canonical.indexOf("/") + 1, canonical.lastIndexOf(":"));
|
|
26
|
+
const tag = canonical.slice(canonical.lastIndexOf(":") + 1);
|
|
27
|
+
const root = `${directory}/docker/registry/v2`;
|
|
28
|
+
const repo = `${root}/repositories/${repository}`;
|
|
29
|
+
const link = async (path: string, value: string) => {
|
|
30
|
+
await mkdir(dirname(path), { recursive: true });
|
|
31
|
+
await writeFile(path, value);
|
|
32
|
+
};
|
|
33
|
+
for (const [id, descriptor] of image.allowed) {
|
|
34
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(id)) throw new Error(`invalid registry seed digest ${id}`);
|
|
35
|
+
const hex = id.slice(7);
|
|
36
|
+
const info = await stat(`${contentDirectory}/${hex}`);
|
|
37
|
+
if (!info.isFile() || info.size !== descriptor.size) throw new Error(`missing or incomplete registry seed blob ${id}`);
|
|
38
|
+
const destination = `${root}/blobs/sha256/${hex.slice(0, 2)}/${hex}/data`;
|
|
39
|
+
await mkdir(dirname(destination), { recursive: true });
|
|
40
|
+
try { await symlink(`${mountedContent}/${hex}`, destination); }
|
|
41
|
+
catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; }
|
|
42
|
+
await link(`${repo}/_layers/sha256/${hex}/link`, id);
|
|
43
|
+
if (image.metadata.has(id)) await link(`${repo}/_manifests/revisions/sha256/${hex}/link`, id);
|
|
44
|
+
}
|
|
45
|
+
const id = image.target.digest;
|
|
46
|
+
await link(`${repo}/_manifests/tags/${tag}/index/sha256/${id.slice(7)}/link`, id);
|
|
47
|
+
await link(`${repo}/_manifests/tags/${tag}/current/link`, id);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export async function seedRegistry(
|
|
51
|
+
service: string, contentDirectory: string, images: readonly PreparedComponentImage[],
|
|
52
|
+
): Promise<void> {
|
|
53
|
+
// Sequential metadata updates avoid races when several tags share blobs.
|
|
54
|
+
// Builds themselves still run concurrently with all environment preparation.
|
|
55
|
+
for (const image of images) {
|
|
56
|
+
await image.ready;
|
|
57
|
+
const built = await hostImage(image.source);
|
|
58
|
+
await seedRegistryImage(registryDirectory(service), contentDirectory, image.reference, built);
|
|
59
|
+
console.log(`[k3s-registry] ${service}: ${image.reference} @ ${built.target.digest} (cached blobs, no export)`);
|
|
60
|
+
}
|
|
61
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1250,7 +1250,8 @@ export type BuildImage = (
|
|
|
1250
1250
|
) & {
|
|
1251
1251
|
/** Full image reference, e.g. "k8s.internal:5000/runtime:spectest".
|
|
1252
1252
|
* Optional for a service; required for a Kubernetes build. This names a
|
|
1253
|
-
* local image
|
|
1253
|
+
* local image. In k3s, builds targeting `registry.host` are also published
|
|
1254
|
+
* to that component's registry; other references are only preloaded. */
|
|
1254
1255
|
reference?: string;
|
|
1255
1256
|
};
|
|
1256
1257
|
|