@specific.dev/spectest 0.30.0 → 0.32.0

package/dist/components/k3s.js

@@ -3,7 +3,7 @@ import { spawn as nodeSpawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { readFile, unlink } from "node:fs/promises";
-import { AppsV1Api, CoreV1Api, KubeConfig, KubernetesObjectApi, ResponseContext, ServerConfiguration, createConfiguration, loadAllYaml, } from "@kubernetes/client-node";
+import { AppsV1Api, BatchV1Api, CoreV1Api, KubeConfig, KubernetesObjectApi, PatchStrategy, ResponseContext, ServerConfiguration, createConfiguration, loadAllYaml, } from "@kubernetes/client-node";
 import { Observable } from "@kubernetes/client-node/dist/gen/rxjsStub.js";
 import { dnsName, provides, SELF_SERVICE_TOKEN } from "../index.js";
 import { readRaw, readTag, wrap } from "../inspect.js";
@@ -696,42 +696,377 @@ spec:
       - name: data
         emptyDir: {}
 `;
+// ────────────────────────────────────────────────────────────────────────
+// Apply — server-side, converging
+//
+// `kubectl apply --server-side --force-conflicts`, over the API. Two
+// properties make a real manifest set work where plain `create` didn't:
+//
+//   * create-or-update. SSA PATCHes with `application/apply-patch+yaml`,
+//     so re-applying an edited manifest converges instead of failing
+//     `AlreadyExists` — which is what a `dependsOn` child re-applying over
+//     its parent's state, or a second apply of a shared base, needs.
+//   * ordering by retry, not by sorting. A CR whose CRD is in the same
+//     manifest, or a resource an admission webhook (cert-manager, CNPG)
+//     rejects while its own pod is still starting, fails the first round
+//     and lands on a later one. Only errors that read as "not yet" are
+//     retried; a genuinely malformed object throws immediately.
+// ────────────────────────────────────────────────────────────────────────
+/** Default SSA field manager. Stable across runs on purpose: server-side
+ *  apply keys ownership on it, so a changing value would make every run
+ *  conflict with the previous one's fields. */
+const APPLY_FIELD_MANAGER = "spectest";
+const APPLY_TIMEOUT_MS = 120_000;
+const WAIT_TIMEOUT_MS = 120_000;
+const WAIT_INTERVAL_MS = 500;
 /**
- * Wait for a Deployment to reach its desired ready-replica count,
- * polling once a second up to `timeoutMs`. Throws with the last-seen
- * status (plus kube-system pod diagnostics) on timeout.
+ * API-server responses that mean "this document can't land *yet*" — the
+ * CRD isn't established, an admission webhook's own pod isn't serving,
+ * the namespace is still being created, or the write raced another
+ * writer. Everything else (a schema violation, a bad field, RBAC) is a
+ * real failure and is thrown on the first round.
  */
-async function waitForDeployment(clusterName, helpers, deployment, timeoutMs) {
+const TRANSIENT_APPLY_PATTERNS = [
+    /no matches for kind/i,
+    // The client's own message when it can't resolve a kind to a resource
+    // path — i.e. the CRD that defines it isn't established yet. This is
+    // the "CR next to its CRD in one manifest" case, so it MUST be retried.
+    /Failed to fetch resource metadata/i,
+    /could not find the requested resource/i,
+    /the server (?:could not find|does not recognize)/i,
+    /failed calling webhook/i,
+    /webhook .* denied the request: .*(?:not ready|unavailable|connection refused)/i,
+    /connection refused/i,
+    /no endpoints available/i,
+    /context deadline exceeded/i,
+    /etcdserver: (?:leader changed|request timed out)/i,
+    /the object has been modified/i,
+    /namespaces? "[^"]+" not found/i,
+    /Internal error occurred/i,
+    /(?:^|\n)HTTP-Code: (?:429|500|503|504)/,
+];
+function errorText(err) {
+    return err?.message ?? String(err);
+}
+function isTransientApplyError(err) {
+    const msg = errorText(err);
+    return TRANSIENT_APPLY_PATTERNS.some((re) => re.test(msg));
+}
+function describeDoc(doc) {
+    const ns = doc.metadata?.namespace;
+    const name = doc.metadata?.name ?? doc.metadata?.generateName ?? "?";
+    return `${doc.kind ?? "?"}/${name}${ns ? ` (ns ${ns})` : ""}`;
+}
+/** CRDs and Namespaces go first so the common "CR next to its CRD in one
+ *  file" case lands in round one rather than costing a retry. Everything
+ *  else keeps input order (results are always returned in input order). */
+function applyRank(doc) {
+    return doc.kind === "CustomResourceDefinition" || doc.kind === "Namespace"
+        ? 0
+        : 1;
+}
+/**
+ * `KubernetesObjectApi` with its resource-discovery lookup exposed, so
+ * `apply` can tell a namespaced kind from a cluster-scoped one before
+ * defaulting `metadata.namespace` (setting it on a cluster-scoped object
+ * puts a namespace in the body the API server then rejects).
+ */
+class ResourceAwareObjectApi extends KubernetesObjectApi {
+    async resourceInfo(apiVersion, kind) {
+        try {
+            return await this.resource(apiVersion, kind);
+        }
+        catch {
+            // Discovery failure (the CRD isn't established yet) — the caller
+            // treats "unknown" as "leave the document alone"; the apply that
+            // follows fails transiently and is retried once discovery works.
+            return undefined;
+        }
+    }
+}
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Default a document's namespace, but only for kinds that have one. */
+async function withDefaultNamespace(deps, doc, namespace) {
+    if (!namespace || doc.metadata?.namespace)
+        return doc;
+    const info = await deps.meta.resourceInfo(doc.apiVersion ?? "v1", doc.kind);
+    // Unknown kind (CRD not established yet) or cluster-scoped: don't touch it.
+    if (!info?.namespaced)
+        return doc;
+    return { ...doc, metadata: { ...doc.metadata, namespace } };
+}
+async function applyOne(deps, doc, opts) {
+    const spec = await withDefaultNamespace(deps, doc, opts.namespace);
+    // Server-side apply addresses the object by name; a `generateName`-only
+    // document has none, so it can only ever be created.
+    if (!spec.metadata?.name) {
+        return deps.objects.create(spec, undefined, undefined, opts.fieldManager);
+    }
+    return deps.objects.patch(spec, undefined, undefined, opts.fieldManager, opts.force, PatchStrategy.ServerSideApply);
+}
+async function applyManifest(deps, manifest, opts) {
+    const docs = loadAllYaml(manifest).filter((d) => !!d && typeof d === "object" && typeof d.kind === "string");
+    const fieldManager = opts?.fieldManager ?? APPLY_FIELD_MANAGER;
+    const force = opts?.force ?? true;
+    const timeoutMs = opts?.timeoutMs ?? APPLY_TIMEOUT_MS;
     const deadline = Date.now() + timeoutMs;
-    let lastErr;
-    while (Date.now() < deadline) {
+    const out = new Array(docs.length);
+    let pending = docs
+        .map((doc, index) => ({ doc, index }))
+        .sort((a, b) => applyRank(a.doc) - applyRank(b.doc) || a.index - b.index);
+    let backoff = 0;
+    let stalled = [];
+    while (pending.length > 0) {
+        const failed = [];
+        stalled = [];
+        for (const item of pending) {
+            try {
+                out[item.index] = await applyOne(deps, item.doc, {
+                    namespace: opts?.namespace,
+                    fieldManager,
+                    force,
+                });
+            }
+            catch (err) {
+                if (!isTransientApplyError(err)) {
+                    throw new Error(`k3s(${deps.clusterName}): applying ${describeDoc(item.doc)} failed: ${errorText(err)}`);
+                }
+                failed.push(item);
+                stalled.push(`${describeDoc(item.doc)}: ${errorText(err).replace(/\s+/g, " ").slice(0, 300)}`);
+            }
+        }
+        if (failed.length === 0)
+            break;
+        if (Date.now() >= deadline) {
+            throw new Error(`k3s(${deps.clusterName}): apply did not converge within ${timeoutMs / 1000}s — ` +
+                `${failed.length} of ${docs.length} document(s) still failing:\n  ${stalled.join("\n  ")}`);
+        }
+        // Nothing landed this round: whatever is missing (a webhook pod, a
+        // CRD's establishment) needs wall-clock, so back off before retrying.
+        // Progress resets the backoff — a long CRD chain shouldn't decay.
+        if (failed.length === pending.length) {
+            await sleep(Math.min(2_000, 250 * 2 ** backoff));
+            backoff += 1;
+        }
+        else {
+            backoff = 0;
+        }
+        pending = failed;
+    }
+    return out;
+}
+const ROLLOUT_KINDS = {
+    deployment: "Deployment",
+    deployments: "Deployment",
+    deploy: "Deployment",
+    statefulset: "StatefulSet",
+    statefulsets: "StatefulSet",
+    sts: "StatefulSet",
+    daemonset: "DaemonSet",
+    daemonsets: "DaemonSet",
+    ds: "DaemonSet",
+};
+function parseRolloutTarget(target, namespace) {
+    if (typeof target === "string") {
+        const slash = target.indexOf("/");
+        const kindPart = slash === -1 ? "deployment" : target.slice(0, slash);
+        const name = slash === -1 ? target : target.slice(slash + 1);
+        const kind = ROLLOUT_KINDS[kindPart.toLowerCase()];
+        if (!kind) {
+            throw new Error(`waitForRollout("${target}"): unknown workload kind "${kindPart}" — ` +
+                `use deployment, statefulset or daemonset (a bare name is a Deployment).`);
+        }
+        if (!name)
+            throw new Error(`waitForRollout("${target}"): missing name.`);
+        return { kind, name, namespace: namespace ?? "default" };
+    }
+    const kind = ROLLOUT_KINDS[String(target.kind ?? "").toLowerCase()];
+    const name = target.metadata?.name;
+    if (!kind || !name) {
+        throw new Error(`waitForRollout(): expected a Deployment/StatefulSet/DaemonSet object with metadata.name, got ` +
+            `${JSON.stringify({ kind: target.kind, name })}.`);
+    }
+    return {
+        kind,
+        name,
+        namespace: namespace ?? target.metadata?.namespace ?? "default",
+    };
+}
+/** Rollout state of one workload: settled, or why not. */
+async function rolloutStatus(deps, ref) {
+    const { name, namespace } = ref;
+    try {
+        if (ref.kind === "Deployment") {
+            const d = (await deps.client.apps.readNamespacedDeployment({ name, namespace })).unwrap();
+            const want = d.spec?.replicas ?? 1;
+            const st = d.status ?? {};
+            if ((st.observedGeneration ?? 0) < (d.metadata?.generation ?? 0)) {
+                return { done: false, reason: "controller has not observed the latest update yet" };
+            }
+            if (want === 0) {
+                return {
+                    done: (st.replicas ?? 0) === 0,
+                    reason: `scaling down: ${st.replicas ?? 0} replica(s) remaining`,
+                };
+            }
+            const updated = st.updatedReplicas ?? 0;
+            const available = st.availableReplicas ?? 0;
+            const total = st.replicas ?? 0;
+            if (updated < want) {
+                return { done: false, reason: `${updated}/${want} replica(s) updated` };
+            }
+            if (total > updated) {
+                return { done: false, reason: `${total - updated} old replica(s) still terminating` };
+            }
+            return {
+                done: available >= want,
+                reason: `${available}/${want} replica(s) available`,
+            };
+        }
+        if (ref.kind === "StatefulSet") {
+            const s = (await deps.client.apps.readNamespacedStatefulSet({ name, namespace })).unwrap();
+            const want = s.spec?.replicas ?? 1;
+            const st = s.status ?? {};
+            if ((st.observedGeneration ?? 0) < (s.metadata?.generation ?? 0)) {
+                return { done: false, reason: "controller has not observed the latest update yet" };
+            }
+            const ready = st.readyReplicas ?? 0;
+            const updated = st.updatedReplicas ?? 0;
+            if (st.updateRevision && st.currentRevision !== st.updateRevision) {
+                return {
+                    done: false,
+                    reason: `rolling update in progress (${updated}/${want} pod(s) updated)`,
+                };
+            }
+            return { done: ready >= want, reason: `${ready}/${want} pod(s) ready` };
+        }
+        const ds = (await deps.client.apps.readNamespacedDaemonSet({ name, namespace })).unwrap();
+        const st = ds.status ?? {};
+        if ((st.observedGeneration ?? 0) < (ds.metadata?.generation ?? 0)) {
+            return { done: false, reason: "controller has not observed the latest update yet" };
+        }
+        const want = st.desiredNumberScheduled ?? 0;
+        const ready = st.numberReady ?? 0;
+        const updated = st.updatedNumberScheduled ?? 0;
+        if (updated < want) {
+            return { done: false, reason: `${updated}/${want} node(s) updated` };
+        }
+        return { done: want > 0 && ready >= want, reason: `${ready}/${want} node(s) ready` };
+    }
+    catch (err) {
+        const msg = errorText(err);
+        return {
+            done: false,
+            reason: /not found|HTTP-Code: 404/i.test(msg)
+                ? `${ref.kind} ${name} does not exist (yet)`
+                : msg.replace(/\s+/g, " ").slice(0, 300),
+        };
+    }
+}
+async function waitForRollout(deps, target, opts) {
+    const ref = parseRolloutTarget(target, opts?.namespace);
+    const timeoutMs = opts?.timeoutMs ?? WAIT_TIMEOUT_MS;
+    let last = "";
+    try {
+        await deps.poll(`${ref.kind.toLowerCase()}/${ref.name} rolled out`, async () => {
+            const st = await rolloutStatus(deps, ref);
+            last = st.reason;
+            return st.done || null;
+        }, { timeoutMs, intervalMs: opts?.intervalMs ?? WAIT_INTERVAL_MS });
+    }
+    catch (err) {
+        const diag = await collectDiagnostics(deps.client, ref.namespace);
+        throw new Error(`k3s(${deps.clusterName}): ${ref.kind}/${ref.name} did not roll out within ` +
+            `${timeoutMs / 1000}s${last ? ` — ${last}` : ""}.\n${diag}\n(${errorText(err)})`);
+    }
+}
+/** Pod names in a namespace, optionally filtered by label selector. */
+async function podNames(client, namespace, labelSelector) {
+    const pods = (await client.core.listNamespacedPod({
+        namespace,
+        ...(labelSelector ? { labelSelector } : {}),
+    })).unwrap();
+    return (pods.items ?? [])
+        .map((p) => p.metadata?.name)
+        .filter((n) => !!n);
+}
+async function readLogs(client, opts) {
+    const namespace = opts.namespace ?? "default";
+    const tailLines = opts.tailLines ?? 200;
+    if (!opts.pod && !opts.selector) {
+        throw new Error("ctx.svc.<k3s>.logs(): pass either `pod` or `selector`.");
+    }
+    const names = opts.pod
+        ? [opts.pod]
+        : await podNames(client, namespace, opts.selector);
+    const chunks = [];
+    for (const name of names) {
         try {
-            // `.unwrap()` recovers the plain object — the client wraps its result in
-            // every context now (provenance-free here, since this internal poll runs
-            // during setup with no active recorder). We're reading for control flow,
-            // not asserting, so go straight to raw.
-            const dep = (await helpers.client.apps.readNamespacedDeployment({
-                name: deployment,
-                namespace: "kube-system",
+            const text = (await client.core.readNamespacedPodLog({
+                name,
+                namespace,
+                tailLines,
+                ...(opts.container ? { container: opts.container } : {}),
             })).unwrap();
-            const ready = dep.status?.readyReplicas ?? 0;
-            const want = dep.spec?.replicas ?? 1;
-            if (ready >= want && want > 0)
-                return;
-            lastErr = `${deployment} Deployment exists but only ${ready}/${want} replicas Ready`;
+            chunks.push(names.length > 1 ? `==> ${name} <==\n${text}` : String(text));
         }
         catch (err) {
-            const msg = err?.message ?? String(err);
-            lastErr = /not found|404/i.test(msg)
-                ? `${deployment} Deployment does not exist yet`
-                : msg;
+            chunks.push(`==> ${name} <==\n(reading logs failed: ${errorText(err)})`);
         }
-        // 250ms: the two sequential rollout waits in setup sit on the cold
-        // start's critical path, and a 1s poll wasted up to ~2s of it.
-        await new Promise((r) => setTimeout(r, 250));
     }
-    const diag = await collectTraefikDiagnostics(helpers);
-    throw new Error(`k3s(${clusterName}): ${deployment} did not reach Ready within ${timeoutMs / 1000}s. ${lastErr ?? ""}\n${diag}`);
+    return chunks.join("\n");
+}
+async function waitForJob(deps, name, opts) {
+    const namespace = opts?.namespace ?? "default";
+    const timeoutMs = opts?.timeoutMs ?? WAIT_TIMEOUT_MS;
+    let last = "";
+    try {
+        // The predicate returns the RAW job; `poll` wraps the winning value, so
+        // the returned object carries provenance to the wait step (double-
+        // wrapping a proxy would not).
+        return (await deps.poll(`job/${name} completed`, async () => {
+            let job;
+            try {
+                job = (await deps.client.batch.readNamespacedJob({ name, namespace })).unwrap();
+            }
+            catch (err) {
+                const msg = errorText(err);
+                if (!/not found|HTTP-Code: 404/i.test(msg))
+                    throw err;
+                last = `Job ${name} does not exist (yet)`;
+                return null;
+            }
+            const conditions = job.status?.conditions ?? [];
+            const failure = conditions.find((c) => c.type === "Failed" && c.status === "True");
+            if (failure) {
+                const logs = await readLogs(deps.client, {
+                    namespace,
+                    selector: `job-name=${name}`,
+                }).catch((e) => `(reading pod logs failed: ${errorText(e)})`);
+                throw new Error(`k3s(${deps.clusterName}): Job ${name} failed` +
+                    `${failure.reason ? ` (${failure.reason})` : ""}` +
+                    `${failure.message ? `: ${failure.message}` : ""}\n${logs}`);
+            }
+            const want = job.spec?.completions ?? 1;
+            const succeeded = job.status?.succeeded ?? 0;
+            const complete = conditions.some((c) => c.type === "Complete" && c.status === "True");
+            if (complete || succeeded >= want)
+                return job;
+            last = `${succeeded}/${want} completion(s), ${job.status?.active ?? 0} pod(s) active`;
+            return null;
+        }, { timeoutMs, intervalMs: opts?.intervalMs ?? WAIT_INTERVAL_MS }));
+    }
+    catch (err) {
+        // A Job that failed already carries its pods' logs — don't bury it.
+        if (/Job .* failed/.test(errorText(err)))
+            throw err;
+        const logs = await readLogs(deps.client, {
+            namespace,
+            selector: `job-name=${name}`,
+        }).catch(() => "");
+        throw new Error(`k3s(${deps.clusterName}): Job ${name} did not complete within ${timeoutMs / 1000}s` +
+            `${last ? ` — ${last}` : ""}.\n${logs}\n(${errorText(err)})`);
+    }
 }
 /**
  * Post-Ready setup. Apply the Traefik manifest (hostNetwork) and, when
@@ -778,26 +1113,30 @@ async function setupK3sCluster(name, helpers, opts) {
         await helpers.apply(REGISTRY_MANIFEST);
     // Both rollouts proceed independently inside the cluster — wait on them
     // concurrently (they used to serialize, wasting up to a rollout's tail).
-    const waits = opts.traefik
-        ? [waitForDeployment(name, helpers, "traefik", 120_000)]
-        : [];
-    if (opts.registry) {
-        waits.push(waitForDeployment(name, helpers, "spectest-registry", 120_000));
-    }
+    // 250ms polling: these two sit on the cold start's critical path, where
+    // the default 500ms interval wastes real time.
+    const wait = (deployment) => helpers.waitForRollout(deployment, {
+        namespace: "kube-system",
+        timeoutMs: 120_000,
+        intervalMs: 250,
+    });
+    const waits = opts.traefik ? [wait("traefik")] : [];
+    if (opts.registry)
+        waits.push(wait("spectest-registry"));
     await Promise.all(waits);
 }
 /**
- * Snapshot of kube-system state, dumped on traefik-wait timeout. With
- * the static install, the failure surface is just "did our Deployment
- * schedule and become Ready?" — pod listing covers that.
+ * Snapshot of a namespace's state, dumped whenever a wait times out: pod
+ * phases, the container reasons/messages behind them, and recent Warning
+ * events. That set answers the question a rollout timeout actually raises —
+ * did the pods schedule, did the image pull, did the container crash — so
+ * the failure is diagnosable without a second round-trip to the cluster.
  */
-async function collectTraefikDiagnostics(helpers) {
+async function collectDiagnostics(client, namespace) {
     const lines = [];
     try {
-        const pods = await helpers.client.core.listNamespacedPod({
-            namespace: "kube-system",
-        });
-        lines.push(`kube-system pods (${pods.items.length}):`);
+        const pods = await client.core.listNamespacedPod({ namespace });
+        lines.push(`${namespace} pods (${pods.items.length}):`);
         for (const p of pods.items) {
             const phase = p.status?.phase ?? "?";
             const cs = p.status?.containerStatuses ?? [];
@@ -824,9 +1163,7 @@ async function collectTraefikDiagnostics(helpers) {
     // container status even settles (FailedPull / Failed / BackOff), with the
     // raw containerd message attached. Best-effort: never let diagnostics throw.
     try {
-        const events = await helpers.client.core.listNamespacedEvent({
-            namespace: "kube-system",
-        });
+        const events = await client.core.listNamespacedEvent({ namespace });
         const warnings = (events.items ?? [])
             .filter((e) => e.type === "Warning")
             .map((e) => ({
@@ -836,7 +1173,7 @@ async function collectTraefikDiagnostics(helpers) {
         }))
             .filter((e) => e.message);
         if (warnings.length) {
-            lines.push(`kube-system Warning events (${warnings.length}):`);
+            lines.push(`${namespace} Warning events (${warnings.length}):`);
             // Keep the tail — newest events are appended last by the API.
             for (const w of warnings.slice(-12)) {
                 lines.push(`  ${w.obj} [${w.reason}] ${w.message}`);
@@ -1047,7 +1384,7 @@ export function k3s(opts = {}) {
                 traefik: traefikEnabled,
             });
         },
-        helpers: async ({ name, exec }) => {
+        helpers: async ({ name, exec, poll }) => {
             // Read the cluster's kubeconfig and address the API server by its
             // auto-assigned `<name>.internal` hostname on spectest-net. TLS
             // verification is off (see the K3sHelpers docstring), so the
@@ -1079,24 +1416,28 @@ export function k3s(opts = {}) {
             });
             const core = withTagging(new CoreV1Api(config));
             const apps = withTagging(new AppsV1Api(config));
-            const objects = withTagging(new KubernetesObjectApi(config));
-            const apply = async (manifest) => {
-                const docs = loadAllYaml(manifest);
-                const out = [];
-                for (const doc of docs) {
-                    if (!doc || typeof doc !== "object" || !("kind" in doc))
-                        continue;
-                    // `objects.create` is wrapped by `withTagging`, so each returned
-                    // object already carries the back-reference to its create call.
-                    const created = await objects.create(doc);
-                    out.push(created);
-                }
-                return out;
+            const batch = withTagging(new BatchV1Api(config));
+            // The generic dynamic API keeps its type parameters (see
+            // {@link TaggedObjectApi}); `withTagging` only rewrites the runtime
+            // values, so the cast re-states what the proxy actually returns.
+            const objectApi = new ResourceAwareObjectApi(config);
+            const objects = withTagging(objectApi);
+            const client = { core, apps, batch, objects };
+            const deps = {
+                clusterName: name,
+                client,
+                objects,
+                meta: objectApi,
+                poll,
             };
             return {
                 kubeconfig,
-                client: { core, apps, objects },
-                apply,
+                client,
+                apply: (manifest, applyOpts) => applyManifest(deps, manifest, applyOpts),
+                waitForRollout: (target, waitOpts) => waitForRollout(deps, target, waitOpts),
+                waitForJob: (jobName, waitOpts) => waitForJob(deps, jobName, waitOpts),
+                logs: async (logOpts) => wrap(await readLogs(client, logOpts), undefined),
+                list: (listOpts) => objects.list(listOpts.apiVersion, listOpts.kind, listOpts.namespace, undefined, undefined, undefined, listOpts.fieldSelector, listOpts.labelSelector, listOpts.limit),
             };
         },
     };