@rulebricks/cli 2.1.7 → 2.3.2

package/dist/lib/kubernetes.js

@@ -194,12 +194,72 @@ function parseMemoryToGi(memory) {
     };
     return value * (multipliers[unit] ?? 1 / 1024 ** 3);
 }
+function roundUpForEligibility(value) {
+    return Math.ceil(value);
+}
+function normalizeNodeArchitecture(architecture) {
+    if (architecture === "amd64" || architecture === "x86_64")
+        return "amd64";
+    if (architecture === "arm64" || architecture === "aarch64")
+        return "arm64";
+    return null;
+}
+function summarizeNodeArchitecture(architectures) {
+    if (architectures.size === 0)
+        return "unknown";
+    if (architectures.size > 1)
+        return "mixed";
+    return architectures.has("arm64") ? "arm64" : "amd64";
+}
+async function getStorageClasses() {
+    try {
+        const { stdout } = await execa("kubectl", ["get", "storageclass", "-o", "json"], { timeout: 15000 });
+        const data = JSON.parse(stdout);
+        return (data.items ?? [])
+            .map((storageClass) => {
+            const annotations = storageClass.metadata?.annotations ?? {};
+            return {
+                name: storageClass.metadata?.name || "",
+                provisioner: storageClass.provisioner || "",
+                isDefault: annotations["storageclass.kubernetes.io/is-default-class"] ===
+                    "true" ||
+                    annotations["storageclass.beta.kubernetes.io/is-default-class"] ===
+                        "true",
+                volumeBindingMode: storageClass.volumeBindingMode,
+                allowVolumeExpansion: storageClass.allowVolumeExpansion,
+            };
+        })
+            .filter((storageClass) => storageClass.name);
+    }
+    catch {
+        return [];
+    }
+}
+async function getPersistentStorageCapacityGi(storageClassName) {
+    if (!storageClassName)
+        return undefined;
+    try {
+        const { stdout } = await execa("kubectl", ["get", "csistoragecapacity", "-A", "-o", "json"], { timeout: 15000 });
+        const data = JSON.parse(stdout);
+        const capacities = data.items
+            ?.filter((item) => item.storageClassName === storageClassName)
+            .map((item) => parseMemoryToGi(item.capacity || "0"))
+            .filter((capacity) => capacity > 0) ?? [];
+        if (capacities.length === 0)
+            return undefined;
+        return capacities.reduce((sum, capacity) => sum + capacity, 0);
+    }
+    catch {
+        return undefined;
+    }
+}
 /**
- * Infers the closest internal Rulebricks sizing tier from the current cluster.
- * This is used only for existing clusters, where the CLI is not responsible for
- * provisioning node pools but still needs app/Kafka/worker Helm sizing values.
+ * Inspects the current cluster's node architecture, schedulable capacity, and
+ * storage classes. The CLI uses this to keep Helm values compatible with the
+ * Kubernetes resources the user has already made available (storage class, ARM
+ * tolerations, etc.); workload sizing itself follows the chart defaults.
  */
-export async function inferClusterTier() {
+export async function inferClusterCapabilities() {
     try {
         const { stdout } = await execa("kubectl", ["get", "nodes", "-o", "json"], {
             timeout: 15000,
@@ -208,17 +268,42 @@ export async function inferClusterTier() {
         const schedulableNodes = data.items?.filter((node) => !node.spec?.unschedulable) ?? [];
         let totalCpu = 0;
         let totalMemoryGi = 0;
+        let arm64TolerationRequired = false;
+        const architectures = new Set();
         for (const node of schedulableNodes) {
             totalCpu += parseCpuToCores(node.status?.allocatable?.cpu || "0");
             totalMemoryGi += parseMemoryToGi(node.status?.allocatable?.memory || "0");
+            const architecture = normalizeNodeArchitecture(node.status?.nodeInfo?.architecture ||
+                node.metadata?.labels?.["kubernetes.io/arch"] ||
+                node.metadata?.labels?.["beta.kubernetes.io/arch"]);
+            if (architecture) {
+                architectures.add(architecture);
+            }
+            if (architecture === "arm64" &&
+                node.spec?.taints?.some((taint) => taint.key === "kubernetes.io/arch" &&
+                    taint.value === "arm64" &&
+                    taint.effect === "NoSchedule")) {
+                arm64TolerationRequired = true;
+            }
         }
-        if (totalCpu >= 40 && totalMemoryGi >= 80)
-            return "large";
-        if (totalCpu >= 16 && totalMemoryGi >= 32)
-            return "medium";
-        if (totalCpu > 0 && totalMemoryGi > 0)
-            return "small";
-        return null;
+        const storageClasses = await getStorageClasses();
+        const defaultStorageClass = storageClasses.find((storageClass) => storageClass.isDefault) ??
+            storageClasses[0];
+        const totalPersistentStorageGi = await getPersistentStorageCapacityGi(defaultStorageClass?.name);
+        return {
+            nodeArchitecture: summarizeNodeArchitecture(architectures),
+            arm64TolerationRequired,
+            schedulableNodeCount: schedulableNodes.length,
+            totalCpuCores: totalCpu,
+            totalMemoryGi,
+            eligibleCpuCores: roundUpForEligibility(totalCpu),
+            eligibleMemoryGi: roundUpForEligibility(totalMemoryGi),
+            totalPersistentStorageGi,
+            storageClasses,
+            defaultStorageClass,
+            storageClass: defaultStorageClass?.name,
+            storageProvisioner: defaultStorageClass?.provisioner,
+        };
     }
     catch {
         return null;
@@ -437,6 +522,203 @@ export async function streamLogs(podName, namespace = DEFAULT_NAMESPACE, options
     }
     await execa("kubectl", args, { stdio: "inherit" });
 }
+export async function execInPod(namespace, podName, container, args) {
+    const kubectlArgs = ["exec", "-n", namespace, podName];
+    if (container) {
+        kubectlArgs.push("-c", container);
+    }
+    kubectlArgs.push("--", ...args);
+    try {
+        const { stdout } = await execa("kubectl", kubectlArgs);
+        return stdout;
+    }
+    catch (error) {
+        throw new Error(`Failed to exec into pod ${podName}:\n${getErrorMessage(error)}`);
+    }
+}
+export async function runEphemeralJob(options) {
+    const { name, namespace, serviceAccountName, image, command, env = [], volumeMounts = [], volumes = [], initContainers = [], labels = {}, backoffLimit = 0, timeoutSeconds = 3600, } = options;
+    const podSpec = {
+        restartPolicy: "Never",
+        serviceAccountName,
+        containers: [
+            {
+                name: "job",
+                image,
+                imagePullPolicy: "IfNotPresent",
+                command,
+                env,
+                volumeMounts,
+            },
+        ],
+        volumes,
+    };
+    if (initContainers.length > 0) {
+        podSpec.initContainers = initContainers;
+    }
+    const manifest = {
+        apiVersion: "batch/v1",
+        kind: "Job",
+        metadata: {
+            name,
+            namespace,
+            labels,
+        },
+        spec: {
+            backoffLimit,
+            template: {
+                metadata: {
+                    labels,
+                },
+                spec: podSpec,
+            },
+        },
+    };
+    try {
+        await execa("kubectl", [
+            "delete",
+            "job",
+            name,
+            "-n",
+            namespace,
+            "--ignore-not-found=true",
+        ]);
+        await execa("kubectl", ["apply", "-f", "-"], {
+            input: JSON.stringify(manifest),
+        });
+        await execa("kubectl", [
+            "wait",
+            "--for=condition=complete",
+            `job/${name}`,
+            "-n",
+            namespace,
+            `--timeout=${timeoutSeconds}s`,
+        ]);
+        const logs = await getJobLogs(name, namespace);
+        return { jobName: name, logs };
+    }
+    catch (error) {
+        const logs = await getJobLogs(name, namespace).catch(() => "");
+        const failed = await isJobFailed(name, namespace).catch(() => false);
+        if (failed) {
+            throw new Error(`Job ${name} failed:\n${logs || getErrorMessage(error)}`);
+        }
+        throw new Error(`Job ${name} did not complete:\n${logs || getErrorMessage(error)}`);
+    }
+}
+export async function createJobFromCronJob(namespace, cronJobName, jobName) {
+    try {
+        await execa("kubectl", [
+            "delete",
+            "job",
+            jobName,
+            "-n",
+            namespace,
+            "--ignore-not-found=true",
+        ]);
+        await execa("kubectl", [
+            "create",
+            "job",
+            jobName,
+            "-n",
+            namespace,
+            `--from=cronjob/${cronJobName}`,
+        ]);
+    }
+    catch (error) {
+        throw new Error(`Failed to create backup job:\n${getErrorMessage(error)}`);
+    }
+}
+export async function waitForJobComplete(namespace, jobName, timeoutSeconds = 3600) {
+    try {
+        await execa("kubectl", [
+            "wait",
+            "--for=condition=complete",
+            `job/${jobName}`,
+            "-n",
+            namespace,
+            `--timeout=${timeoutSeconds}s`,
+        ]);
+        return await getJobLogs(jobName, namespace);
+    }
+    catch (error) {
+        const logs = await getJobLogs(jobName, namespace).catch(() => "");
+        const failed = await isJobFailed(jobName, namespace).catch(() => false);
+        if (failed) {
+            throw new Error(`Job ${jobName} failed:\n${logs || getErrorMessage(error)}`);
+        }
+        throw new Error(`Timed out waiting for job ${jobName}:\n${logs || getErrorMessage(error)}`);
+    }
+}
+export async function getJobLogs(jobName, namespace) {
+    const { stdout } = await execa("kubectl", [
+        "logs",
+        `job/${jobName}`,
+        "-n",
+        namespace,
+        "--all-containers=true",
+    ]);
+    return stdout;
+}
+async function isJobFailed(jobName, namespace) {
+    const { stdout } = await execa("kubectl", [
+        "get",
+        "job",
+        jobName,
+        "-n",
+        namespace,
+        "-o",
+        "jsonpath={.status.failed}",
+    ]);
+    return Number.parseInt(stdout || "0", 10) > 0;
+}
+export async function scaleDeployment(namespace, name, replicas) {
+    try {
+        await execa("kubectl", [
+            "scale",
+            "deployment",
+            name,
+            "-n",
+            namespace,
+            `--replicas=${replicas}`,
+        ]);
+    }
+    catch (error) {
+        throw new Error(`Failed to scale deployment ${name}:\n${getErrorMessage(error)}`);
+    }
+}
+export async function waitForDeploymentReady(namespace, name, timeoutSeconds = 600) {
+    try {
+        await execa("kubectl", [
+            "rollout",
+            "status",
+            `deployment/${name}`,
+            "-n",
+            namespace,
+            `--timeout=${timeoutSeconds}s`,
+        ]);
+    }
+    catch (error) {
+        throw new Error(`Deployment ${name} is not ready:\n${getErrorMessage(error)}`);
+    }
+}
+export async function getDeploymentReplicas(namespace, name) {
+    try {
+        const { stdout } = await execa("kubectl", [
+            "get",
+            "deployment",
+            name,
+            "-n",
+            namespace,
+            "-o",
+            "jsonpath={.spec.replicas}",
+        ]);
+        return Number.parseInt(stdout || "0", 10);
+    }
+    catch {
+        return null;
+    }
+}
 /**
  * Colors for multi-pod log prefixes
  */
@@ -682,45 +964,111 @@ export async function deletePVCs(namespace, options = {}) {
         }
     }
 }
+// Custom resources whose operator sets a finalizer that only that operator can
+// clear. When the operator is uninstalled with the release, those finalizers are
+// never removed and wedge the namespace (and the CRD) in Terminating forever.
+// Observed blockers: KEDA ScaledObjects, cert-manager ACME Challenges/Orders, and
+// Strimzi Kafka resources.
+const FINALIZER_BLOCKING_CR_TYPES = [
+    "scaledobjects.keda.sh",
+    "scaledjobs.keda.sh",
+    "challenges.acme.cert-manager.io",
+    "orders.acme.cert-manager.io",
+    "certificaterequests.cert-manager.io",
+    "certificates.cert-manager.io",
+    "kafkatopics.kafka.strimzi.io",
+    "kafkausers.kafka.strimzi.io",
+    "kafkanodepools.kafka.strimzi.io",
+    "kafkas.kafka.strimzi.io",
+];
 /**
- * Removes finalizers from KEDA ScaledObjects to prevent namespace deletion from hanging.
- * KEDA finalizers wait for the KEDA controller to clean up, but if KEDA is being deleted
- * with the namespace, this causes a deadlock.
+ * Strips finalizers from the custom resources whose controllers are torn down
+ * with the release, so the namespace can finalize instead of hanging in
+ * Terminating (NamespaceFinalizersRemaining). Best-effort per type — a missing
+ * CRD (feature disabled) or already-gone object is fine.
  */
-export async function removeKedaFinalizers(namespace) {
-    try {
-        // Get all ScaledObjects in the namespace
-        const { stdout } = await execa("kubectl", [
-            "get",
-            "scaledobjects.keda.sh",
-            "-n",
-            namespace,
-            "-o",
-            "jsonpath={.items[*].metadata.name}",
-        ], { timeout: 15000 });
-        const scaledObjects = stdout.split(" ").filter(Boolean);
-        // Patch each ScaledObject to remove finalizers
-        for (const name of scaledObjects) {
-            try {
-                await execa("kubectl", [
-                    "patch",
-                    "scaledobject",
-                    name,
-                    "-n",
-                    namespace,
-                    "-p",
-                    '{"metadata":{"finalizers":null}}',
-                    "--type=merge",
-                ], { timeout: 15000 });
+export async function removeBlockingFinalizers(namespace) {
+    for (const resourceType of FINALIZER_BLOCKING_CR_TYPES) {
+        try {
+            const { stdout } = await execa("kubectl", [
+                "get",
+                resourceType,
+                "-n",
+                namespace,
+                "-o",
+                "jsonpath={.items[*].metadata.name}",
+            ], { timeout: 15000 });
+            const names = stdout.split(" ").filter(Boolean);
+            for (const name of names) {
+                try {
+                    await execa("kubectl", [
+                        "patch",
+                        resourceType,
+                        name,
+                        "-n",
+                        namespace,
+                        "-p",
+                        '{"metadata":{"finalizers":null}}',
+                        "--type=merge",
+                    ], { timeout: 15000 });
+                }
+                catch {
+                    // Ignore — object might already be deleted.
+                }
             }
-            catch {
-                // Ignore errors - object might already be deleted
+        }
+        catch {
+            // Ignore — this CRD might not be installed (feature disabled).
+        }
+    }
+}
+/**
+ * Deletes aggregated APIServices (apiregistration.k8s.io) whose backing service
+ * lives in the given namespace.
+ *
+ * Why this matters for teardown: an aggregated API (e.g. KEDA's
+ * v1beta1.external.metrics.k8s.io, prometheus-adapter's custom.metrics.k8s.io,
+ * etc.) is served by an in-namespace Service. When the namespace is torn down
+ * that Service disappears and the (cluster-scoped) APIService goes Unavailable
+ * with ServiceNotFound. The namespace controller must enumerate every API group
+ * to delete a namespace's contents, so a single broken APIService makes its
+ * discovery step fail and wedges the namespace in Terminating forever
+ * (NamespaceDeletionDiscoveryFailure) - which then rejects any reinstall into
+ * that namespace ("being terminated").
+ *
+ * Deleting these APIServices up front (they are going away with the namespace
+ * anyway) keeps discovery healthy so the namespace can finalize. This is
+ * generalized to ALL APIServices backed by the target namespace, not just KEDA,
+ * and is safe: cluster APIs backed by other namespaces (e.g. metrics-server in
+ * kube-system) are never matched. Listing APIService objects is served directly
+ * by kube-apiserver, so this also works to rescue an already-stuck namespace.
+ *
+ * Returns the names of the APIServices that were deleted.
+ */
+export async function cleanupNamespaceAPIServices(namespace) {
+    const deleted = [];
+    try {
+        const { stdout } = await execa("kubectl", ["get", "apiservices", "-o", "json"], { timeout: 30000 });
+        const parsed = JSON.parse(stdout);
+        for (const item of parsed.items ?? []) {
+            const name = item.metadata?.name;
+            if (!name)
+                continue;
+            if (item.spec?.service?.namespace === namespace) {
+                try {
+                    await execa("kubectl", ["delete", "apiservice", name, "--ignore-not-found"], { timeout: 30000 });
+                    deleted.push(name);
+                }
+                catch {
+                    // Best-effort: a single failure should not block teardown.
+                }
             }
         }
     }
     catch {
-        // Ignore errors - KEDA CRDs might not be installed
+        // Best-effort: if APIServices can't be listed, don't block the destroy.
     }
+    return deleted;
 }
 /**
  * Checks if a namespace exists
@@ -735,98 +1083,144 @@ export async function namespaceExists(namespace) {
     }
 }
 /**
- * Waits for cluster to be accessible with retries.
- * EKS IAM authentication can take time to propagate after cluster creation.
+ * Removes this release's leftovers in the kube-system namespace. The
+ * kube-prometheus-stack prometheus-operator creates a "<release>-...-kubelet"
+ * Service there at runtime (via its --kubelet-service flag); it lives OUTSIDE the
+ * release namespace and is operator-created (not chart-templated), so
+ * `helm uninstall` never deletes it and one accumulates per deployment. Also
+ * sweeps any helm-labeled kube-system objects (exporter Services/Endpoints) a
+ * partial uninstall may have stranded. Scoped strictly to this release; matched
+ * by the release-name prefix so a coexisting deployment's kubelet Service is
+ * never touched. Best-effort — never blocks teardown.
  */
-export async function waitForClusterAccess(maxRetries = 30, delayMs = 10000) {
-    for (let attempt = 1; attempt <= maxRetries; attempt++) {
-        try {
-            await execa("kubectl", ["cluster-info"]);
-            return; // Success
-        }
-        catch (error) {
-            if (attempt === maxRetries) {
-                throw new Error(`Cluster not accessible after ${maxRetries} attempts. ` +
-                    `EKS IAM authentication may not have propagated yet. ` +
-                    `Please wait a few minutes and try again.\n${getErrorMessage(error)}`);
+export async function cleanupKubeSystemLeftovers(releaseName) {
+    // 1) helm-labeled kube-system objects from this release (only present if a
+    //    prior uninstall didn't finish): the kube-prometheus-stack exporter
+    //    Services (coredns/kube-controller-manager/etc.) and their Endpoints.
+    try {
+        await execa("kubectl", [
+            "delete",
+            "service,endpoints",
+            "-n",
+            "kube-system",
+            "-l",
+            `release=${releaseName}`,
+            "--ignore-not-found",
+        ], { timeout: 30000 });
+    }
+    catch {
+        // best-effort
+    }
+    // 2) the operator-created kubelet Service, matched by name (it carries no
+    //    reliable per-release label). Name is "<release>-<kube-prometheus>-kubelet"
+    //    (the middle segment is truncated by the helm fullname template). The
+    //    trailing "-" in the prefix guard prevents matching a sibling whose name
+    //    is a prefix of this one (e.g. az-p0 vs az-p055).
+    try {
+        const { stdout } = await execa("kubectl", [
+            "get",
+            "service",
+            "-n",
+            "kube-system",
+            "-o",
+            "jsonpath={.items[*].metadata.name}",
+        ], { timeout: 15000 });
+        const targets = stdout
+            .split(" ")
+            .filter(Boolean)
+            .filter((n) => n.startsWith(`${releaseName}-`) && n.endsWith("-kubelet"));
+        for (const name of targets) {
+            try {
+                await execa("kubectl", ["delete", "service", name, "-n", "kube-system", "--ignore-not-found"], { timeout: 30000 });
+            }
+            catch {
+                // best-effort
             }
-            // Wait before next retry
-            await sleep(delayMs);
         }
     }
+    catch {
+        // best-effort
+    }
 }
 /**
- * Creates default StorageClass for the cloud provider.
- * Should be called after kubeconfig is configured and cluster is accessible.
- */
-export async function createDefaultStorageClass(provider) {
-    // First wait for cluster to be accessible
-    await waitForClusterAccess();
-    let storageClassYaml;
-    switch (provider) {
-        case "aws":
-            storageClassYaml = `
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: gp3
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-provisioner: ebs.csi.aws.com
-reclaimPolicy: Delete
-volumeBindingMode: WaitForFirstConsumer
-parameters:
-  type: gp3
-  encrypted: "true"
-`;
-            break;
-        case "gcp":
-            storageClassYaml = `
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: pd-ssd
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-provisioner: pd.csi.storage.gke.io
-reclaimPolicy: Delete
-volumeBindingMode: WaitForFirstConsumer
-parameters:
-  type: pd-ssd
-`;
-            break;
-        case "azure":
-            storageClassYaml = `
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: managed-premium
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-provisioner: disk.csi.azure.com
-reclaimPolicy: Delete
-volumeBindingMode: WaitForFirstConsumer
-parameters:
-  skuName: Premium_LRS
-`;
-            break;
-        default:
-            throw new Error(`Unsupported cloud provider: ${provider}`);
-    }
+ * True only when no OTHER Rulebricks deployment remains on the cluster (besides
+ * `releaseName`). Gates deletion of cluster-SHARED resources (CRDs) so tearing
+ * down one deployment never cascade-deletes another deployment's custom
+ * resources. Deployments are named `rulebricks-<name>` for both the namespace and
+ * the helm release (see getNamespace/getReleaseName), so the "rulebricks-" prefix
+ * is a sound cluster-side signal. Fails CLOSED (returns false) if the cluster
+ * can't be enumerated — we never purge shared resources on uncertainty.
+ */
+export async function isLastRulebricksDeployment(releaseName) {
     try {
-        await execa("kubectl", ["apply", "-f", "-"], {
-            input: storageClassYaml,
+        // Authoritative: helm releases cluster-wide.
+        const { stdout } = await execa("helm", ["list", "-A", "-o", "json"], {
+            timeout: 30000,
         });
+        const releases = JSON.parse(stdout);
+        const otherReleases = releases.filter((r) => typeof r.name === "string" &&
+            r.name.startsWith("rulebricks-") &&
+            r.name !== releaseName);
+        if (otherReleases.length > 0)
+            return false;
+        // Cross-check namespaces in case a release secret is gone but the ns lingers
+        // (namespace name == release name by convention).
+        const { stdout: nsOut } = await execa("kubectl", ["get", "namespaces", "-o", "jsonpath={.items[*].metadata.name}"], { timeout: 15000 });
+        const otherNamespaces = nsOut
+            .split(" ")
+            .filter(Boolean)
+            .filter((n) => n.startsWith("rulebricks-") && n !== releaseName);
+        return otherNamespaces.length === 0;
     }
-    catch (error) {
-        throw new Error(`Failed to create StorageClass:\n${getErrorMessage(error)}`);
+    catch {
+        return false; // fail closed — do not purge shared resources on uncertainty
+    }
+}
+// CRD API-group suffixes the chart ships in crds/ dirs (cert-manager + keda from
+// the parent crds/, strimzi + kube-prometheus-stack from subchart crds/). helm
+// NEVER deletes crds/ contents on uninstall, so they leak and accumulate.
+const RULEBRICKS_CRD_GROUP_SUFFIXES = [
+    ".strimzi.io", // kafka.strimzi.io, core.strimzi.io
+    "cert-manager.io", // cert-manager.io, acme.cert-manager.io
+    ".keda.sh", // keda.sh, eventing.keda.sh
+    "monitoring.coreos.com", // kube-prometheus-stack
+];
+/**
+ * Deletes the cluster-scoped CRDs the chart installs from crds/ dirs (cert-
+ * manager, keda, strimzi, kube-prometheus-stack). CLUSTER-SHARED: deleting a CRD
+ * cascade-deletes every custom resource of that kind across ALL namespaces, so
+ * callers MUST gate this on isLastRulebricksDeployment() (or an explicit
+ * operator --purge) — never call it while another Rulebricks deployment exists.
+ * Best-effort, non-blocking; returns the CRD names removed.
+ */
+export async function deleteRulebricksCRDs() {
+    const deleted = [];
+    try {
+        const { stdout } = await execa("kubectl", ["get", "crd", "-o", "jsonpath={.items[*].metadata.name}"], { timeout: 30000 });
+        const targets = stdout
+            .split(" ")
+            .filter(Boolean)
+            .filter((name) => RULEBRICKS_CRD_GROUP_SUFFIXES.some((suffix) => name.endsWith(suffix)));
+        for (const name of targets) {
+            try {
+                await execa("kubectl", ["delete", "crd", name, "--ignore-not-found", "--wait=false"], { timeout: 30000 });
+                deleted.push(name);
+            }
+            catch {
+                // best-effort: a single CRD failure should not block teardown
+            }
+        }
     }
+    catch {
+        // best-effort: if CRDs can't be listed, don't block the destroy
+    }
+    return deleted;
 }
 /**
  * Extracts the version tag from a Docker image string.
  * E.g., "rulebricks/rulebricks:v1.5.8" -> "v1.5.8"
  */
-function extractImageTag(image) {
+export function extractImageTag(image) {
     if (!image)
         return null;
     const parts = image.split(":");
@@ -834,51 +1228,88 @@ function extractImageTag(image) {
         return null;
     return parts[parts.length - 1];
 }
-/**
- * Gets the actual deployed image versions from Kubernetes deployments.
- * Queries the app and HPS deployments to get their current image tags.
- *
- * @param releaseName - The Helm release name (e.g., "rulebricks")
- * @param namespace - The Kubernetes namespace
- * @returns DeployedVersions with app and HPS versions, or null if not found
- */
-export async function getDeployedImageVersions(releaseName, namespace) {
-    const result = {
-        appVersion: null,
-        hpsVersion: null,
-    };
-    // Get app deployment image
+export function extractImageDigest(imageId) {
+    const digest = imageId.split("@").pop();
+    return digest?.startsWith("sha256:") ? digest : null;
+}
+async function getWorkloadImage(workloadType, name, namespace) {
     try {
-        const { stdout: appImage } = await execa("kubectl", [
+        const { stdout } = await execa("kubectl", [
             "get",
-            "deployment",
-            `${releaseName}-app`,
+            workloadType,
+            name,
             "-n",
             namespace,
             "-o",
             "jsonpath={.spec.template.spec.containers[0].image}",
         ]);
-        result.appVersion = extractImageTag(appImage.trim());
+        return stdout.trim() || null;
     }
     catch {
-        // Deployment may not exist or cluster not accessible
+        return null;
     }
-    // Get HPS deployment image
+}
+async function getPodImageDigests(releaseName, workloadName, namespace, containerName) {
     try {
-        const { stdout: hpsImage } = await execa("kubectl", [
+        const { stdout } = await execa("kubectl", [
             "get",
-            "deployment",
-            `${releaseName}-hps`,
+            "pods",
             "-n",
             namespace,
+            "-l",
+            `app.kubernetes.io/name=${workloadName},app.kubernetes.io/instance=${releaseName}`,
             "-o",
-            "jsonpath={.spec.template.spec.containers[0].image}",
+            "json",
         ]);
-        result.hpsVersion = extractImageTag(hpsImage.trim());
+        const data = JSON.parse(stdout);
+        return Array.from(new Set((data.items || [])
+            .flatMap((pod) => pod.status?.containerStatuses || [])
+            .filter((status) => status.name === containerName)
+            .map((status) => extractImageDigest(status.imageID || ""))
+            .filter((digest) => Boolean(digest))));
     }
     catch {
-        // Deployment may not exist or cluster not accessible
+        return [];
     }
+}
+/**
+ * Gets actual deployed image tags and running image digests from Kubernetes.
+ * HPS runs as StatefulSets, so digest checks inspect the pods behind those sets.
+ *
+ * @param releaseName - The Helm release name (e.g., "rulebricks")
+ * @param namespace - The Kubernetes namespace
+ * @returns DeployedVersions with app and HPS versions, or null if not found
+ */
+export async function getDeployedImageVersions(releaseName, namespace) {
+    const result = {
+        appVersion: null,
+        hpsVersion: null,
+        hpsWorkerVersion: null,
+        appDigest: null,
+        hpsDigests: [],
+        hpsWorkerDigests: [],
+    };
+    const appName = `${releaseName}-app`;
+    const hpsName = `${releaseName}-hps`;
+    const hpsWorkerName = `${releaseName}-hps-worker`;
+    const [appImage, hpsImage, hpsWorkerImage] = await Promise.all([
+        getWorkloadImage("deployment", appName, namespace),
+        getWorkloadImage("statefulset", hpsName, namespace),
+        getWorkloadImage("statefulset", hpsWorkerName, namespace),
+    ]);
+    result.appVersion = appImage ? extractImageTag(appImage) : null;
+    result.hpsVersion = hpsImage ? extractImageTag(hpsImage) : null;
+    result.hpsWorkerVersion = hpsWorkerImage
+        ? extractImageTag(hpsWorkerImage)?.replace(/^worker-/, "") || null
+        : null;
+    const [appDigests, hpsDigests, hpsWorkerDigests] = await Promise.all([
+        getPodImageDigests(releaseName, appName, namespace, "app"),
+        getPodImageDigests(releaseName, hpsName, namespace, "hps"),
+        getPodImageDigests(releaseName, hpsWorkerName, namespace, "hps-worker"),
+    ]);
+    result.appDigest = appDigests[0] || null;
+    result.hpsDigests = hpsDigests;
+    result.hpsWorkerDigests = hpsWorkerDigests;
     return result;
 }
 /**