alchemy 2.0.0-beta.64 → 2.0.0-beta.65

package/src/AWS/EC2/LingeringEnis.ts

@@ -0,0 +1,169 @@
+import * as ec2 from "@distilled.cloud/aws/ec2";
+import * as Effect from "effect/Effect";
+import * as Result from "effect/Result";
+
+/**
+ * Shared teardown accelerator for network interfaces that AWS releases
+ * *asynchronously* after their owning resource is deleted.
+ *
+ * The canonical case is a VPC-attached Lambda Function: `DeleteFunction`
+ * returns immediately, but the function's Hyperplane ENIs stay `in-use` in
+ * their subnets/security groups for several minutes, and Lambda's own reaper
+ * may not delete the detached ENI for up to ~20 minutes. Until the ENI is
+ * gone, `DeleteSubnet`/`DeleteSecurityGroup` fail with `DependencyViolation`
+ * — historically forcing users to run `destroy` twice, minutes apart.
+ *
+ * `retryWhileLingeringEnis` wraps a subnet/security-group delete call:
+ * on every `DependencyViolation` it observes the ENIs still occupying the
+ * resource, explicitly deletes any *detached* (status `available`) Lambda
+ * ENI instead of waiting for Lambda's reaper, and — while the blockers are
+ * exclusively Lambda ENIs that are provably on their way out — keeps waiting
+ * on an extended budget that covers AWS's worst-case release window. Any
+ * other lingering dependency keeps today's shorter budget.
+ */
+
+/**
+ * ENIs that their owning service tears down asynchronously and that are safe
+ * for us to delete once detached. Only Lambda Hyperplane ENIs qualify today:
+ * every other managed type (NAT gateways, VPC endpoints, ELB, EFS mount
+ * targets, ECS task ENIs) is deleted by the API call that deletes its owner
+ * — and their owners are proper alchemy resources ordered ahead of the
+ * subnet/SG by the deletion graph.
+ *
+ * Matching note (observed live): while attached, a Hyperplane ENI reports
+ * `InterfaceType: "lambda"` — but the moment Lambda detaches it, the type
+ * flips to plain `"interface"` and only the description (`AWS Lambda VPC
+ * ENI-{functionName}`) still identifies it. Match both, like Terraform does.
+ */
+const isReapableEni = (eni: ec2.NetworkInterface): boolean =>
+  eni.InterfaceType === "lambda" ||
+  (eni.Description?.startsWith("AWS Lambda VPC ENI") ?? false);
+
+export interface LingeringEniScope {
+  /** ENI filter naming the resource being deleted. */
+  readonly name: "subnet-id" | "group-id";
+  readonly value: string;
+}
+
+/**
+ * Observe ENIs still occupying the resource and delete any detached
+ * reapable ones. Best-effort by design: the reaper only accelerates the
+ * outer DependencyViolation retry, so an account that cannot describe or
+ * delete ENIs degrades to plain waiting instead of failing the destroy.
+ */
+const reapLingeringEnis = Effect.fn(function* (
+  scope: LingeringEniScope,
+  session: { note: (note: string) => Effect.Effect<void> },
+) {
+  const described = yield* ec2
+    .describeNetworkInterfaces({
+      Filters: [{ Name: scope.name, Values: [scope.value] }],
+    })
+    .pipe(Effect.catch(() => Effect.succeed({ NetworkInterfaces: [] })));
+
+  const lingering = (described.NetworkInterfaces ?? []).filter(isReapableEni);
+
+  let reaped = 0;
+  let pending = 0;
+  for (const eni of lingering) {
+    if (eni.Status !== "available" || eni.NetworkInterfaceId === undefined) {
+      pending += 1;
+      continue;
+    }
+    const outcome = yield* ec2
+      .deleteNetworkInterface({
+        NetworkInterfaceId: eni.NetworkInterfaceId,
+      })
+      .pipe(
+        Effect.as("deleted" as const),
+        // Already gone counts as progress; anything else (raced back to
+        // in-use, throttle, missing IAM permission) must not fail the
+        // destroy — reaping is purely an accelerator, so degrade to the
+        // outer retry's plain waiting.
+        Effect.catchTag("InvalidNetworkInterfaceID.NotFound", () =>
+          Effect.succeed("deleted" as const),
+        ),
+        Effect.catch(() => Effect.succeed("pending" as const)),
+      );
+    if (outcome === "deleted") {
+      yield* session.note(
+        `Deleted detached Lambda ENI ${eni.NetworkInterfaceId}`,
+      );
+      reaped += 1;
+    } else {
+      pending += 1;
+    }
+  }
+
+  return {
+    /** Reapable ENIs still occupying the resource after this pass. */
+    pendingReapable: pending,
+    /** Detached reapable ENIs actually deleted this pass. */
+    reaped,
+  };
+});
+
+/**
+ * Retry `deleteCall` while it fails with a dependency violation, reaping
+ * lingering ENIs between attempts.
+ *
+ * - Base budget (~12 min) matches the historical subnet schedule: fast
+ *   exponential start, capped at 30-second steps.
+ * - While the observed blockers include reapable ENIs (attached or just
+ *   reaped), the budget extends to ~25 min — Lambda's documented worst-case
+ *   ENI release window — because those blockers are guaranteed to clear.
+ */
+export const retryWhileLingeringEnis = <A, E extends { _tag: string }, R>(
+  deleteCall: Effect.Effect<A, E, R>,
+  options: {
+    scope: LingeringEniScope;
+    isDependencyViolation: (error: E) => boolean;
+    session: { note: (note: string) => Effect.Effect<void> };
+  },
+) =>
+  Effect.gen(function* () {
+    const BASE_BUDGET_MILLIS = 12 * 60 * 1000;
+    const EXTENDED_BUDGET_MILLIS = 25 * 60 * 1000;
+
+    let elapsedMillis = 0;
+    let sawReapable = false;
+
+    for (let attempt = 1; ; attempt++) {
+      const result = yield* Effect.result(deleteCall);
+      if (Result.isSuccess(result)) {
+        return result.success;
+      }
+      const error = result.failure;
+      if (!options.isDependencyViolation(error)) {
+        return yield* Effect.fail(error);
+      }
+
+      const { pendingReapable, reaped } = yield* reapLingeringEnis(
+        options.scope,
+        options.session,
+      );
+      if (reaped > 0) {
+        // An ENI just left — the next attempt has a real chance; skip the
+        // backoff sleep and try immediately.
+        continue;
+      }
+      sawReapable ||= pendingReapable > 0;
+
+      const budget = sawReapable ? EXTENDED_BUDGET_MILLIS : BASE_BUDGET_MILLIS;
+      if (elapsedMillis >= budget) {
+        return yield* Effect.fail(error);
+      }
+
+      yield* options.session.note(
+        pendingReapable > 0
+          ? `Waiting for AWS to release ${pendingReapable} lingering ENI(s)... (attempt ${attempt})`
+          : `Waiting for dependencies to clear... (attempt ${attempt})`,
+      );
+
+      // Fast exponential start capped at 30-second steps (the historical
+      // subnet schedule).
+      const delayMillis = Math.min(1000 * 1.5 ** (attempt - 1), 30_000);
+      yield* Effect.sleep(delayMillis);
+      elapsedMillis += delayMillis;
+    }
+  });

package/src/AWS/EC2/SecurityGroup.ts

@@ -11,6 +11,7 @@ import type { AccountID } from "../Environment.ts";
 import { AWSEnvironment } from "../Environment.ts";
 import type { Providers } from "../Providers.ts";
 import type { RegionID } from "../Region.ts";
+import { retryWhileLingeringEnis } from "./LingeringEnis.ts";
 import type { VpcId } from "./Vpc.ts";
 
 export type SecurityGroupId<ID extends string = string> = `sg-${ID}`;
@@ -681,34 +682,29 @@ export const SecurityGroupProvider = () =>
 
           yield* session.note(`Deleting Security Group: ${groupId}`);
 
-          yield* ec2
-            .deleteSecurityGroup({
-              GroupId: groupId,
-              DryRun: false,
-            })
-            .pipe(
-              Effect.catchTag("InvalidGroup.NotFound", () => Effect.void),
-              // Retry on dependency violations (e.g., ENIs still using the security group)
-              Effect.retry({
-                while: (e) => {
-                  return (
-                    e._tag === "DependencyViolation" ||
-                    (e._tag === "ValidationError" &&
-                      e.message?.includes("DependencyViolation"))
-                  );
-                },
-                schedule: Schedule.max([
-                  Schedule.fixed(5000),
-                  Schedule.recurs(30),
-                ]).pipe(
-                  Schedule.tap(({ attempt }) =>
-                    session.note(
-                      `Waiting for dependencies to clear... (attempt ${attempt})`,
-                    ),
-                  ),
-                ),
-              }),
-            );
+          // DependencyViolation means ENIs still reference the group — ALB,
+          // ECS task, or VPC-attached Lambda ENIs release minutes after the
+          // owning resource is deleted. Lambda Hyperplane ENIs are reaped
+          // explicitly between attempts (they otherwise linger up to ~20
+          // minutes and used to force a second `destroy` run).
+          yield* retryWhileLingeringEnis(
+            ec2
+              .deleteSecurityGroup({
+                GroupId: groupId,
+                DryRun: false,
+              })
+              .pipe(
+                Effect.catchTag("InvalidGroup.NotFound", () => Effect.void),
+              ),
+            {
+              scope: { name: "group-id", value: groupId },
+              isDependencyViolation: (e) =>
+                e._tag === "DependencyViolation" ||
+                (e._tag === "ValidationError" &&
+                  (e.message?.includes("DependencyViolation") ?? false)),
+              session,
+            },
+          );
 
           yield* session.note(`Security Group ${groupId} deleted`);
         }),

package/src/AWS/EC2/Subnet.ts

@@ -7,6 +7,7 @@ import * as Stream from "effect/Stream";
 
 import type { ScopedPlanStatusSession } from "../../Cli/Cli.ts";
 import { isResolved, somePropsAreDifferent } from "../../Diff.ts";
+import { retryWhileLingeringEnis } from "./LingeringEnis.ts";
 import * as Provider from "../../Provider.ts";
 import { Resource } from "../../Resource.ts";
 import type { Providers } from "../Providers.ts";
@@ -597,40 +598,28 @@ export const SubnetProvider = () =>
 
           yield* session.note(`Deleting subnet: ${subnetId}`);
 
-          // 1. Attempt to delete subnet
-          yield* ec2
-            .deleteSubnet({
-              SubnetId: subnetId,
-              DryRun: false,
-            })
-            .pipe(
-              Effect.tapError(Effect.logDebug),
-              Effect.catchTag("InvalidSubnetID.NotFound", () => Effect.void),
-              // Retry on dependency violations (resources still being deleted).
-              // ENIs from a just-deleted ALB or CloudFront VPC origin can take
-              // several minutes to detach after the owning resource is gone, so
-              // budget ~12 min (fast exponential start, capped at 30s steps).
-              Effect.retry({
-                while: (e) => {
-                  // DependencyViolation means there are still dependent resources
-                  // This can happen if ENIs/instances are being deleted concurrently
-                  return e._tag === "DependencyViolation";
-                },
-                schedule: Schedule.max([
-                  Schedule.min([
-                    Schedule.exponential(1000, 1.5),
-                    Schedule.spaced("30 seconds"),
-                  ]),
-                  Schedule.recurs(30),
-                ]).pipe(
-                  Schedule.tap(({ attempt }) =>
-                    session.note(
-                      `Waiting for dependencies to clear... (attempt ${attempt})`,
-                    ),
-                  ),
-                ),
-              }),
-            );
+          // 1. Attempt to delete subnet. DependencyViolation means resources
+          // still occupy it — ENIs from a just-deleted ALB, CloudFront VPC
+          // origin, or a VPC-attached Lambda can take minutes to release
+          // after the owning resource is gone. Lambda Hyperplane ENIs are
+          // reaped explicitly between attempts (they otherwise linger up to
+          // ~20 minutes and used to force a second `destroy` run).
+          yield* retryWhileLingeringEnis(
+            ec2
+              .deleteSubnet({
+                SubnetId: subnetId,
+                DryRun: false,
+              })
+              .pipe(
+                Effect.tapError(Effect.logDebug),
+                Effect.catchTag("InvalidSubnetID.NotFound", () => Effect.void),
+              ),
+            {
+              scope: { name: "subnet-id", value: subnetId },
+              isDependencyViolation: (e) => e._tag === "DependencyViolation",
+              session,
+            },
+          );
 
           // 2. Wait for subnet to be fully deleted
           yield* waitForSubnetDeleted(subnetId, session);

package/src/AWS/EKS/Addon.ts

@@ -161,7 +161,11 @@ class AddonNotReady extends Data.TaggedError("AddonNotReady")<{
   readonly clusterName: string;
   readonly addonName: string;
   readonly status: string | undefined;
-}> {}
+}> {
+  override get message(): string {
+    return `addon '${this.clusterName}/${this.addonName}' is not ACTIVE (status: ${this.status ?? "absent"})`;
+  }
+}
 
 class AddonStillExists extends Data.TaggedError("AddonStillExists")<{
   readonly clusterName: string;
@@ -233,8 +237,14 @@ export const AddonProvider = () =>
           }
         }),
         read: Effect.fn(function* ({ id, olds }) {
+          const clusterName = olds.clusterName as string | undefined;
+          // A crashed prior run can persist a row before its unresolved
+          // inputs were stripped — nothing observable yet.
+          if (clusterName === undefined || olds.addonName === undefined) {
+            return undefined;
+          }
           const state = yield* readAddon({
-            clusterName: olds.clusterName as string,
+            clusterName,
             addonName: olds.addonName,
           });
           if (!state) return undefined;
@@ -404,11 +414,14 @@ const waitForAddonActive = Effect.fn(function* ({
     }),
     Effect.retry({
       while: (error) => error instanceof AddonNotReady,
-      // Flat 5s polls, ~10 min budget (uncapped exponential sleeps for
+      // Flat 5s polls, ~20 min budget (uncapped exponential sleeps for
       // multi-minute stretches late in the wait — looks like a deadlock).
+      // Node-bound addons (e.g. HyperPod task governance) stay DEGRADED
+      // until nodes join and pull images, which can take most of the
+      // budget when the addon installs alongside its node group.
       schedule: Schedule.max([
         Schedule.spaced("5 seconds"),
-        Schedule.recurs(120),
+        Schedule.recurs(240),
       ]),
     }),
   );

package/src/AWS/EKS/Deployment.ts

@@ -35,6 +35,12 @@ import {
   deleteObjects,
   type KubernetesClusterConnection,
 } from "./internal/client.ts";
+import {
+  hyperpodNamespace,
+  hyperpodNodeSelector,
+  hyperpodWorkloadLabels,
+  type HyperPodWorkloadProps,
+} from "./HyperPod.ts";
 import {
   toKubernetesObjectRef,
   type KubernetesObjectDefinition,
@@ -76,9 +82,17 @@ export interface DeploymentPropsBase extends PlatformProps {
   /**
    * Kubernetes namespace to deploy into. The namespace must already exist
    * (Auto Mode clusters ship a `default` namespace).
-   * @default "default"
+   * @default "default" (or `hyperpod-ns-<team>` when `hyperpod.quota` is set)
    */
   namespace?: string;
+  /**
+   * Run on SageMaker HyperPod nodes attached to this EKS cluster: pin to an
+   * instance group, keep off unhealthy nodes, and optionally submit through
+   * HyperPod task governance by passing the team's
+   * `AWS.SageMaker.ComputeQuota` (which derives the namespace and Kueue
+   * labels).
+   */
+  hyperpod?: HyperPodWorkloadProps;
   /**
    * HTTP port exposed by the container and the Service.
    * @default 3000
@@ -539,9 +553,11 @@ export const DeploymentProvider = () =>
           ) {
             return { action: "replace" } as const;
           }
+          const effectiveNamespace = (props: DeploymentProps) =>
+            props.namespace ?? hyperpodNamespace(props.hyperpod) ?? "default";
           if (
-            olds.namespace !== undefined &&
-            (olds.namespace ?? "default") !== (news.namespace ?? "default")
+            olds.cluster?.clusterName !== undefined &&
+            effectiveNamespace(olds) !== effectiveNamespace(news)
           ) {
             return { action: "replace" } as const;
           }
@@ -585,7 +601,8 @@ export const DeploymentProvider = () =>
         }) {
           const connection = toConnection(news.cluster);
           const clusterName = news.cluster.clusterName;
-          const namespace = news.namespace ?? "default";
+          const namespace =
+            news.namespace ?? hyperpodNamespace(news.hyperpod) ?? "default";
           const port = news.port ?? 3000;
           const serviceType = news.serviceType ?? "LoadBalancer";
 
@@ -651,7 +668,10 @@ export const DeploymentProvider = () =>
           // get no region env var — inject it so the bootstrap's
           // `Region.fromEnv()` resolves inside the pod.
           const { region } = yield* AWSEnvironment.current;
-          const labels = news.labels ?? { "app.kubernetes.io/name": baseName };
+          const labels = {
+            ...(news.labels ?? { "app.kubernetes.io/name": baseName }),
+            ...hyperpodWorkloadLabels(news.hyperpod),
+          };
           const containerEnv = {
             ...bindingEnv,
             ...alchemyEnv,
@@ -670,6 +690,7 @@ export const DeploymentProvider = () =>
               metadata: { labels },
               spec: {
                 serviceAccountName,
+                nodeSelector: hyperpodNodeSelector(news.hyperpod),
                 containers: [
                   {
                     name: baseName,

package/src/AWS/EKS/HyperPod.ts

@@ -0,0 +1,97 @@
+/**
+ * First-class SageMaker HyperPod scheduling for EKS workloads.
+ *
+ * HyperPod nodes are ordinary EKS nodes carrying well-known labels, and
+ * HyperPod task governance rides on Kueue conventions. `AWS.EKS.Job` and
+ * `AWS.EKS.Deployment` accept these props under `hyperpod:` and derive the
+ * node selector, namespace, and Kueue labels — no manual label wiring.
+ */
+
+/** The well-known node label carrying the HyperPod instance-group name. */
+export const HYPERPOD_INSTANCE_GROUP_LABEL =
+  "sagemaker.amazonaws.com/instance-group-name";
+
+/** The well-known node label carrying HyperPod's node health verdict. */
+export const HYPERPOD_NODE_HEALTH_LABEL =
+  "sagemaker.amazonaws.com/node-health-status";
+
+/** Kueue label selecting the task-governance queue. */
+export const KUEUE_QUEUE_NAME_LABEL = "kueue.x-k8s.io/queue-name";
+
+/** Kueue label selecting the task-governance priority class. */
+export const KUEUE_PRIORITY_CLASS_LABEL = "kueue.x-k8s.io/priority-class";
+
+export interface HyperPodWorkloadProps {
+  /**
+   * Pin the workload to a HyperPod instance group (matches the
+   * `sagemaker.amazonaws.com/instance-group-name` node label). Reference
+   * the group through the cluster's attributes —
+   * `hyperpod.instanceGroups.workers` — so the workload is connected to
+   * the cluster through the resource graph. A plain name string also
+   * works.
+   */
+  instanceGroup?: string | { InstanceGroupName?: string };
+  /**
+   * Only schedule onto nodes that passed HyperPod health checks
+   * (`sagemaker.amazonaws.com/node-health-status: Schedulable`).
+   * @default true
+   */
+  healthyNodesOnly?: boolean;
+  /**
+   * Submit through HyperPod task governance under this team's quota — pass
+   * the `AWS.SageMaker.ComputeQuota` resource. Derives the
+   * `hyperpod-ns-<team>` namespace and the Kueue queue label (both
+   * materialized by the quota), and orders the workload after it.
+   */
+  quota?: {
+    /** The quota's team name (`ComputeQuota.teamName`). */
+    teamName: string;
+  };
+  /**
+   * The task-governance priority class (a `PriorityClass` name from the
+   * cluster's `AWS.SageMaker.ClusterSchedulerConfig`).
+   */
+  priorityClass?: string;
+}
+
+/** @internal The `hyperpod-ns-<team>` namespace, when governed. */
+export const hyperpodNamespace = (
+  hyperpod: HyperPodWorkloadProps | undefined,
+): string | undefined =>
+  hyperpod?.quota !== undefined
+    ? `hyperpod-ns-${hyperpod.quota.teamName}`
+    : undefined;
+
+/** @internal Kueue labels for the workload object. */
+export const hyperpodWorkloadLabels = (
+  hyperpod: HyperPodWorkloadProps | undefined,
+): Record<string, string> => ({
+  ...(hyperpod?.quota !== undefined
+    ? {
+        [KUEUE_QUEUE_NAME_LABEL]: `hyperpod-ns-${hyperpod.quota.teamName}-localqueue`,
+      }
+    : {}),
+  ...(hyperpod?.priorityClass !== undefined
+    ? { [KUEUE_PRIORITY_CLASS_LABEL]: `${hyperpod.priorityClass}-priority` }
+    : {}),
+});
+
+/** @internal The instance-group name from either reference form. */
+const instanceGroupName = (
+  group: string | { InstanceGroupName?: string } | undefined,
+): string | undefined =>
+  typeof group === "string" ? group : group?.InstanceGroupName;
+
+/** @internal Node selector pinning pods onto HyperPod nodes. */
+export const hyperpodNodeSelector = (
+  hyperpod: HyperPodWorkloadProps | undefined,
+): Record<string, string> | undefined => {
+  if (hyperpod === undefined) return undefined;
+  const group = instanceGroupName(hyperpod.instanceGroup);
+  return {
+    ...(hyperpod.healthyNodesOnly !== false
+      ? { [HYPERPOD_NODE_HEALTH_LABEL]: "Schedulable" }
+      : {}),
+    ...(group !== undefined ? { [HYPERPOD_INSTANCE_GROUP_LABEL]: group } : {}),
+  };
+};

package/src/AWS/EKS/Job.ts

@@ -35,6 +35,12 @@ import {
 import { AWSEnvironment } from "../Environment.ts";
 import type { PolicyStatement } from "../IAM/Policy.ts";
 import type { Providers } from "../Providers.ts";
+import {
+  hyperpodNamespace,
+  hyperpodNodeSelector,
+  hyperpodWorkloadLabels,
+  type HyperPodWorkloadProps,
+} from "./HyperPod.ts";
 import { reconcileObjects, deleteObjects } from "./internal/client.ts";
 import type {
   KubernetesObjectDefinition,
@@ -75,9 +81,17 @@ export interface JobPropsBase extends PlatformProps {
   /**
    * Kubernetes namespace to run in. The namespace must already exist (Auto
    * Mode clusters ship a `default` namespace).
-   * @default "default"
+   * @default "default" (or `hyperpod-ns-<team>` when `hyperpod.quota` is set)
    */
   namespace?: string;
+  /**
+   * Run on SageMaker HyperPod nodes attached to this EKS cluster: pin to an
+   * instance group, keep off unhealthy nodes, and optionally submit through
+   * HyperPod task governance by passing the team's
+   * `AWS.SageMaker.ComputeQuota` (which derives the namespace and Kueue
+   * labels).
+   */
+  hyperpod?: HyperPodWorkloadProps;
   /**
    * Number of retries before the Job is marked failed (Kubernetes
    * `backoffLimit`).
@@ -472,9 +486,11 @@ export const JobProvider = () =>
           ) {
             return { action: "replace" } as const;
           }
+          const effectiveNamespace = (props: JobProps) =>
+            props.namespace ?? hyperpodNamespace(props.hyperpod) ?? "default";
           if (
-            olds.namespace !== undefined &&
-            (olds.namespace ?? "default") !== (news.namespace ?? "default")
+            olds.cluster?.clusterName !== undefined &&
+            effectiveNamespace(olds) !== effectiveNamespace(news)
           ) {
             return { action: "replace" } as const;
           }
@@ -515,7 +531,8 @@ export const JobProvider = () =>
         }) {
           const connection = toConnection(news.cluster);
           const clusterName = news.cluster.clusterName;
-          const namespace = news.namespace ?? "default";
+          const namespace =
+            news.namespace ?? hyperpodNamespace(news.hyperpod) ?? "default";
 
           const baseName = yield* toBaseName(id, news);
           const serviceAccountName = output?.serviceAccountName ?? baseName;
@@ -569,7 +586,10 @@ export const JobProvider = () =>
           });
 
           // Synthesize the Kubernetes objects.
-          const labels = news.labels ?? { "app.kubernetes.io/name": baseName };
+          const labels = {
+            ...(news.labels ?? { "app.kubernetes.io/name": baseName }),
+            ...hyperpodWorkloadLabels(news.hyperpod),
+          };
           const containerEnv = {
             ...bindingEnv,
             ...alchemyEnv,
@@ -586,6 +606,7 @@ export const JobProvider = () =>
               metadata: { labels },
               spec: {
                 serviceAccountName,
+                nodeSelector: hyperpodNodeSelector(news.hyperpod),
                 restartPolicy: news.restartPolicy ?? "Never",
                 containers: [
                   {
@@ -620,9 +641,13 @@ export const JobProvider = () =>
           // hash of the spec, and a spec change applies a NEW Job (which
           // runs) while `reconcileObjects` deletes the previous one.
           // CronJobs are mutable and keep the stable base name.
+          // Kubernetes rejects Job names over 63 characters (the API
+          // stamps the name into the batch.kubernetes.io/job-name pod
+          // label, and label values cap at 63) — truncate the base so the
+          // content-address suffix always fits.
           const jobName = news.schedule
-            ? baseName
-            : `${baseName}-${(yield* sha256Object(jobSpec)).slice(0, 8)}`;
+            ? baseName.slice(0, 52).replace(/-+$/, "")
+            : `${baseName.slice(0, 54).replace(/-+$/, "")}-${(yield* sha256Object(jobSpec)).slice(0, 8)}`;
 
           const workloadObject: KubernetesObjectDefinition = news.schedule
             ? {