openlattice-cloudrun 0.0.3

package/src/cloudrun-provider.ts

@@ -0,0 +1,867 @@
+import type {
+  ComputeProvider,
+  ComputeSpec,
+  ExecOpts,
+  ExecResult,
+  ExtensionMap,
+  HealthStatus,
+  LogEntry,
+  LogOpts,
+  NetworkExtension,
+  ProviderCapabilities,
+  ProviderNode,
+  ProviderNodeStatus,
+} from "openlattice";
+import type {
+  CloudRunProviderConfig,
+  CloudRunService,
+  CloudRunOperation,
+  CloudRunExecution,
+  CloudLoggingEntry,
+} from "./types";
+import { GcpAuthManager } from "./auth";
+
+export class CloudRunProvider implements ComputeProvider {
+  readonly name = "cloudrun";
+  readonly capabilities: ProviderCapabilities = {
+    restart: true,
+    pause: false,
+    snapshot: false,
+    gpu: false,
+    logs: true,
+    tailscale: false,
+    coldStartMs: 3000,
+    maxConcurrent: 0,
+    architectures: ["x86_64"],
+    persistentStorage: false,
+  };
+
+  private readonly config: CloudRunProviderConfig;
+  private readonly region: string;
+  private readonly auth: GcpAuthManager;
+
+  constructor(config: CloudRunProviderConfig) {
+    if (!config.projectId) {
+      throw new Error("[cloudrun] projectId is required");
+    }
+    this.config = config;
+    this.region = config.region ?? "us-central1";
+    this.auth = new GcpAuthManager({
+      authMethod: config.authMethod,
+      serviceAccountKeyPath: config.serviceAccountKeyPath,
+      accessToken: config.accessToken,
+    });
+  }
+
+  // ── Required methods ────────────────────────────────────────────
+
+  async provision(spec: ComputeSpec): Promise<ProviderNode> {
+    const serviceId = spec.name ?? `ol-svc-${Date.now()}`;
+    const port =
+      spec.network?.ports && spec.network.ports.length > 0
+        ? spec.network.ports[0].port
+        : 3000;
+
+    const labels = sanitizeLabels({
+      ...this.config.defaultLabels,
+      ...spec.labels,
+      "openlattice-managed": "true",
+    });
+
+    const cpuString = String(spec.cpu?.cores ?? 1);
+    const memoryMiB = spec.memory ? spec.memory.sizeGiB * 1024 : 512;
+    const memoryString = `${memoryMiB}Mi`;
+
+    const serviceBody = {
+      template: {
+        containers: [
+          {
+            image: spec.runtime.image,
+            ports: [{ containerPort: port }],
+            env: spec.runtime.env
+              ? Object.entries(spec.runtime.env).map(([name, value]) => ({
+                  name,
+                  value,
+                }))
+              : undefined,
+            command: spec.runtime.command,
+            resources: {
+              limits: {
+                cpu: cpuString,
+                memory: memoryString,
+              },
+            },
+            startupProbe: {
+              httpGet: { path: "/health", port },
+              initialDelaySeconds: 0,
+              periodSeconds: 3,
+              failureThreshold: 10,
+            },
+            livenessProbe: {
+              httpGet: { path: "/health", port },
+              periodSeconds: 30,
+            },
+          },
+        ],
+        scaling: {
+          minInstanceCount: this.config.minInstances ?? 0,
+          maxInstanceCount: this.config.maxInstances ?? 1,
+        },
+        maxInstanceRequestConcurrency: this.config.concurrency ?? 80,
+        ...(this.config.serviceAccount
+          ? { serviceAccount: this.config.serviceAccount }
+          : {}),
+        ...(this.config.vpcConnector
+          ? {
+              vpcAccess: {
+                connector: this.config.vpcConnector,
+              },
+            }
+          : {}),
+      },
+      labels,
+    };
+
+    // Create the service
+    const operation = await this.cloudRunRequest<CloudRunOperation>(
+      "POST",
+      `/services?serviceId=${encodeURIComponent(serviceId)}`,
+      serviceBody
+    );
+
+    // Poll the operation until done
+    if (operation.name && !operation.done) {
+      await this.pollOperation(operation.name);
+    }
+
+    // Get the service to retrieve its URI
+    const service = await this.cloudRunRequest<CloudRunService>(
+      "GET",
+      `/services/${encodeURIComponent(serviceId)}`
+    );
+
+    const serviceUri = service.uri ?? "";
+    let host = "";
+    try {
+      host = new URL(serviceUri).hostname;
+    } catch {
+      host = serviceUri;
+    }
+
+    return {
+      externalId: serviceId,
+      endpoints: serviceUri
+        ? [
+            {
+              type: "https",
+              host,
+              port: 443,
+              url: serviceUri,
+            },
+          ]
+        : [],
+      metadata: {
+        publicUrl: serviceUri,
+        cloudRunProject: this.config.projectId,
+        cloudRunRegion: this.region,
+      },
+    };
+  }
+
+  async exec(
+    externalId: string,
+    command: string[],
+    opts?: ExecOpts
+  ): Promise<ExecResult> {
+    // Cloud Run services don't support exec directly.
+    // Use Cloud Run Jobs: create a one-off job, run it, retrieve output from logs.
+    const jobId = sanitizeJobId(`${externalId}-exec-${Date.now()}`);
+
+    // Get the service to find the image and env from its template
+    const serviceRaw = await this.cloudRunRequest<Record<string, any>>(
+      "GET",
+      `/services/${encodeURIComponent(externalId)}`
+    );
+    const image =
+      serviceRaw?.template?.containers?.[0]?.image ?? "alpine:latest";
+    const env = serviceRaw?.template?.containers?.[0]?.env;
+
+    // Build the full command with cwd and env support
+    let cmd = command;
+    if (opts?.cwd || opts?.env) {
+      const parts: string[] = [];
+      if (opts.cwd) {
+        parts.push(`cd ${opts.cwd}`);
+      }
+      if (opts.env) {
+        const envEntries = Object.entries(opts.env);
+        for (const [k, v] of envEntries) {
+          parts.push(`export ${k}='${(v as string).replace(/'/g, "'\\''")}'`);
+        }
+      }
+      parts.push(command.join(" "));
+      cmd = ["sh", "-c", parts.join(" && ")];
+    }
+
+    // Create a Cloud Run Job
+    const jobBody = {
+      template: {
+        template: {
+          containers: [
+            {
+              image,
+              command: cmd,
+              env,
+            },
+          ],
+          maxRetries: 0,
+          timeout: opts?.timeoutMs
+            ? `${Math.ceil(opts.timeoutMs / 1000)}s`
+            : "600s",
+        },
+      },
+      launchStage: "GA",
+    };
+
+    // Create the job
+    await this.cloudRunJobRequest<CloudRunOperation>(
+      "POST",
+      `/jobs?jobId=${encodeURIComponent(jobId)}`,
+      jobBody
+    );
+
+    // Run the job
+    const runOp = await this.cloudRunJobRequest<CloudRunOperation>(
+      "POST",
+      `/jobs/${encodeURIComponent(jobId)}:run`
+    );
+
+    // Poll the run operation
+    if (runOp.name && !runOp.done) {
+      await this.pollOperation(runOp.name);
+    }
+
+    // Retrieve logs from Cloud Logging
+    const { stdout, stderr } = await this.fetchJobLogs(jobId);
+
+    opts?.onStdout?.(stdout);
+    opts?.onStderr?.(stderr);
+
+    // Determine exit code from execution status (before cleanup)
+    const executions = await this.cloudRunJobRequest<{
+      executions?: CloudRunExecution[];
+    }>(
+      "GET",
+      `/jobs/${encodeURIComponent(jobId)}/executions`
+    ).catch(() => ({ executions: [] }));
+
+    const execution = executions.executions?.[0];
+    const succeeded = execution?.conditions?.some(
+      (c) =>
+        c.type === "Completed" && c.state === "CONDITION_SUCCEEDED"
+    );
+
+    // Clean up the job (best effort, after we've read status)
+    try {
+      await this.cloudRunJobRequest<void>(
+        "DELETE",
+        `/jobs/${encodeURIComponent(jobId)}`
+      );
+    } catch {
+      // Best effort cleanup
+    }
+
+    return {
+      exitCode: succeeded ? 0 : 1,
+      stdout,
+      stderr,
+    };
+  }
+
+  async destroy(externalId: string): Promise<void> {
+    try {
+      await this.cloudRunRequest<void>(
+        "DELETE",
+        `/services/${encodeURIComponent(externalId)}`
+      );
+    } catch (err: unknown) {
+      if (!isNotFound(err)) {
+        throw new Error(
+          `[cloudrun] destroy failed: ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+    }
+  }
+
+  async inspect(externalId: string): Promise<ProviderNodeStatus> {
+    try {
+      const service = await this.cloudRunRequest<CloudRunService>(
+        "GET",
+        `/services/${encodeURIComponent(externalId)}`
+      );
+
+      return {
+        status: mapCloudRunState(service),
+      };
+    } catch (err: unknown) {
+      if (isNotFound(err)) {
+        return { status: "terminated" };
+      }
+      return { status: "unknown" };
+    }
+  }
+
+  // ── Optional: stop / start ──────────────────────────────────────
+
+  async stop(externalId: string): Promise<void> {
+    try {
+      await this.cloudRunRequest<CloudRunOperation>(
+        "PATCH",
+        `/services/${encodeURIComponent(externalId)}`,
+        {
+          traffic: [
+            {
+              percent: 0,
+              type: "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST",
+            },
+          ],
+        }
+      );
+    } catch (err: unknown) {
+      if (!isNotFound(err)) {
+        throw new Error(
+          `[cloudrun] stop failed: ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+    }
+  }
+
+  async start(externalId: string): Promise<void> {
+    await this.cloudRunRequest<CloudRunOperation>(
+      "PATCH",
+      `/services/${encodeURIComponent(externalId)}`,
+      {
+        traffic: [
+          {
+            percent: 100,
+            type: "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST",
+          },
+        ],
+      }
+    );
+  }
+
+  // ── Optional: logs ──────────────────────────────────────────────
+
+  logs(externalId: string, opts?: LogOpts): AsyncIterable<LogEntry> {
+    const self = this;
+
+    if (opts?.follow) {
+      // Follow mode: poll Cloud Logging periodically for new entries
+      return {
+        [Symbol.asyncIterator]() {
+          let done = false;
+          let buffer: LogEntry[] = [];
+          let lastTimestamp: Date | undefined = opts?.since;
+          const pollIntervalMs = 2000;
+
+          return {
+            async next(): Promise<IteratorResult<LogEntry>> {
+              while (!done) {
+                if (buffer.length > 0) {
+                  return { value: buffer.shift()!, done: false };
+                }
+
+                const entries = await self.fetchLogs(externalId, {
+                  ...opts,
+                  since: lastTimestamp,
+                  tail: opts?.tail,
+                });
+
+                if (entries.length > 0) {
+                  // Advance past the latest timestamp to avoid duplicates
+                  lastTimestamp = new Date(
+                    entries[entries.length - 1].timestamp.getTime() + 1
+                  );
+                  buffer = entries;
+                  continue;
+                }
+
+                // No new entries — wait before polling again
+                await new Promise((r) => setTimeout(r, pollIntervalMs));
+              }
+
+              return { value: undefined as any, done: true };
+            },
+            async return() {
+              done = true;
+              buffer = [];
+              return { value: undefined as any, done: true };
+            },
+          };
+        },
+      };
+    }
+
+    // Non-follow mode: single batch fetch
+    return {
+      [Symbol.asyncIterator]() {
+        let done = false;
+        let buffer: LogEntry[] = [];
+        let initialized = false;
+
+        return {
+          async next(): Promise<IteratorResult<LogEntry>> {
+            if (buffer.length > 0) {
+              return { value: buffer.shift()!, done: false };
+            }
+            if (done) {
+              return { value: undefined as any, done: true };
+            }
+
+            if (!initialized) {
+              initialized = true;
+              const entries = await self.fetchLogs(externalId, opts);
+              buffer = entries;
+              if (buffer.length > 0) {
+                return { value: buffer.shift()!, done: false };
+              }
+              done = true;
+              return { value: undefined as any, done: true };
+            }
+
+            done = true;
+            return { value: undefined as any, done: true };
+          },
+          async return() {
+            done = true;
+            buffer = [];
+            return { value: undefined as any, done: true };
+          },
+        };
+      },
+    };
+  }
+
+  // ── Optional: healthCheck ─────────────────────────────────────
+
+  async healthCheck(): Promise<HealthStatus> {
+    const start = Date.now();
+    try {
+      await this.cloudRunRequest<{ services?: CloudRunService[] }>(
+        "GET",
+        "/services"
+      );
+      return {
+        healthy: true,
+        latencyMs: Date.now() - start,
+      };
+    } catch (err: unknown) {
+      return {
+        healthy: false,
+        latencyMs: Date.now() - start,
+        message: `[cloudrun] API unreachable: ${err instanceof Error ? err.message : String(err)}`,
+      };
+    }
+  }
+
+  // ── Optional: getCost ─────────────────────────────────────────
+
+  async getCost(
+    externalId: string
+  ): Promise<{ totalUsd: number }> {
+    // Estimate cost based on Cloud Run pricing.
+    // This is a rough estimate; actual costs depend on usage patterns.
+    try {
+      const serviceRaw = await this.cloudRunRequest<Record<string, any>>(
+        "GET",
+        `/services/${encodeURIComponent(externalId)}`
+      );
+
+      const cpuCores = parseFloat(
+        serviceRaw?.template?.containers?.[0]?.resources?.limits?.cpu ?? "1"
+      );
+      const memoryStr: string =
+        serviceRaw?.template?.containers?.[0]?.resources?.limits?.memory ??
+        "512Mi";
+      const memoryGiB = parseMemoryToGiB(memoryStr);
+
+      // Assume the service has been running since creation
+      // Cloud Run pricing (per-second when active):
+      // CPU: $0.00002400/vCPU-second
+      // Memory: $0.00000250/GiB-second
+      const createTime = serviceRaw?.createTime;
+      const uptimeSeconds = createTime
+        ? (Date.now() - new Date(createTime).getTime()) / 1000
+        : 0;
+
+      const cpuCost = cpuCores * uptimeSeconds * 0.000024;
+      const memoryCost = memoryGiB * uptimeSeconds * 0.0000025;
+      const totalUsd = Math.round((cpuCost + memoryCost) * 100) / 100;
+
+      return { totalUsd };
+    } catch {
+      return { totalUsd: 0 };
+    }
+  }
+
+  // ── Optional: extensions ──────────────────────────────────────
+
+  getExtension<K extends keyof ExtensionMap>(
+    externalId: string,
+    extension: K
+  ): ExtensionMap[K] | undefined {
+    if (extension === "network") {
+      return this.createNetworkExtension(externalId) as ExtensionMap[K];
+    }
+    return undefined;
+  }
+
+  // ── Private helpers ───────────────────────────────────────────
+
+  private async cloudRunRequest<T>(
+    method: string,
+    path: string,
+    body?: unknown
+  ): Promise<T> {
+    const base = `https://run.googleapis.com/v2/projects/${this.config.projectId}/locations/${this.region}`;
+    const url = `${base}${path}`;
+    const token = await this.auth.getToken();
+    const maxRetries = 3;
+
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      const response = await fetch(url, {
+        method,
+        headers: {
+          Authorization: `Bearer ${token}`,
+          "Content-Type": "application/json",
+        },
+        body: body ? JSON.stringify(body) : undefined,
+      });
+
+      // Retry on 429 with exponential backoff
+      if (response.status === 429 && attempt < maxRetries) {
+        const backoffMs = Math.pow(2, attempt) * 1000;
+        await new Promise((r) => setTimeout(r, backoffMs));
+        continue;
+      }
+
+      if (!response.ok) {
+        const text = await response.text();
+        const err = new Error(
+          `[cloudrun] ${method} ${path} failed (${response.status}): ${text}`
+        );
+        (err as any).statusCode = response.status;
+        throw err;
+      }
+
+      if (
+        response.status === 204 ||
+        response.headers.get("content-length") === "0"
+      ) {
+        return undefined as T;
+      }
+
+      return response.json() as Promise<T>;
+    }
+
+    throw new Error(
+      `[cloudrun] ${method} ${path} failed: max retries exceeded`
+    );
+  }
+
+  private async cloudRunJobRequest<T>(
+    method: string,
+    path: string,
+    body?: unknown
+  ): Promise<T> {
+    const base = `https://run.googleapis.com/v2/projects/${this.config.projectId}/locations/${this.region}`;
+    const url = `${base}${path}`;
+    const token = await this.auth.getToken();
+
+    const response = await fetch(url, {
+      method,
+      headers: {
+        Authorization: `Bearer ${token}`,
+        "Content-Type": "application/json",
+      },
+      body: body ? JSON.stringify(body) : undefined,
+    });
+
+    if (!response.ok) {
+      const text = await response.text();
+      const err = new Error(
+        `[cloudrun] ${method} ${path} failed (${response.status}): ${text}`
+      );
+      (err as any).statusCode = response.status;
+      throw err;
+    }
+
+    if (
+      response.status === 204 ||
+      response.headers.get("content-length") === "0"
+    ) {
+      return undefined as T;
+    }
+
+    return response.json() as Promise<T>;
+  }
+
+  private async pollOperation(
+    operationName: string,
+    maxWaitMs: number = 120_000
+  ): Promise<void> {
+    const url = `https://run.googleapis.com/v2/${operationName}`;
+    const startTime = Date.now();
+    let pollIntervalMs = 1000;
+
+    while (Date.now() - startTime < maxWaitMs) {
+      // Refresh token each iteration to avoid expiry on long operations
+      const token = await this.auth.getToken();
+      const response = await fetch(url, {
+        headers: {
+          Authorization: `Bearer ${token}`,
+        },
+      });
+
+      if (!response.ok) {
+        throw new Error(
+          `[cloudrun] poll operation failed (${response.status}): ${await response.text()}`
+        );
+      }
+
+      const op = (await response.json()) as CloudRunOperation;
+
+      if (op.done) {
+        if (op.error) {
+          throw new Error(
+            `[cloudrun] operation failed: ${op.error.message ?? "unknown error"}`
+          );
+        }
+        return;
+      }
+
+      await new Promise((r) => setTimeout(r, pollIntervalMs));
+      pollIntervalMs = Math.min(pollIntervalMs * 1.5, 5000);
+    }
+
+    throw new Error("[cloudrun] operation timed out");
+  }
+
+  private async fetchLogs(
+    serviceId: string,
+    opts?: LogOpts
+  ): Promise<LogEntry[]> {
+    const token = await this.auth.getToken();
+    const filter = [
+      `resource.type="cloud_run_revision"`,
+      `resource.labels.service_name="${serviceId}"`,
+      `resource.labels.project_id="${this.config.projectId}"`,
+    ];
+
+    if (opts?.since) {
+      filter.push(`timestamp>="${opts.since.toISOString()}"`);
+    }
+
+    const body: Record<string, unknown> = {
+      resourceNames: [`projects/${this.config.projectId}`],
+      filter: filter.join(" AND "),
+      orderBy: opts?.follow ? "timestamp asc" : "timestamp desc",
+      pageSize: opts?.tail ?? 100,
+    };
+
+    const response = await fetch(
+      "https://logging.googleapis.com/v2/entries:list",
+      {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${token}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(body),
+      }
+    );
+
+    if (!response.ok) {
+      return [];
+    }
+
+    const data = (await response.json()) as {
+      entries?: CloudLoggingEntry[];
+    };
+
+    return (data.entries ?? []).map((entry) => ({
+      timestamp: new Date(entry.timestamp ?? Date.now()),
+      message:
+        entry.textPayload ?? JSON.stringify(entry.jsonPayload ?? {}),
+      stream:
+        entry.severity === "ERROR" || entry.severity === "CRITICAL"
+          ? ("stderr" as const)
+          : ("stdout" as const),
+    }));
+  }
+
+  private async fetchJobLogs(
+    jobId: string
+  ): Promise<{ stdout: string; stderr: string }> {
+    const token = await this.auth.getToken();
+    const filter = [
+      `resource.type="cloud_run_job"`,
+      `resource.labels.job_name="${jobId}"`,
+      `resource.labels.project_id="${this.config.projectId}"`,
+    ];
+
+    const body = {
+      resourceNames: [`projects/${this.config.projectId}`],
+      filter: filter.join(" AND "),
+      orderBy: "timestamp asc",
+      pageSize: 1000,
+    };
+
+    const response = await fetch(
+      "https://logging.googleapis.com/v2/entries:list",
+      {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${token}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(body),
+      }
+    );
+
+    if (!response.ok) {
+      return { stdout: "", stderr: "" };
+    }
+
+    const data = (await response.json()) as {
+      entries?: CloudLoggingEntry[];
+    };
+
+    const stdoutLines: string[] = [];
+    const stderrLines: string[] = [];
+
+    for (const entry of data.entries ?? []) {
+      const message =
+        entry.textPayload ?? JSON.stringify(entry.jsonPayload ?? {});
+      if (
+        entry.severity === "ERROR" ||
+        entry.severity === "CRITICAL"
+      ) {
+        stderrLines.push(message);
+      } else {
+        stdoutLines.push(message);
+      }
+    }
+
+    return {
+      stdout: stdoutLines.join("\n"),
+      stderr: stderrLines.join("\n"),
+    };
+  }
+
+  private createNetworkExtension(
+    externalId: string
+  ): NetworkExtension {
+    const projectId = this.config.projectId;
+    const region = this.region;
+    return {
+      async getUrl(port: number): Promise<string> {
+        // Cloud Run services have a single public HTTPS endpoint.
+        // All traffic goes through the Google-managed TLS proxy on port 443.
+        if (port === 443 || port === 80) {
+          return `https://${externalId}-${projectId}.${region}.run.app`;
+        }
+        // Non-standard ports are not directly supported by Cloud Run.
+        // Return the base URL — the container port is mapped via the service config.
+        return `https://${externalId}-${projectId}.${region}.run.app`;
+      },
+    };
+  }
+}
+
+// ── Utility functions ─────────────────────────────────────────────
+
+function mapCloudRunState(
+  service: CloudRunService
+): "running" | "stopped" | "paused" | "terminated" | "unknown" {
+  const condition = service.terminalCondition;
+  if (condition?.state === "CONDITION_SUCCEEDED") {
+    // Check if traffic is routed — if 0% traffic, treat as stopped
+    const totalTraffic =
+      service.traffic?.reduce((sum, t) => sum + (t.percent ?? 0), 0) ?? 100;
+    if (totalTraffic === 0) {
+      return "stopped";
+    }
+    return "running";
+  }
+  if (condition?.state === "CONDITION_FAILED") {
+    return "stopped";
+  }
+  // Service exists but condition not yet resolved
+  if (service.reconciling) {
+    return "running";
+  }
+  return "unknown";
+}
+
+function sanitizeLabels(
+  labels: Record<string, string>
+): Record<string, string> {
+  const sanitized: Record<string, string> = {};
+  let count = 0;
+  for (const [key, value] of Object.entries(labels)) {
+    if (count >= 64) break;
+    const safeKey = key
+      .toLowerCase()
+      .replace(/[^a-z0-9_-]/g, "_")
+      .slice(0, 63);
+    const safeValue = value
+      .toLowerCase()
+      .replace(/[^a-z0-9_-]/g, "_")
+      .slice(0, 63);
+    sanitized[safeKey] = safeValue;
+    count++;
+  }
+  return sanitized;
+}
+
+function parseMemoryToGiB(memStr: string): number {
+  const match = memStr.match(/^(\d+)(Mi|Gi|M|G)?$/);
+  if (!match) return 0.5;
+  const value = parseInt(match[1], 10);
+  const unit = match[2] ?? "Mi";
+  switch (unit) {
+    case "Gi":
+    case "G":
+      return value;
+    case "Mi":
+    case "M":
+      return value / 1024;
+    default:
+      return value / 1024;
+  }
+}
+
+/**
+ * Sanitize a job ID to meet Cloud Run constraints:
+ * must match [a-z]([-a-z0-9]*[a-z0-9])?, max 49 characters.
+ */
+function sanitizeJobId(raw: string): string {
+  const sanitized = raw
+    .toLowerCase()
+    .replace(/[^a-z0-9-]/g, "-")
+    .replace(/^[^a-z]+/, "")       // must start with a letter
+    .replace(/-+$/g, "")           // must not end with a hyphen
+    .replace(/-{2,}/g, "-")        // collapse consecutive hyphens
+    .slice(0, 49);
+  // Trim trailing hyphens after slicing
+  return sanitized.replace(/-+$/, "") || "ol-exec-job";
+}
+
+function isNotFound(err: unknown): boolean {
+  if (err && typeof err === "object") {
+    return (err as any).statusCode === 404;
+  }
+  return false;
+}