lagora-cli 1.1.0

package/dist/scripts/sdk-runtime-kubernetes-job-KOWL4ITV.js

@@ -0,0 +1,479 @@
+import {
+  sdkPackageIndexEnv
+} from "./chunk-2KTLCUFI.js";
+import {
+  SDK_PREPARE_SOURCE_PATHS,
+  sdkPrepareCacheEnv,
+  sdkPrepareCacheFingerprint,
+  sdkPrepareCacheStatePath
+} from "./chunk-UQ6I6VTY.js";
+
+// lib/server/sdk-runtime-kubernetes-job.ts
+import { randomUUID } from "node:crypto";
+
+// lib/server/kubernetes-client.ts
+import { readFile } from "node:fs/promises";
+import https from "node:https";
+import path from "node:path";
+var SERVICE_ACCOUNT_ROOT = "/var/run/secrets/kubernetes.io/serviceaccount";
+async function inClusterKubernetesClient() {
+  const endpoint = kubernetesApiEndpointFromEnv(process.env);
+  const namespace = (await readFile(path.join(SERVICE_ACCOUNT_ROOT, "namespace"), "utf8")).trim();
+  const token = (await readFile(path.join(SERVICE_ACCOUNT_ROOT, "token"), "utf8")).trim();
+  const ca = await readFile(path.join(SERVICE_ACCOUNT_ROOT, "ca.crt"));
+  return {
+    namespace,
+    request: async (input) => await kubernetesRequest({ ...endpoint, token, ca, ...input })
+  };
+}
+function kubernetesApiEndpointFromEnv(env) {
+  return {
+    host: env.AGORA_KUBERNETES_API_HOST ?? "kubernetes.default.svc",
+    port: env.KUBERNETES_SERVICE_PORT_HTTPS ?? env.KUBERNETES_SERVICE_PORT ?? "443"
+  };
+}
+async function podLogs(client, jobName, container, tailLines = 4e3) {
+  const pods = await listJobPods(client, jobName);
+  if (typeof pods === "string") return pods;
+  const name = readString(readRecord(pods, "metadata"), "name");
+  if (!name) return "Job pod was not found.";
+  const logs = await client.request({
+    method: "GET",
+    path: `/api/v1/namespaces/${client.namespace}/pods/${name}/log?container=${encodeURIComponent(container)}&tailLines=${tailLines}`
+  });
+  if (logs.statusCode >= 400) return `Failed to read job logs: HTTP ${logs.statusCode}`;
+  return logs.body;
+}
+async function podFailureReason(client, jobName) {
+  const pod = await listJobPods(client, jobName);
+  if (typeof pod === "string") return pod;
+  return podFailureDetails(pod);
+}
+async function listJobPods(client, jobName) {
+  const pods = await client.request({
+    method: "GET",
+    path: `/api/v1/namespaces/${client.namespace}/pods?labelSelector=job-name%3D${encodeURIComponent(jobName)}`
+  });
+  if (pods.statusCode >= 400) return `Failed to list job pods: HTTP ${pods.statusCode}`;
+  const items = readArray(readRecord(JSON.parse(pods.body), void 0), "items");
+  const first = items[0];
+  return isRecord(first) ? first : "Job pod was not found.";
+}
+function podFailureDetails(pod) {
+  const status = readRecord(pod, "status");
+  const lines = [];
+  for (const key of ["phase", "reason", "message"]) {
+    const value = readString(status, key);
+    if (value) lines.push(`pod ${key}: ${value}`);
+  }
+  for (const item of readArray(status, "containerStatuses")) {
+    if (!isRecord(item)) continue;
+    const name = readString(item, "name") ?? "container";
+    lines.push(...containerStateDetails(name, readRecord(item, "state")));
+    lines.push(...containerStateDetails(`${name} last state`, readRecord(item, "lastState")));
+  }
+  return lines.join("\n") || "Pod failed without a Kubernetes status reason.";
+}
+function containerStateDetails(name, state) {
+  for (const phase of ["terminated", "waiting"]) {
+    const detail = readRecord(state, phase);
+    if (Object.keys(detail).length > 0) return [`${name} ${phase}: ${formatState(detail)}`];
+  }
+  return Object.keys(readRecord(state, "running")).length > 0 ? [`${name} running`] : [];
+}
+function formatState(state) {
+  const parts = [
+    readString(state, "reason"),
+    readNumberValue(state, "exitCode"),
+    readNumberValue(state, "signal"),
+    readString(state, "message")
+  ].filter(Boolean);
+  return parts.length > 0 ? parts.join(", ") : "no reason reported";
+}
+function kubernetesRequest(input) {
+  const body = input.body === void 0 ? void 0 : JSON.stringify(input.body);
+  return new Promise((resolve, reject) => {
+    const request = https.request({
+      host: input.host,
+      port: input.port,
+      path: input.path,
+      method: input.method,
+      ca: input.ca,
+      headers: {
+        Authorization: `Bearer ${input.token}`,
+        ...body === void 0 ? {} : { "Content-Type": "application/json", "Content-Length": Buffer.byteLength(body) }
+      }
+    }, (response) => {
+      response.setEncoding("utf8");
+      let data = "";
+      response.on("data", (chunk) => {
+        data += chunk;
+      });
+      response.on("end", () => {
+        resolve({ statusCode: response.statusCode ?? 0, body: data });
+      });
+    });
+    request.on("error", reject);
+    if (body !== void 0) request.write(body);
+    request.end();
+  });
+}
+function readRecord(value, key) {
+  const candidate = key === void 0 ? value : isRecord(value) ? value[key] : void 0;
+  return isRecord(candidate) ? candidate : {};
+}
+function readArray(record, key) {
+  const value = record[key];
+  return Array.isArray(value) ? value : [];
+}
+function readString(record, key) {
+  const value = record[key];
+  return typeof value === "string" && value.trim() ? value.trim() : void 0;
+}
+function readNumber(record, key) {
+  const value = record[key];
+  return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+function readNumberValue(record, key) {
+  const value = record[key];
+  return typeof value === "number" && Number.isFinite(value) ? `${key} ${value}` : void 0;
+}
+function isRecord(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+// lib/server/sdk-runtime-job-script.ts
+import path2 from "node:path";
+function sdkPrepareShellScript(input) {
+  const parent = path2.dirname(input.settings.preparedRoot);
+  const progressParent = path2.dirname(input.progressPath);
+  const liveLogParent = path2.dirname(input.liveLogPath);
+  const resultParent = path2.dirname(input.resultPath);
+  const cacheStatePath = sdkPrepareCacheStatePath(input.settings);
+  const cacheEnv = sdkPrepareCacheEnv(input.settings);
+  const cacheFingerprint = sdkPrepareCacheFingerprint(input.settings);
+  return [
+    "set -eu",
+    `mkdir -p ${shellQuote(parent)} ${shellQuote(progressParent)} ${shellQuote(liveLogParent)} ${shellQuote(resultParent)} ${shellQuote(path2.dirname(cacheStatePath))}`,
+    `progress_file=${shellQuote(input.progressPath)}`,
+    `live_log_file=${shellQuote(input.liveLogPath)}`,
+    `result_file=${shellQuote(input.resultPath)}`,
+    `cache_state_file=${shellQuote(cacheStatePath)}`,
+    `cache_fingerprint=${shellQuote(cacheFingerprint)}`,
+    `run_id=${shellQuote(input.runId)}`,
+    `runtime_root=${shellQuote(input.settings.runtimeRoot)}`,
+    `prepared_root=${shellQuote(input.settings.preparedRoot)}`,
+    `status_root=${shellQuote(input.statusRoot ?? input.settings.preparedRoot)}`,
+    `python_path=${shellQuote(input.settings.pythonPath)}`,
+    `repo_url=${shellQuote(input.settings.repoUrl)}`,
+    `sdk_ref=${shellQuote(input.settings.branch)}`,
+    `smoke_script=${shellQuote(process.env.AGORA_SDK_PREPARE_SMOKE_SCRIPT ?? "/app/scripts/sdk-runtime-smoke.py")}`,
+    `smoke_stages=${shellQuote(process.env.AGORA_SDK_PREPARE_SMOKE_STAGES ?? "MLIR,CORE_IR")}`,
+    `export PATH=${shellQuote(cacheEnv.PATH)}`,
+    `export LD_LIBRARY_PATH=${shellQuote(cacheEnv.LD_LIBRARY_PATH)}`,
+    `export CC=${shellQuote(cacheEnv.CC)}`,
+    `export CXX=${shellQuote(cacheEnv.CXX)}`,
+    `export CMAKE_PREFIX_PATH=${shellQuote(cacheEnv.CMAKE_PREFIX_PATH)}`,
+    `export LLVM_DIR=${shellQuote(cacheEnv.LLVM_DIR)}`,
+    `export MLIR_DIR=${shellQuote(cacheEnv.MLIR_DIR)}`,
+    `export OpenMP_ROOT=${shellQuote(cacheEnv.OpenMP_ROOT)}`,
+    `export UV_CACHE_DIR=${shellQuote(cacheEnv.UV_CACHE_DIR)}`,
+    `export UV_PYTHON_INSTALL_DIR=${shellQuote(cacheEnv.UV_PYTHON_INSTALL_DIR)}`,
+    `export CCACHE_DIR=${shellQuote(cacheEnv.CCACHE_DIR)}`,
+    `export CCACHE_BASEDIR=${shellQuote(cacheEnv.CCACHE_BASEDIR)}`,
+    `export UV_LINK_MODE=${shellQuote(cacheEnv.UV_LINK_MODE)}`,
+    `export UV_LOCK_TIMEOUT=${shellQuote(cacheEnv.UV_LOCK_TIMEOUT)}`,
+    "current_step=start",
+    "current_message='Preparing SDK source worktree'",
+    "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)",
+    'export AGORA_STARTED_AT="$started_at"',
+    `progress() { current_step=$1; current_message=$2; printf '{"at":"%s","runId":"%s","step":"%s","message":"%s"}\\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$run_id" "$1" "$2" >> "$progress_file"; }`,
+    `write_runtime_status() { AGORA_STATUS=$1 AGORA_COMPLETED_AT=$2 AGORA_MESSAGE=$3 AGORA_COMMIT_SHA=\${commit_sha:-} AGORA_REPO_URL="$repo_url" AGORA_BRANCH="$sdk_ref" AGORA_RUNTIME_ROOT="$runtime_root" AGORA_PREPARED_ROOT="$prepared_root" AGORA_STATUS_ROOT="$status_root" AGORA_PYTHON_PATH="$python_path" node <<'NODE'
+const fs = require('node:fs');
+const path = require('node:path');
+const status = {
+  status: process.env.AGORA_STATUS,
+  repoUrl: process.env.AGORA_REPO_URL,
+  branch: process.env.AGORA_BRANCH,
+  runtimeRoot: path.resolve(process.env.AGORA_RUNTIME_ROOT),
+  preparedRoot: path.resolve(process.env.AGORA_PREPARED_ROOT),
+  pythonPath: process.env.AGORA_PYTHON_PATH,
+  startedAt: process.env.AGORA_STARTED_AT,
+  completedAt: process.env.AGORA_COMPLETED_AT,
+  message: process.env.AGORA_MESSAGE,
+};
+if (process.env.AGORA_COMMIT_SHA) status.commitSha = process.env.AGORA_COMMIT_SHA;
+for (const root of new Set([process.env.AGORA_PREPARED_ROOT, process.env.AGORA_STATUS_ROOT])) {
+  fs.mkdirSync(root, { recursive: true });
+  fs.writeFileSync(path.join(root, 'runtime.json'), \`\${JSON.stringify(status, null, 2)}\\n\`);
+}
+NODE
+}`,
+    'run_smoke_probe() { PYTHONPATH="$prepared_root/legato_aten_lib${PYTHONPATH:+:$PYTHONPATH}" "$python_path" "$smoke_script" --repo-root "$prepared_root" --output-root "$prepared_root/.agora/smoke" --stages "$smoke_stages"; }',
+    `finish() { code=$?; trap - EXIT; completed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ); if [ "$code" -ne 0 ]; then message=$(printf 'SDK prepare job failed during %s' "$current_step"); progress failed "$message"; write_runtime_status failed "$completed_at" "$message"; printf '{"status":"failed","exitCode":%s,"completedAt":"%s","message":"%s"}\\n' "$code" "$completed_at" "$message" > "$result_file"; else write_runtime_status ready "$completed_at" "SDK prepare job completed"; printf '{"status":"ready","exitCode":0,"completedAt":"%s","message":"SDK prepare job completed"}\\n' "$completed_at" > "$result_file"; fi; exit "$code"; }`,
+    "trap finish EXIT",
+    `printf '\\n===== SDK prepare run %s started at %s =====\\n' "$run_id" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$live_log_file"`,
+    'progress start "Preparing SDK source worktree"',
+    'exec >> "$live_log_file" 2>&1',
+    'progress git-auth "Configuring GitHub authentication"',
+    `printf '\\n[%s] Configuring GitHub authentication\\n' "$(date -u +%H:%M:%S)"`,
+    `if [ -n "\${AGORA_GITHUB_TOKEN:-}" ]; then git config --global http.https://github.com/.extraheader "AUTHORIZATION: basic $(printf 'x-access-token:%s' "$AGORA_GITHUB_TOKEN" | base64 | tr -d '\\n')"; fi`,
+    "git config --global url.https://github.com/.insteadOf git@github.com:",
+    'progress sync-source "Syncing SDK source"',
+    `printf '\\n[%s] Syncing SDK source %s#%s\\n' "$(date -u +%H:%M:%S)" ${shellQuote(input.settings.repoUrl)} ${shellQuote(input.settings.branch)}`,
+    `if [ -d ${shellQuote(path2.join(input.settings.preparedRoot, ".git"))} ]; then`,
+    `  git -C ${shellQuote(input.settings.preparedRoot)} remote set-url origin ${shellQuote(input.settings.repoUrl)}`,
+    `  git -C ${shellQuote(input.settings.preparedRoot)} reset --hard`,
+    `  git -C ${shellQuote(input.settings.preparedRoot)} clean -ffd -e .venv/`,
+    "else",
+    `  if [ -d ${shellQuote(input.settings.preparedRoot)} ] && [ "$(find ${shellQuote(input.settings.preparedRoot)} -mindepth 1 -maxdepth 1 | head -n 1)" ]; then backup=$(printf '%s.pre-sdk-job-%s' ${shellQuote(input.settings.preparedRoot)} "$(date +%s)"); mv ${shellQuote(input.settings.preparedRoot)} "$backup"; fi`,
+    `  git clone --no-checkout ${shellQuote(input.settings.repoUrl)} ${shellQuote(input.settings.preparedRoot)}`,
+    "fi",
+    `git -C ${shellQuote(input.settings.preparedRoot)} fetch origin ${shellQuote(input.settings.branch)}`,
+    `git -C ${shellQuote(input.settings.preparedRoot)} checkout --force --detach FETCH_HEAD`,
+    `git -C ${shellQuote(input.settings.preparedRoot)} reset --hard FETCH_HEAD`,
+    `cd ${shellQuote(input.settings.preparedRoot)}`,
+    // A prepare killed mid-clone -- a pod restart during a deploy is enough --
+    // leaves a submodule whose HEAD cannot be resolved ("Unable to find current
+    // revision in submodule path"), and every later prepare dies on the same
+    // wreckage. It outlives the pod on the shared volume, so without this repair
+    // one interrupted prepare blocks every SDK update from then on.
+    `printf '\\n[%s] Syncing SDK submodules\\n' "$(date -u +%H:%M:%S)"`,
+    "git submodule sync --recursive || true",
+    "if ! git submodule update --init --recursive; then",
+    `  printf '\\n[%s] Submodule update failed; discarding submodule state and retrying from scratch\\n' "$(date -u +%H:%M:%S)"`,
+    "  git submodule deinit -f --all || true",
+    "  rm -rf .git/modules",
+    "  git submodule update --init --recursive",
+    "fi",
+    "commit_sha=$(git rev-parse HEAD)",
+    `source_tree_listing=$(git ls-tree HEAD -- ${SDK_PREPARE_SOURCE_PATHS.map(shellQuote).join(" ")})`,
+    `source_fingerprint=$(SOURCE_TREE_LISTING="$source_tree_listing" node <<'NODE'`,
+    "const crypto = require('node:crypto');",
+    "const input = `sdk-prepare-source-v1\\n${process.env.SOURCE_TREE_LISTING || ''}`;",
+    "process.stdout.write(crypto.createHash('sha256').update(input).digest('hex'));",
+    "NODE",
+    ")",
+    'mkdir -p "$UV_CACHE_DIR" "$CCACHE_DIR" "$(dirname "$cache_state_file")"',
+    `printf '\\ncommitSha: %s\\nsourceFingerprint: %s\\npreparedRoot: %s\\ncacheState: %s\\n' "$commit_sha" "$source_fingerprint" "$PWD" "$cache_state_file"`,
+    `printf '\\n[%s] Runtime storage before prepare\\n' "$(date -u +%H:%M:%S)"`,
+    'df -h "$runtime_root" || true',
+    'du -sh "$runtime_root"/.agora/cache "$runtime_root"/.agora/cache/uv "$runtime_root"/.agora/cache/ccache 2>/dev/null || true',
+    `printf '\\n[%s] Runtime volume usage breakdown\\n' "$(date -u +%H:%M:%S)"`,
+    'du -h -d 1 "$(dirname "$runtime_root")" 2>/dev/null | sort -h | tail -n 40 || true',
+    `printf '\\n[%s] SDK runtime root usage breakdown\\n' "$(date -u +%H:%M:%S)"`,
+    'du -h -d 1 "$runtime_root" 2>/dev/null | sort -h | tail -n 60 || true',
+    `find "$UV_CACHE_DIR" -mindepth 1 -maxdepth 1 -type d -name '.tmp*' -exec rm -rf {} + 2>/dev/null || true`,
+    `printf '\\n[%s] ccache stats before prepare\\n' "$(date -u +%H:%M:%S)"`,
+    "ccache --show-stats --verbose || true",
+    'progress cache-check "Checking prepared SDK runtime cache"',
+    `printf '\\n[%s] Checking prepared SDK runtime cache\\n' "$(date -u +%H:%M:%S)"`,
+    'if [ -f "$cache_state_file" ]; then',
+    `  if AGORA_CACHE_STATE_FILE="$cache_state_file" AGORA_CACHE_REPO_URL=${shellQuote(input.settings.repoUrl)} AGORA_CACHE_BRANCH=${shellQuote(input.settings.branch)} AGORA_CACHE_COMMIT="$commit_sha" AGORA_CACHE_SOURCE_FINGERPRINT="$source_fingerprint" AGORA_CACHE_FINGERPRINT="$cache_fingerprint" AGORA_CACHE_PREPARED_ROOT=${shellQuote(input.settings.preparedRoot)} node <<'NODE'`,
+    "const fs = require('node:fs');",
+    "const path = require('node:path');",
+    "const state = JSON.parse(fs.readFileSync(process.env.AGORA_CACHE_STATE_FILE, 'utf8'));",
+    "const ok = state.status === 'ready'",
+    "  && state.repoUrl === process.env.AGORA_CACHE_REPO_URL",
+    "  && state.branch === process.env.AGORA_CACHE_BRANCH",
+    "  && state.fingerprint === process.env.AGORA_CACHE_FINGERPRINT",
+    "  && (state.sourceFingerprint ? state.sourceFingerprint === process.env.AGORA_CACHE_SOURCE_FINGERPRINT : state.commitSha === process.env.AGORA_CACHE_COMMIT)",
+    "  && path.resolve(state.preparedRoot) === path.resolve(process.env.AGORA_CACHE_PREPARED_ROOT);",
+    "process.exit(ok ? 0 : 1);",
+    "NODE",
+    "  then",
+    "    if run_smoke_probe; then",
+    `      AGORA_CACHE_STATE_FILE="$cache_state_file" AGORA_CACHE_REPO_URL=${shellQuote(input.settings.repoUrl)} AGORA_CACHE_BRANCH=${shellQuote(input.settings.branch)} AGORA_CACHE_COMMIT="$commit_sha" AGORA_CACHE_SOURCE_FINGERPRINT="$source_fingerprint" AGORA_CACHE_FINGERPRINT="$cache_fingerprint" AGORA_CACHE_PREPARED_ROOT=${shellQuote(input.settings.preparedRoot)} node <<'NODE'`,
+    "const fs = require('node:fs');",
+    "const path = require('node:path');",
+    "const statePath = process.env.AGORA_CACHE_STATE_FILE;",
+    "const state = {",
+    "  status: 'ready',",
+    "  repoUrl: process.env.AGORA_CACHE_REPO_URL,",
+    "  branch: process.env.AGORA_CACHE_BRANCH,",
+    "  commitSha: process.env.AGORA_CACHE_COMMIT,",
+    "  fingerprint: process.env.AGORA_CACHE_FINGERPRINT,",
+    "  sourceFingerprint: process.env.AGORA_CACHE_SOURCE_FINGERPRINT,",
+    "  preparedRoot: path.resolve(process.env.AGORA_CACHE_PREPARED_ROOT),",
+    "  updatedAt: new Date().toISOString(),",
+    "};",
+    "fs.writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\\n`);",
+    "NODE",
+    `      printf '\\n[%s] SDK runtime cache hit; prepare command skipped\\n' "$(date -u +%H:%M:%S)"`,
+    '      progress complete "SDK runtime cache hit"',
+    "      exit 0",
+    "    fi",
+    `    printf '\\n[%s] SDK runtime cache marker found, but smoke probe failed\\n' "$(date -u +%H:%M:%S)"`,
+    "  fi",
+    "fi",
+    'progress install "Installing SDK runtime dependencies"',
+    `printf '\\n[%s] Running prepare command\\n%s\\n' "$(date -u +%H:%M:%S)" ${shellQuote(input.settings.prepareCommand)}`,
+    "set +e",
+    `sh -lc ${shellQuote(input.settings.prepareCommand)}`,
+    "prepare_code=$?",
+    "set -e",
+    `if [ "$prepare_code" -ne 0 ]; then printf '\\n[%s] Prepare command failed with exit %s\\n' "$(date -u +%H:%M:%S)" "$prepare_code"; exit "$prepare_code"; fi`,
+    `printf '\\n[%s] ccache stats after prepare\\n' "$(date -u +%H:%M:%S)"`,
+    "ccache --show-stats --verbose || true",
+    'progress commit "Recording SDK commit"',
+    `printf '\\n[%s] Recording SDK commit\\n' "$(date -u +%H:%M:%S)"`,
+    `printf '%s\\n' "$commit_sha"`,
+    'progress probe "Verifying SDK runtime lowering smoke"',
+    `printf '\\n[%s] Verifying SDK runtime lowering smoke\\n' "$(date -u +%H:%M:%S)"`,
+    "run_smoke_probe",
+    'progress cache-write "Recording SDK runtime cache state"',
+    `printf '\\n[%s] Recording SDK runtime cache state\\n' "$(date -u +%H:%M:%S)"`,
+    `AGORA_CACHE_STATE_FILE="$cache_state_file" AGORA_CACHE_REPO_URL=${shellQuote(input.settings.repoUrl)} AGORA_CACHE_BRANCH=${shellQuote(input.settings.branch)} AGORA_CACHE_COMMIT="$commit_sha" AGORA_CACHE_SOURCE_FINGERPRINT="$source_fingerprint" AGORA_CACHE_FINGERPRINT="$cache_fingerprint" AGORA_CACHE_PREPARED_ROOT=${shellQuote(input.settings.preparedRoot)} node <<'NODE'`,
+    "const fs = require('node:fs');",
+    "const path = require('node:path');",
+    "const statePath = process.env.AGORA_CACHE_STATE_FILE;",
+    "const state = {",
+    "  status: 'ready',",
+    "  repoUrl: process.env.AGORA_CACHE_REPO_URL,",
+    "  branch: process.env.AGORA_CACHE_BRANCH,",
+    "  commitSha: process.env.AGORA_CACHE_COMMIT,",
+    "  fingerprint: process.env.AGORA_CACHE_FINGERPRINT,",
+    "  sourceFingerprint: process.env.AGORA_CACHE_SOURCE_FINGERPRINT,",
+    "  preparedRoot: path.resolve(process.env.AGORA_CACHE_PREPARED_ROOT),",
+    "  updatedAt: new Date().toISOString(),",
+    "};",
+    "fs.writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\\n`);",
+    "NODE",
+    'progress complete "SDK prepare command completed"'
+  ].join("\n");
+}
+function shellQuote(value) {
+  return `'${value.replace(/'/g, "'\\''")}'`;
+}
+
+// lib/server/sdk-runtime-kubernetes-job-spec.ts
+function sdkPrepareJobSpec(input) {
+  return {
+    apiVersion: "batch/v1",
+    kind: "Job",
+    metadata: {
+      name: input.jobName,
+      namespace: input.namespace,
+      labels: { app: "legato-dev-agora", "agora.hyperaccel.net/job": "sdk-prepare" }
+    },
+    spec: {
+      ttlSecondsAfterFinished: 3600,
+      activeDeadlineSeconds: 14400,
+      backoffLimit: 0,
+      template: {
+        metadata: {
+          labels: { app: "legato-dev-agora", "agora.hyperaccel.net/job": "sdk-prepare" },
+          annotations: { "sidecar.istio.io/inject": "false" }
+        },
+        spec: {
+          restartPolicy: "Never",
+          serviceAccountName: "legato-dev-agora",
+          imagePullSecrets: [{ name: "harbor-secret" }],
+          affinity: {
+            podAffinity: {
+              requiredDuringSchedulingIgnoredDuringExecution: [{
+                labelSelector: { matchLabels: { app: "legato-dev-agora" } },
+                topologyKey: "kubernetes.io/hostname"
+              }]
+            }
+          },
+          containers: [{
+            name: "sdk-prepare",
+            image: input.image,
+            imagePullPolicy: "Always",
+            command: ["sh", "-lc", sdkPrepareShellScript(input)],
+            env: [
+              {
+                name: "AGORA_GITHUB_TOKEN",
+                valueFrom: { secretKeyRef: { name: input.secretName, key: "token", optional: true } }
+              },
+              // Forwarded from the web pod: the prepare command installs the SDK
+              // team's published wheels when an index is configured, and builds
+              // from source when it is not.
+              ...sdkPackageIndexEnv(process.env, input.pin)
+            ],
+            volumeMounts: [{ name: "data", mountPath: "/data" }],
+            resources: {
+              requests: { cpu: "500m", memory: "1Gi" },
+              limits: { cpu: "2000m", memory: "4Gi" }
+            }
+          }],
+          volumes: [{ name: "data", persistentVolumeClaim: { claimName: input.pvcName } }]
+        }
+      }
+    }
+  };
+}
+
+// lib/server/sdk-runtime-kubernetes-job.ts
+var JOB_TIMEOUT_MS = 144e5;
+var POLL_INTERVAL_MS = 2e3;
+async function runSdkPrepareJob(settings, run, statusRoot = settings.preparedRoot, pin) {
+  const client = await inClusterKubernetesClient();
+  const jobName = sdkPrepareJobName(settings);
+  const spec = sdkPrepareJobSpec({
+    settings,
+    statusRoot,
+    namespace: client.namespace,
+    jobName,
+    image: process.env.AGORA_SDK_PREPARE_JOB_IMAGE ?? "cr.hyperaccel.net/hyperaccel/legato-dev-agora:latest",
+    secretName: process.env.AGORA_GITHUB_SECRET_NAME ?? "legato-dev-agora-github",
+    pvcName: process.env.AGORA_DATA_PVC_NAME ?? "legato-dev-agora-data",
+    runId: run.runId,
+    progressPath: run.progressPath,
+    liveLogPath: run.liveLogPath,
+    resultPath: run.resultPath,
+    pin
+  });
+  const created = await client.request({ method: "POST", path: `/apis/batch/v1/namespaces/${client.namespace}/jobs`, body: spec });
+  if (created.statusCode >= 400) {
+    return { succeeded: false, logs: created.body, message: `Kubernetes SDK prepare job creation failed: HTTP ${created.statusCode}` };
+  }
+  return await waitForJob(client, jobName, readPositiveInt(process.env.AGORA_SDK_PREPARE_JOB_TIMEOUT_MS, JOB_TIMEOUT_MS));
+}
+async function waitForJob(client, jobName, timeoutMs) {
+  const started = Date.now();
+  while (Date.now() - started < timeoutMs) {
+    const job = await client.request({ method: "GET", path: `/apis/batch/v1/namespaces/${client.namespace}/jobs/${jobName}` });
+    if (job.statusCode >= 400) return { succeeded: false, logs: "", message: `Kubernetes job lookup failed: HTTP ${job.statusCode}` };
+    const status = readRecord(readRecord(JSON.parse(job.body), "status"), void 0);
+    if (readNumber(status, "succeeded") > 0) return { succeeded: true, logs: await podLogs(client, jobName, "sdk-prepare") };
+    if (readNumber(status, "failed") > 0) {
+      const details = await podFailureReason(client, jobName);
+      return {
+        succeeded: false,
+        logs: joinLogSections(await podLogs(client, jobName, "sdk-prepare"), details ? `Kubernetes failure details:
+${details}` : ""),
+        message: details ? `SDK prepare job failed: ${firstLine(details)}` : "SDK prepare job failed."
+      };
+    }
+    await delay(POLL_INTERVAL_MS);
+  }
+  return { succeeded: false, logs: await podLogs(client, jobName, "sdk-prepare"), message: "SDK prepare job timed out." };
+}
+function sdkPrepareJobName(settings) {
+  const suffix = randomUUID().replace(/-/g, "").slice(0, 10);
+  const branch = settings.branch.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 24) || "branch";
+  return `sdk-prepare-${branch}-${suffix}`.slice(0, 63);
+}
+function joinLogSections(...sections) {
+  return sections.map((section) => section.trim()).filter(Boolean).join("\n\n");
+}
+function firstLine(text) {
+  return text.split("\n").map((line) => line.trim()).find(Boolean) ?? "unknown Kubernetes failure";
+}
+function readPositiveInt(value, fallback) {
+  if (!value) return fallback;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+async function delay(ms) {
+  await new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+export {
+  kubernetesApiEndpointFromEnv,
+  runSdkPrepareJob,
+  sdkPrepareJobSpec
+};

package/dist/scripts/sdk-runtime-smoke.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+SMOKE_KERNEL = '''#!/usr/bin/env python3
+from __future__ import annotations
+
+from pathlib import Path
+import os
+import sys
+
+
+def _prepare_imports(repo_root: Path) -> None:
+    for path in (repo_root, repo_root / "legato" / "src"):
+        text = str(path)
+        if text not in sys.path:
+            sys.path.insert(0, text)
+
+
+def _prepare_bertha(ctx):
+    import legato.model.bertha as bertha_model
+
+    return bertha_model.Bertha(ctx, "default", 32, False, 8, 128 * pow(1024, 3))
+
+
+def run(output_type_name: str, output_root: Path) -> None:
+    import torch
+
+    import legato
+    import legato.model.bertha as bertha
+    from legato.language import arith
+
+    @legato.compile(
+        backend=_prepare_bertha,
+        param_kinds={"tensor_a": "universal", "tensor_b": "top", "tensor_out": "core"},
+    )
+    def agora_runtime_smoke_kernel(
+        tensor_a: legato.get_model().tensor_type(
+            legato.types.float("bfloat16"), (32, (-1, 8), (-1, 32)), "shared_dram"
+        ),
+        tensor_b: legato.get_model().tensor_type(
+            legato.types.float("bfloat16"), (32, (-1, 32), 128), "shared_dram"
+        ),
+        tensor_out: legato.get_model().tensor_type(
+            legato.types.float("bfloat16"), ((-1, 256), 128), "shared_dram"
+        ),
+    ):
+        device = legato.get_context().get_device()
+        seq_len = legato.tensor.get_dimension(tensor_a, 1)
+        sram_a_type = legato.get_model().tensor_type(
+            legato.types.float("bfloat16"), ((-1, 8), (-1, 64)), "row_major"
+        )
+        sram_b_type = legato.get_model().tensor_type(
+            legato.types.float("bfloat16"), ((-1, 32), 128), "row_major"
+        )
+        zero = arith.cast(0, legato.types.index())
+        one = arith.cast(1, legato.types.index())
+        batch_end = arith.cast(32, legato.types.index())
+
+        with device.get_top():
+            for i in legato.range(zero, batch_end):
+                a_i_dram = legato.tensor.view(
+                    tensor_a,
+                    (legato.range(i, i + one), legato.range(0, seq_len), legato.range(0, seq_len)),
+                )
+                b_i_dram = legato.tensor.view(
+                    tensor_b,
+                    (legato.range(i, i + one), legato.range(0, seq_len), legato.range(0, 128)),
+                )
+                legato.tensor.request_load(a_i_dram, device.get_core(0), "sram")
+                legato.tensor.request_load(b_i_dram, device.get_core(0), "sram")
+
+        with device.get_core(0):
+            for i in legato.range(zero, batch_end):
+                loaded_a_i = legato.tensor.receive_type(sram_a_type, 0, (seq_len, seq_len), "sram")
+                loaded_b_i = legato.tensor.receive_type(sram_b_type, 1, (seq_len,), "sram")
+                z_i = loaded_a_i @ loaded_b_i
+                row_start_i = seq_len * i
+                row_end_i = seq_len * (i + one)
+                out_i = legato.tensor.view(
+                    tensor_out,
+                    (legato.range(row_start_i, row_end_i), legato.range(0, 128)),
+                )
+                legato.tensor.memcpy(out_i, z_i)
+
+    seq_len = 8
+    tensor_a = torch.randn(32, seq_len, seq_len, dtype=torch.bfloat16)
+    tensor_b = torch.randn(32, seq_len, 128, dtype=torch.bfloat16)
+    tensor_out = torch.zeros(32 * seq_len, 128, dtype=torch.bfloat16)
+    output_type = getattr(legato.OutputType, output_type_name)
+    with legato.session(output_type=output_type, output_path=str(output_root)):
+        agora_runtime_smoke_kernel(tensor_a, legato.Arg(tensor_b, bertha.top), legato.Arg(tensor_out, bertha.core(0)))
+
+
+def main() -> int:
+    repo_root = Path(os.environ.get("AGORA_SDK_SMOKE_REPO_ROOT", Path.cwd())).resolve()
+    output_type = os.environ["AGORA_SDK_SMOKE_OUTPUT_TYPE"]
+    output_root = Path(os.environ["AGORA_SDK_SMOKE_OUTPUT_ROOT"]).resolve()
+    os.chdir(repo_root)
+    _prepare_imports(repo_root)
+    run(output_type, output_root)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+'''
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Run prepared SDK lowering smoke")
+    parser.add_argument("--repo-root", required=True)
+    parser.add_argument("--output-root", required=True)
+    parser.add_argument("--stages", default="MLIR")
+    parser.add_argument("--runner", default=str(Path(__file__).resolve().with_name("legato-lowering-runner.py")))
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    repo_root = Path(args.repo_root).resolve()
+    output_root = Path(args.output_root).resolve()
+    smoke_root = repo_root / ".agora" / "smoke"
+    kernel_path = smoke_root / "agora_runtime_smoke.py"
+    kernel_path.parent.mkdir(parents=True, exist_ok=True)
+    output_root.mkdir(parents=True, exist_ok=True)
+    kernel_path.write_text(SMOKE_KERNEL, encoding="utf-8")
+
+    stages = [stage.strip() for stage in args.stages.split(",") if stage.strip()]
+    if not stages:
+        stages = ["MLIR"]
+
+    print(f"[SDK_SMOKE] repo_root={repo_root}", flush=True)
+    print(f"[SDK_SMOKE] stages={','.join(stages)}", flush=True)
+    for stage in stages:
+        stage_output = output_root / stage.lower()
+        stage_output.mkdir(parents=True, exist_ok=True)
+        env = {
+            **dict(os.environ),
+            "AGORA_SDK_SMOKE_REPO_ROOT": str(repo_root),
+            "AGORA_SDK_SMOKE_OUTPUT_TYPE": stage,
+            "AGORA_SDK_SMOKE_OUTPUT_ROOT": str(stage_output),
+        }
+        command = [
+            sys.executable,
+            str(Path(args.runner).resolve()),
+            "--kernel",
+            str(kernel_path),
+            "--output-root",
+            str(output_root),
+            "--stages",
+            stage,
+        ]
+        print(f"[SDK_SMOKE] stage={stage} status=running", flush=True)
+        subprocess.run(command, cwd=repo_root, env=env, check=True)
+        print(f"[SDK_SMOKE] stage={stage} status=success", flush=True)
+    print("[SDK_SMOKE] lowering_smoke=success", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())