@dtechvision/fabrik-runtime 0.1.0

package/README.md

@@ -0,0 +1,182 @@
+# @dtechvision/fabrik-runtime
+
+Shared TypeScript utilities for Fabrik workflow pods.
+
+- **Credential pool** — read from mounted `/etc/fabrik/credentials`, rotate on failure, notify operators
+- **Codex auth rotation** — rotate among `auth.json` / `*.auth.json` credentials for Codex-backed workflows
+- **K8s jobs** — dispatch child verification jobs from a running workflow
+- **JJ shell** — deterministic JJ/Git snapshot, bookmark push, workspace prep
+
+## Import Surface
+
+Workflows should import from `@dtechvision/fabrik-runtime/...`.
+
+- `@dtechvision/fabrik-runtime/credential-pool`
+- `@dtechvision/fabrik-runtime/codex-auth`
+- `@dtechvision/fabrik-runtime/jj-shell`
+- `@dtechvision/fabrik-runtime/k8s-jobs`
+
+For in-cluster Fabrik runs, the Smithers runtime image ships this package in its `node_modules`.
+For local workflow development in another repo, add the package as a dependency from a release or local path.
+
+## Installation
+
+Install from npm:
+
+```bash
+bun add @dtechvision/fabrik-runtime
+```
+
+or:
+
+```bash
+npm install @dtechvision/fabrik-runtime
+```
+
+Smithers workflows also need their normal workflow dependencies in the consuming repo:
+
+```bash
+bun add smithers-orchestrator zod
+```
+
+or:
+
+```bash
+npm install smithers-orchestrator zod
+```
+
+Package releases follow the same `v*` tag version as the Fabrik CLI release flow.
+
+## Smithers Integration
+
+Use the package from ordinary Smithers workflows:
+
+```ts
+/** @jsxImportSource smithers-orchestrator */
+import { createSmithers, Task, Workflow } from "smithers-orchestrator";
+import { z } from "zod";
+import { withCodexAuthPoolEnv } from "@dtechvision/fabrik-runtime/codex-auth";
+import { prepareWorkspaces } from "@dtechvision/fabrik-runtime/jj-shell";
+
+const { smithers, outputs } = createSmithers(
+  {
+    report: z.object({
+      codexHomeSet: z.boolean(),
+      jjHelpersLoaded: z.boolean(),
+    }),
+  },
+  { dbPath: process.env.SMITHERS_DB_PATH ?? ".smithers/runtime-check.db" },
+);
+
+export default smithers(() => (
+  <Workflow name="runtime-package-check">
+    <Task id="verify" output={outputs.report}>
+      {async () => {
+        const env = withCodexAuthPoolEnv({});
+        return {
+          codexHomeSet: typeof env.CODEX_HOME === "string" && env.CODEX_HOME.length > 0,
+          jjHelpersLoaded: typeof prepareWorkspaces === "function",
+        };
+      }}
+    </Task>
+  </Workflow>
+));
+```
+
+Run it locally with Smithers from a repo that has installed:
+
+- `@dtechvision/fabrik-runtime`
+- `smithers-orchestrator`
+- `zod`
+
+Then:
+
+```bash
+bunx smithers run path/to/workflow.tsx --run-id runtime-package-check
+```
+
+The workflow file should live in the consuming project tree so normal Node/Bun package resolution can find the installed dependencies.
+
+## Credentials
+
+Operators manage `fabrik-credentials` in `fabrik-system` via kubectl. The CLI mirrors it into the run namespace at dispatch time. The secret is directory-mounted (no subPath) at `/etc/fabrik/credentials/` so running pods observe file replacements.
+
+```ts
+import { injectCredentialEnv } from "@dtechvision/fabrik-runtime/credential-pool";
+
+// Reads /etc/fabrik/credentials/ANTHROPIC_API_KEY → process.env.ANTHROPIC_API_KEY
+injectCredentialEnv("ANTHROPIC_API_KEY");
+```
+
+For file-pool rotation (e.g. multiple Codex auth files):
+
+```ts
+import { CredentialFilePool } from "@dtechvision/fabrik-runtime/credential-pool";
+
+const pool = new CredentialFilePool({
+  prefix: "codex-auth",
+  extension: ".json",
+  activeDir: "/tmp/codex-active",
+  activeFilename: "auth.json",
+  agent: "codex",
+});
+pool.init();
+
+// On auth failure:
+const rotated = await pool.handleError(err);
+```
+
+For Codex-specific rotation, use the higher-level helper:
+
+```ts
+import { createCodexAgentWithPool } from "@dtechvision/fabrik-runtime/codex-auth";
+
+const codex = createCodexAgentWithPool({
+  model: "gpt-5",
+  cwd: process.cwd(),
+  env: {},
+});
+```
+
+## Local Verification
+
+Runtime package tests:
+
+```bash
+cd src/fabrik-runtime
+bun test ./src
+```
+
+Repo-wide CLI and workflow verification:
+
+```bash
+make verify-cli
+make verify-cli-k3d
+```
+
+Focused runtime-package k3d import verification:
+
+```bash
+cd src/fabrik-cli
+FABRIK_K3D_E2E=1 FABRIK_K3D_CLUSTER=dev-single \
+  go test ./internal/run -run TestK3dWorkflowRuntimePackageImports -timeout 10m -v
+```
+
+The complex sample in [examples/complex/README.md](/Users/samuel/git/local-isolated-ralph/examples/complex/README.md) shows how workflow code consumes the package surface in practice.
+
+Local Smithers CLI verification:
+
+```bash
+bunx smithers run path/to/workflow.tsx --run-id runtime-package-check
+```
+
+The expected result is a successful run whose output reports:
+
+- `codexHomeSet: true`
+- `jjHelpersLoaded: true`
+
+## Precedence
+
+1. Fabrik runtime metadata (`SMITHERS_*`, `FABRIK_*`, `KUBERNETES_*`)
+2. Project env (`fabrik-env-<project>-<env>`) via `envFrom`
+3. Shared credentials (`fabrik-credentials`) via file mount

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@dtechvision/fabrik-runtime",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts",
+    "./credential-pool": "./src/credential-pool.ts",
+    "./codex-auth": "./src/codex-auth.ts",
+    "./k8s-jobs": "./src/k8s-jobs.ts",
+    "./jj-shell": "./src/jj-shell.ts"
+  },
+  "files": [
+    "src/index.ts",
+    "src/credential-pool.ts",
+    "src/codex-auth.ts",
+    "src/k8s-jobs.ts",
+    "src/jj-shell.ts",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/SamuelLHuber/local-isolated-ralph.git",
+    "directory": "src/fabrik-runtime"
+  },
+  "license": "MIT",
+  "scripts": {
+    "test": "bun test ./src"
+  },
+  "dependencies": {
+    "smithers-orchestrator": "0.9.1"
+  },
+  "devDependencies": {
+    "bun-types": "1.2.12"
+  }
+}

package/src/codex-auth.ts

@@ -0,0 +1,231 @@
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, resolve } from "node:path";
+import { CodexAgent } from "smithers-orchestrator";
+import {
+  classifyFailure,
+  getCredentialMountPath,
+  type FailureKind,
+} from "./credential-pool";
+
+const DEFAULT_CODEX_DIR = resolve(process.env.HOME ?? "", ".codex");
+
+function getCodexAuthSourceDir(): string {
+  const sourceDir =
+    process.env.CODEX_AUTH_SOURCE_DIR ??
+    process.env.FABRIK_SHARED_CREDENTIALS_DIR ??
+    (existsSync(getCredentialMountPath()) ? getCredentialMountPath() : DEFAULT_CODEX_DIR);
+  return resolve(sourceDir);
+}
+
+export const CODEX_AUTH_HOME = resolve(
+  process.env.CODEX_AUTH_HOME ?? resolve(tmpdir(), "codex-auth-pool"),
+);
+
+const NOTIFY_WEBHOOK_URL = process.env.CODEX_AUTH_NOTIFY_WEBHOOK_URL?.trim() ?? "";
+const NOTIFY_CLUSTER = process.env.KUBERNETES_NAMESPACE?.trim() ?? "";
+const NOTIFY_RUN_ID = process.env.SMITHERS_RUN_ID?.trim() ?? "";
+
+const AUTH_ROTATE_PATTERN =
+  /no last agent message|usage limit|quota|rate limit|insufficient (?:credits|balance|quota)|payment required|billing|exceeded.*(quota|limit)|not signed in|please run 'codex login'|unauthorized|authentication required|authentication failed|forbidden|invalid (?:api key|token|credentials)|expired (?:token|credentials)/i;
+const AUTH_REFRESH_REUSED_PATTERN =
+  /refresh_token_reused|refresh token has already been used|could not be refreshed because your refresh token was already used/i;
+
+const listAuthFiles = (): string[] => {
+  const sourceDir = getCodexAuthSourceDir();
+  if (!existsSync(sourceDir)) return [];
+  return readdirSync(sourceDir)
+    .filter((name) => name.endsWith(".auth.json") || name === "auth.json")
+    .map((name) => resolve(sourceDir, name))
+    .sort();
+};
+
+const ensureCodexHome = () => {
+  if (!existsSync(CODEX_AUTH_HOME)) {
+    mkdirSync(CODEX_AUTH_HOME, { recursive: true });
+  }
+};
+
+let authPool = listAuthFiles();
+let authIndex = 0;
+let activeAuth = "";
+const authFailures = new Map<string, FailureKind>();
+
+export function resetCodexAuthStateForTests(): void {
+  authPool = [];
+  authIndex = 0;
+  activeAuth = "";
+  authFailures.clear();
+}
+
+const setActiveAuth = (authPath: string, reason: string) => {
+  ensureCodexHome();
+  const authContents = readFileSync(authPath, "utf8");
+  writeFileSync(resolve(CODEX_AUTH_HOME, "auth.json"), authContents, "utf8");
+  const previous = activeAuth ? ` from ${basename(activeAuth)}` : "";
+  activeAuth = authPath;
+  console.error(
+    `[fabrik-runtime] codex auth rotation${previous} -> ${basename(authPath)} (${reason})`,
+  );
+};
+
+const initAuthPool = () => {
+  ensureCodexHome();
+  authPool = listAuthFiles();
+  if (authPool.length === 0 || activeAuth) return;
+  const defaultAuth = resolve(getCodexAuthSourceDir(), "auth.json");
+  if (existsSync(defaultAuth)) {
+    setActiveAuth(defaultAuth, "initial");
+    return;
+  }
+  setActiveAuth(authPool[0]!, "initial");
+};
+
+const logAuthSummary = () => {
+  const total = authPool.length;
+  const failed = [...authFailures.entries()].map(
+    ([path, status]) => `${basename(path)}:${status}`,
+  );
+  const failedCount = authFailures.size;
+  const remaining = Math.max(total - failedCount, 0);
+  const active = activeAuth ? basename(activeAuth) : "none";
+  console.error(
+    `[fabrik-runtime] codex auth pool summary: total=${total} failed=${failedCount} remaining=${remaining} active=${active}`,
+  );
+  if (failed.length > 0) {
+    console.error(`[fabrik-runtime] failed auths: ${failed.join(", ")}`);
+  }
+};
+
+const rotateAuth = (reason: string): boolean => {
+  authPool = listAuthFiles();
+  if (authPool.length === 0) return false;
+  for (let i = 0; i < authPool.length; i += 1) {
+    const next = authPool[authIndex % authPool.length];
+    authIndex += 1;
+    if (next && next !== activeAuth && !authFailures.has(next)) {
+      setActiveAuth(next, reason);
+      logAuthSummary();
+      return true;
+    }
+  }
+  console.error("[fabrik-runtime] no codex auth left to rotate to");
+  logAuthSummary();
+  return false;
+};
+
+export const withCodexAuthPoolEnv = (env: Record<string, string>) => ({
+  ...env,
+  CODEX_HOME: CODEX_AUTH_HOME,
+});
+
+export type AuthFailureKind = FailureKind;
+
+export type AuthFailureEvent = {
+  authPath: string;
+  authName: string;
+  reason: string;
+  kind: AuthFailureKind;
+  message: string;
+  clusterNamespace?: string;
+  runId?: string;
+};
+
+export type RotatingCodexAgentOptions = {
+  onAuthFailure?: (event: AuthFailureEvent) => void | Promise<void>;
+};
+
+const notifyAuthFailure = async (
+  event: AuthFailureEvent,
+  onAuthFailure?: RotatingCodexAgentOptions["onAuthFailure"],
+) => {
+  if (onAuthFailure) {
+    await onAuthFailure(event);
+  }
+  if (!NOTIFY_WEBHOOK_URL) return;
+  try {
+    const response = await fetch(NOTIFY_WEBHOOK_URL, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(event),
+    });
+    if (!response.ok) {
+      console.error(
+        `[fabrik-runtime] codex auth notification failed: webhook status ${response.status}`,
+      );
+    }
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    console.error(`[fabrik-runtime] codex auth notification failed: ${message}`);
+  }
+};
+
+export const createCodexAgentWithPool = (
+  opts: ConstructorParameters<typeof CodexAgent>[0],
+  rotationOpts: RotatingCodexAgentOptions = {},
+) =>
+  new RotatingCodexAgent(
+    new CodexAgent({
+      ...opts,
+      env: withCodexAuthPoolEnv(opts.env ?? {}),
+    }),
+    rotationOpts,
+  );
+
+export class RotatingCodexAgent {
+  private readonly inner: CodexAgent;
+  private readonly onAuthFailure?: RotatingCodexAgentOptions["onAuthFailure"];
+
+  constructor(inner: CodexAgent, opts: RotatingCodexAgentOptions = {}) {
+    this.inner = inner;
+    this.onAuthFailure = opts.onAuthFailure;
+  }
+
+  get id() {
+    return this.inner.id;
+  }
+
+  get tools() {
+    return this.inner.tools;
+  }
+
+  async generate(args: Parameters<CodexAgent["generate"]>[0]) {
+    initAuthPool();
+    const attempts = Math.max(authPool.length, 1);
+    let lastError: unknown = null;
+    for (let i = 0; i < attempts; i += 1) {
+      try {
+        return await this.inner.generate(args);
+      } catch (err) {
+        lastError = err;
+        const message = err instanceof Error ? err.message : String(err);
+        if (!AUTH_ROTATE_PATTERN.test(message)) {
+          throw err;
+        }
+        if (activeAuth) {
+          const kind = classifyFailure(message);
+          authFailures.set(activeAuth, kind);
+          if (AUTH_REFRESH_REUSED_PATTERN.test(message)) {
+            console.error("[fabrik-runtime] codex refresh token reused; re-auth required");
+          }
+          await notifyAuthFailure(
+            {
+              authPath: activeAuth,
+              authName: basename(activeAuth),
+              reason: "codex generate failed and rotation was requested",
+              kind,
+              message,
+              clusterNamespace: NOTIFY_CLUSTER || undefined,
+              runId: NOTIFY_RUN_ID || undefined,
+            },
+            this.onAuthFailure,
+          );
+        }
+        if (!rotateAuth("codex auth / usage failure")) {
+          break;
+        }
+      }
+    }
+    throw lastError ?? new Error("Codex auth pool exhausted");
+  }
+}

package/src/credential-pool.ts

@@ -0,0 +1,266 @@
+/**
+ * Generic credential pool for Fabrik workflow pods.
+ *
+ * Credentials are managed by operators via kubectl and mounted into pods
+ * at /etc/fabrik/credentials as a Kubernetes Secret directory mount.
+ * This module reads from that mount, provides pool rotation for agents
+ * that support multiple credential files, and emits structured failure
+ * notifications without exposing secret contents.
+ *
+ * Architecture:
+ * - Operators create/update `fabrik-credentials` secret in `fabrik-system`
+ *   via kubectl (e.g. `kubectl create secret generic fabrik-credentials
+ *   --from-file=ANTHROPIC_API_KEY=./key.txt --from-literal=OPENAI_API_KEY=sk-...`)
+ * - Fabrik CLI mirrors the secret into the run namespace at dispatch time
+ * - The secret is directory-mounted (no subPath) at CREDENTIAL_MOUNT_PATH
+ *   so running pods observe file replacements for rotation
+ * - This module reads credential files from that mount directory
+ *
+ * Supported credential layouts:
+ * - Flat env-var keys: /etc/fabrik/credentials/ANTHROPIC_API_KEY (file contains the value)
+ * - Codex auth pool: /etc/fabrik/credentials/codex-auth.json,
+ *   /etc/fabrik/credentials/codex-auth-2.json, etc.
+ * - Claude Code: /etc/fabrik/credentials/ANTHROPIC_API_KEY
+ * - Pi: /etc/fabrik/credentials/FIREWORKS_API_KEY or provider config files
+ */
+import { existsSync, readFileSync, readdirSync, writeFileSync, mkdirSync } from "node:fs";
+import { basename, resolve } from "node:path";
+
+/** Default mount path for the fabrik-credentials directory. */
+export function getCredentialMountPath(): string {
+  return process.env.FABRIK_CREDENTIAL_PATH ?? "/etc/fabrik/credentials";
+}
+
+/** @deprecated Use getCredentialMountPath() for dynamic resolution. */
+export const CREDENTIAL_MOUNT_PATH = "/etc/fabrik/credentials";
+
+// ---------------------------------------------------------------------------
+// Failure classification
+// ---------------------------------------------------------------------------
+
+export type FailureKind =
+  | "refresh_token_reused"
+  | "usage_limit"
+  | "auth_invalid"
+  | "unknown";
+
+export type FailureEvent = {
+  credentialName: string;
+  kind: FailureKind;
+  message: string;
+  agent: string;
+  namespace?: string;
+  runId?: string;
+};
+
+const REFRESH_REUSED =
+  /refresh_token_reused|refresh token has already been used|could not be refreshed because your refresh token was already used/i;
+const USAGE_LIMIT =
+  /no last agent message|usage limit|quota|rate limit|insufficient (?:credits|balance|quota)|payment required|billing|exceeded.*(quota|limit)/i;
+const AUTH_INVALID =
+  /not signed in|please run.*login|unauthorized|authentication required|authentication failed|forbidden|invalid (?:api key|token|credentials)|expired (?:token|credentials)|Not logged in/i;
+
+export function classifyFailure(message: string): FailureKind {
+  if (REFRESH_REUSED.test(message)) return "refresh_token_reused";
+  if (AUTH_INVALID.test(message)) return "auth_invalid";
+  if (USAGE_LIMIT.test(message)) return "usage_limit";
+  return "unknown";
+}
+
+/** Returns true if the error message indicates an auth/credential problem
+ * that credential rotation might fix. */
+export function isRotatableFailure(message: string): boolean {
+  return classifyFailure(message) !== "unknown";
+}
+
+// ---------------------------------------------------------------------------
+// Notification
+// ---------------------------------------------------------------------------
+
+const NOTIFY_WEBHOOK = process.env.FABRIK_CREDENTIAL_NOTIFY_WEBHOOK?.trim() ?? "";
+
+export async function notifyFailure(event: FailureEvent): Promise<void> {
+  console.error(
+    `[fabrik-runtime] credential failure: ${event.credentialName} kind=${event.kind} agent=${event.agent}`,
+  );
+  if (!NOTIFY_WEBHOOK) return;
+  try {
+    const resp = await fetch(NOTIFY_WEBHOOK, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(event),
+    });
+    if (!resp.ok) {
+      console.error(`[fabrik-runtime] notification webhook returned ${resp.status}`);
+    }
+  } catch (err) {
+    console.error(`[fabrik-runtime] notification failed: ${err instanceof Error ? err.message : err}`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Credential reading from mounted directory
+// ---------------------------------------------------------------------------
+
+/** Read a single credential value from the mounted directory. */
+export function readCredential(name: string): string | null {
+  const mountPath = getCredentialMountPath();
+  const path = resolve(mountPath, name);
+  if (!existsSync(path)) return null;
+  return readFileSync(path, "utf8").trim();
+}
+
+/** List all credential file names in the mounted directory. */
+export function listCredentials(): string[] {
+  const mountPath = getCredentialMountPath();
+  if (!existsSync(mountPath)) return [];
+  return readdirSync(mountPath)
+    .filter((name) => !name.startsWith(".") && name !== "..timestamp_of_last_update")
+    .sort();
+}
+
+/** Read all credentials as a key→value map. */
+export function readAllCredentials(): Record<string, string> {
+  const result: Record<string, string> = {};
+  for (const name of listCredentials()) {
+    const value = readCredential(name);
+    if (value !== null) result[name] = value;
+  }
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// File pool rotation (for agents that use auth files like Codex)
+// ---------------------------------------------------------------------------
+
+export type PoolOptions = {
+  /** Glob pattern to match pool files, e.g. "codex-auth" matches codex-auth*.json */
+  prefix: string;
+  /** Extension to match, e.g. ".json" */
+  extension?: string;
+  /** Directory to write the active credential file to */
+  activeDir: string;
+  /** Filename for the active credential, e.g. "auth.json" */
+  activeFilename: string;
+  /** Agent name for failure events */
+  agent: string;
+};
+
+export class CredentialFilePool {
+  private pool: string[] = [];
+  private index = 0;
+  private active = "";
+  private failures = new Map<string, FailureKind>();
+  private readonly opts: PoolOptions;
+
+  constructor(opts: PoolOptions) {
+    this.opts = opts;
+  }
+
+  /** Scan the credential mount for pool files and activate the first one. */
+  init(): void {
+    this.pool = this.scanPool();
+    if (this.pool.length === 0) return;
+    if (this.active && this.pool.includes(this.active)) return;
+    this.activate(this.pool[0]!, "initial");
+  }
+
+  /** Number of available (non-failed) credentials. */
+  get available(): number {
+    return this.pool.filter((p) => !this.failures.has(p)).length;
+  }
+
+  get activeName(): string {
+    return this.active ? basename(this.active) : "";
+  }
+
+  /** Try to rotate to the next unfailed credential. Returns false if exhausted. */
+  rotate(reason: string): boolean {
+    this.pool = this.scanPool();
+    if (this.pool.length === 0) return false;
+    for (let i = 0; i < this.pool.length; i++) {
+      const next = this.pool[this.index % this.pool.length]!;
+      this.index++;
+      if (next !== this.active && !this.failures.has(next)) {
+        this.activate(next, reason);
+        return true;
+      }
+    }
+    console.error(`[fabrik-runtime] ${this.opts.agent} credential pool exhausted`);
+    return false;
+  }
+
+  /** Mark the current credential as failed and optionally notify. */
+  async markFailed(message: string): Promise<void> {
+    if (!this.active) return;
+    const kind = classifyFailure(message);
+    this.failures.set(this.active, kind);
+    await notifyFailure({
+      credentialName: basename(this.active),
+      kind,
+      message,
+      agent: this.opts.agent,
+      namespace: process.env.KUBERNETES_NAMESPACE?.trim(),
+      runId: process.env.SMITHERS_RUN_ID?.trim(),
+    });
+  }
+
+  /** Handle an agent error: mark failed, try rotate, throw if exhausted. */
+  async handleError(err: unknown): Promise<boolean> {
+    const message = err instanceof Error ? err.message : String(err);
+    if (!isRotatableFailure(message)) return false;
+    await this.markFailed(message);
+    return this.rotate("credential failure");
+  }
+
+  private scanPool(): string[] {
+    const mountPath = getCredentialMountPath();
+    if (!existsSync(mountPath)) return [];
+    const ext = this.opts.extension ?? ".json";
+    return readdirSync(mountPath)
+      .filter((name) => name.startsWith(this.opts.prefix) && name.endsWith(ext))
+      .map((name) => resolve(mountPath, name))
+      .sort();
+  }
+
+  private activate(path: string, reason: string): void {
+    mkdirSync(this.opts.activeDir, { recursive: true });
+    const contents = readFileSync(path, "utf8");
+    writeFileSync(resolve(this.opts.activeDir, this.opts.activeFilename), contents, "utf8");
+    const prev = this.active ? ` from ${basename(this.active)}` : "";
+    this.active = path;
+    console.error(
+      `[fabrik-runtime] ${this.opts.agent} credential${prev} -> ${basename(path)} (${reason})`,
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Env-var credential helpers for agents that use env vars
+// ---------------------------------------------------------------------------
+
+/**
+ * Read a credential from the mounted directory and set it as an env var.
+ * This is the standard pattern for agents that use env vars for auth
+ * (Claude Code ANTHROPIC_API_KEY, Pi FIREWORKS_API_KEY, etc.)
+ */
+export function injectCredentialEnv(credentialName: string, envVar?: string): boolean {
+  const value = readCredential(credentialName);
+  if (value === null) return false;
+  process.env[envVar ?? credentialName] = value;
+  return true;
+}
+
+/**
+ * Inject all credentials from the mounted directory as env vars.
+ * File names become env var names, file contents become values.
+ */
+export function injectAllCredentialEnvs(): string[] {
+  const injected: string[] = [];
+  for (const name of listCredentials()) {
+    if (injectCredentialEnv(name)) {
+      injected.push(name);
+    }
+  }
+  return injected;
+}

package/src/index.ts

@@ -0,0 +1,48 @@
+/**
+ * fabrik-runtime — shared TypeScript utilities for Fabrik workflow pods.
+ *
+ * This package provides:
+ * - Credential pool management (read from mounted K8s secrets, rotate, notify)
+ * - K8s job helpers (dispatch child verification jobs)
+ * - Deterministic JJ/Git shell operations
+ */
+
+export {
+  CREDENTIAL_MOUNT_PATH,
+  getCredentialMountPath,
+  classifyFailure,
+  isRotatableFailure,
+  notifyFailure,
+  readCredential,
+  listCredentials,
+  readAllCredentials,
+  injectCredentialEnv,
+  injectAllCredentialEnvs,
+  CredentialFilePool,
+  type FailureKind,
+  type FailureEvent,
+  type PoolOptions,
+} from "./credential-pool";
+
+export {
+  CODEX_AUTH_HOME,
+  withCodexAuthPoolEnv,
+  createCodexAgentWithPool,
+  RotatingCodexAgent,
+  type AuthFailureKind,
+  type AuthFailureEvent,
+  type RotatingCodexAgentOptions,
+} from "./codex-auth";
+
+export {
+  runVerificationJob,
+  buildVerificationJobManifest,
+  type VerificationResult,
+} from "./k8s-jobs";
+
+export {
+  prepareWorkspaces,
+  snapshotChange,
+  pushBookmark,
+  type ReportOutput,
+} from "./jj-shell";

package/src/jj-shell.ts

@@ -0,0 +1,212 @@
+/**
+ * Deterministic JJ shell operations for workflow-owned progress tracking.
+ *
+ * These commands have fixed semantics and should not be delegated to the
+ * coding agent. Keeping them here makes workspace creation, snapshotting, and
+ * bookmark pushes reproducible across runs.
+ */
+import { $ } from "bun";
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+
+export type ReportOutput = {
+  ticketId: string;
+  status: "done" | "partial" | "blocked";
+  summary: string;
+};
+
+type JjResult = {
+  ok: boolean;
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+};
+
+async function jj(args: string[], cwd: string): Promise<JjResult> {
+  const result = await $`jj ${args}`.cwd(cwd).nothrow().quiet();
+  return {
+    ok: result.exitCode === 0,
+    stdout: result.stdout.toString().trim(),
+    stderr: result.stderr.toString().trim(),
+    exitCode: result.exitCode,
+  };
+}
+
+export async function prepareWorkspaces(
+  repoRoot: string,
+  workspacesDir: string,
+  ticketIds: readonly string[],
+): Promise<ReportOutput> {
+  await $`mkdir -p ${workspacesDir}`.quiet();
+
+  const created: string[] = [];
+  const skipped: string[] = [];
+  const errors: string[] = [];
+
+  for (const ticketId of ticketIds) {
+    const wsPath = resolve(workspacesDir, ticketId);
+
+    if (existsSync(wsPath)) {
+      const check = await jj(["status"], wsPath);
+      if (check.ok) {
+        skipped.push(ticketId);
+        continue;
+      }
+    }
+
+    const result = await jj(
+      ["workspace", "add", wsPath, "--name", ticketId],
+      repoRoot,
+    );
+    if (result.ok) {
+      created.push(ticketId);
+      continue;
+    }
+
+    const fallback = await jj(
+      ["workspace", "add", ticketId, wsPath],
+      repoRoot,
+    );
+    if (fallback.ok) {
+      created.push(ticketId);
+      continue;
+    }
+
+    errors.push(`${ticketId}: ${result.stderr || fallback.stderr}`);
+  }
+
+  const parts: string[] = [];
+  if (created.length > 0) parts.push(`Created: ${created.join(", ")}`);
+  if (skipped.length > 0) parts.push(`Existing: ${skipped.join(", ")}`);
+  if (errors.length > 0) parts.push(`Errors: ${errors.join("; ")}`);
+
+  return {
+    ticketId: "prepare-workspaces",
+    status: errors.length > 0 ? "partial" : "done",
+    summary: parts.join(". ") || "No workspaces to prepare.",
+  };
+}
+
+export async function snapshotChange(
+  workspacePath: string,
+  ticketId: string,
+  phase: string,
+): Promise<ReportOutput> {
+  const status = await jj(["status"], workspacePath);
+  if (!status.ok) {
+    return {
+      ticketId,
+      status: "blocked",
+      summary: `jj status failed: ${status.stderr}`,
+    };
+  }
+
+  const hasChanges = !status.stdout.includes("The working copy is clean");
+  const message = `${ticketId}: ${phase}`;
+
+  const describe = await jj(["describe", "-m", message], workspacePath);
+  if (!describe.ok) {
+    return {
+      ticketId,
+      status: "blocked",
+      summary: `jj describe failed: ${describe.stderr}`,
+    };
+  }
+
+  const newChange = await jj(["new"], workspacePath);
+  if (!newChange.ok) {
+    return {
+      ticketId,
+      status: "blocked",
+      summary: `jj new failed: ${newChange.stderr}`,
+    };
+  }
+
+  return {
+    ticketId,
+    status: "done",
+    summary: hasChanges
+      ? `Snapshotted: "${message}"`
+      : `Described (no file changes): "${message}"`,
+  };
+}
+
+export async function pushBookmark(
+  workspacePath: string,
+  bookmarkName: string,
+  ticketId: string,
+): Promise<ReportOutput> {
+  const targetRev = "@-";
+  const track = await jj(
+    ["bookmark", "track", bookmarkName, "--remote", "origin"],
+    workspacePath,
+  );
+  const trackSummary =
+    track.ok || track.stderr === ""
+      ? ""
+      : ` Tracking remote bookmark reported: ${track.stderr}`;
+
+  const targetCommit = await jj(
+    ["log", "-r", targetRev, "--no-graph", "-T", "commit_id"],
+    workspacePath,
+  );
+  if (!targetCommit.ok || !targetCommit.stdout) {
+    return {
+      ticketId,
+      status: "blocked",
+      summary: `Failed to resolve target revision for bookmark push: ${targetCommit.stderr}`,
+    };
+  }
+
+  const move = await jj(
+    ["bookmark", "set", bookmarkName, "-r", targetRev, "--allow-backwards"],
+    workspacePath,
+  );
+
+  if (!move.ok) {
+    const create = await jj(
+      ["bookmark", "create", "-r", targetRev, bookmarkName],
+      workspacePath,
+    );
+    if (!create.ok) {
+      return {
+        ticketId,
+        status: "blocked",
+        summary: `Failed to set bookmark '${bookmarkName}': ${create.stderr}`,
+      };
+    }
+  }
+
+  const push = await jj(
+    ["git", "push", "--bookmark", bookmarkName],
+    workspacePath,
+  );
+  if (!push.ok) {
+    return {
+      ticketId,
+      status: "blocked",
+      summary: `Bookmark set but push failed: ${push.stderr}${trackSummary}`,
+    };
+  }
+
+  const remote = await $`git ls-remote origin refs/heads/${bookmarkName}`
+    .cwd(workspacePath)
+    .nothrow()
+    .quiet();
+  const remoteCommit = remote.stdout.toString().trim().split(/\s+/)[0] ?? "";
+  if (remote.exitCode !== 0 || remoteCommit !== targetCommit.stdout) {
+    return {
+      ticketId,
+      status: "blocked",
+      summary:
+        `Bookmark push returned success but remote ${bookmarkName} is ${remoteCommit || "missing"} instead of ${targetCommit.stdout}.` +
+        trackSummary,
+    };
+  }
+
+  return {
+    ticketId,
+    status: "done",
+    summary: `Pushed bookmark '${bookmarkName}' to origin at ${targetCommit.stdout}.${trackSummary}`,
+  };
+}

package/src/k8s-jobs.ts

@@ -0,0 +1,313 @@
+import https from "node:https";
+import { readFileSync } from "node:fs";
+
+const SERVICE_ACCOUNT_TOKEN = "/var/run/secrets/kubernetes.io/serviceaccount/token";
+const SERVICE_ACCOUNT_CA = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt";
+
+export type VerificationResult = {
+  passed: boolean;
+  jobName: string;
+  podName: string;
+  commands: string[];
+  logs: string;
+  summary: string;
+};
+
+type VerificationJobOptions = {
+  name: string;
+  image: string;
+  namespace: string;
+  serviceAccountName: string;
+  pvcName: string;
+  nodeName: string;
+  workspacePath: string;
+  commands: string[];
+  cleanupCommands?: string[];
+  labels?: Record<string, string>;
+  timeoutSeconds?: number;
+};
+
+type K8sObjectMeta = {
+  name: string;
+  namespace: string;
+};
+
+type PodList = {
+  items?: Array<{
+    metadata?: {
+      name?: string;
+      labels?: Record<string, string>;
+    };
+  }>;
+};
+
+type JobStatus = {
+  status?: {
+    succeeded?: number;
+    failed?: number;
+  };
+};
+
+function requiredEnv(name: string): string {
+  const value = process.env[name]?.trim();
+  if (!value) {
+    throw new Error(`Missing required Kubernetes environment variable ${name}.`);
+  }
+  return value;
+}
+
+function k8sRequest(
+  method: string,
+  path: string,
+  body?: string,
+  contentType = "application/json",
+): Promise<string> {
+  const token = readFileSync(SERVICE_ACCOUNT_TOKEN, "utf8").trim();
+  const ca = readFileSync(SERVICE_ACCOUNT_CA);
+  const host = requiredEnv("KUBERNETES_SERVICE_HOST");
+  const port = process.env.KUBERNETES_SERVICE_PORT_HTTPS?.trim() || "443";
+
+  return new Promise((resolve, reject) => {
+    const req = https.request(
+      {
+        host,
+        port,
+        method,
+        path,
+        ca,
+        headers: {
+          Authorization: `Bearer ${token}`,
+          Accept: "application/json",
+          ...(body
+            ? {
+                "Content-Type": contentType,
+                "Content-Length": Buffer.byteLength(body),
+              }
+            : {}),
+        },
+      },
+      (res) => {
+        let data = "";
+        res.setEncoding("utf8");
+        res.on("data", (chunk) => {
+          data += chunk;
+        });
+        res.on("end", () => {
+          const statusCode = res.statusCode ?? 500;
+          if (statusCode >= 200 && statusCode < 300) {
+            resolve(data);
+            return;
+          }
+          reject(
+            new Error(
+              `Kubernetes API ${method} ${path} failed with ${statusCode}: ${data}`,
+            ),
+          );
+        });
+      },
+    );
+
+    req.on("error", reject);
+    if (body) req.write(body);
+    req.end();
+  });
+}
+
+async function createJob(namespace: string, manifest: unknown): Promise<K8sObjectMeta> {
+  const response = await k8sRequest(
+    "POST",
+    `/apis/batch/v1/namespaces/${namespace}/jobs`,
+    JSON.stringify(manifest),
+  );
+  const parsed = JSON.parse(response) as { metadata?: K8sObjectMeta };
+  if (!parsed.metadata?.name || !parsed.metadata?.namespace) {
+    throw new Error("Kubernetes API create job response did not include metadata.");
+  }
+  return parsed.metadata;
+}
+
+async function getJob(namespace: string, jobName: string): Promise<JobStatus> {
+  const response = await k8sRequest(
+    "GET",
+    `/apis/batch/v1/namespaces/${namespace}/jobs/${jobName}`,
+  );
+  return JSON.parse(response) as JobStatus;
+}
+
+async function listPodsForJob(namespace: string, jobName: string): Promise<string[]> {
+  const response = await k8sRequest(
+    "GET",
+    `/api/v1/namespaces/${namespace}/pods?labelSelector=${encodeURIComponent(`job-name=${jobName}`)}`,
+  );
+  const parsed = JSON.parse(response) as PodList;
+  return (parsed.items ?? [])
+    .map((item) => item.metadata?.name?.trim() ?? "")
+    .filter((name) => name !== "");
+}
+
+async function getPodLogs(namespace: string, podName: string): Promise<string> {
+  return await k8sRequest(
+    "GET",
+    `/api/v1/namespaces/${namespace}/pods/${podName}/log?container=fabrik`,
+    undefined,
+    "text/plain",
+  );
+}
+
+async function deleteJob(namespace: string, jobName: string): Promise<void> {
+  await k8sRequest(
+    "DELETE",
+    `/apis/batch/v1/namespaces/${namespace}/jobs/${jobName}?propagationPolicy=Background`,
+  );
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function buildVerifierScript(
+  commands: readonly string[],
+  cleanupCommands: readonly string[] = [],
+): string {
+  const lines = ["set -euo pipefail"];
+  if (cleanupCommands.length > 0) {
+    lines.push("cleanup() {");
+    for (const command of cleanupCommands) {
+      lines.push(`  ${command}`);
+    }
+    lines.push("}");
+    lines.push("trap cleanup EXIT");
+  }
+  lines.push(...commands);
+  return lines.join("\n");
+}
+
+export function buildVerificationJobManifest(options: VerificationJobOptions) {
+  return {
+    apiVersion: "batch/v1",
+    kind: "Job",
+    metadata: {
+      name: options.name,
+      namespace: options.namespace,
+      labels: {
+        "fabrik.sh/managed-by": "fabrik",
+        "fabrik.sh/phase": "verify",
+        "fabrik.sh/task": options.name,
+        ...(options.labels ?? {}),
+      },
+    },
+    spec: {
+      ttlSecondsAfterFinished: 3600,
+      backoffLimit: 0,
+      template: {
+        metadata: {
+          labels: {
+            "fabrik.sh/managed-by": "fabrik",
+            "fabrik.sh/phase": "verify",
+            "fabrik.sh/task": options.name,
+            ...(options.labels ?? {}),
+          },
+        },
+        spec: {
+          serviceAccountName: options.serviceAccountName,
+          restartPolicy: "Never",
+          nodeName: options.nodeName,
+          containers: [
+            {
+              name: "fabrik",
+              image: options.image,
+              imagePullPolicy: "IfNotPresent",
+              command: [
+                "sh",
+                "-lc",
+                buildVerifierScript(
+                  options.commands,
+                  options.cleanupCommands ?? [],
+                ),
+              ],
+              env: [
+                { name: "FABRIK_RUN_IMAGE", value: options.image },
+                { name: "KUBERNETES_NAMESPACE", value: options.namespace },
+                { name: "FABRIK_WORKSPACE_PVC", value: options.pvcName },
+                { name: "KUBERNETES_NODE_NAME", value: options.nodeName },
+              ],
+              workingDir: options.workspacePath,
+              volumeMounts: [
+                {
+                  name: "workspace",
+                  mountPath: "/workspace",
+                },
+              ],
+            },
+          ],
+          volumes: [
+            {
+              name: "workspace",
+              persistentVolumeClaim: {
+                claimName: options.pvcName,
+              },
+            },
+          ],
+        },
+      },
+    },
+  };
+}
+
+export async function runVerificationJob(
+  options: VerificationJobOptions,
+): Promise<VerificationResult> {
+  const timeoutMs = (options.timeoutSeconds ?? 900) * 1000;
+  const manifest = buildVerificationJobManifest(options);
+
+  const created = await createJob(options.namespace, manifest);
+  const startedAt = Date.now();
+  let podName = "";
+  let logs = "";
+
+  try {
+    for (;;) {
+      if (Date.now()-startedAt > timeoutMs) {
+        throw new Error(`Timed out waiting for verification job ${created.name}.`);
+      }
+
+      const job = await getJob(options.namespace, created.name);
+      const pods = await listPodsForJob(options.namespace, created.name);
+      if (pods.length > 0) {
+        podName = pods[0]!;
+      }
+
+      if ((job.status?.succeeded ?? 0) > 0) {
+        if (podName) {
+          logs = await getPodLogs(options.namespace, podName);
+        }
+        return {
+          passed: true,
+          jobName: created.name,
+          podName,
+          commands: [...options.commands],
+          logs,
+          summary: `Verification job ${created.name} succeeded.`,
+        };
+      }
+
+      if ((job.status?.failed ?? 0) > 0) {
+        if (podName) {
+          logs = await getPodLogs(options.namespace, podName);
+        }
+        return {
+          passed: false,
+          jobName: created.name,
+          podName,
+          commands: [...options.commands],
+          logs,
+          summary: `Verification job ${created.name} failed.`,
+        };
+      }
+
+      await sleep(2000);
+    }
+  } finally {
+    await deleteJob(options.namespace, created.name).catch(() => undefined);
+  }
+}