@mycadet/credential-executor 0.0.1 → 0.12.6-dev.202610080043.b727a33

package/src/migrations/runner.ts

@@ -0,0 +1,152 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
+
+import type { SecureKeyBackend } from "@mycadet/credential-storage";
+
+import { getLogger } from "../logger.js";
+import type { CesMigration, CesMigrationStatus } from "./types.js";
+
+const log = getLogger("ces-migrations");
+
+// ---------------------------------------------------------------------------
+// Checkpoint file
+// ---------------------------------------------------------------------------
+
+type CheckpointFile = {
+  applied: Record<string, { appliedAt: string; status?: CesMigrationStatus }>;
+};
+
+function getCheckpointPath(cesDataRoot: string): string {
+  return join(cesDataRoot, ".ces-migrations.json");
+}
+
+function loadCheckpoints(cesDataRoot: string): CheckpointFile {
+  const path = getCheckpointPath(cesDataRoot);
+  if (!existsSync(path)) {
+    return { applied: {} };
+  }
+  try {
+    const raw = readFileSync(path, "utf-8");
+    const data = JSON.parse(raw);
+    if (
+      typeof data === "object" &&
+      data != null &&
+      typeof data.applied === "object" &&
+      data.applied != null
+    ) {
+      return data as CheckpointFile;
+    }
+    log.warn(
+      "CES migration checkpoint file has unexpected structure; treating as fresh state",
+    );
+  } catch {
+    log.warn(
+      "CES migration checkpoint file is malformed; treating as fresh state",
+    );
+  }
+  return { applied: {} };
+}
+
+function saveCheckpoints(
+  cesDataRoot: string,
+  checkpoints: CheckpointFile,
+): void {
+  const path = getCheckpointPath(cesDataRoot);
+  mkdirSync(dirname(path), { recursive: true });
+  const tmpPath = path + ".tmp";
+  writeFileSync(tmpPath, JSON.stringify(checkpoints, null, 2) + "\n", "utf-8");
+  renameSync(tmpPath, path);
+}
+
+// ---------------------------------------------------------------------------
+// Runner
+// ---------------------------------------------------------------------------
+
+/**
+ * Run all pending CES migrations in registry order.
+ *
+ * - Skips migrations that already have a checkpoint entry (`"completed"` or
+ *   `"failed"`). Only `"started"` and `"rolling_back"` entries are cleared
+ *   and re-run on the next startup (crash recovery — migrations must be
+ *   idempotent).
+ * - Marks failed migrations as `"failed"` and continues startup; a failed
+ *   migration does not block the RPC server from starting.
+ *
+ * @param cesDataRoot  The CES-private data root (from `getCesDataRoot(mode)`).
+ *   The checkpoint file is stored here as `.ces-migrations.json`.
+ * @param backend  The active `SecureKeyBackend` instance, passed directly to
+ *   each migration's `run()` function.
+ * @param migrations  Ordered list of migrations from the registry.
+ */
+export async function runCesMigrations(
+  cesDataRoot: string,
+  backend: SecureKeyBackend,
+  migrations: CesMigration[],
+): Promise<void> {
+  // Validate uniqueness.
+  const seen = new Set<string>();
+  for (const m of migrations) {
+    if (seen.has(m.id)) {
+      throw new Error(`Duplicate CES migration id: "${m.id}"`);
+    }
+    seen.add(m.id);
+  }
+
+  const checkpoints = loadCheckpoints(cesDataRoot);
+
+  // Clear any interrupted checkpoints so they re-run.
+  for (const [id, entry] of Object.entries(checkpoints.applied)) {
+    if (entry.status === "started" || entry.status === "rolling_back") {
+      log.warn(
+        `CES migration "${id}" was interrupted during a previous run; will re-run`,
+      );
+      delete checkpoints.applied[id];
+    }
+  }
+
+  for (const migration of migrations) {
+    if (checkpoints.applied[migration.id]) {
+      continue;
+    }
+
+    log.info(
+      `Running CES migration: ${migration.id} — ${migration.description}`,
+    );
+
+    // Mark as started before executing (crash recovery observability).
+    checkpoints.applied[migration.id] = {
+      appliedAt: new Date().toISOString(),
+      status: "started",
+    };
+    saveCheckpoints(cesDataRoot, checkpoints);
+
+    try {
+      await migration.run(backend);
+    } catch (error) {
+      log.error(
+        { migrationId: migration.id, error },
+        `CES migration failed: ${migration.id} — marking as failed and continuing`,
+      );
+      checkpoints.applied[migration.id] = {
+        appliedAt: new Date().toISOString(),
+        status: "failed",
+      };
+      saveCheckpoints(cesDataRoot, checkpoints);
+      continue;
+    }
+
+    checkpoints.applied[migration.id] = {
+      appliedAt: new Date().toISOString(),
+      status: "completed",
+    };
+    saveCheckpoints(cesDataRoot, checkpoints);
+
+    log.info(`CES migration completed: ${migration.id}`);
+  }
+}

package/src/migrations/types.ts

@@ -0,0 +1,54 @@
+import type { SecureKeyBackend } from "@mycadet/credential-storage";
+
+/**
+ * A single CES data migration.
+ *
+ * Migrations run at CES startup — once per installation, tracked via a
+ * checkpoint file in the CES-private data root. They are the right place
+ * for one-time transformations of the credential store (key renames,
+ * format changes, etc.) that must happen before the RPC server accepts
+ * connections.
+ *
+ * **Idempotency**: `run` and `down` must both be safe to re-run. The
+ * runner re-executes any migration whose checkpoint was left in `"started"`
+ * state (i.e. the process crashed mid-migration).
+ *
+ * **Ordering**: Migrations are executed in registry order. Never reorder
+ * or remove an entry from the registry once it has been released.
+ */
+export interface CesMigration {
+  /**
+   * Unique identifier used as the checkpoint key.
+   * Convention: `"NNN-short-description"`, e.g. `"001-no-op"`.
+   * Must be unique across all registered migrations.
+   */
+  id: string;
+
+  /** Human-readable description logged when the migration runs. */
+  description: string;
+
+  /**
+   * Apply the migration.
+   *
+   * Receives the active `SecureKeyBackend` so the migration can read,
+   * write, and delete credential store entries without re-opening the
+   * encrypted store.
+   */
+  run(backend: SecureKeyBackend): void | Promise<void>;
+
+  /**
+   * Reverse the migration (best-effort).
+   *
+   * Some migrations are forward-only (e.g. dropping a key that is no
+   * longer used). In those cases, implement `down` as a no-op and document
+   * why reversal is not meaningful.
+   */
+  down(backend: SecureKeyBackend): void | Promise<void>;
+}
+
+/** Checkpoint status values written to `.ces-migrations.json`. */
+export type CesMigrationStatus =
+  | "started"
+  | "completed"
+  | "rolling_back"
+  | "failed";

package/src/paths.ts

@@ -0,0 +1,140 @@
+/**
+ * CES private data-root layout.
+ *
+ * Defines the directory structure for CES-private durable state (grants,
+ * audit logs, tool store). This state is never stored on the assistant-visible
+ * workspace/data root — it lives in a CES-only path that the assistant process
+ * cannot read or write.
+ *
+ * Two modes:
+ *
+ * - **Local**: CES private state lives under the Cadet root's `protected/`
+ *   directory at `<rootDir>/protected/credential-executor/`.
+ *
+ * - **Managed**: CES private state lives at `/ces-data` by default
+ *   (overridable via `CES_DATA_DIR` env var). The assistant container never
+ *   sees this path.
+ *
+ * The assistant-visible data root (where workspace, embeddings, etc. live)
+ * is a separate path and must never be used for CES-private writes.
+ */
+
+import { join } from "node:path";
+import { homedir } from "node:os";
+
+import { resolveIpcEndpoint } from "@mycadet/ipc-server-utils";
+
+// ---------------------------------------------------------------------------
+// Mode detection
+// ---------------------------------------------------------------------------
+
+export type CesMode = "local" | "managed";
+
+/**
+ * Determine the CES operating mode from the environment.
+ *
+ * `CES_MODE=managed` is set explicitly in managed container entrypoints.
+ * Everything else defaults to local.
+ */
+export function getCesMode(): CesMode {
+  return process.env["CES_MODE"] === "managed" ? "managed" : "local";
+}
+
+// ---------------------------------------------------------------------------
+// Root directory helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve the CES security directory.
+ *
+ * Priority:
+ * 1. `CREDENTIAL_SECURITY_DIR` env var (set by the platform template for the
+ *    CES container — `/ces-security` in managed mode)
+ * 2. Default: `~/.cadet/protected` (local mode shares the filesystem with
+ *    the gateway)
+ */
+export function getSecurityDir(): string {
+  return (
+    process.env["CREDENTIAL_SECURITY_DIR"]?.trim() ||
+    join(homedir(), ".cadet", "protected")
+  );
+}
+
+/**
+ * Default managed CES data root.
+ *
+ * Defaults to `/ces-data` — the platform template provisions this path
+ * via the `CES_DATA_DIR` env var and a dedicated PVC.
+ */
+const DEFAULT_MANAGED_CES_DATA_ROOT = "/ces-data";
+
+/**
+ * Return the CES-private data root.
+ *
+ * - Local: `<securityDir>/credential-executor/`
+ * - Managed: `CES_DATA_DIR` env var, or `/ces-data` by default
+ */
+export function getCesDataRoot(mode?: CesMode): string {
+  const resolvedMode = mode ?? getCesMode();
+  if (resolvedMode === "managed") {
+    return process.env["CES_DATA_DIR"] ?? DEFAULT_MANAGED_CES_DATA_ROOT;
+  }
+  return join(getSecurityDir(), "credential-executor");
+}
+
+// ---------------------------------------------------------------------------
+// Subdirectory layout
+// ---------------------------------------------------------------------------
+
+/** Directory for CES log files. */
+export function getCesLogDir(mode?: CesMode): string {
+  return join(getCesDataRoot(mode), "logs");
+}
+
+// ---------------------------------------------------------------------------
+// CES socket path (local sibling and managed sidecar)
+// ---------------------------------------------------------------------------
+
+/** Default directory for the CES Unix socket. */
+const BOOTSTRAP_SOCKET_DIR = "/run/ces-bootstrap";
+
+/**
+ * Return the path to the CES Unix socket used in every environment.
+ *
+ * Local siblings and managed sidecars share this resolver. The assistant
+ * dials the same path CES binds.
+ *
+ * Uses `CES_BOOTSTRAP_SOCKET_DIR` when set, otherwise `/run/ces-bootstrap`.
+ * `ces.sock` is resolved through `resolveIpcEndpoint` so macOS AF_UNIX
+ * limits and Windows named pipes stay consistent.
+ *
+ * The pod template and local CLI export `CES_BOOTSTRAP_SOCKET_DIR`.
+ */
+export function getBootstrapSocketPath(): string {
+  const dir =
+    process.env["CES_BOOTSTRAP_SOCKET_DIR"]?.trim() || BOOTSTRAP_SOCKET_DIR;
+  return resolveIpcEndpoint("ces", { workspaceDir: dir }).path;
+}
+
+// ---------------------------------------------------------------------------
+// Health port (managed mode only)
+// ---------------------------------------------------------------------------
+
+/** Default health probe port for managed CES. */
+const DEFAULT_HEALTH_PORT = 8090;
+
+/**
+ * Return the health probe port for managed mode.
+ *
+ * Health probes are served on a dedicated HTTP port, separate from the
+ * Unix socket command transport. This ensures liveness/readiness probes
+ * work without a Unix socket client.
+ */
+export function getHealthPort(): number {
+  const envPort = process.env["CES_HEALTH_PORT"];
+  if (envPort) {
+    const parsed = parseInt(envPort, 10);
+    if (!isNaN(parsed) && parsed > 0) return parsed;
+  }
+  return DEFAULT_HEALTH_PORT;
+}

package/src/records/handlers.ts

@@ -0,0 +1,55 @@
+/**
+ * RPC handlers for CES credential metadata.
+ */
+
+import { CesRpcMethod } from "@mycadet/service-contracts/credential-rpc";
+
+import type { RpcHandlerRegistry } from "../server.js";
+import { getMetadataStore, type CesMetadataStore } from "./metadata-store.js";
+
+export function buildRecordHandlers(): RpcHandlerRegistry {
+  const handlers: RpcHandlerRegistry = {};
+
+  handlers[CesRpcMethod.GetCredentialRecord] = (async (req: {
+    account: string;
+  }) => {
+    const record = getMetadataStore().getByAccount(req.account);
+    return { found: record !== undefined, record };
+  }) as (typeof handlers)[string];
+
+  handlers[CesRpcMethod.SetCredentialRecord] = (async (req: {
+    account: string;
+    record: Parameters<CesMetadataStore["setByAccount"]>[1];
+  }) => {
+    const ok = getMetadataStore().setByAccount(req.account, req.record);
+    return { ok };
+  }) as (typeof handlers)[string];
+
+  handlers[CesRpcMethod.DeleteCredentialRecord] = (async (req: {
+    account: string;
+  }) => {
+    const result = getMetadataStore().deleteByAccount(req.account);
+    return { result };
+  }) as (typeof handlers)[string];
+
+  handlers[CesRpcMethod.ListCredentialRecords] = (async () => {
+    return { records: getMetadataStore().list() };
+  }) as (typeof handlers)[string];
+
+  handlers[CesRpcMethod.BulkSetCredentialRecords] = (async (req: {
+    records: Array<{
+      account: string;
+      record: Parameters<CesMetadataStore["setByAccount"]>[1];
+    }>;
+  }) => {
+    const store = getMetadataStore();
+    const results = [];
+    for (const { account, record } of req.records) {
+      const ok = store.setByAccount(account, record);
+      results.push({ account, ok });
+    }
+    return { results };
+  }) as (typeof handlers)[string];
+
+  return handlers;
+}

package/src/records/metadata-store.ts

@@ -0,0 +1,187 @@
+/**
+ * CES-owned credential metadata store.
+ *
+ * Persists non-secret identity and policy (`allowedTools`, `allowedDomains`,
+ * `injectionTemplates`, plus catalog fields) at
+ * `<cesDataRoot>/metadata.json`. Secret values stay in `keys.enc`.
+ */
+
+import { randomUUID } from "node:crypto";
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
+
+import type { CredentialRecord } from "@mycadet/service-contracts/credential-rpc";
+
+import { getCesDataRoot } from "../paths.js";
+
+export const METADATA_FILENAME = "metadata.json";
+
+const CURRENT_VERSION = 5;
+
+interface MetadataFile {
+  version: typeof CURRENT_VERSION;
+  credentials: CredentialRecord[];
+}
+
+export function getMetadataPath(cesDataRoot: string): string {
+  return join(cesDataRoot, METADATA_FILENAME);
+}
+
+export function parseCredentialAccount(
+  account: string,
+): { service: string; field: string } | undefined {
+  const prefix = "credential/";
+  if (!account.startsWith(prefix)) {
+    return undefined;
+  }
+  const rest = account.slice(prefix.length);
+  const slashIdx = rest.lastIndexOf("/");
+  if (slashIdx < 1 || slashIdx === rest.length - 1) {
+    return undefined;
+  }
+  return {
+    service: rest.slice(0, slashIdx),
+    field: rest.slice(slashIdx + 1),
+  };
+}
+
+export function accountForRecord(
+  record: Pick<CredentialRecord, "service" | "field">,
+): string {
+  return `credential/${record.service}/${record.field}`;
+}
+
+function isRecord(value: unknown): value is CredentialRecord {
+  if (typeof value !== "object" || value == null) {
+    return false;
+  }
+  const record = value as Record<string, unknown>;
+  return (
+    typeof record.credentialId === "string" &&
+    typeof record.service === "string" &&
+    typeof record.field === "string" &&
+    Array.isArray(record.allowedTools) &&
+    Array.isArray(record.allowedDomains) &&
+    typeof record.createdAt === "number" &&
+    typeof record.updatedAt === "number"
+  );
+}
+
+export class CesMetadataStore {
+  constructor(private readonly metadataPath: string) {}
+
+  getPath(): string {
+    return this.metadataPath;
+  }
+
+  getByAccount(account: string): CredentialRecord | undefined {
+    const parsed = parseCredentialAccount(account);
+    if (!parsed) {
+      return undefined;
+    }
+    return this.load().credentials.find(
+      (record) =>
+        record.service === parsed.service && record.field === parsed.field,
+    );
+  }
+
+  setByAccount(account: string, record: CredentialRecord): boolean {
+    const parsed = parseCredentialAccount(account);
+    if (!parsed) {
+      return false;
+    }
+    if (parsed.service !== record.service || parsed.field !== record.field) {
+      return false;
+    }
+    const data = this.load();
+    const copy: CredentialRecord = { ...record };
+    const idx = data.credentials.findIndex(
+      (entry) =>
+        entry.service === record.service && entry.field === record.field,
+    );
+    if (idx === -1) {
+      data.credentials.push(copy);
+    } else {
+      data.credentials[idx] = copy;
+    }
+    this.save(data);
+    return true;
+  }
+
+  deleteByAccount(account: string): "deleted" | "not-found" | "error" {
+    const parsed = parseCredentialAccount(account);
+    if (!parsed) {
+      return "error";
+    }
+    const data = this.load();
+    const idx = data.credentials.findIndex(
+      (entry) =>
+        entry.service === parsed.service && entry.field === parsed.field,
+    );
+    if (idx === -1) {
+      return "not-found";
+    }
+    data.credentials.splice(idx, 1);
+    this.save(data);
+    return "deleted";
+  }
+
+  list(): Array<{ account: string; record: CredentialRecord }> {
+    return this.load().credentials.map((record) => ({
+      account: accountForRecord(record),
+      record,
+    }));
+  }
+
+  private load(): MetadataFile {
+    if (!existsSync(this.metadataPath)) {
+      return { version: CURRENT_VERSION, credentials: [] };
+    }
+    try {
+      const parsed = JSON.parse(readFileSync(this.metadataPath, "utf-8")) as {
+        credentials?: unknown;
+      };
+      const credentials = Array.isArray(parsed.credentials)
+        ? parsed.credentials.filter(isRecord)
+        : [];
+      return { version: CURRENT_VERSION, credentials };
+    } catch {
+      return { version: CURRENT_VERSION, credentials: [] };
+    }
+  }
+
+  private save(data: MetadataFile): void {
+    const dir = dirname(this.metadataPath);
+    if (!existsSync(dir)) {
+      mkdirSync(dir, { recursive: true });
+    }
+    const tmpPath = join(dir, `.tmp-${randomUUID()}`);
+    writeFileSync(tmpPath, JSON.stringify(data, null, 2), "utf-8");
+    renameSync(tmpPath, this.metadataPath);
+  }
+}
+
+let processStore: CesMetadataStore | undefined;
+
+export function initMetadataStore(cesDataRoot: string): CesMetadataStore {
+  processStore = new CesMetadataStore(getMetadataPath(cesDataRoot));
+  return processStore;
+}
+
+export function getMetadataStore(): CesMetadataStore {
+  if (!processStore) {
+    processStore = new CesMetadataStore(getMetadataPath(getCesDataRoot()));
+  }
+  return processStore;
+}
+
+/** Test-only: drop the process store so the next init uses a fresh path. */
+export function resetMetadataStoreForTests(): void {
+  processStore = undefined;
+}