okengine 0.10.3 → 0.11.0

package/src/drivers/vault-aws-secrets-manager.ts

@@ -0,0 +1,333 @@
+/**
+ * AWS Secrets Manager backend for the `managed` vault driver.
+ *
+ * Reads are snapshotted at `open` (the {@link VaultBag} surface is
+ * synchronous); writes and deletes go straight to the API and are settled by
+ * `close`.
+ *
+ * `@aws-sdk/client-secrets-manager` is an optional peer, imported dynamically
+ * so a project on platform-injected secrets or another provider never pays for it.
+ * Tests inject an {@link AwsSecretsManagerClient} and skip the SDK entirely.
+ *
+ * SDK errors are never re-thrown: their messages can echo request payloads,
+ * which for this API means secret material. Every failure is translated into
+ * a {@link VaultError} with a message this module authored.
+ */
+
+import { VaultError } from "../elements/vault/errors.ts";
+import type { VaultBag } from "./vault-types.ts";
+
+/** Package name of the optional peer, resolved at call time. */
+const AWS_SM_PACKAGE = "@aws-sdk/client-secrets-manager";
+
+/** AWS error name for a secret id that does not exist. */
+const NOT_FOUND = "ResourceNotFoundException";
+
+/**
+ * Secret-name-addressed client the bag talks to.
+ *
+ * Names are bag names (`STRIPE_KEY`), never fully-qualified AWS secret ids —
+ * the prefix belongs to the client, so the bag stays prefix-agnostic.
+ */
+export interface AwsSecretsManagerClient {
+  /** Bag names visible to this credential under the configured prefix. */
+  list(): Promise<readonly string[]>;
+  /**
+   * Read one secret's string value.
+   *
+   * @param name - Bag name
+   */
+  get(name: string): Promise<string | undefined>;
+  /**
+   * Create or overwrite one secret.
+   *
+   * @param name - Bag name
+   * @param value - Cleartext
+   */
+  put(name: string, value: string): Promise<void>;
+  /**
+   * Delete one secret. A missing secret is not an error.
+   *
+   * @param name - Bag name
+   */
+  remove(name: string): Promise<void>;
+  /** Release the underlying SDK client. */
+  close?(): Promise<void>;
+}
+
+/** Options for {@link openAwsSecretsManagerBag}. */
+export interface OpenAwsSecretsManagerOptions {
+  /** AWS region. Falls back to the SDK's own resolution chain. */
+  readonly region?: string;
+  /** Secret-name prefix scoping this app (`oke/prod/`). */
+  readonly prefix?: string;
+  /** Injected client for tests — skips the SDK import entirely. */
+  readonly client?: AwsSecretsManagerClient;
+  /**
+   * Declared names, used as the read set when `ListSecrets` is denied so a
+   * least-privilege credential still boots.
+   */
+  readonly secrets?: Readonly<Record<string, string>>;
+}
+
+/**
+ * Open a secret bag backed by AWS Secrets Manager.
+ *
+ * An unreachable or unauthorized API is fatal: the bag never silently
+ * degrades to a seed-only view, because that turns a credentials outage into
+ * a "missing secret contract" message pointing at the wrong thing.
+ *
+ * @param options - Region / prefix / injected client / declared names
+ * @throws VaultError `MISSING_PEER` when the SDK is absent and no client is injected
+ * @throws VaultError `BACKEND_ERROR` when the API rejects the read
+ */
+export async function openAwsSecretsManagerBag(
+  options: OpenAwsSecretsManagerOptions = {},
+): Promise<VaultBag> {
+  const client = options.client ?? (await connectSecretsManager(options.region, options.prefix));
+  const map = new Map<string, string>(
+    Object.entries(options.secrets ?? {}).filter(
+      (e): e is [string, string] => typeof e[1] === "string",
+    ),
+  );
+
+  let listed: readonly string[] = [];
+  try {
+    listed = await client.list();
+  } catch (error) {
+    // ListSecrets is commonly denied on least-privilege roles; only fail when
+    // there is no declared read set to fall back to.
+    if (map.size === 0) throw asVaultError(error, "vault: aws secrets manager list failed");
+  }
+
+  const names = listed.length > 0 ? listed : [...map.keys()];
+  for (const name of names) {
+    let value: string | undefined;
+    try {
+      value = await client.get(name);
+    } catch (error) {
+      throw asVaultError(error, `vault: aws secrets manager read of "${name}" failed`);
+    }
+    if (value !== undefined) map.set(name, value);
+  }
+
+  const pending: Promise<void>[] = [];
+  let failure: VaultError | undefined;
+  /**
+   * Run a write in the background, holding its failure for `close`.
+   *
+   * @param work - In-flight API call
+   * @param message - Safe message when it rejects
+   */
+  function track(work: Promise<void>, message: string): void {
+    pending.push(
+      work.catch((error: unknown) => {
+        failure ??= asVaultError(error, message);
+      }),
+    );
+  }
+
+  return {
+    driverId: "managed",
+    get(name) {
+      return map.get(name);
+    },
+    names() {
+      return [...map.keys()];
+    },
+    set(name, value) {
+      map.set(name, value);
+      track(client.put(name, value), `vault: aws secrets manager write of "${name}" failed`);
+    },
+    delete(name) {
+      const had = map.delete(name);
+      track(client.remove(name), `vault: aws secrets manager delete of "${name}" failed`);
+      return had;
+    },
+    async close() {
+      await Promise.all(pending);
+      await client.close?.();
+      if (failure) throw failure;
+    },
+  };
+}
+
+/** Response fields this module reads off the SDK. */
+interface SecretsManagerResult {
+  readonly SecretString?: string;
+  readonly SecretList?: readonly { readonly Name?: string }[];
+  readonly NextToken?: string;
+}
+
+/** Structural SDK client — the SDK's `SecretsManagerClient` satisfies it. */
+interface SdkClientLike {
+  send(command: unknown): Promise<SecretsManagerResult>;
+  destroy?(): void;
+}
+
+/** Constructors pulled off the dynamically imported module. */
+interface SecretsManagerBindings {
+  readonly SecretsManagerClient: new (config: { region?: string }) => SdkClientLike;
+  readonly ListSecretsCommand: new (input: {
+    Filters?: { Key: string; Values: string[] }[];
+    MaxResults?: number;
+    NextToken?: string;
+  }) => unknown;
+  readonly GetSecretValueCommand: new (input: { SecretId: string }) => unknown;
+  readonly PutSecretValueCommand: new (input: {
+    SecretId: string;
+    SecretString: string;
+  }) => unknown;
+  readonly CreateSecretCommand: new (input: { Name: string; SecretString: string }) => unknown;
+  readonly DeleteSecretCommand: new (input: {
+    SecretId: string;
+    ForceDeleteWithoutRecovery: boolean;
+  }) => unknown;
+}
+
+/**
+ * Build a real client over the optional AWS SDK peer.
+ *
+ * @param region - AWS region
+ * @param prefix - Secret-name prefix scoping this app
+ * @throws VaultError `MISSING_PEER` when the SDK is absent
+ */
+async function connectSecretsManager(
+  region: string | undefined,
+  prefix: string | undefined,
+): Promise<AwsSecretsManagerClient> {
+  const sdk = await loadSecretsManager();
+  const at = prefix ?? "";
+  const client = new sdk.SecretsManagerClient(region === undefined ? {} : { region });
+
+  return {
+    async list() {
+      const names: string[] = [];
+      let token: string | undefined;
+      do {
+        const page: SecretsManagerResult = await client.send(
+          new sdk.ListSecretsCommand({
+            ...(at === "" ? {} : { Filters: [{ Key: "name", Values: [at] }] }),
+            MaxResults: 100,
+            ...(token === undefined ? {} : { NextToken: token }),
+          }),
+        );
+        for (const entry of page.SecretList ?? []) {
+          const id = entry.Name;
+          if (id === undefined || !id.startsWith(at)) continue;
+          names.push(id.slice(at.length));
+        }
+        token = page.NextToken;
+      } while (token !== undefined);
+      return names;
+    },
+    async get(name) {
+      try {
+        const result = await client.send(
+          new sdk.GetSecretValueCommand({ SecretId: `${at}${name}` }),
+        );
+        return result.SecretString;
+      } catch (error) {
+        if (errorName(error) === NOT_FOUND) return undefined;
+        throw error;
+      }
+    },
+    async put(name, value) {
+      try {
+        await client.send(
+          new sdk.PutSecretValueCommand({ SecretId: `${at}${name}`, SecretString: value }),
+        );
+      } catch (error) {
+        if (errorName(error) !== NOT_FOUND) throw error;
+        await client.send(
+          new sdk.CreateSecretCommand({ Name: `${at}${name}`, SecretString: value }),
+        );
+      }
+    },
+    async remove(name) {
+      try {
+        await client.send(
+          new sdk.DeleteSecretCommand({
+            SecretId: `${at}${name}`,
+            ForceDeleteWithoutRecovery: true,
+          }),
+        );
+      } catch (error) {
+        if (errorName(error) !== NOT_FOUND) throw error;
+      }
+    },
+    async close() {
+      client.destroy?.();
+    },
+  };
+}
+
+/**
+ * Import the optional `@aws-sdk/client-secrets-manager` peer.
+ *
+ * @throws VaultError `MISSING_PEER` with the exact install command
+ */
+async function loadSecretsManager(): Promise<SecretsManagerBindings> {
+  // Indirect specifier: the peer is optional, so it must not become a
+  // static dependency of the module graph.
+  const specifier: string = AWS_SM_PACKAGE;
+  let mod: unknown;
+  try {
+    mod = await import(specifier);
+  } catch {
+    throw new VaultError(
+      "MISSING_PEER",
+      `vault: managed provider "aws-secrets-manager" needs the optional peer \`${AWS_SM_PACKAGE}\` (bun add ${AWS_SM_PACKAGE})`,
+    );
+  }
+  const bindings = mod as Partial<SecretsManagerBindings>;
+  if (
+    !bindings.SecretsManagerClient ||
+    !bindings.ListSecretsCommand ||
+    !bindings.GetSecretValueCommand ||
+    !bindings.PutSecretValueCommand ||
+    !bindings.CreateSecretCommand ||
+    !bindings.DeleteSecretCommand
+  ) {
+    throw new VaultError(
+      "MISSING_PEER",
+      `vault: \`${AWS_SM_PACKAGE}\` is installed but does not export the SecretsManager client and commands`,
+    );
+  }
+  return {
+    SecretsManagerClient: bindings.SecretsManagerClient,
+    ListSecretsCommand: bindings.ListSecretsCommand,
+    GetSecretValueCommand: bindings.GetSecretValueCommand,
+    PutSecretValueCommand: bindings.PutSecretValueCommand,
+    CreateSecretCommand: bindings.CreateSecretCommand,
+    DeleteSecretCommand: bindings.DeleteSecretCommand,
+  };
+}
+
+/**
+ * AWS error discriminator (`ResourceNotFoundException`, …).
+ *
+ * Reads only `name`, never `message`: the latter can quote request payloads.
+ *
+ * @param error - Thrown value
+ */
+function errorName(error: unknown): string | undefined {
+  if (typeof error !== "object" || error === null) return undefined;
+  const name = (error as { name?: unknown }).name;
+  return typeof name === "string" ? name : undefined;
+}
+
+/**
+ * Translate a foreign failure into a secret-free {@link VaultError}.
+ *
+ * @param error - Thrown value
+ * @param message - Safe message authored by this module
+ */
+function asVaultError(error: unknown, message: string): VaultError {
+  if (error instanceof VaultError) return error;
+  const name = errorName(error);
+  if (name === "AccessDeniedException" || name === "UnrecognizedClientException") {
+    return new VaultError("PERMISSION_DENIED", `${message} — credentials denied`);
+  }
+  return new VaultError("BACKEND_ERROR", message);
+}

package/src/drivers/vault-builtin.test.ts

@@ -0,0 +1,110 @@
+/**
+ * `vault` driver — adapter wiring and the never-fail-boot degradation path.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { connectPglite } from "./pglite.ts";
+import { builtinVaultDriver, openBuiltinVaultAdapter, VAULT_BAG_ADAPTER } from "./vault-builtin.ts";
+import type { BuiltinVaultAdapter } from "../elements/vault/builtin-adapter.ts";
+
+describe("builtin vault driver", () => {
+  test("openBuiltinVaultAdapter unseals an injected connection", async () => {
+    const connection = await connectPglite({ url: "memory://vault-driver-test" });
+    try {
+      const first = await openBuiltinVaultAdapter({ connection, env: {} });
+      const init = await first.adapter.initialize();
+      await first.adapter.unseal(init.masterKey);
+      await first.adapter.set("prod/api/stripe", "sk_live_x");
+
+      // A second open over the same connection unseals from the master key.
+      const second = await openBuiltinVaultAdapter({
+        connection,
+        env: { OKE_VAULT_MASTER_KEY: init.masterKey },
+      });
+      expect(second.adapter.getUnsealer()).not.toBeNull();
+      expect((await second.adapter.get("prod/api/stripe"))?.value).toBe("sk_live_x");
+    } finally {
+      await connection.close();
+    }
+  });
+
+  test("open with no SQL configured degrades to the seed bag", async () => {
+    const bag = await builtinVaultDriver.open({
+      env: {},
+      secrets: { APP_NAME: "demo" },
+    });
+    try {
+      expect(bag.driverId).toBe("vault");
+      expect(bag.get("APP_NAME")).toBe("demo");
+      expect(bag.get("MISSING")).toBeUndefined();
+      expect(bag.names()).toEqual(["APP_NAME"]);
+    } finally {
+      await bag.close?.();
+    }
+  });
+
+  test("bag close auto-seals the adapter (SIGTERM → bootResult.close path)", async () => {
+    const connection = await connectPglite({ url: "memory://vault-autoseal" });
+    try {
+      const staging = await openBuiltinVaultAdapter({ connection, env: {} });
+      const init = await staging.adapter.initialize();
+      await staging.adapter.unseal(init.masterKey);
+      await staging.adapter.set("prod/api/stripe", "sk_live_z");
+      await staging.adapter.seal();
+
+      const bag = await builtinVaultDriver.open({
+        connection,
+        env: { OKE_VAULT_MASTER_KEY: init.masterKey },
+      });
+      const held = (bag as unknown as Record<symbol, BuiltinVaultAdapter | undefined>)[
+        VAULT_BAG_ADAPTER
+      ];
+      expect(held).toBeDefined();
+      expect(held!.getUnsealer()).not.toBeNull();
+      expect(bag.get("prod/api/stripe")).toBe("sk_live_z");
+
+      await bag.close?.();
+      expect(held!.getUnsealer()).toBeNull();
+      await expect(held!.get("prod/api/stripe")).rejects.toThrow(/sealed/i);
+    } finally {
+      await connection.close();
+    }
+  });
+
+  test("open snapshots live secrets from an initialized vault", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "oke-vault-driver-"));
+    try {
+      const staging = await connectPglite({ url: dir });
+      const opened = await openBuiltinVaultAdapter({ connection: staging, env: {} });
+      const init = await opened.adapter.initialize();
+      await opened.adapter.unseal(init.masterKey);
+      await opened.adapter.set("STRIPE_KEY", "sk_live_y");
+      await staging.close();
+
+      const bag = await builtinVaultDriver.open({
+        url: dir,
+        env: { OKE_VAULT_MASTER_KEY: init.masterKey },
+      });
+      try {
+        expect(bag.names()).toEqual(["STRIPE_KEY"]);
+        expect(bag.get("STRIPE_KEY")).toBe("sk_live_y");
+      } finally {
+        await bag.close?.();
+      }
+
+      // Without the master key the vault stays sealed and the bag is empty —
+      // boot then reports ordinary gaps instead of a driver failure.
+      const sealed = await builtinVaultDriver.open({ url: dir, env: {} });
+      try {
+        expect(sealed.names()).toEqual([]);
+      } finally {
+        await sealed.close?.();
+      }
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});

package/src/drivers/vault-builtin.ts

@@ -0,0 +1,184 @@
+/**
+ * `vault` driver — okengine's own encrypted-at-rest secret store.
+ *
+ * The built-in Vault is asynchronous and path-addressed
+ * ({@link BuiltinVaultAdapter}), while the boot resolution chain wants a
+ * synchronous name→value {@link VaultBag}. This driver bridges the two: at
+ * `open` it unseals with `OKE_VAULT_MASTER_KEY`, snapshots every live secret
+ * into a bag, and keeps the adapter around for write-through.
+ *
+ * Boot must never die because the operator has not run `oke vault init` yet,
+ * so an uninitialized, sealed, or unreachable backend degrades to the seed
+ * bag — the Vault runtime then reports the missing contracts as ordinary
+ * gaps, with the same message every other driver produces.
+ */
+
+import {
+  createBuiltinVaultAdapter,
+  sqlConnectionAsExec,
+  type BuiltinVaultAdapter,
+} from "../elements/vault/builtin-adapter.ts";
+import type { SqlConnection } from "./types.ts";
+import type { VaultBag, VaultDriver, VaultOpenOptions } from "./vault-types.ts";
+
+/** Environment variable holding the base64 master key. */
+export const VAULT_MASTER_KEY_ENV = "OKE_VAULT_MASTER_KEY";
+
+/** Test-only handle to the adapter a bag is sealing on {@link VaultBag.close}. */
+export const VAULT_BAG_ADAPTER = Symbol.for("oke.vault.bagAdapter");
+
+/** Options for {@link openBuiltinVaultAdapter}. */
+export interface OpenBuiltinVaultOptions {
+  /** Postgres URL. Falls back to `DATABASE_URL` / `OKE_STORE_SQL_URL`, then PGlite. */
+  readonly url?: string;
+  /** Pre-opened SQL connection — wins over `url`. */
+  readonly connection?: SqlConnection;
+  /** Base64 master key to unseal with. Defaults to `OKE_VAULT_MASTER_KEY`. */
+  readonly masterKey?: string;
+  /** Injected env map for tests. */
+  readonly env?: Readonly<Record<string, string | undefined>>;
+  /** DEKs re-wrapped per master-rotation batch. */
+  readonly kekRewrapBatchSize?: number;
+}
+
+/** An adapter plus the connection this module opened for it, if any. */
+export interface OpenedBuiltinVault {
+  /** The built-in adapter, unsealed when a valid master key was available. */
+  readonly adapter: BuiltinVaultAdapter;
+  /** Close the underlying connection when this module opened it. */
+  close(): Promise<void>;
+}
+
+/**
+ * Open the built-in Vault adapter over SQL, unsealing when a master key is
+ * available.
+ *
+ * Used by the CLI (`oke vault …`) and Console, which need the full
+ * path-addressed surface rather than a flat bag.
+ *
+ * @param options - URL / connection / master key / batch size
+ */
+export async function openBuiltinVaultAdapter(
+  options: OpenBuiltinVaultOptions = {},
+): Promise<OpenedBuiltinVault> {
+  const env = options.env ?? process.env;
+  const url = options.url ?? env.DATABASE_URL ?? env.OKE_STORE_SQL_URL;
+
+  let owned: SqlConnection | undefined;
+  let connection = options.connection;
+  if (!connection) {
+    owned = await connectVaultSql(url);
+    connection = owned;
+  }
+
+  const adapter = createBuiltinVaultAdapter({
+    db: sqlConnectionAsExec(connection),
+    ...(options.kekRewrapBatchSize === undefined
+      ? {}
+      : { kekRewrapBatchSize: options.kekRewrapBatchSize }),
+  });
+
+  const masterKey = options.masterKey ?? env[VAULT_MASTER_KEY_ENV];
+  if (masterKey !== undefined && masterKey.trim().length > 0) {
+    await adapter.unseal(masterKey);
+  }
+
+  return {
+    adapter,
+    async close() {
+      await owned?.close();
+    },
+  };
+}
+
+/**
+ * Built-in Vault driver. Reads resolve from a snapshot taken at `open`.
+ */
+export const builtinVaultDriver: VaultDriver = {
+  id: "vault",
+  async open(options: VaultOpenOptions = {}): Promise<VaultBag> {
+    const map = new Map<string, string>(
+      Object.entries(options.secrets ?? {}).filter(
+        (e): e is [string, string] => typeof e[1] === "string",
+      ),
+    );
+
+    const env = options.env ?? process.env;
+    const url = options.url ?? env.DATABASE_URL ?? env.OKE_STORE_SQL_URL;
+
+    let opened: OpenedBuiltinVault | undefined;
+    // With no SQL configured there is nothing to snapshot, and spinning up a
+    // throwaway PGlite instance would cost every boot ~1s for an empty bag.
+    if (options.connection !== undefined || url !== undefined) {
+      try {
+        opened = await openBuiltinVaultAdapter({
+          env,
+          ...(options.connection !== undefined ? { connection: options.connection } : { url }),
+        });
+        if (opened.adapter.getUnsealer() !== null) {
+          for (const entry of await opened.adapter.list()) {
+            const secret = await opened.adapter.get(entry.path);
+            if (secret) map.set(entry.path, secret.value);
+          }
+        }
+      } catch {
+        // Uninitialized / sealed / unreachable: fall back to the seed bag so
+        // boot reports missing contracts instead of a driver stack trace.
+        await opened?.close().catch(() => undefined);
+        opened = undefined;
+      }
+    }
+
+    const adapter = opened?.adapter;
+    return {
+      driverId: "vault",
+      ...(adapter === undefined ? {} : { [VAULT_BAG_ADAPTER]: adapter }),
+      get(name) {
+        return map.get(name);
+      },
+      names() {
+        return [...map.keys()];
+      },
+      set(name, value) {
+        map.set(name, value);
+        if (adapter && adapter.getUnsealer() !== null) {
+          void adapter.set(name, value).catch(() => undefined);
+        }
+      },
+      delete(name) {
+        const had = map.delete(name);
+        if (adapter && adapter.getUnsealer() !== null) {
+          void adapter.delete(name).catch(() => undefined);
+        }
+        return had;
+      },
+      async close() {
+        // Auto-seal on bag close (app stop / SIGTERM → bootResult.close → vault.close).
+        // Clears the in-memory master key before the SQL connection drops.
+        try {
+          await adapter?.seal();
+        } catch {
+          // Already sealed / never unsealed — ignore.
+        }
+        await opened?.close();
+      },
+    } as VaultBag;
+  },
+};
+
+/**
+ * Open the SQL connection backing the built-in Vault.
+ *
+ * A real Postgres URL uses the `postgres` driver; anything else (including
+ * no URL at all) uses PGlite so local dev and tests work with no services.
+ *
+ * @param url - Resolved SQL URL, when configured
+ */
+async function connectVaultSql(url: string | undefined): Promise<SqlConnection> {
+  if (url !== undefined && /^postgres(ql)?:\/\//.test(url)) {
+    const { connectPostgres } = await import("./postgres.ts");
+    return connectPostgres({ url });
+  }
+  const { connectPglite } = await import("./pglite.ts");
+  return connectPglite(url === undefined ? {} : { url });
+}

package/src/drivers/vault-driver-removal.test.ts

@@ -16,6 +16,10 @@ const FORBIDDEN_DRIVER = ["so", "ps"].join("");
 const FORBIDDEN_SECRETS_SAAS = ["infi", "sical"].join("");
 /** Forbidden optional peer, split so this file does not match itself. */
 const FORBIDDEN_AGE = ["age", "-", "encryption"].join("");
+/** Forbidden external vault driver id, split so this file does not match itself. */
+const FORBIDDEN_EXT_VAULT = ["open", "bao"].join("");
+/** Forbidden vault-provider alias, split so this file does not match itself. */
+const FORBIDDEN_VAULT_ALIAS = ["hashi", "corp"].join("");
 
 /** Lockfiles regenerated by `bun install` — not source residue. */
 const LOCKFILE_PREFIXES = [
@@ -62,4 +66,13 @@ describe("removed vault driver gate", () => {
   test("tracked tree has zero matches for the removed age peer outside lockfiles", () => {
     assertZeroGitGrep(FORBIDDEN_AGE, LOCKFILE_PREFIXES);
   });
+
+  test("tracked tree has zero matches for the removed external vault driver", () => {
+    // Community-plugin notes in changelog / site docs may mention the name.
+    assertZeroGitGrep(FORBIDDEN_EXT_VAULT, ["changelog.md:", "site/content/docs/"]);
+  });
+
+  test("tracked tree has zero matches for the removed vault alias outside docs notes", () => {
+    assertZeroGitGrep(FORBIDDEN_VAULT_ALIAS, ["changelog.md:", "site/content/docs/"]);
+  });
 });