@monster-orange/den 0.0.0 → 0.2.0

package/src/shared/results/result-writer.ts

@@ -0,0 +1,168 @@
+/**
+ * Result Writer
+ * Writes test results to JSONL (JSON Lines) format
+ */
+
+import {
+  appendFile,
+  writeFile,
+  readFile,
+  mkdir,
+  rename,
+  unlink,
+} from "fs/promises";
+import { resolve, join } from "path";
+import type { TestResult, Pillar } from "../types/index.js";
+
+export class ResultWriter {
+  private resultsDir: string;
+
+  constructor(resultsDir: string = "./results") {
+    this.resultsDir = resolve(resultsDir);
+  }
+
+  /**
+   * Write a single test result to JSONL file
+   */
+  async writeResult(result: TestResult): Promise<void> {
+    const filename = this.getFilename(result.pillar, result.environment);
+
+    try {
+      // Ensure results directory exists
+      await mkdir(this.resultsDir, { recursive: true });
+
+      // Append result as JSON line
+      const line = JSON.stringify(result) + "\n";
+      await appendFile(filename, line);
+
+      // Update index
+      await this.updateIndex(result.pillar, result.environment);
+    } catch (error) {
+      throw new Error(`Failed to write result: ${(error as Error).message}`);
+    }
+  }
+
+  /**
+   * Write multiple test results in batch
+   */
+  async writeResults(results: TestResult[]): Promise<void> {
+    if (results.length === 0) {
+      return;
+    }
+
+    // Group results by pillar and environment
+    const grouped = this.groupResults(results);
+
+    for (const [key, groupResults] of Object.entries(grouped)) {
+      const [pillar, environment] = key.split(":");
+      const filename = this.getFilename(pillar as Pillar, environment);
+
+      try {
+        // Ensure results directory exists
+        await mkdir(this.resultsDir, { recursive: true });
+
+        // Write all results as JSON lines
+        const lines =
+          groupResults.map((r) => JSON.stringify(r)).join("\n") + "\n";
+        await appendFile(filename, lines);
+
+        // Update index
+        await this.updateIndex(pillar as Pillar, environment);
+      } catch (error) {
+        throw new Error(
+          `Failed to write results for ${pillar}/${environment}: ${(error as Error).message}`,
+        );
+      }
+    }
+  }
+
+  /**
+   * Update results index for quick lookups
+   */
+  private async updateIndex(
+    pillar: Pillar,
+    environment: string,
+  ): Promise<void> {
+    const indexPath = join(this.resultsDir, "index.json");
+
+    try {
+      let index: Record<string, unknown> = {};
+
+      // Load existing index. Distinguish "no index yet" (ENOENT → start fresh)
+      // from a corrupt/unreadable existing file — clobbering a corrupt index
+      // with {} would silently wipe every other pillar/env entry.
+      let content: string | undefined;
+      try {
+        content = await readFile(indexPath, "utf-8");
+      } catch (e) {
+        if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e;
+      }
+      if (content !== undefined) {
+        const parsed = JSON.parse(content) as unknown;
+        if (
+          parsed === null ||
+          typeof parsed !== "object" ||
+          Array.isArray(parsed)
+        ) {
+          throw new Error("existing index.json is not an object");
+        }
+        index = parsed as Record<string, unknown>;
+      }
+
+      const key = `${pillar}:${environment}`;
+      const date = new Date().toISOString().split("T")[0];
+
+      // Update index entry
+      index[key] = {
+        pillar,
+        environment,
+        lastUpdated: new Date().toISOString(),
+        latestFile: this.getFilename(pillar, environment),
+        date,
+      };
+
+      // Write atomically (temp + rename) so a crash/concurrent worker can't
+      // leave a half-written index.json.
+      const tmp = `${indexPath}.${process.pid}.tmp`;
+      try {
+        await writeFile(tmp, JSON.stringify(index, null, 2));
+        await rename(tmp, indexPath);
+      } catch (err) {
+        // Don't leave an orphaned temp file behind when the write/rename fails.
+        await unlink(tmp).catch(() => {});
+        throw err;
+      }
+    } catch (error) {
+      // Index update is non-critical, just log error
+      console.warn(`Failed to update index: ${(error as Error).message}`);
+    }
+  }
+
+  /**
+   * Get filename for results
+   */
+  private getFilename(pillar: string, environment: string): string {
+    const date = new Date().toISOString().split("T")[0];
+    return join(this.resultsDir, `${pillar}-${environment}-${date}.jsonl`);
+  }
+
+  /**
+   * Group results by pillar and environment
+   */
+  private groupResults(results: TestResult[]): Record<string, TestResult[]> {
+    return results.reduce(
+      (acc, result) => {
+        const key = `${result.pillar}:${result.environment}`;
+        if (!acc[key]) {
+          acc[key] = [];
+        }
+        acc[key].push(result);
+        return acc;
+      },
+      {} as Record<string, TestResult[]>,
+    );
+  }
+}
+
+// Export singleton instance
+export const resultWriter = new ResultWriter();

package/src/shared/secrets/doppler.ts

@@ -0,0 +1,126 @@
+/**
+ * Doppler strategy — reads/writes flat keys with the `doppler` CLI.
+ * Selected with `secrets.provider: doppler` (the default).
+ *
+ * Project/config come from `secrets.doppler.{project,config}` in
+ * config/den.yml, else from the directory's `doppler setup` scope,
+ * else from DOPPLER_PROJECT / DOPPLER_CONFIG / DOPPLER_TOKEN.
+ */
+import { execFile, spawn } from "child_process";
+import { promisify } from "util";
+import { flatKeyFor } from "./keys.js";
+import type { PreflightResult, SecretRef, SecretsProvider } from "./types.js";
+
+import { product } from "../product.js";
+const execFileAsync = promisify(execFile);
+
+export interface DopplerOptions {
+  project?: string;
+  config?: string;
+  keys?: Record<string, string>;
+  timeoutMs?: number;
+}
+
+export class DopplerProvider implements SecretsProvider {
+  readonly name = "doppler" as const;
+  private readonly timeout: number;
+
+  constructor(private readonly opts: DopplerOptions = {}) {
+    this.timeout = opts.timeoutMs ?? 30_000;
+  }
+
+  /** `--project/--config` flags when configured explicitly. */
+  scopeArgs(): string[] {
+    const a: string[] = [];
+    if (this.opts.project) a.push("--project", this.opts.project);
+    if (this.opts.config) a.push("--config", this.opts.config);
+    return a;
+  }
+
+  keyFor(ref: SecretRef): string {
+    return flatKeyFor(ref, this.opts.keys);
+  }
+
+  async isReady(): Promise<boolean> {
+    try {
+      await execFileAsync("doppler", ["me", "--json"], {
+        timeout: this.timeout,
+      });
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async get(ref: SecretRef): Promise<string> {
+    const key = this.keyFor(ref);
+    const { stdout } = await execFileAsync(
+      "doppler",
+      ["secrets", "get", key, "--plain", ...this.scopeArgs()],
+      { timeout: this.timeout },
+    );
+    return String(stdout).trim();
+  }
+
+  async tryGet(ref: SecretRef): Promise<string | null> {
+    try {
+      const v = await this.get(ref);
+      return v.length > 0 ? v : null;
+    } catch {
+      return null;
+    }
+  }
+
+  /** Value goes over stdin so it never appears in the process table. */
+  async set(ref: SecretRef, value: string): Promise<void> {
+    const key = this.keyFor(ref);
+    await new Promise<void>((resolve, reject) => {
+      const child = spawn(
+        "doppler",
+        ["secrets", "set", key, "--silent", ...this.scopeArgs()],
+        { stdio: ["pipe", "ignore", "pipe"] },
+      );
+      let stderr = "";
+      child.stderr?.on("data", (d) => (stderr += String(d)));
+      child.on("error", reject);
+      child.on("close", (code) =>
+        code === 0
+          ? resolve()
+          : reject(
+              new Error(`doppler secrets set ${key} failed: ${stderr.trim()}`),
+            ),
+      );
+      child.stdin?.end(value);
+    });
+  }
+
+  async preflight(): Promise<PreflightResult> {
+    if (!(await this.isReady())) {
+      return {
+        ok: false,
+        message:
+          "Doppler CLI is not installed or not logged in.\n" +
+          "Install: brew install dopplerhq/cli/doppler  then: doppler login",
+      };
+    }
+    try {
+      await execFileAsync(
+        "doppler",
+        ["secrets", "--only-names", "--json", ...this.scopeArgs()],
+        { timeout: this.timeout },
+      );
+      return { ok: true };
+    } catch {
+      return {
+        ok: false,
+        message:
+          "Doppler is logged in but no project/config is selected.\n" +
+          `Run \`doppler setup\` in the repo, or set secrets.doppler.project/config in ${product().configFile}.`,
+      };
+    }
+  }
+
+  runPrefix(): string[] {
+    return ["doppler", "run", ...this.scopeArgs(), "--"];
+  }
+}

package/src/shared/secrets/env.ts

@@ -0,0 +1,48 @@
+/**
+ * Environment-variable strategy — reads flat keys from process.env.
+ * Selected with `secrets.provider: env`. Intended for CI and for engineers who
+ * inject secrets themselves (direnv, `doppler run -- den …`, CI secrets).
+ */
+import { flatKeyFor } from "./keys.js";
+import type { PreflightResult, SecretRef, SecretsProvider } from "./types.js";
+
+export class EnvProvider implements SecretsProvider {
+  readonly name = "env" as const;
+
+  constructor(
+    private readonly keys: Record<string, string> = {},
+    private readonly env: NodeJS.ProcessEnv = process.env,
+  ) {}
+
+  keyFor(ref: SecretRef): string {
+    return flatKeyFor(ref, this.keys);
+  }
+
+  async isReady(): Promise<boolean> {
+    return true;
+  }
+
+  async get(ref: SecretRef): Promise<string> {
+    const key = this.keyFor(ref);
+    const v = this.env[key];
+    if (!v) throw new Error(`Secret ${key} is not set in the environment`);
+    return v;
+  }
+
+  async tryGet(ref: SecretRef): Promise<string | null> {
+    return this.env[this.keyFor(ref)] || null;
+  }
+
+  async set(ref: SecretRef, value: string): Promise<void> {
+    // Process-local only: the env backend has no durable store.
+    this.env[this.keyFor(ref)] = value;
+  }
+
+  async preflight(): Promise<PreflightResult> {
+    return { ok: true };
+  }
+
+  runPrefix(): string[] {
+    return [];
+  }
+}

package/src/shared/secrets/index.ts

@@ -0,0 +1,197 @@
+/**
+ * Secrets strategy selection.
+ *
+ * Resolution order for the backend:
+ *   1. DEN_SECRETS_PROVIDER env var (doppler | 1password | env)
+ *   2. `secrets.provider` in config/local/user.yml (per-engineer)
+ *   3. `secrets.provider` in config/den.yml (team default)
+ *   4. "doppler"
+ *
+ * Adding a backend = implement SecretsProvider and register it in FACTORIES.
+ */
+import { OP_ACCOUNT_DEFAULT, OP_CLI_TIMEOUT_MS } from "../constants.js";
+import { readDenConfig, readUserConfig } from "../den-config.js";
+import { DopplerProvider } from "./doppler.js";
+import { EnvProvider } from "./env.js";
+import { OnePasswordProvider } from "./onepassword.js";
+import type {
+  SecretRef,
+  SecretsProvider,
+  SecretsProviderName,
+} from "./types.js";
+
+import { envKey } from "../product.js";
+export * from "./types.js";
+export { flatKeyFor, toEnvName, refPath } from "./keys.js";
+export { DopplerProvider } from "./doppler.js";
+export { EnvProvider } from "./env.js";
+export { OnePasswordProvider } from "./onepassword.js";
+
+export interface SecretsConfig {
+  provider?: SecretsProviderName;
+  /**
+   * When true, `den up` / `make web` refuse to start until the backend is
+   * ready. Default false: a missing backend is a warning, because each repo's
+   * own `make up` manages its own secrets and den only needs this backend
+   * for test logins and the saved sudo password.
+   */
+  required?: boolean;
+  doppler?: { project?: string; config?: string };
+  onepassword?: { account?: string };
+  /** `"vault/item/field": FLAT_KEY` overrides for doppler/env. */
+  keys?: Record<string, string>;
+  /**
+   * Where the engine's own named secrets live, overriding the built-in
+   * defaults field by field (see {@link namedSecretRef}). Names in use:
+   * `sudo` (the saved local-dev sudo password) and `journeys_agent_key`
+   * (the journeys pillar's agent API key).
+   *
+   *   refs:
+   *     journeys_agent_key: { vault: Shared, item: CI Keys }
+   */
+  refs?: Record<string, Partial<SecretRef>>;
+  /**
+   * The per-engineer developer keys `make den-setup-1password` collects
+   * into the developer-keys item (see scripts/setup-1password.ts). Default:
+   * only the keys the engine itself uses.
+   */
+  developer_keys?: DeveloperKeysConfig;
+}
+
+/** One developer key the setup script prompts for. */
+export interface DeveloperKeyDef {
+  /** Field label in the item (= the env-var name). */
+  field: string;
+  description?: string;
+  /** Where to create one. */
+  url?: string;
+  required?: boolean;
+}
+
+export interface DeveloperKeysConfig {
+  /**
+   * A dotenv file (relative to the den root, e.g. an app checkout's
+   * `.env`) whose values are offered as defaults.
+   */
+  env_file?: string;
+  fields?: DeveloperKeyDef[];
+}
+
+export const SECRETS_PROVIDERS: readonly SecretsProviderName[] = [
+  "doppler",
+  "1password",
+  "env",
+];
+
+const FACTORIES: Record<
+  SecretsProviderName,
+  (c: SecretsConfig) => SecretsProvider
+> = {
+  doppler: (c) => new DopplerProvider({ ...c.doppler, keys: c.keys }),
+  "1password": (c) =>
+    new OnePasswordProvider({
+      account: c.onepassword?.account ?? OP_ACCOUNT_DEFAULT,
+      timeoutMs: OP_CLI_TIMEOUT_MS,
+    }),
+  env: (c) => new EnvProvider(c.keys),
+};
+
+function isProviderName(v: unknown): v is SecretsProviderName {
+  return typeof v === "string" && (SECRETS_PROVIDERS as string[]).includes(v);
+}
+
+/** Effective secrets config (team file, then per-engineer override). */
+export function readSecretsConfig(): SecretsConfig {
+  let team: SecretsConfig = {};
+  let user: SecretsConfig = {};
+  try {
+    team = readDenConfig().secrets ?? {};
+  } catch {
+    /* unreadable config → defaults */
+  }
+  try {
+    user = readUserConfig().secrets ?? {};
+  } catch {
+    /* no user config */
+  }
+  return {
+    ...team,
+    ...user,
+    doppler: { ...team.doppler, ...user.doppler },
+    onepassword: { ...team.onepassword, ...user.onepassword },
+    keys: { ...team.keys, ...user.keys },
+    refs: { ...team.refs, ...user.refs },
+  };
+}
+
+/**
+ * The engine-named secret `name`: `defaults` with any configured
+ * `secrets.refs.<name>` fields laid over them. The engine never hard-wires
+ * an org's vault or item names; the defaults are neutral placeholders.
+ */
+export function namedSecretRef(
+  name: string,
+  defaults: SecretRef,
+  cfg: SecretsConfig = readSecretsConfig(),
+): SecretRef {
+  const override = cfg.refs?.[name] ?? {};
+  const pick = (v: unknown, d: string): string =>
+    typeof v === "string" && v.trim() ? v.trim() : d;
+  const ref: SecretRef = {
+    vault: pick(override.vault, defaults.vault),
+    item: pick(override.item, defaults.item),
+    field: pick(override.field, defaults.field),
+  };
+  const key = override.key ?? defaults.key;
+  if (key) ref.key = key;
+  return ref;
+}
+
+export function resolveProviderName(
+  cfg: SecretsConfig,
+  env: NodeJS.ProcessEnv = process.env,
+): SecretsProviderName {
+  const fromEnv = env.DEN_SECRETS_PROVIDER;
+  if (fromEnv) {
+    if (!isProviderName(fromEnv)) {
+      throw new Error(
+        `${envKey("SECRETS_PROVIDER")}="${fromEnv}" is not one of ${SECRETS_PROVIDERS.join(", ")}`,
+      );
+    }
+    return fromEnv;
+  }
+  if (cfg.provider !== undefined && !isProviderName(cfg.provider)) {
+    throw new Error(
+      `secrets.provider "${String(cfg.provider)}" is not one of ${SECRETS_PROVIDERS.join(", ")}`,
+    );
+  }
+  return cfg.provider ?? "doppler";
+}
+
+export function createSecretsProvider(
+  name: SecretsProviderName,
+  cfg: SecretsConfig = {},
+): SecretsProvider {
+  return FACTORIES[name](cfg);
+}
+
+let cached: SecretsProvider | null = null;
+let cachedKey = "";
+
+/** The configured backend. Re-resolves when DEN_SECRETS_PROVIDER changes. */
+export function getSecretsProvider(): SecretsProvider {
+  const cfg = readSecretsConfig();
+  const name = resolveProviderName(cfg);
+  const key = `${name}|${JSON.stringify(cfg)}`;
+  if (!cached || key !== cachedKey) {
+    cached = createSecretsProvider(name, cfg);
+    cachedKey = key;
+  }
+  return cached;
+}
+
+/** Test hook. */
+export function resetSecretsProviderCache(): void {
+  cached = null;
+  cachedKey = "";
+}

package/src/shared/secrets/keys.ts

@@ -0,0 +1,33 @@
+import type { SecretRef } from "./types.js";
+
+/** Upper-snake a free-form label: "Test User (Auth0)" → "TEST_USER_AUTH0". */
+export function toEnvName(label: string): string {
+  return label
+    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+    .replace(/[^A-Za-z0-9]+/g, "_")
+    .replace(/^_+|_+$/g, "")
+    .toUpperCase();
+}
+
+/** Canonical `vault/item/field` string used as the key of `secrets.keys`. */
+export function refPath(ref: SecretRef): string {
+  return `${ref.vault}/${ref.item}/${ref.field}`;
+}
+
+/**
+ * Resolve the flat key a key/value backend should use for `ref`:
+ *   1. `ref.key` when the caller pinned one;
+ *   2. the `secrets.keys` override map (`"vault/item/field": KEY`);
+ *   3. the field itself when it already looks like an env var (API keys);
+ *   4. otherwise `ITEM_FIELD` upper-snaked.
+ */
+export function flatKeyFor(
+  ref: SecretRef,
+  overrides: Record<string, string> = {},
+): string {
+  if (ref.key) return ref.key;
+  const mapped = overrides[refPath(ref)];
+  if (mapped) return mapped;
+  if (/^[A-Z][A-Z0-9_]*$/.test(ref.field)) return ref.field;
+  return toEnvName(`${ref.item} ${ref.field}`);
+}

package/src/shared/secrets/onepassword.ts

@@ -0,0 +1,75 @@
+/**
+ * 1Password strategy — reads/writes item fields with the `op` CLI.
+ * Selected with `secrets.provider: 1password`.
+ */
+import { execFile } from "child_process";
+import { promisify } from "util";
+import type { PreflightResult, SecretRef, SecretsProvider } from "./types.js";
+
+import { product } from "../product.js";
+const execFileAsync = promisify(execFile);
+
+export interface OnePasswordOptions {
+  account?: string;
+  timeoutMs?: number;
+}
+
+export class OnePasswordProvider implements SecretsProvider {
+  readonly name = "1password" as const;
+  private readonly timeout: number;
+
+  constructor(opts: OnePasswordOptions = {}) {
+    this.timeout = opts.timeoutMs ?? 60_000;
+    if (opts.account) process.env.OP_ACCOUNT ||= opts.account;
+  }
+
+  async isReady(): Promise<boolean> {
+    try {
+      await execFileAsync("op", ["whoami"], { timeout: this.timeout });
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async get(ref: SecretRef): Promise<string> {
+    // Direct CLI args rather than an op:// URI — URIs break on "(" "," etc.
+    const args = ["item", "get", ref.item];
+    if (ref.vault) args.push("--vault", ref.vault);
+    args.push("--fields", `label=${ref.field}`, "--reveal");
+    const { stdout } = await execFileAsync("op", args, {
+      timeout: this.timeout,
+    });
+    return String(stdout).trim();
+  }
+
+  async tryGet(ref: SecretRef): Promise<string | null> {
+    try {
+      const v = await this.get(ref);
+      return v.length > 0 ? v : null;
+    } catch {
+      return null;
+    }
+  }
+
+  async set(ref: SecretRef, value: string): Promise<void> {
+    const args = ["item", "edit", ref.item];
+    if (ref.vault) args.push("--vault", ref.vault);
+    args.push(`${ref.field}[password]=${value}`);
+    await execFileAsync("op", args, { timeout: this.timeout });
+  }
+
+  async preflight(): Promise<PreflightResult> {
+    if (await this.isReady()) return { ok: true };
+    return {
+      ok: false,
+      message:
+        "1Password CLI is not signed in. Run: eval $(op signin)\n" +
+        `Or switch backends with secrets.provider in ${product().configFile}.`,
+    };
+  }
+
+  runPrefix(): string[] {
+    return [];
+  }
+}

package/src/shared/secrets/types.ts

@@ -0,0 +1,52 @@
+/**
+ * Secrets strategy — the contract every secrets backend implements.
+ *
+ * Den never talks to a vault directly. Callers describe WHAT they need as
+ * a {@link SecretRef} (a logical vault / item / field triple) and the active
+ * {@link SecretsProvider} decides HOW to fetch it: 1Password item fields,
+ * Doppler flat keys, or plain environment variables. The backend is selected
+ * by configuration (see `resolveSecretsProvider`), never hard-wired.
+ */
+
+/** A logical pointer to one secret value. */
+export interface SecretRef {
+  /** Logical vault/namespace (1Password vault; ignored by flat-key backends). */
+  vault: string;
+  /** Logical item within the vault (1Password item name or ID). */
+  item: string;
+  /** Field within the item (e.g. "password", "username", "OPENAI_API_KEY"). */
+  field: string;
+  /**
+   * Flat key for key/value backends (Doppler, env). When omitted it is taken
+   * from the `secrets.keys` map in config/den.yml, else derived — see
+   * `flatKeyFor`.
+   */
+  key?: string;
+}
+
+export type SecretsProviderName = "doppler" | "1password" | "env";
+
+export interface PreflightResult {
+  ok: boolean;
+  /** Human-readable remediation when !ok. */
+  message?: string;
+}
+
+export interface SecretsProvider {
+  readonly name: SecretsProviderName;
+  /** True when the backend CLI is installed and authenticated. Never throws. */
+  isReady(): Promise<boolean>;
+  /** Read a secret. Throws when it cannot be resolved. */
+  get(ref: SecretRef): Promise<string>;
+  /** Read a secret, returning null instead of throwing. */
+  tryGet(ref: SecretRef): Promise<string | null>;
+  /** Write (create or overwrite) a secret. Throws on failure. */
+  set(ref: SecretRef, value: string): Promise<void>;
+  /** Check the backend is usable before `den up`/`restart`. */
+  preflight(): Promise<PreflightResult>;
+  /**
+   * argv prefix that runs a child command with secrets injected as env vars,
+   * e.g. `["doppler","run","--"]`. Empty when the backend injects nothing.
+   */
+  runPrefix(): string[];
+}