@monster-orange/den 0.0.0-stage → 0.1.0

package/src/shared/secrets/index.ts

@@ -0,0 +1,197 @@
+/**
+ * Secrets strategy selection.
+ *
+ * Resolution order for the backend:
+ *   1. DEN_SECRETS_PROVIDER env var (doppler | 1password | env)
+ *   2. `secrets.provider` in config/local/user.yml (per-engineer)
+ *   3. `secrets.provider` in config/den.yml (team default)
+ *   4. "doppler"
+ *
+ * Adding a backend = implement SecretsProvider and register it in FACTORIES.
+ */
+import { OP_ACCOUNT_DEFAULT, OP_CLI_TIMEOUT_MS } from "../constants.js";
+import { readDenConfig, readUserConfig } from "../den-config.js";
+import { DopplerProvider } from "./doppler.js";
+import { EnvProvider } from "./env.js";
+import { OnePasswordProvider } from "./onepassword.js";
+import type {
+  SecretRef,
+  SecretsProvider,
+  SecretsProviderName,
+} from "./types.js";
+
+import { envKey } from "../product.js";
+export * from "./types.js";
+export { flatKeyFor, toEnvName, refPath } from "./keys.js";
+export { DopplerProvider } from "./doppler.js";
+export { EnvProvider } from "./env.js";
+export { OnePasswordProvider } from "./onepassword.js";
+
+export interface SecretsConfig {
+  provider?: SecretsProviderName;
+  /**
+   * When true, `den up` / `make web` refuse to start until the backend is
+   * ready. Default false: a missing backend is a warning, because each repo's
+   * own `make up` manages its own secrets and den only needs this backend
+   * for test logins and the saved sudo password.
+   */
+  required?: boolean;
+  doppler?: { project?: string; config?: string };
+  onepassword?: { account?: string };
+  /** `"vault/item/field": FLAT_KEY` overrides for doppler/env. */
+  keys?: Record<string, string>;
+  /**
+   * Where the engine's own named secrets live, overriding the built-in
+   * defaults field by field (see {@link namedSecretRef}). Names in use:
+   * `sudo` (the saved local-dev sudo password) and `journeys_agent_key`
+   * (the journeys pillar's agent API key).
+   *
+   *   refs:
+   *     journeys_agent_key: { vault: Shared, item: CI Keys }
+   */
+  refs?: Record<string, Partial<SecretRef>>;
+  /**
+   * The per-engineer developer keys `make den-setup-1password` collects
+   * into the developer-keys item (see scripts/setup-1password.ts). Default:
+   * only the keys the engine itself uses.
+   */
+  developer_keys?: DeveloperKeysConfig;
+}
+
+/** One developer key the setup script prompts for. */
+export interface DeveloperKeyDef {
+  /** Field label in the item (= the env-var name). */
+  field: string;
+  description?: string;
+  /** Where to create one. */
+  url?: string;
+  required?: boolean;
+}
+
+export interface DeveloperKeysConfig {
+  /**
+   * A dotenv file (relative to the den root, e.g. an app checkout's
+   * `.env`) whose values are offered as defaults.
+   */
+  env_file?: string;
+  fields?: DeveloperKeyDef[];
+}
+
+export const SECRETS_PROVIDERS: readonly SecretsProviderName[] = [
+  "doppler",
+  "1password",
+  "env",
+];
+
+const FACTORIES: Record<
+  SecretsProviderName,
+  (c: SecretsConfig) => SecretsProvider
+> = {
+  doppler: (c) => new DopplerProvider({ ...c.doppler, keys: c.keys }),
+  "1password": (c) =>
+    new OnePasswordProvider({
+      account: c.onepassword?.account ?? OP_ACCOUNT_DEFAULT,
+      timeoutMs: OP_CLI_TIMEOUT_MS,
+    }),
+  env: (c) => new EnvProvider(c.keys),
+};
+
+function isProviderName(v: unknown): v is SecretsProviderName {
+  return typeof v === "string" && (SECRETS_PROVIDERS as string[]).includes(v);
+}
+
+/** Effective secrets config (team file, then per-engineer override). */
+export function readSecretsConfig(): SecretsConfig {
+  let team: SecretsConfig = {};
+  let user: SecretsConfig = {};
+  try {
+    team = readDenConfig().secrets ?? {};
+  } catch {
+    /* unreadable config → defaults */
+  }
+  try {
+    user = readUserConfig().secrets ?? {};
+  } catch {
+    /* no user config */
+  }
+  return {
+    ...team,
+    ...user,
+    doppler: { ...team.doppler, ...user.doppler },
+    onepassword: { ...team.onepassword, ...user.onepassword },
+    keys: { ...team.keys, ...user.keys },
+    refs: { ...team.refs, ...user.refs },
+  };
+}
+
+/**
+ * The engine-named secret `name`: `defaults` with any configured
+ * `secrets.refs.<name>` fields laid over them. The engine never hard-wires
+ * an org's vault or item names; the defaults are neutral placeholders.
+ */
+export function namedSecretRef(
+  name: string,
+  defaults: SecretRef,
+  cfg: SecretsConfig = readSecretsConfig(),
+): SecretRef {
+  const override = cfg.refs?.[name] ?? {};
+  const pick = (v: unknown, d: string): string =>
+    typeof v === "string" && v.trim() ? v.trim() : d;
+  const ref: SecretRef = {
+    vault: pick(override.vault, defaults.vault),
+    item: pick(override.item, defaults.item),
+    field: pick(override.field, defaults.field),
+  };
+  const key = override.key ?? defaults.key;
+  if (key) ref.key = key;
+  return ref;
+}
+
+export function resolveProviderName(
+  cfg: SecretsConfig,
+  env: NodeJS.ProcessEnv = process.env,
+): SecretsProviderName {
+  const fromEnv = env.DEN_SECRETS_PROVIDER;
+  if (fromEnv) {
+    if (!isProviderName(fromEnv)) {
+      throw new Error(
+        `${envKey("SECRETS_PROVIDER")}="${fromEnv}" is not one of ${SECRETS_PROVIDERS.join(", ")}`,
+      );
+    }
+    return fromEnv;
+  }
+  if (cfg.provider !== undefined && !isProviderName(cfg.provider)) {
+    throw new Error(
+      `secrets.provider "${String(cfg.provider)}" is not one of ${SECRETS_PROVIDERS.join(", ")}`,
+    );
+  }
+  return cfg.provider ?? "doppler";
+}
+
+export function createSecretsProvider(
+  name: SecretsProviderName,
+  cfg: SecretsConfig = {},
+): SecretsProvider {
+  return FACTORIES[name](cfg);
+}
+
+let cached: SecretsProvider | null = null;
+let cachedKey = "";
+
+/** The configured backend. Re-resolves when DEN_SECRETS_PROVIDER changes. */
+export function getSecretsProvider(): SecretsProvider {
+  const cfg = readSecretsConfig();
+  const name = resolveProviderName(cfg);
+  const key = `${name}|${JSON.stringify(cfg)}`;
+  if (!cached || key !== cachedKey) {
+    cached = createSecretsProvider(name, cfg);
+    cachedKey = key;
+  }
+  return cached;
+}
+
+/** Test hook. */
+export function resetSecretsProviderCache(): void {
+  cached = null;
+  cachedKey = "";
+}

package/src/shared/secrets/keys.ts

@@ -0,0 +1,33 @@
+import type { SecretRef } from "./types.js";
+
+/** Upper-snake a free-form label: "Test User (Auth0)" → "TEST_USER_AUTH0". */
+export function toEnvName(label: string): string {
+  return label
+    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+    .replace(/[^A-Za-z0-9]+/g, "_")
+    .replace(/^_+|_+$/g, "")
+    .toUpperCase();
+}
+
+/** Canonical `vault/item/field` string used as the key of `secrets.keys`. */
+export function refPath(ref: SecretRef): string {
+  return `${ref.vault}/${ref.item}/${ref.field}`;
+}
+
+/**
+ * Resolve the flat key a key/value backend should use for `ref`:
+ *   1. `ref.key` when the caller pinned one;
+ *   2. the `secrets.keys` override map (`"vault/item/field": KEY`);
+ *   3. the field itself when it already looks like an env var (API keys);
+ *   4. otherwise `ITEM_FIELD` upper-snaked.
+ */
+export function flatKeyFor(
+  ref: SecretRef,
+  overrides: Record<string, string> = {},
+): string {
+  if (ref.key) return ref.key;
+  const mapped = overrides[refPath(ref)];
+  if (mapped) return mapped;
+  if (/^[A-Z][A-Z0-9_]*$/.test(ref.field)) return ref.field;
+  return toEnvName(`${ref.item} ${ref.field}`);
+}

package/src/shared/secrets/onepassword.ts

@@ -0,0 +1,75 @@
+/**
+ * 1Password strategy — reads/writes item fields with the `op` CLI.
+ * Selected with `secrets.provider: 1password`.
+ */
+import { execFile } from "child_process";
+import { promisify } from "util";
+import type { PreflightResult, SecretRef, SecretsProvider } from "./types.js";
+
+import { product } from "../product.js";
+const execFileAsync = promisify(execFile);
+
+export interface OnePasswordOptions {
+  account?: string;
+  timeoutMs?: number;
+}
+
+export class OnePasswordProvider implements SecretsProvider {
+  readonly name = "1password" as const;
+  private readonly timeout: number;
+
+  constructor(opts: OnePasswordOptions = {}) {
+    this.timeout = opts.timeoutMs ?? 60_000;
+    if (opts.account) process.env.OP_ACCOUNT ||= opts.account;
+  }
+
+  async isReady(): Promise<boolean> {
+    try {
+      await execFileAsync("op", ["whoami"], { timeout: this.timeout });
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async get(ref: SecretRef): Promise<string> {
+    // Direct CLI args rather than an op:// URI — URIs break on "(" "," etc.
+    const args = ["item", "get", ref.item];
+    if (ref.vault) args.push("--vault", ref.vault);
+    args.push("--fields", `label=${ref.field}`, "--reveal");
+    const { stdout } = await execFileAsync("op", args, {
+      timeout: this.timeout,
+    });
+    return String(stdout).trim();
+  }
+
+  async tryGet(ref: SecretRef): Promise<string | null> {
+    try {
+      const v = await this.get(ref);
+      return v.length > 0 ? v : null;
+    } catch {
+      return null;
+    }
+  }
+
+  async set(ref: SecretRef, value: string): Promise<void> {
+    const args = ["item", "edit", ref.item];
+    if (ref.vault) args.push("--vault", ref.vault);
+    args.push(`${ref.field}[password]=${value}`);
+    await execFileAsync("op", args, { timeout: this.timeout });
+  }
+
+  async preflight(): Promise<PreflightResult> {
+    if (await this.isReady()) return { ok: true };
+    return {
+      ok: false,
+      message:
+        "1Password CLI is not signed in. Run: eval $(op signin)\n" +
+        `Or switch backends with secrets.provider in ${product().configFile}.`,
+    };
+  }
+
+  runPrefix(): string[] {
+    return [];
+  }
+}

package/src/shared/secrets/types.ts

@@ -0,0 +1,52 @@
+/**
+ * Secrets strategy — the contract every secrets backend implements.
+ *
+ * Den never talks to a vault directly. Callers describe WHAT they need as
+ * a {@link SecretRef} (a logical vault / item / field triple) and the active
+ * {@link SecretsProvider} decides HOW to fetch it: 1Password item fields,
+ * Doppler flat keys, or plain environment variables. The backend is selected
+ * by configuration (see `resolveSecretsProvider`), never hard-wired.
+ */
+
+/** A logical pointer to one secret value. */
+export interface SecretRef {
+  /** Logical vault/namespace (1Password vault; ignored by flat-key backends). */
+  vault: string;
+  /** Logical item within the vault (1Password item name or ID). */
+  item: string;
+  /** Field within the item (e.g. "password", "username", "OPENAI_API_KEY"). */
+  field: string;
+  /**
+   * Flat key for key/value backends (Doppler, env). When omitted it is taken
+   * from the `secrets.keys` map in config/den.yml, else derived — see
+   * `flatKeyFor`.
+   */
+  key?: string;
+}
+
+export type SecretsProviderName = "doppler" | "1password" | "env";
+
+export interface PreflightResult {
+  ok: boolean;
+  /** Human-readable remediation when !ok. */
+  message?: string;
+}
+
+export interface SecretsProvider {
+  readonly name: SecretsProviderName;
+  /** True when the backend CLI is installed and authenticated. Never throws. */
+  isReady(): Promise<boolean>;
+  /** Read a secret. Throws when it cannot be resolved. */
+  get(ref: SecretRef): Promise<string>;
+  /** Read a secret, returning null instead of throwing. */
+  tryGet(ref: SecretRef): Promise<string | null>;
+  /** Write (create or overwrite) a secret. Throws on failure. */
+  set(ref: SecretRef, value: string): Promise<void>;
+  /** Check the backend is usable before `den up`/`restart`. */
+  preflight(): Promise<PreflightResult>;
+  /**
+   * argv prefix that runs a child command with secrets injected as env vars,
+   * e.g. `["doppler","run","--"]`. Empty when the backend injects nothing.
+   */
+  runPrefix(): string[];
+}

package/src/shared/service-url.ts

@@ -0,0 +1,119 @@
+import type {
+  Environment,
+  RawManifest,
+  ServiceHealthConfig,
+} from "./types/index.js";
+
+export interface ResolveManifestBaseUrlOptions {
+  manifest: RawManifest;
+  repoName: string;
+  serviceName: string;
+  environment: Environment;
+  baseUrlKey: "api" | "web";
+}
+
+export interface ResolveServiceBaseUrlOptions {
+  baseUrlKey: "api" | "web";
+  configuredUrl: string;
+  manifestUrl?: string | null;
+  overrideUrl?: string | null;
+}
+
+function healthUrlForConfig(healthConfig?: ServiceHealthConfig): string | null {
+  if (!healthConfig) {
+    return null;
+  }
+
+  if ("type" in healthConfig && healthConfig.type === "shell") {
+    return null;
+  }
+
+  return healthConfig.url;
+}
+
+function normalizeServiceBaseUrl(
+  healthUrl: string,
+  baseUrlKey: "api" | "web",
+): string {
+  if (baseUrlKey !== "api") {
+    return healthUrl;
+  }
+
+  const url = new URL(healthUrl);
+  url.pathname =
+    url.pathname.replace(/\/(healthz|readyz|version)\/?$/, "") || "/";
+
+  const normalized = url.toString();
+  return normalized.endsWith("/") ? normalized.slice(0, -1) : normalized;
+}
+
+/**
+ * Derive a base URL from the manifest's `services.*.health.<env>` entry.
+ *
+ * Lookup order: `services[serviceName]`, then `services[repoName]`, then the
+ * manifest's declared `entrypoint`.
+ *
+ * The entrypoint step matters because `serviceName` here is the *pillar module*
+ * name (smoke passes `svcName` from `smoke.<name>`, perf passes
+ * `moduleConfig.service`), which need not match any `services:` key. A repo
+ * whose modules are named for what they test rather than for the service —
+ * `tier1`, `search`, `ingest` — matched neither of the first two lookups, so
+ * this returned null and callers silently fell back to
+ * `config/environments/<env>.json -> baseUrls.api`, i.e. shop's API. Repos
+ * appeared to work only when a module happened to share the service's name.
+ *
+ * `entrypoint` is precisely the manifest's statement of which service is the
+ * front door, and `service-runner.ts` already resolves an ambiguous subsystem
+ * the same way.
+ *
+ * For `baseUrlKey === "api"`, strips trailing `/healthz`, `/readyz`, or
+ * `/version` so the health endpoint becomes a usable API root.
+ */
+export function resolveManifestBaseUrl({
+  manifest,
+  repoName,
+  serviceName,
+  environment,
+  baseUrlKey,
+}: ResolveManifestBaseUrlOptions): string | null {
+  const entrypointConfig = manifest.entrypoint
+    ? manifest.services?.[manifest.entrypoint]
+    : undefined;
+  const serviceConfig =
+    manifest.services?.[serviceName] ??
+    manifest.services?.[repoName] ??
+    entrypointConfig;
+  const healthConfig = serviceConfig?.health?.[environment];
+  const healthUrl = healthUrlForConfig(healthConfig);
+  if (!healthUrl) {
+    return null;
+  }
+
+  return normalizeServiceBaseUrl(healthUrl, baseUrlKey);
+}
+
+/**
+ * Precedence-based URL resolver shared across pillars.
+ *
+ * 1. Explicit override (`BASE_API_URL` env, only for `baseUrlKey === "api"`)
+ * 2. Manifest-derived URL from `services.*.health.<env>`
+ * 3. Configured URL from `config/environments/<env>.json`
+ */
+export function resolveServiceBaseUrl({
+  baseUrlKey,
+  configuredUrl,
+  manifestUrl,
+  overrideUrl,
+}: ResolveServiceBaseUrlOptions): string {
+  const trimmedOverride = baseUrlKey === "api" ? overrideUrl?.trim() : "";
+  if (trimmedOverride) {
+    return trimmedOverride;
+  }
+
+  const trimmedManifestUrl = manifestUrl?.trim();
+  if (trimmedManifestUrl) {
+    return trimmedManifestUrl;
+  }
+
+  return configuredUrl;
+}

package/src/shared/smoke/branch-resolver.ts

@@ -0,0 +1,141 @@
+/**
+ * Deployed branch/SHA resolver
+ *
+ * Uses the GitHub Deployments API (via `gh`) to find the exact commit SHA
+ * that is currently deployed to a given environment.  This is the authoritative
+ * source of truth — more precise than a branch name, and independent of what
+ * happens to be on the local machine.
+ *
+ * Flow:
+ *   1. GET /repos/{owner}/{repo}/deployments?environment={env}&per_page=20
+ *   2. Walk deployments newest-first, fetch each one's statuses
+ *   3. Return the first deployment whose latest status is "success"
+ *
+ * Why not just take deployment[0]?  A deployment record is created when a
+ * deploy *starts*, not when it *succeeds*.  If the most recent deploy is still
+ * in progress or failed, we want the last known-good one.
+ *
+ * Requires: `gh` CLI authenticated (`gh auth login`).
+ */
+
+import { execSync } from "child_process";
+
+export interface DeployedRef {
+  /** Exact commit SHA deployed to this environment. */
+  sha: string;
+  /** Branch the commit was deployed from (informational). */
+  ref: string;
+  /** GitHub environment name. */
+  environment: string;
+  /** ISO timestamp of the deployment. */
+  deployedAt: string;
+  /** GitHub deployment ID. */
+  deploymentId: number;
+}
+
+interface GhDeployment {
+  id: number;
+  sha: string;
+  ref: string;
+  environment: string;
+  created_at: string;
+  statuses_url: string;
+}
+
+interface GhDeploymentStatus {
+  state: string; // "success" | "failure" | "error" | "pending" | "in_progress" | "queued" | "inactive"
+}
+
+function gh(args: string): string {
+  try {
+    return execSync(`gh ${args}`, {
+      stdio: ["ignore", "pipe", "pipe"],
+      encoding: "utf-8",
+    }).trim();
+  } catch (err) {
+    const msg = (err as { stderr?: string }).stderr?.trim() ?? String(err);
+    throw new Error(`gh command failed: ${msg}`);
+  }
+}
+
+/**
+ * Fetch the latest deployment status state for a deployment.
+ * Returns null if no statuses exist yet (deployment just created).
+ */
+function getLatestDeploymentState(
+  owner: string,
+  repo: string,
+  deploymentId: number,
+): string | null {
+  try {
+    const raw = gh(
+      `api "/repos/${owner}/${repo}/deployments/${deploymentId}/statuses?per_page=1"`,
+    );
+    const statuses: GhDeploymentStatus[] = JSON.parse(raw);
+    return statuses[0]?.state ?? null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Resolve the latest *successfully* deployed ref for a repo+environment.
+ *
+ * @param owner       GitHub org/user (e.g. "acme")
+ * @param repo        GitHub repo name (e.g. "api")
+ * @param environment GitHub environment name (e.g. "staging")
+ * @param limit       Max deployments to inspect before giving up (default 20)
+ */
+export async function resolveDeployedRef(
+  owner: string,
+  repo: string,
+  environment: string,
+  limit = 20,
+): Promise<DeployedRef> {
+  // Check gh is available and authenticated
+  try {
+    gh("auth status --active");
+  } catch {
+    throw new Error("GitHub CLI is not authenticated. Run: gh auth login");
+  }
+
+  const raw = gh(
+    `api "/repos/${owner}/${repo}/deployments?environment=${environment}&per_page=${limit}"`,
+  );
+  const deployments: GhDeployment[] = JSON.parse(raw);
+
+  if (!deployments.length) {
+    throw new Error(
+      `No deployments found for ${owner}/${repo} on environment "${environment}". ` +
+        `Has it been deployed via GitHub Actions?`,
+    );
+  }
+
+  // Walk newest-first, find the last successful one
+  for (const d of deployments) {
+    const state = getLatestDeploymentState(owner, repo, d.id);
+    if (state === "success") {
+      return {
+        sha: d.sha,
+        ref: d.ref,
+        environment: d.environment,
+        deployedAt: d.created_at,
+        deploymentId: d.id,
+      };
+    }
+  }
+
+  // No successful deployment found in the window — warn and use most recent
+  const latest = deployments[0];
+  console.warn(
+    `  ⚠  No successful deployment found for ${owner}/${repo}@${environment} ` +
+      `in the last ${limit} deploys. Using most recent (${latest.sha.slice(0, 8)}).`,
+  );
+  return {
+    sha: latest.sha,
+    ref: latest.ref,
+    environment: latest.environment,
+    deployedAt: latest.created_at,
+    deploymentId: latest.id,
+  };
+}

package/src/shared/smoke/git-remote.ts

@@ -0,0 +1,70 @@
+/**
+ * Git remote URL utilities
+ *
+ * Extracts the GitHub owner/repo from a git remote URL so the rest of the
+ * smoke pipeline can call the GitHub API without requiring this info to be
+ * hardcoded in .den.yml.
+ *
+ * Supported URL formats:
+ *   https://github.com/acme/api.git
+ *   https://github.com/acme/api
+ *   git@github.com:acme/api.git
+ */
+
+import { execSync } from "child_process";
+
+export interface GitHubCoords {
+  owner: string;
+  repo: string;
+}
+
+/**
+ * Run `git remote get-url origin` inside the given directory.
+ * Returns the raw remote URL string, or null if the directory is not a git
+ * repo or has no origin remote.
+ */
+export function getGitRemoteOrigin(repoPath: string): string | null {
+  try {
+    return execSync("git remote get-url origin", {
+      cwd: repoPath,
+      stdio: ["ignore", "pipe", "ignore"],
+      encoding: "utf-8",
+    }).trim();
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Parse GitHub owner and repo from a remote URL.
+ * Returns null if the URL is not a recognisable GitHub URL.
+ */
+export function parseGitHubOwnerRepo(remoteUrl: string): GitHubCoords | null {
+  // HTTPS: https://github.com/owner/repo[.git]
+  const httpsMatch = remoteUrl.match(
+    /^https?:\/\/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?$/,
+  );
+  if (httpsMatch) {
+    return { owner: httpsMatch[1], repo: httpsMatch[2] };
+  }
+
+  // SSH: git@github.com:owner/repo[.git]
+  const sshMatch = remoteUrl.match(
+    /^git@github\.com:([^/]+)\/([^/]+?)(?:\.git)?$/,
+  );
+  if (sshMatch) {
+    return { owner: sshMatch[1], repo: sshMatch[2] };
+  }
+
+  return null;
+}
+
+/**
+ * Convenience: get GitHub coords for a local repo path.
+ * Returns null if the remote is absent or not a GitHub remote.
+ */
+export function getGitHubCoords(repoPath: string): GitHubCoords | null {
+  const origin = getGitRemoteOrigin(repoPath);
+  if (!origin) return null;
+  return parseGitHubOwnerRepo(origin);
+}