@monster-orange/den 0.0.0 → 0.2.0

package/src/shared/manifest/read-manifest.ts

@@ -0,0 +1,278 @@
+/**
+ * Manifest reader — supports .den.yml (preferred) and .den.json (fallback).
+ *
+ * Import this helper instead of calling fs.readFileSync('.den.json') directly
+ * so all callers automatically get YAML support.
+ *
+ * A repo's own manifest always wins. When a checkout has none, Den's own
+ * copy is used: `manifests/<github-repo>.den.yml` in the Den repo, keyed by
+ * the checkout's `origin` repo name. That covers repos whose manifest is not
+ * merged yet and repos Den should not commit to.
+ *
+ * Whichever base won, `manifests/<github-repo>.override.yml` (when present)
+ * is deep-merged on top: see merge-override.ts for the rules.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import { execFileSync } from "child_process";
+import { getDistRoot } from "../paths.js";
+import { parse as parseYaml } from "yaml";
+import type { RawManifest } from "../types/index.js";
+
+import { product } from "../product.js";
+import { mergeOverride } from "./merge-override.js";
+/** Names probed in order. First match wins. */
+const MANIFEST_NAMES = [
+  `${product().manifestYml}`,
+  `${product().manifestJson}`,
+] as const;
+
+/** The repo's own manifest file, or null. */
+export function repoManifestPath(repoPath: string): string | null {
+  for (const name of MANIFEST_NAMES) {
+    const p = path.join(repoPath, name);
+    if (fs.existsSync(p)) return p;
+  }
+  return null;
+}
+
+// ── Den-held manifests ───────────────────────────────────────────────────
+// For a repo that doesn't carry a manifest yet (one we don't commit to, or
+// whose PR hasn't merged), den keeps one in manifests/<repo>.den.yml,
+// keyed by the git origin's repo name (falling back to the folder name). The
+// repo's own file always wins.
+
+const originCache = new Map<string, string | null>();
+
+/** Repo name of `repoPath`'s git origin (`…/<name>.git`), cached. */
+export function originRepoName(repoPath: string): string | null {
+  if (originCache.has(repoPath)) return originCache.get(repoPath)!;
+  let name: string | null = null;
+  try {
+    const url = execFileSync(
+      "git",
+      ["-C", repoPath, "config", "--get", "remote.origin.url"],
+      { encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"], timeout: 2000 },
+    ).trim();
+    name =
+      url
+        .replace(/\.git$/, "")
+        .split(/[/:]/)
+        .pop() || null;
+  } catch {
+    /* not a git checkout, or no origin */
+  }
+  originCache.set(repoPath, name);
+  return name;
+}
+
+/** Directory of den-held manifests (override: DEN_MANIFESTS_DIR). */
+export function heldManifestsDir(): string {
+  return process.env.DEN_MANIFESTS_DIR ?? path.join(getDistRoot(), "manifests");
+}
+
+/** The repo name of a `.den-services/<repo>/<branch>` path, else null. */
+function managedCloneRepo(repoPath: string): string | null {
+  const parent = path.dirname(repoPath);
+  return path.basename(path.dirname(parent)) === `${product().servicesDir}`
+    ? path.basename(parent)
+    : null;
+}
+
+/** The names `repoPath` is known by: origin repo, managed clone, folder. */
+function repoKeys(repoPath: string): string[] {
+  return [
+    originRepoName(repoPath),
+    // A managed clone lives at .den-services/<repo>/<branch>,
+    // and need not be cloned yet.
+    managedCloneRepo(repoPath),
+    path.basename(repoPath),
+  ].filter((k): k is string => !!k);
+}
+
+/** The first `manifests/<key><suffix>` that exists, or null. */
+function heldFile(repoPath: string, suffix: string): string | null {
+  for (const key of repoKeys(repoPath)) {
+    const p = path.join(heldManifestsDir(), `${key}${suffix}`);
+    if (fs.existsSync(p)) return p;
+  }
+  return null;
+}
+
+/** The den-held manifest for `repoPath`, or null. */
+export function heldManifestPath(repoPath: string): string | null {
+  return heldFile(repoPath, `${product().manifestYml}`);
+}
+
+/**
+ * The override suffix: `.override` plus the manifest's extension, so a
+ * profile whose manifest is `.acme.yml` reads `manifests/<repo>.override.yml`.
+ */
+export function overrideSuffix(): string {
+  return `.override${path.extname(product().manifestYml) || ".yml"}`;
+}
+
+/**
+ * The override for `repoPath` (`manifests/<repo>.override.yml`), or null.
+ * Matched by the same names, in the same order, as the held manifest.
+ */
+export function overrideManifestPath(repoPath: string): string | null {
+  return heldFile(repoPath, overrideSuffix());
+}
+
+/**
+ * Resolve the base manifest for `repoPath`: the repo's own .den.yml (or
+ * .json), else the one den holds for it. Returns null if neither exists.
+ * An override is layered on top by readManifest; it is never a base.
+ */
+export function resolveManifestPath(repoPath: string): string | null {
+  return repoManifestPath(repoPath) ?? heldManifestPath(repoPath);
+}
+
+/** Where `repoPath`'s base manifest comes from. */
+export function manifestSource(repoPath: string): "repo" | "held" | null {
+  if (repoManifestPath(repoPath)) return "repo";
+  return heldManifestPath(repoPath) ? "held" : null;
+}
+
+// ── Layers ───────────────────────────────────────────────────────────────
+
+/** A layer of a resolved manifest. The base comes first. */
+export type ManifestLayer = "repo" | "held" | "override";
+
+export interface ManifestLoad {
+  /** The merged manifest, or null when none is usable. */
+  manifest: RawManifest | null;
+  /** The layers that applied, base first: e.g. ["repo", "override"]. */
+  layers: ManifestLayer[];
+  /** The base manifest file (repo or held), or null. */
+  baseFile: string | null;
+  /** The base's layer, or null when there is no base. */
+  baseLayer: "repo" | "held" | null;
+  /** The override file, or null. */
+  overrideFile: string | null;
+  /** Why no manifest is usable, when files exist but cannot be used. */
+  error?: string;
+}
+
+/** Parse a manifest file's text: a map, or null for anything else. */
+export function parseManifestText(
+  filePath: string,
+  raw: string,
+): Record<string, unknown> | null {
+  const parsed: unknown = filePath.endsWith(".json")
+    ? JSON.parse(raw)
+    : parseYaml(raw);
+  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
+    return null; // scalar/array/empty manifest is not usable
+  return parsed as Record<string, unknown>;
+}
+
+/** The message for an override that has no base to apply to. */
+export function orphanOverrideMessage(): string {
+  return `an override with no base manifest is not a manifest: add a ${product().manifestYml} to the repo, or to manifests/`;
+}
+
+interface Located {
+  baseFile: string | null;
+  baseLayer: "repo" | "held" | null;
+  overrideFile: string | null;
+}
+
+function locate(repoPath: string): Located {
+  const own = repoManifestPath(repoPath);
+  const held = own ? null : heldManifestPath(repoPath);
+  return {
+    baseFile: own ?? held,
+    baseLayer: own ? "repo" : held ? "held" : null,
+    overrideFile: overrideManifestPath(repoPath),
+  };
+}
+
+/** Parse and merge the located files' texts. Never throws. */
+function combine(
+  at: Located,
+  baseRaw: string | null,
+  overrideRaw: string | null,
+): ManifestLoad {
+  const out: ManifestLoad = { manifest: null, layers: [], ...at };
+  const parse = (file: string, raw: string): Record<string, unknown> => {
+    const parsed = parseManifestText(file, raw);
+    if (!parsed) throw new Error(`${file}: not a map`);
+    return parsed;
+  };
+  if (!at.baseFile || !at.baseLayer || baseRaw === null) {
+    if (at.overrideFile)
+      out.error = `${at.overrideFile}: ${orphanOverrideMessage()}`;
+    return out;
+  }
+  try {
+    let manifest = parse(at.baseFile, baseRaw);
+    const layers: ManifestLayer[] = [at.baseLayer];
+    if (at.overrideFile && overrideRaw !== null) {
+      manifest = mergeOverride(manifest, parse(at.overrideFile, overrideRaw));
+      layers.push("override");
+    }
+    out.manifest = manifest as unknown as RawManifest;
+    out.layers = layers;
+  } catch (e) {
+    out.error = (e as Error).message;
+  }
+  return out;
+}
+
+/** Resolve, read and merge `repoPath`'s manifest layers. Never throws. */
+export function loadManifest(repoPath: string): ManifestLoad {
+  const at = locate(repoPath);
+  try {
+    return combine(
+      at,
+      at.baseFile && fs.readFileSync(at.baseFile, "utf-8"),
+      at.overrideFile && fs.readFileSync(at.overrideFile, "utf-8"),
+    );
+  } catch (e) {
+    return { manifest: null, layers: [], ...at, error: (e as Error).message };
+  }
+}
+
+/** The layers of `repoPath`'s manifest that applied, e.g. ["held", "override"]. */
+export function manifestLayers(repoPath: string): ManifestLayer[] {
+  return loadManifest(repoPath).layers;
+}
+
+/** "repo", "held + override", … ("none" when no manifest applied). */
+export function manifestLayersLabel(layers: readonly ManifestLayer[]): string {
+  return layers.length > 0 ? layers.join(" + ") : "none";
+}
+
+/**
+ * Read and parse the manifest from `repoPath`, its override merged in.
+ * Supports both YAML and JSON.
+ * Returns null if no manifest file exists or parsing fails.
+ */
+export function readManifest(repoPath: string): RawManifest | null {
+  return loadManifest(repoPath).manifest;
+}
+
+/**
+ * Async version of readManifest.
+ */
+export async function readManifestAsync(
+  repoPath: string,
+): Promise<RawManifest | null> {
+  const at = locate(repoPath);
+  try {
+    const { readFile } = await import("fs/promises");
+    return combine(
+      at,
+      at.baseFile && (await readFile(at.baseFile, "utf-8")),
+      at.overrideFile && (await readFile(at.overrideFile, "utf-8")),
+    ).manifest;
+  } catch {
+    return null;
+  }
+}
+
+/** The canonical manifest filename for display/error messages. */
+export const MANIFEST_DISPLAY_NAME = `${product().manifestYml}`;

package/src/shared/manifest/schema.ts

@@ -0,0 +1,313 @@
+/**
+ * Zod schemas for .den.yml manifests.
+ *
+ * Each schema version is defined separately so migrations can validate
+ * inputs and outputs at their respective version boundaries.
+ *
+ * Current version: 1  (all existing manifests without a version field are v1)
+ */
+
+import { z } from "zod";
+import { journeysAppSchema } from "../../pillars/journeys/app-config.js";
+
+export const CURRENT_SCHEMA_VERSION = 1;
+export const KNOWN_VERSIONS = [1] as const;
+
+// ── Shared primitives ─────────────────────────────────────────────────────────
+
+// Command definition: plain string or object with options
+const commandDef = z.union([
+  z.string(),
+  z.object({
+    cmd: z.string(),
+    detachable: z.boolean().optional(),
+    interactive: z.boolean().optional(),
+  }),
+]);
+
+// Health check: HTTP (with or without explicit type) or shell
+//
+// Valid shapes:
+//   { url: "http://..." }                         — implicit HTTP
+//   { type: "http", url: "http://..." }           — explicit HTTP
+//   { type: "shell", cmd: "pg_isready ..." }      — shell
+const healthConfig = z.union([
+  // Shell health check — must have cmd
+  z.object({
+    type: z.literal("shell"),
+    cmd: z.string({
+      required_error: '"cmd" is required for shell health checks',
+    }),
+    label: z.string().optional(),
+  }),
+  // HTTP health check (explicit or implicit type) — must have url
+  z.object({
+    type: z.union([z.literal("http"), z.undefined()]).optional(),
+    url: z.string({
+      required_error: '"url" is required for HTTP health checks',
+    }),
+    label: z.string().optional(),
+  }),
+]);
+
+// `distinct: true` marks a SUBSYSTEM's log as its own listed source in the
+// /logs view (e.g. shop/db — shop runs its own postgres, not infra-stack's).
+// Without it, only a system's primary service log is listed there.
+const logSource = z.union([
+  z.object({ type: z.literal("process"), distinct: z.boolean().optional() }),
+  z.object({
+    type: z.literal("shell"),
+    cmd: z.string(),
+    distinct: z.boolean().optional(),
+  }),
+  z.object({
+    type: z.literal("cloudwatch"),
+    logGroup: z.string(),
+    logStream: z.string().optional(),
+    region: z.string().optional(),
+    distinct: z.boolean().optional(),
+  }),
+]);
+
+// ── Service management config ─────────────────────────────────────────────────
+//
+// Keys for commands/logs/health are environment names (local, dev, …).
+// We accept any string key here — Zod's z.record(z.enum(…)) requires all
+// enum keys to be present (exhaustive), which is too strict for optional env blocks.
+
+// depends_on entry: plain string or object with a `service` field and optional
+// `wait` flag (opts this service's startup into blocking on the dep's health
+// check — otherwise default fire-and-forget / detach-window behavior applies).
+const dependsOnEntry = z.union([
+  z.string(),
+  z.object({
+    service: z.string(),
+    wait: z.boolean().optional(),
+  }),
+]);
+
+const httpUrlString = z
+  .string()
+  .regex(/^https?:\/\/\S+$/, "must be an http(s) URL");
+
+// Placement (R13): where a service runs per env, and where a remote one is.
+// Env names are checked against the org's environments in placement.ts.
+const statusCode = z.number().int().min(100).max(599);
+const serviceTarget = z
+  .object({
+    env: z.string().min(1),
+    account: z.string().min(1).optional(),
+    url: httpUrlString.optional(),
+    expect: z.union([statusCode, z.array(statusCode).min(1)]).optional(),
+  })
+  .strict();
+
+const serviceManagementConfig = z.object({
+  runs: z.record(z.string(), z.enum(["local", "remote"])).optional(),
+  target: serviceTarget.optional(),
+  depends_on: z.array(dependsOnEntry).optional(),
+  commands: z.record(z.string(), z.record(z.string(), commandDef)).optional(),
+  logs: z.record(z.string(), logSource).optional(),
+  health: z.record(z.string(), healthConfig).optional(),
+  // Open-in-new-tab: env → app URL, plus named extras. http(s) only.
+  urls: z.record(z.string(), httpUrlString).optional(),
+  links: z
+    .array(
+      z.object({
+        label: z.string().min(1),
+        urls: z.record(z.string(), httpUrlString),
+      }),
+    )
+    .optional(),
+});
+
+// ── Smoke / perf / synthetics ─────────────────────────────────────────────────
+
+const validAuthRoles = ["admin", "user", "beta"] as const;
+
+const smokeServiceConfig = z.object({
+  testDir: z.string(),
+  testMatch: z.string(),
+  baseUrlKey: z.enum(["api", "web"]),
+  auth: z.array(z.enum(validAuthRoles)),
+  fixtures: z.record(z.string(), z.unknown()).optional(),
+});
+
+const performanceServiceConfig = z.object({
+  enabled: z.boolean().optional(),
+  testDir: z.string(),
+  testMatch: z.string(),
+  baseUrlKey: z.enum(["api", "web"]),
+  service: z.string(),
+  tags: z.array(z.string()).optional(),
+  fixtures: z.record(z.string(), z.unknown()).optional(),
+});
+
+const syntheticsConfig = z.object({
+  testDir: z.string(),
+  testMatch: z.string(),
+  baseUrlKey: z.enum(["api", "web"]),
+  auth: z.array(z.enum(validAuthRoles)),
+});
+
+// Journeys (agent-driven E2E) — top-level like `synthetics:`, one spec home
+// per repo. Deliberately thinner than syntheticsConfig: no baseUrlKey (the
+// journeys runner resolves both web and api base URLs from env config) and
+// no auth array (each journey spec's persona.role declares its own role).
+// `app` describes the app under test (resources, session token, web-origin
+// API feed): see src/pillars/journeys/app-config.ts.
+const journeysConfig = z.object({
+  testDir: z.string(),
+  testMatch: z.string(),
+  app: journeysAppSchema.optional(),
+});
+
+// Security (ZAP scanning) — top-level record keyed by service name, like
+// `smoke:`/`performance:`. Production may only ever list the passive
+// `baseline` mode: the pillar caps it at runtime too, but a manifest that
+// asks for more is rejected up front so the mistake is visible at validation.
+const securityEnvironments = [
+  "local",
+  "dev",
+  "sandbox",
+  "staging",
+  "production",
+] as const;
+const securityModes = ["baseline", "api", "full"] as const;
+
+const securityServiceConfig = z
+  .object({
+    baseUrlKey: z.enum(["api", "web"]).optional(),
+    hosts: z
+      .partialRecord(
+        z.enum(securityEnvironments),
+        z.array(
+          z
+            .string()
+            .url()
+            .refine((u) => /^https?:\/\//.test(u), {
+              message: "host must be an http(s) URL",
+            }),
+        ),
+      )
+      .optional(),
+    openapi: z.string().min(1).optional(),
+    auth: z.array(z.enum(validAuthRoles)).optional(),
+    // Passed to ZAP's `-z` option, which splits on whitespace — a regex with
+    // a literal space would silently become two options, so reject it.
+    exclude: z
+      .array(
+        z
+          .string()
+          .min(1)
+          .refine((r) => !/\s/.test(r), {
+            message: "exclude regex must not contain whitespace (use \\s)",
+          })
+          .refine(
+            (r) => {
+              try {
+                new RegExp(r);
+                return true;
+              } catch {
+                return false;
+              }
+            },
+            { message: "exclude entry is not a valid regular expression" },
+          ),
+      )
+      .optional(),
+    modes: z
+      .partialRecord(
+        z.enum(securityEnvironments),
+        z.array(z.enum(securityModes)),
+      )
+      .optional(),
+  })
+  .refine((c) => c.baseUrlKey !== undefined || c.hosts !== undefined, {
+    message: 'one of "baseUrlKey" or "hosts" is required',
+  })
+  .refine(
+    (c) => (c.modes?.production ?? ["baseline"]).every((m) => m === "baseline"),
+    {
+      message: 'production may only allow the passive "baseline" mode',
+      path: ["modes", "production"],
+    },
+  );
+// What is deployed where. Per Den environment, the first source that answers
+// wins: `version` (a JSON endpoint; `field` names the commit SHA, default
+// "build"), then `deployment` (a GitHub environment's deployments, made by
+// `workflow`), then the newest successful `workflow` run on main.
+const releaseSource = z.object({
+  version: z.string().optional(),
+  field: z.string().optional(),
+  deployment: z.string().optional(),
+});
+const releaseConfig = z.object({
+  workflow: z.string().optional(),
+  environments: z.record(z.string(), releaseSource).optional(),
+});
+
+// The repo's own test suites, run from its checkout. Local only unless
+// `environments` says otherwise; the command gets DEN_ENV and BASE_URL.
+const testSuite = z.object({
+  cmd: z.string(),
+  cwd: z.string().optional(),
+  label: z.string().optional(),
+  environments: z.array(z.string()).optional(),
+});
+
+const githubManifestConfig = z.object({
+  environments: z.record(z.string(), z.string()).optional(),
+});
+
+// Docker Compose repos run behind the edge (see src/shared/stacks/types.ts).
+const composeConfig = z.object({
+  name: z.string().optional(),
+  dir: z.string().optional(),
+  files: z.array(z.string()).optional(),
+  routes: z.record(z.string(), z.string()).optional(),
+  generate: z.array(z.string()).optional(),
+  uses: z.array(z.string()).optional(),
+  env: z
+    .record(z.string(), z.union([z.string(), z.number(), z.boolean()]))
+    .optional(),
+  profiles: z.array(z.string()).optional(),
+  healthchecks: z.record(z.string(), z.array(z.string()).min(1)).optional(),
+  // service → env vars set on that service in the override (wins by key).
+  service_env: z
+    .record(
+      z.string(),
+      z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])),
+    )
+    .optional(),
+});
+
+// ── v1 manifest schema ────────────────────────────────────────────────────────
+
+export const v1ManifestSchema = z.object({
+  // Accept any integer version — the validator handles known/unknown version
+  // warnings separately so that future versions don't fail schema validation.
+  version: z.number().int().positive().optional(),
+  name: z.string({ required_error: "name is required" }),
+  entrypoint: z.string().optional(),
+  services: z.record(z.string(), serviceManagementConfig).optional(),
+  smoke: z.record(z.string(), smokeServiceConfig).optional(),
+  performance: z.record(z.string(), performanceServiceConfig).optional(),
+  synthetics: syntheticsConfig.optional(),
+  journeys: journeysConfig.optional(),
+  security: z.record(z.string(), securityServiceConfig).optional(),
+  github: githubManifestConfig.optional(),
+  compose: composeConfig.optional(),
+  application: z.string().optional(),
+  kind: z.enum(["service", "module", "library"]).optional(),
+  contributes_to: z.array(z.string()).optional(),
+  release: releaseConfig.optional(),
+  tests: z.record(z.string(), testSuite).optional(),
+});
+
+export type V1Manifest = z.infer<typeof v1ManifestSchema>;
+
+/** Map from version number to its Zod schema. */
+export const schemaByVersion: Record<number, z.ZodTypeAny> = {
+  1: v1ManifestSchema,
+};

package/src/shared/manifest/service-links.ts

@@ -0,0 +1,93 @@
+/**
+ * Where a person opens a service, and which org a repo belongs to.
+ *
+ * A service's open URL for an environment is its manifest `urls.<env>`, else
+ * the origin of that environment's HTTP health URL (health checks point at
+ * /readyz or /api/health, never the app, so only the origin is kept). Shell
+ * health checks and services with no HTTP check have no URL.
+ */
+
+import type {
+  Environment,
+  ServiceCatalog,
+  ServiceManagementConfig,
+} from "../types/index.js";
+
+export interface OpenLink {
+  label: string;
+  url: string;
+}
+
+function httpOrigin(raw: string | undefined): string | null {
+  if (!raw) return null;
+  try {
+    const u = new URL(raw);
+    if (u.protocol !== "https:" && u.protocol !== "http:") return null;
+    return `${u.origin}/`;
+  } catch {
+    return null;
+  }
+}
+
+function httpUrl(raw: string | undefined): string | null {
+  if (!raw) return null;
+  try {
+    const u = new URL(raw);
+    return u.protocol === "https:" || u.protocol === "http:" ? u.href : null;
+  } catch {
+    return null;
+  }
+}
+
+export function openUrlFor(
+  svc: ServiceManagementConfig | undefined,
+  env: Environment,
+): string | null {
+  if (!svc) return null;
+  const declared = httpUrl(svc.urls?.[env]);
+  if (declared) return declared;
+  const health = svc.health?.[env];
+  if (!health || ("type" in health && health.type === "shell")) return null;
+  return httpOrigin((health as { url?: string }).url);
+}
+
+export function linksFor(
+  svc: ServiceManagementConfig | undefined,
+  env: Environment,
+): OpenLink[] {
+  const out: OpenLink[] = [];
+  for (const l of svc?.links ?? []) {
+    const url = httpUrl(l.urls?.[env]);
+    if (url && l.label) out.push({ label: l.label, url });
+  }
+  return out;
+}
+
+/** The GitHub owner a repo is catalogued under ("acme"), or "". */
+export function orgOf(repo: string, catalog: ServiceCatalog): string {
+  const gh = catalog[repo]?.gh ?? "";
+  const slash = gh.indexOf("/");
+  return slash > 0 ? gh.slice(0, slash) : "";
+}
+
+export interface OrgDisplay {
+  label?: string;
+  color?: string;
+}
+
+/** Every org in the catalog with its display name, configured ones first. */
+export function listOrgs(
+  catalog: ServiceCatalog,
+  configured: Record<string, OrgDisplay> = {},
+): Array<{ id: string; label: string; color?: string }> {
+  const ids = new Set<string>(Object.keys(configured));
+  for (const repo of Object.keys(catalog)) {
+    const o = orgOf(repo, catalog);
+    if (o) ids.add(o);
+  }
+  return [...ids].map((id) => ({
+    id,
+    label: configured[id]?.label ?? id,
+    ...(configured[id]?.color ? { color: configured[id].color } : {}),
+  }));
+}