@monster-orange/den 0.0.0 → 0.1.0

package/src/shared/manifest/deps.ts

@@ -0,0 +1,27 @@
+/**
+ * `depends_on` entry normalization.
+ *
+ * A dep can be:
+ *   - a string: "shop/db"
+ *   - an object: { service: "shop/db", wait: true }
+ *
+ * The object form lets a service opt into blocking startup until the dep's
+ * health check is green (default behavior is the existing 5-second detach
+ * window — no change for unflagged deps).
+ */
+
+import type { DependsOnEntry } from "../types/index.js";
+
+export interface NormalizedDep {
+  service: string;
+  wait: boolean;
+}
+
+export function depService(dep: DependsOnEntry): string {
+  return typeof dep === "string" ? dep : dep.service;
+}
+
+export function normalizeDep(dep: DependsOnEntry): NormalizedDep {
+  if (typeof dep === "string") return { service: dep, wait: false };
+  return { service: dep.service, wait: dep.wait ?? false };
+}

package/src/shared/manifest/migrate-manifest.ts

@@ -0,0 +1,101 @@
+/**
+ * .den.yml schema migration engine.
+ *
+ * Walks the migration chain from the manifest's current version up to
+ * the requested target version (default: CURRENT_SCHEMA_VERSION).
+ *
+ * Each step is a pure function:  migrate_N_to_M(manifest) → manifest
+ * Adding a new version means:
+ *   1. Bump CURRENT_SCHEMA_VERSION
+ *   2. Add migrate_<old>_to_<new> to the MIGRATION_STEPS map
+ *   3. Add the new version to schema.ts KNOWN_VERSIONS + schemaByVersion
+ */
+
+import type { RawManifest, MigrationResult } from "../types/index.js";
+import { CURRENT_SCHEMA_VERSION } from "./schema.js";
+
+// ── Migration step registry ───────────────────────────────────────────────────
+
+type MigrationFn = (manifest: RawManifest) => {
+  manifest: RawManifest;
+  change: string;
+};
+
+/**
+ * Migration steps keyed by "fromVersion".
+ * migrate_1_to_2 lives at key 1, migrate_2_to_3 at key 2, etc.
+ *
+ * Currently empty — v1 is the only version.
+ * When v2 is introduced, add:
+ *   [1]: migrate_1_to_2,
+ */
+const MIGRATION_STEPS: Record<number, MigrationFn> = {
+  // Future migrations go here, e.g.:
+  // [1]: migrate_1_to_2,
+};
+
+// ── Public API ────────────────────────────────────────────────────────────────
+
+/**
+ * Migrate a manifest to the target schema version.
+ *
+ * @param input        - A parsed manifest object (or any plain object).
+ * @param toVersion    - Target version. Defaults to CURRENT_SCHEMA_VERSION.
+ * @returns MigrationResult with the migrated manifest and a log of changes.
+ * @throws  When toVersion is unknown / unsupported.
+ */
+export function migrateManifest(
+  input: Record<string, unknown> | RawManifest,
+  toVersion: number = CURRENT_SCHEMA_VERSION,
+): MigrationResult {
+  // Validate target version
+  if (toVersion > CURRENT_SCHEMA_VERSION) {
+    throw new Error(
+      `Cannot migrate to unsupported schema version ${toVersion}. ` +
+        `Latest known version is ${CURRENT_SCHEMA_VERSION}.`,
+    );
+  }
+
+  const fromVersion =
+    typeof (input as Record<string, unknown>).version === "number"
+      ? ((input as Record<string, unknown>).version as number)
+      : 1;
+
+  // Never silently downgrade: re-stamping a lower version onto newer-schema
+  // content corrupts the manifest. Fail fast, mirroring the forward guard above.
+  if (fromVersion > toVersion) {
+    throw new Error(
+      `Cannot downgrade manifest from v${fromVersion} to v${toVersion}.`,
+    );
+  }
+
+  const changes: string[] = [];
+  let current: RawManifest = { ...(input as RawManifest) };
+
+  // Walk the chain: fromVersion → fromVersion+1 → … → toVersion
+  for (let v = fromVersion; v < toVersion; v++) {
+    const step = MIGRATION_STEPS[v];
+    if (!step) {
+      throw new Error(
+        `No migration step found from v${v} to v${v + 1}. ` +
+          `This is a bug — please file an issue.`,
+      );
+    }
+    const { manifest: next, change } = step(current);
+    current = next;
+    changes.push(change);
+  }
+
+  // Stamp the output with the target version. RawManifest has no `version`
+  // field (absence means v1), so the stamp is applied as an extra property.
+  current = { ...current, version: toVersion } as RawManifest;
+
+  return {
+    manifest: current,
+    fromVersion,
+    toVersion,
+    changes,
+  };
+}
+
+export { CURRENT_SCHEMA_VERSION };

package/src/shared/manifest/read-manifest.ts

@@ -0,0 +1,160 @@
+/**
+ * Manifest reader — supports .den.yml (preferred) and .den.json (fallback).
+ *
+ * Import this helper instead of calling fs.readFileSync('.den.json') directly
+ * so all callers automatically get YAML support.
+ *
+ * A repo's own manifest always wins. When a checkout has none, Den's own
+ * copy is used: `manifests/<github-repo>.den.yml` in the Den repo, keyed by
+ * the checkout's `origin` repo name. That covers repos whose manifest is not
+ * merged yet and repos Den should not commit to.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import { execFileSync } from "child_process";
+import { getDistRoot } from "../paths.js";
+import { parse as parseYaml } from "yaml";
+import type { RawManifest } from "../types/index.js";
+
+import { product } from "../product.js";
+/** Names probed in order. First match wins. */
+const MANIFEST_NAMES = [
+  `${product().manifestYml}`,
+  `${product().manifestJson}`,
+] as const;
+
+/** The repo's own manifest file, or null. */
+export function repoManifestPath(repoPath: string): string | null {
+  for (const name of MANIFEST_NAMES) {
+    const p = path.join(repoPath, name);
+    if (fs.existsSync(p)) return p;
+  }
+  return null;
+}
+
+// ── Den-held manifests ───────────────────────────────────────────────────
+// For a repo that doesn't carry a manifest yet (one we don't commit to, or
+// whose PR hasn't merged), den keeps one in manifests/<repo>.den.yml,
+// keyed by the git origin's repo name (falling back to the folder name). The
+// repo's own file always wins.
+
+const originCache = new Map<string, string | null>();
+
+/** Repo name of `repoPath`'s git origin (`…/<name>.git`), cached. */
+export function originRepoName(repoPath: string): string | null {
+  if (originCache.has(repoPath)) return originCache.get(repoPath)!;
+  let name: string | null = null;
+  try {
+    const url = execFileSync(
+      "git",
+      ["-C", repoPath, "config", "--get", "remote.origin.url"],
+      { encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"], timeout: 2000 },
+    ).trim();
+    name =
+      url
+        .replace(/\.git$/, "")
+        .split(/[/:]/)
+        .pop() || null;
+  } catch {
+    /* not a git checkout, or no origin */
+  }
+  originCache.set(repoPath, name);
+  return name;
+}
+
+/** Directory of den-held manifests (override: DEN_MANIFESTS_DIR). */
+export function heldManifestsDir(): string {
+  return process.env.DEN_MANIFESTS_DIR ?? path.join(getDistRoot(), "manifests");
+}
+
+/** The repo name of a `.den-services/<repo>/<branch>` path, else null. */
+function managedCloneRepo(repoPath: string): string | null {
+  const parent = path.dirname(repoPath);
+  return path.basename(path.dirname(parent)) === `${product().servicesDir}`
+    ? path.basename(parent)
+    : null;
+}
+
+/** The den-held manifest for `repoPath`, or null. */
+export function heldManifestPath(repoPath: string): string | null {
+  const keys = [
+    originRepoName(repoPath),
+    // A managed clone lives at .den-services/<repo>/<branch>,
+    // and need not be cloned yet.
+    managedCloneRepo(repoPath),
+    path.basename(repoPath),
+  ];
+  for (const key of keys) {
+    if (!key) continue;
+    const p = path.join(heldManifestsDir(), `${key}${product().manifestYml}`);
+    if (fs.existsSync(p)) return p;
+  }
+  return null;
+}
+
+/**
+ * Resolve the manifest for `repoPath`: the repo's own .den.yml (or
+ * .json), else the one den holds for it. Returns null if neither exists.
+ */
+export function resolveManifestPath(repoPath: string): string | null {
+  return repoManifestPath(repoPath) ?? heldManifestPath(repoPath);
+}
+
+/** Where `repoPath`'s manifest comes from. */
+export function manifestSource(repoPath: string): "repo" | "held" | null {
+  if (repoManifestPath(repoPath)) return "repo";
+  return heldManifestPath(repoPath) ? "held" : null;
+}
+
+/**
+ * Read and parse the manifest from `repoPath`.
+ * Supports both YAML and JSON.
+ * Returns null if no manifest file exists or parsing fails.
+ */
+export function readManifest(repoPath: string): RawManifest | null {
+  const filePath = resolveManifestPath(repoPath);
+  if (!filePath) return null;
+  try {
+    const raw = fs.readFileSync(filePath, "utf-8");
+    const parsed = filePath.endsWith(".yml") ? parseYaml(raw) : JSON.parse(raw);
+    if (
+      parsed === null ||
+      typeof parsed !== "object" ||
+      Array.isArray(parsed)
+    ) {
+      return null; // scalar/array/empty manifest is not usable
+    }
+    return parsed as RawManifest;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Async version of readManifest.
+ */
+export async function readManifestAsync(
+  repoPath: string,
+): Promise<RawManifest | null> {
+  const filePath = resolveManifestPath(repoPath);
+  if (!filePath) return null;
+  try {
+    const { readFile } = await import("fs/promises");
+    const raw = await readFile(filePath, "utf-8");
+    const parsed = filePath.endsWith(".yml") ? parseYaml(raw) : JSON.parse(raw);
+    if (
+      parsed === null ||
+      typeof parsed !== "object" ||
+      Array.isArray(parsed)
+    ) {
+      return null; // scalar/array/empty manifest is not usable
+    }
+    return parsed as RawManifest;
+  } catch {
+    return null;
+  }
+}
+
+/** The canonical manifest filename for display/error messages. */
+export const MANIFEST_DISPLAY_NAME = `${product().manifestYml}`;

package/src/shared/manifest/schema.ts

@@ -0,0 +1,299 @@
+/**
+ * Zod schemas for .den.yml manifests.
+ *
+ * Each schema version is defined separately so migrations can validate
+ * inputs and outputs at their respective version boundaries.
+ *
+ * Current version: 1  (all existing manifests without a version field are v1)
+ */
+
+import { z } from "zod";
+import { journeysAppSchema } from "../../pillars/journeys/app-config.js";
+
+export const CURRENT_SCHEMA_VERSION = 1;
+export const KNOWN_VERSIONS = [1] as const;
+
+// ── Shared primitives ─────────────────────────────────────────────────────────
+
+// Command definition: plain string or object with options
+const commandDef = z.union([
+  z.string(),
+  z.object({
+    cmd: z.string(),
+    detachable: z.boolean().optional(),
+    interactive: z.boolean().optional(),
+  }),
+]);
+
+// Health check: HTTP (with or without explicit type) or shell
+//
+// Valid shapes:
+//   { url: "http://..." }                         — implicit HTTP
+//   { type: "http", url: "http://..." }           — explicit HTTP
+//   { type: "shell", cmd: "pg_isready ..." }      — shell
+const healthConfig = z.union([
+  // Shell health check — must have cmd
+  z.object({
+    type: z.literal("shell"),
+    cmd: z.string({
+      required_error: '"cmd" is required for shell health checks',
+    }),
+    label: z.string().optional(),
+  }),
+  // HTTP health check (explicit or implicit type) — must have url
+  z.object({
+    type: z.union([z.literal("http"), z.undefined()]).optional(),
+    url: z.string({
+      required_error: '"url" is required for HTTP health checks',
+    }),
+    label: z.string().optional(),
+  }),
+]);
+
+// `distinct: true` marks a SUBSYSTEM's log as its own listed source in the
+// /logs view (e.g. shop/db — shop runs its own postgres, not infra-stack's).
+// Without it, only a system's primary service log is listed there.
+const logSource = z.union([
+  z.object({ type: z.literal("process"), distinct: z.boolean().optional() }),
+  z.object({
+    type: z.literal("shell"),
+    cmd: z.string(),
+    distinct: z.boolean().optional(),
+  }),
+  z.object({
+    type: z.literal("cloudwatch"),
+    logGroup: z.string(),
+    logStream: z.string().optional(),
+    region: z.string().optional(),
+    distinct: z.boolean().optional(),
+  }),
+]);
+
+// ── Service management config ─────────────────────────────────────────────────
+//
+// Keys for commands/logs/health are environment names (local, dev, …).
+// We accept any string key here — Zod's z.record(z.enum(…)) requires all
+// enum keys to be present (exhaustive), which is too strict for optional env blocks.
+
+// depends_on entry: plain string or object with a `service` field and optional
+// `wait` flag (opts this service's startup into blocking on the dep's health
+// check — otherwise default fire-and-forget / detach-window behavior applies).
+const dependsOnEntry = z.union([
+  z.string(),
+  z.object({
+    service: z.string(),
+    wait: z.boolean().optional(),
+  }),
+]);
+
+const httpUrlString = z
+  .string()
+  .regex(/^https?:\/\/\S+$/, "must be an http(s) URL");
+
+const serviceManagementConfig = z.object({
+  depends_on: z.array(dependsOnEntry).optional(),
+  commands: z.record(z.string(), z.record(z.string(), commandDef)).optional(),
+  logs: z.record(z.string(), logSource).optional(),
+  health: z.record(z.string(), healthConfig).optional(),
+  // Open-in-new-tab: env → app URL, plus named extras. http(s) only.
+  urls: z.record(z.string(), httpUrlString).optional(),
+  links: z
+    .array(
+      z.object({
+        label: z.string().min(1),
+        urls: z.record(z.string(), httpUrlString),
+      }),
+    )
+    .optional(),
+});
+
+// ── Smoke / perf / synthetics ─────────────────────────────────────────────────
+
+const validAuthRoles = ["admin", "user", "beta"] as const;
+
+const smokeServiceConfig = z.object({
+  testDir: z.string(),
+  testMatch: z.string(),
+  baseUrlKey: z.enum(["api", "web"]),
+  auth: z.array(z.enum(validAuthRoles)),
+  fixtures: z.record(z.string(), z.unknown()).optional(),
+});
+
+const performanceServiceConfig = z.object({
+  enabled: z.boolean().optional(),
+  testDir: z.string(),
+  testMatch: z.string(),
+  baseUrlKey: z.enum(["api", "web"]),
+  service: z.string(),
+  tags: z.array(z.string()).optional(),
+  fixtures: z.record(z.string(), z.unknown()).optional(),
+});
+
+const syntheticsConfig = z.object({
+  testDir: z.string(),
+  testMatch: z.string(),
+  baseUrlKey: z.enum(["api", "web"]),
+  auth: z.array(z.enum(validAuthRoles)),
+});
+
+// Journeys (agent-driven E2E) — top-level like `synthetics:`, one spec home
+// per repo. Deliberately thinner than syntheticsConfig: no baseUrlKey (the
+// journeys runner resolves both web and api base URLs from env config) and
+// no auth array (each journey spec's persona.role declares its own role).
+// `app` describes the app under test (resources, session token, web-origin
+// API feed): see src/pillars/journeys/app-config.ts.
+const journeysConfig = z.object({
+  testDir: z.string(),
+  testMatch: z.string(),
+  app: journeysAppSchema.optional(),
+});
+
+// Security (ZAP scanning) — top-level record keyed by service name, like
+// `smoke:`/`performance:`. Production may only ever list the passive
+// `baseline` mode: the pillar caps it at runtime too, but a manifest that
+// asks for more is rejected up front so the mistake is visible at validation.
+const securityEnvironments = [
+  "local",
+  "dev",
+  "sandbox",
+  "staging",
+  "production",
+] as const;
+const securityModes = ["baseline", "api", "full"] as const;
+
+const securityServiceConfig = z
+  .object({
+    baseUrlKey: z.enum(["api", "web"]).optional(),
+    hosts: z
+      .partialRecord(
+        z.enum(securityEnvironments),
+        z.array(
+          z
+            .string()
+            .url()
+            .refine((u) => /^https?:\/\//.test(u), {
+              message: "host must be an http(s) URL",
+            }),
+        ),
+      )
+      .optional(),
+    openapi: z.string().min(1).optional(),
+    auth: z.array(z.enum(validAuthRoles)).optional(),
+    // Passed to ZAP's `-z` option, which splits on whitespace — a regex with
+    // a literal space would silently become two options, so reject it.
+    exclude: z
+      .array(
+        z
+          .string()
+          .min(1)
+          .refine((r) => !/\s/.test(r), {
+            message: "exclude regex must not contain whitespace (use \\s)",
+          })
+          .refine(
+            (r) => {
+              try {
+                new RegExp(r);
+                return true;
+              } catch {
+                return false;
+              }
+            },
+            { message: "exclude entry is not a valid regular expression" },
+          ),
+      )
+      .optional(),
+    modes: z
+      .partialRecord(
+        z.enum(securityEnvironments),
+        z.array(z.enum(securityModes)),
+      )
+      .optional(),
+  })
+  .refine((c) => c.baseUrlKey !== undefined || c.hosts !== undefined, {
+    message: 'one of "baseUrlKey" or "hosts" is required',
+  })
+  .refine(
+    (c) => (c.modes?.production ?? ["baseline"]).every((m) => m === "baseline"),
+    {
+      message: 'production may only allow the passive "baseline" mode',
+      path: ["modes", "production"],
+    },
+  );
+// What is deployed where. Per Den environment, the first source that answers
+// wins: `version` (a JSON endpoint; `field` names the commit SHA, default
+// "build"), then `deployment` (a GitHub environment's deployments, made by
+// `workflow`), then the newest successful `workflow` run on main.
+const releaseSource = z.object({
+  version: z.string().optional(),
+  field: z.string().optional(),
+  deployment: z.string().optional(),
+});
+const releaseConfig = z.object({
+  workflow: z.string().optional(),
+  environments: z.record(z.string(), releaseSource).optional(),
+});
+
+// The repo's own test suites, run from its checkout. Local only unless
+// `environments` says otherwise; the command gets DEN_ENV and BASE_URL.
+const testSuite = z.object({
+  cmd: z.string(),
+  cwd: z.string().optional(),
+  label: z.string().optional(),
+  environments: z.array(z.string()).optional(),
+});
+
+const githubManifestConfig = z.object({
+  environments: z.record(z.string(), z.string()).optional(),
+});
+
+// Docker Compose repos run behind the edge (see src/shared/stacks/types.ts).
+const composeConfig = z.object({
+  name: z.string().optional(),
+  dir: z.string().optional(),
+  files: z.array(z.string()).optional(),
+  routes: z.record(z.string(), z.string()).optional(),
+  generate: z.array(z.string()).optional(),
+  uses: z.array(z.string()).optional(),
+  env: z
+    .record(z.string(), z.union([z.string(), z.number(), z.boolean()]))
+    .optional(),
+  profiles: z.array(z.string()).optional(),
+  healthchecks: z.record(z.string(), z.array(z.string()).min(1)).optional(),
+  // service → env vars set on that service in the override (wins by key).
+  service_env: z
+    .record(
+      z.string(),
+      z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])),
+    )
+    .optional(),
+});
+
+// ── v1 manifest schema ────────────────────────────────────────────────────────
+
+export const v1ManifestSchema = z.object({
+  // Accept any integer version — the validator handles known/unknown version
+  // warnings separately so that future versions don't fail schema validation.
+  version: z.number().int().positive().optional(),
+  name: z.string({ required_error: "name is required" }),
+  entrypoint: z.string().optional(),
+  services: z.record(z.string(), serviceManagementConfig).optional(),
+  smoke: z.record(z.string(), smokeServiceConfig).optional(),
+  performance: z.record(z.string(), performanceServiceConfig).optional(),
+  synthetics: syntheticsConfig.optional(),
+  journeys: journeysConfig.optional(),
+  security: z.record(z.string(), securityServiceConfig).optional(),
+  github: githubManifestConfig.optional(),
+  compose: composeConfig.optional(),
+  application: z.string().optional(),
+  kind: z.enum(["service", "module", "library"]).optional(),
+  contributes_to: z.array(z.string()).optional(),
+  release: releaseConfig.optional(),
+  tests: z.record(z.string(), testSuite).optional(),
+});
+
+export type V1Manifest = z.infer<typeof v1ManifestSchema>;
+
+/** Map from version number to its Zod schema. */
+export const schemaByVersion: Record<number, z.ZodTypeAny> = {
+  1: v1ManifestSchema,
+};

package/src/shared/manifest/service-links.ts

@@ -0,0 +1,93 @@
+/**
+ * Where a person opens a service, and which org a repo belongs to.
+ *
+ * A service's open URL for an environment is its manifest `urls.<env>`, else
+ * the origin of that environment's HTTP health URL (health checks point at
+ * /readyz or /api/health, never the app, so only the origin is kept). Shell
+ * health checks and services with no HTTP check have no URL.
+ */
+
+import type {
+  Environment,
+  ServiceCatalog,
+  ServiceManagementConfig,
+} from "../types/index.js";
+
+export interface OpenLink {
+  label: string;
+  url: string;
+}
+
+function httpOrigin(raw: string | undefined): string | null {
+  if (!raw) return null;
+  try {
+    const u = new URL(raw);
+    if (u.protocol !== "https:" && u.protocol !== "http:") return null;
+    return `${u.origin}/`;
+  } catch {
+    return null;
+  }
+}
+
+function httpUrl(raw: string | undefined): string | null {
+  if (!raw) return null;
+  try {
+    const u = new URL(raw);
+    return u.protocol === "https:" || u.protocol === "http:" ? u.href : null;
+  } catch {
+    return null;
+  }
+}
+
+export function openUrlFor(
+  svc: ServiceManagementConfig | undefined,
+  env: Environment,
+): string | null {
+  if (!svc) return null;
+  const declared = httpUrl(svc.urls?.[env]);
+  if (declared) return declared;
+  const health = svc.health?.[env];
+  if (!health || ("type" in health && health.type === "shell")) return null;
+  return httpOrigin((health as { url?: string }).url);
+}
+
+export function linksFor(
+  svc: ServiceManagementConfig | undefined,
+  env: Environment,
+): OpenLink[] {
+  const out: OpenLink[] = [];
+  for (const l of svc?.links ?? []) {
+    const url = httpUrl(l.urls?.[env]);
+    if (url && l.label) out.push({ label: l.label, url });
+  }
+  return out;
+}
+
+/** The GitHub owner a repo is catalogued under ("acme"), or "". */
+export function orgOf(repo: string, catalog: ServiceCatalog): string {
+  const gh = catalog[repo]?.gh ?? "";
+  const slash = gh.indexOf("/");
+  return slash > 0 ? gh.slice(0, slash) : "";
+}
+
+export interface OrgDisplay {
+  label?: string;
+  color?: string;
+}
+
+/** Every org in the catalog with its display name, configured ones first. */
+export function listOrgs(
+  catalog: ServiceCatalog,
+  configured: Record<string, OrgDisplay> = {},
+): Array<{ id: string; label: string; color?: string }> {
+  const ids = new Set<string>(Object.keys(configured));
+  for (const repo of Object.keys(catalog)) {
+    const o = orgOf(repo, catalog);
+    if (o) ids.add(o);
+  }
+  return [...ids].map((id) => ({
+    id,
+    label: configured[id]?.label ?? id,
+    ...(configured[id]?.color ? { color: configured[id].color } : {}),
+  }));
+}