@monster-orange/den 0.2.2 → 0.2.4

package/src/ui/server/single-flight.ts

@@ -0,0 +1,53 @@
+/**
+ * A short-lived, keyed cache whose builds are single-flight: callers that ask
+ * for the same key while a build is running share it instead of starting their
+ * own, and a settled result is reused for `ttlMs`. A failed build is never
+ * cached, so the next caller retries.
+ *
+ * For server routes the UI polls from every open tab (/api/services,
+ * /api/health): N tabs cost one build, not N.
+ */
+export interface SingleFlight<T> {
+  /** The cached or in-flight value for `key`; `fresh` forces a new build. */
+  get(key: string, build: () => Promise<T>, fresh?: boolean): Promise<T>;
+  /** Drop every entry (after a change the cached value may not reflect). */
+  clear(): void;
+}
+
+export function singleFlight<T>(
+  ttlMs: number,
+  now: () => number = Date.now,
+): SingleFlight<T> {
+  const entries = new Map<
+    string,
+    { promise: Promise<T>; settledAt: number | null }
+  >();
+  return {
+    get(key, build, fresh = false) {
+      const hit = entries.get(key);
+      if (
+        !fresh &&
+        hit &&
+        (hit.settledAt === null || now() - hit.settledAt < ttlMs)
+      )
+        return hit.promise;
+      const entry: { promise: Promise<T>; settledAt: number | null } = {
+        promise: Promise.resolve().then(build),
+        settledAt: null,
+      };
+      entries.set(key, entry);
+      entry.promise.then(
+        () => {
+          entry.settledAt = now();
+        },
+        () => {
+          if (entries.get(key) === entry) entries.delete(key);
+        },
+      );
+      return entry.promise;
+    },
+    clear() {
+      entries.clear();
+    },
+  };
+}

package/src/ui/server/workspace-routes.ts

@@ -12,7 +12,8 @@ import { localCommandsFor } from "../../shared/stacks/manifest-commands.js";
 import * as http from "http";
 import * as fs from "fs";
 import * as path from "path";
-import { execSync } from "child_process";
+import { execFile, execSync } from "child_process";
+import { singleFlight } from "./single-flight.js";
 import { readBody, sendJson, send400 } from "./http-helpers.js";
 import type { ResolvedService } from "../../shared/types/index.js";
 import type { EditorManager } from "./editor-manager.js";
@@ -44,6 +45,7 @@ import {
   checkServiceHealth,
   claimHealthyService,
   runServiceCommand,
+  prepareServiceSecretEnv,
   resolveService,
   hasRunningOpctlContainers,
   releaseWorkspaceDaemons,
@@ -105,6 +107,229 @@ import { product } from "../../shared/product.js";
 // cwd-started-elsewhere server would resolve and persist garbage paths.
 const ROOT = getDistRoot();
 
+/** `git <args>` in `cwd`, trimmed stdout; null on any failure. Never blocks. */
+function gitOut(cwd: string, args: string[]): Promise<string | null> {
+  return new Promise((resolve) => {
+    execFile(
+      "git",
+      args,
+      { cwd, encoding: "utf-8", timeout: GIT_REMOTE_TIMEOUT_MS },
+      (err, stdout) => resolve(err ? null : stdout.trim()),
+    );
+  });
+}
+
+/**
+ * The repos Local Dev's "add" picker offers: local checkouts under
+ * repos_root with a manifest, managed clones, and catalog-only entries.
+ */
+async function buildAvailableRepos(wsOverride: string | null) {
+  const reposRoot = getReposRoot();
+  const wsFile = wsOverride ? readWorkspacesFile() : null;
+  const ws =
+    wsOverride && wsFile?.workspaces[wsOverride]
+      ? { config: wsFile.workspaces[wsOverride] }
+      : getActiveWorkspace();
+  const catalog = readCatalog();
+  const existingAddresses = new Set(Object.keys(ws?.config.services ?? {}));
+
+  // Build a set of gh slugs already represented in the workspace, so that
+  // alternate local paths for the same repo (e.g. TestWorkspace/shop) are
+  // correctly flagged as already-added.
+  const existingGhSlugs = new Set<string>();
+  for (const addr of existingAddresses) {
+    const { repoName } = parseAddress(addr, catalog);
+    const gh = catalog[repoName]?.gh;
+    if (gh) existingGhSlugs.add(gh.toLowerCase());
+  }
+
+  // Git remote origin for a local path, as "owner/repo" (null on failure).
+  // Async and in parallel: one execSync per repo here used to hold the
+  // event loop for the whole walk.
+  const getRemoteGh = async (absPath: string): Promise<string | null> => {
+    const remote = await gitOut(absPath, ["remote", "get-url", "origin"]);
+    // Normalise SSH and HTTPS GitHub URLs → "owner/repo"
+    const m =
+      remote?.match(/github\.com[:/]([^/]+\/[^/]+?)(?:\.git)?$/) ?? null;
+    return m ? m[1].toLowerCase() : null;
+  };
+  /** Lookups still in flight; each fills in its entry when it lands. */
+  const pending: Promise<void>[] = [];
+  const isAdded = (relPath: string, gh: string | null): boolean =>
+    existingAddresses.has(relPath) ||
+    (!!gh && existingGhSlugs.has(gh.toLowerCase()));
+
+  type RepoEntry = {
+    name: string;
+    relPath: string;
+    absPath: string | null;
+    hasManifest: boolean;
+    alreadyAdded: boolean;
+    source: "local" | "catalog" | "managed";
+    gh: string | null;
+    currentBranch?: string | null;
+  };
+  const entries: RepoEntry[] = [];
+  const seenRelPaths = new Set<string>();
+
+  // Walk repos_root for checkouts with their own manifest. A git checkout is
+  // never descended into, manifest or not: the walk used to recurse into
+  // every directory without the manifest file, so a distribution whose
+  // repos use held manifests (manifests/<repo>.yml) had each repo's whole
+  // tree walked to depth 4 — seconds of readdirSync on every call. All of it
+  // is async now: the event loop serves other requests meanwhile.
+  const exists = (p: string): Promise<boolean> =>
+    fs.promises.access(p).then(
+      () => true,
+      () => false,
+    );
+  const walkDir = async (
+    dir: string,
+    relPrefix: string,
+    depth: number,
+  ): Promise<void> => {
+    if (depth > 4) return; // safety limit
+    let dirents: fs.Dirent[];
+    try {
+      dirents = await fs.promises.readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    const children = dirents.filter(
+      (d) =>
+        (d.isDirectory() || d.isSymbolicLink()) &&
+        !d.name.startsWith(".") && // .git, .github, .DS_Store, …
+        d.name !== "node_modules" &&
+        path.join(dir, d.name) !== ROOT, // den itself
+    );
+    // One directory's children are checked together; the descent stays
+    // depth-first in readdir order, so the list keeps its old order.
+    const checked = await Promise.all(
+      children.map(async (d) => {
+        const absPath = path.join(dir, d.name);
+        const [hasManifest, isCheckout] = await Promise.all([
+          exists(path.join(absPath, `${product().manifestYml}`)),
+          exists(path.join(absPath, ".git")),
+        ]);
+        return { d, absPath, hasManifest, isCheckout };
+      }),
+    );
+    for (const { d, absPath, hasManifest, isCheckout } of checked) {
+      const relPath = relPrefix ? `${relPrefix}/${d.name}` : d.name;
+      if (hasManifest) {
+        const known = catalog[relPath]?.gh ?? null;
+        // Already added if the exact address is in the workspace, OR if
+        // a repo with the same GitHub remote is already present.
+        const entry: RepoEntry = {
+          name: d.name,
+          relPath,
+          absPath,
+          hasManifest: true,
+          alreadyAdded: isAdded(relPath, known),
+          source: "local",
+          gh: known,
+        };
+        entries.push(entry);
+        if (!known)
+          pending.push(
+            getRemoteGh(absPath).then((gh) => {
+              entry.gh = gh;
+              entry.alreadyAdded = isAdded(relPath, gh);
+            }),
+          );
+        seenRelPaths.add(relPath);
+      } else if (!isCheckout) {
+        // Not a repo: look for repos in its subdirectories.
+        await walkDir(absPath, relPath, depth + 1);
+      }
+    }
+  };
+  await walkDir(reposRoot, "", 0);
+
+  // Managed clones live at .den-services/<repo>/<branch>/
+  // Walk two levels: repo dir → branch dir (each branch dir is a full clone).
+  // Multiple workspaces pointing to the same repo+branch share the same clone.
+  const readDirs = async (dir: string): Promise<string[]> => {
+    try {
+      return (await fs.promises.readdir(dir, { withFileTypes: true }))
+        .filter((d) => d.isDirectory())
+        .map((d) => d.name);
+    } catch {
+      return []; // the managed-clones dir does not exist yet
+    }
+  };
+  for (const repoName of await readDirs(DEN_SERVICES_DIR)) {
+    const repoDirPath = path.join(DEN_SERVICES_DIR, repoName);
+    const clones = await Promise.all(
+      (await readDirs(repoDirPath)).map(async (branchName) => {
+        const absPath = path.join(repoDirPath, branchName);
+        const [isGit, hasManifest] = await Promise.all([
+          exists(path.join(absPath, ".git")),
+          exists(path.join(absPath, `${product().manifestYml}`)),
+        ]);
+        return { branchName, absPath, isGit, hasManifest };
+      }),
+    );
+    for (const { branchName, absPath, isGit, hasManifest } of clones) {
+      // Only treat as a branch clone if it's actually a git repo
+      if (!isGit) continue;
+      const key = `${repoName}::${branchName}`; // unique per repo+branch
+      if (seenRelPaths.has(key)) continue;
+      const known = catalog[repoName]?.gh ?? null;
+      const entry: RepoEntry = {
+        name: repoName,
+        relPath: repoName, // catalog key used as workspace address
+        absPath,
+        hasManifest,
+        alreadyAdded: isAdded(repoName, known),
+        source: "managed",
+        gh: known,
+        currentBranch: branchName,
+      };
+      entries.push(entry);
+      pending.push(
+        Promise.all([
+          known ? known : getRemoteGh(absPath),
+          gitOut(absPath, ["rev-parse", "--abbrev-ref", "HEAD"]),
+        ]).then(([gh, branch]) => {
+          entry.gh = gh;
+          entry.alreadyAdded = isAdded(repoName, gh);
+          entry.currentBranch = branch ?? branchName;
+        }),
+      );
+      seenRelPaths.add(key);
+      seenRelPaths.add(repoName); // prevent catalog fallback from duplicating
+    }
+  }
+
+  // Catalog-only entries — repos den knows about but not found locally.
+  for (const [relPath, entry] of Object.entries(catalog)) {
+    if (seenRelPaths.has(relPath)) continue;
+    const name = relPath.includes("/")
+      ? relPath.slice(relPath.lastIndexOf("/") + 1)
+      : relPath;
+    const alreadyAdded =
+      existingAddresses.has(relPath) ||
+      (!!entry.gh && existingGhSlugs.has(entry.gh.toLowerCase()));
+    entries.push({
+      name,
+      relPath,
+      absPath: null,
+      hasManifest: false,
+      alreadyAdded,
+      source: "catalog",
+      gh: entry.gh,
+    });
+  }
+
+  await Promise.all(pending);
+  return entries;
+}
+
+type AvailableRepos = Awaited<ReturnType<typeof buildAvailableRepos>>;
+/** Shared by concurrent requests, reused briefly; cleared by any mutation. */
+const availableReposCache = singleFlight<AvailableRepos>(2_000);
+
 /**
  * Handle all /api/workspaces/* and /api/workspace/* routes.
  *
@@ -119,6 +344,12 @@ export async function handleWorkspaceRoutes(
   editors: EditorManager,
   github: GitHubService,
 ): Promise<void> {
+  // A mutation may change what available-repos would list.
+  if (
+    (req.method !== "GET" && req.method !== "HEAD") ||
+    /\/(exec|sync)$/.test(pathname)
+  )
+    availableReposCache.clear();
   // ── Workspace cleanup helpers ─────────────────────────────────────────────
   /**
    * Compute which clones can be deleted when removing `wsName`.
@@ -438,187 +669,16 @@ export async function handleWorkspaceRoutes(
   //   name     — leaf directory name (display label)
   //   absPath  — absolute filesystem path (stored as linkedPath for dev-managed repos)
   if (pathname === "/api/workspaces/available-repos" && req.method === "GET") {
-    const reposRoot = getReposRoot();
     const wsOverride = url.searchParams.get("workspace");
-    const wsFile = wsOverride ? readWorkspacesFile() : null;
-    const ws =
-      wsOverride && wsFile?.workspaces[wsOverride]
-        ? { config: wsFile.workspaces[wsOverride] }
-        : getActiveWorkspace();
-    const catalog = readCatalog();
-    const existingAddresses = new Set(Object.keys(ws?.config.services ?? {}));
-
-    // Build a set of gh slugs already represented in the workspace, so that
-    // alternate local paths for the same repo (e.g. TestWorkspace/shop) are
-    // correctly flagged as already-added.
-    const existingGhSlugs = new Set<string>();
-    for (const addr of existingAddresses) {
-      const { repoName } = parseAddress(addr, catalog);
-      const gh = catalog[repoName]?.gh;
-      if (gh) existingGhSlugs.add(gh.toLowerCase());
-    }
-
-    // Get git remote origin for a local path (fast, silent on failure).
-    const getRemoteGh = (absPath: string): string | null => {
-      try {
-        const remote = execSync("git remote get-url origin", {
-          cwd: absPath,
-          encoding: "utf-8",
-          stdio: ["ignore", "pipe", "ignore"],
-          timeout: GIT_REMOTE_TIMEOUT_MS,
-        }).trim();
-        // Normalise SSH and HTTPS GitHub URLs → "owner/repo"
-        const m =
-          remote.match(/github\.com[:/]([^/]+\/[^/]+?)(?:\.git)?$/) ?? null;
-        return m ? m[1].toLowerCase() : null;
-      } catch {
-        return null;
-      }
-    };
-
-    type RepoEntry = {
-      name: string;
-      relPath: string;
-      absPath: string | null;
-      hasManifest: boolean;
-      alreadyAdded: boolean;
-      source: "local" | "catalog" | "managed";
-      gh: string | null;
-      currentBranch?: string | null;
-    };
-    const entries: RepoEntry[] = [];
-    const seenRelPaths = new Set<string>();
-
-    // Recursively walk repos_root, stopping at directories that have .den.yml.
-    // We don't recurse into repo directories themselves.
-    const walkDir = (dir: string, relPrefix: string, depth: number): void => {
-      if (depth > 4) return; // safety limit
-      let dirents: fs.Dirent[];
-      try {
-        dirents = fs.readdirSync(dir, { withFileTypes: true });
-      } catch {
-        return;
-      }
-
-      for (const d of dirents) {
-        if (!d.isDirectory() && !d.isSymbolicLink()) continue;
-        if (d.name.startsWith(".")) continue; // skip .git, .github, .DS_Store, etc.
-        const absPath = path.join(dir, d.name);
-        if (absPath === ROOT) continue; // skip den itself
-        const relPath = relPrefix ? `${relPrefix}/${d.name}` : d.name;
-        const hasManifest = fs.existsSync(
-          path.join(absPath, `${product().manifestYml}`),
-        );
-        if (hasManifest) {
-          const gh = catalog[relPath]?.gh ?? getRemoteGh(absPath);
-          // Already added if the exact address is in the workspace, OR if
-          // a repo with the same GitHub remote is already present.
-          const alreadyAdded =
-            existingAddresses.has(relPath) ||
-            (!!gh && existingGhSlugs.has(gh.toLowerCase()));
-          entries.push({
-            name: d.name,
-            relPath,
-            absPath,
-            hasManifest: true,
-            alreadyAdded,
-            source: "local",
-            gh: gh ?? null,
-          });
-          seenRelPaths.add(relPath);
-          // Don't recurse into repo directories.
-        } else {
-          // Not a repo — recurse to find repos in subdirectories.
-          walkDir(absPath, relPath, depth + 1);
-        }
-      }
-    };
-    walkDir(reposRoot, "", 0);
-
-    // Managed clones live at .den-services/<repo>/<branch>/
-    // Walk two levels: repo dir → branch dir (each branch dir is a full clone).
-    // Multiple workspaces pointing to the same repo+branch share the same clone.
-    const getCurrentBranch = (absPath: string): string | null => {
-      try {
-        return execSync("git rev-parse --abbrev-ref HEAD", {
-          cwd: absPath,
-          encoding: "utf-8",
-          stdio: ["ignore", "pipe", "ignore"],
-          timeout: GIT_REMOTE_TIMEOUT_MS,
-        }).trim();
-      } catch {
-        return null;
-      }
-    };
-    try {
-      const repoDirs = fs.readdirSync(DEN_SERVICES_DIR, {
-        withFileTypes: true,
-      });
-      for (const repoDir of repoDirs) {
-        if (!repoDir.isDirectory()) continue;
-        const repoName = repoDir.name;
-        const repoDirPath = path.join(DEN_SERVICES_DIR, repoName);
-        let branchDirs: fs.Dirent[];
-        try {
-          branchDirs = fs.readdirSync(repoDirPath, { withFileTypes: true });
-        } catch {
-          continue;
-        }
-        for (const branchDir of branchDirs) {
-          if (!branchDir.isDirectory()) continue;
-          const branchName = branchDir.name;
-          const absPath = path.join(repoDirPath, branchName);
-          // Only treat as a branch clone if it's actually a git repo
-          if (!fs.existsSync(path.join(absPath, ".git"))) continue;
-          const key = `${repoName}::${branchName}`; // unique per repo+branch
-          if (seenRelPaths.has(key)) continue;
-          const hasManifest = fs.existsSync(
-            path.join(absPath, `${product().manifestYml}`),
-          );
-          const gh = catalog[repoName]?.gh ?? getRemoteGh(absPath);
-          const alreadyAdded =
-            existingAddresses.has(repoName) ||
-            (!!gh && existingGhSlugs.has(gh.toLowerCase()));
-          const currentBranch = getCurrentBranch(absPath) ?? branchName;
-          entries.push({
-            name: repoName,
-            relPath: repoName, // catalog key used as workspace address
-            absPath,
-            hasManifest,
-            alreadyAdded,
-            source: "managed",
-            gh: gh ?? null,
-            currentBranch,
-          });
-          seenRelPaths.add(key);
-          seenRelPaths.add(repoName); // prevent catalog fallback from duplicating
-        }
-      }
-    } catch {
-      /* the managed-clones dir does not exist yet */
-    }
-
-    // Catalog-only entries — repos den knows about but not found locally.
-    for (const [relPath, entry] of Object.entries(catalog)) {
-      if (seenRelPaths.has(relPath)) continue;
-      const name = relPath.includes("/")
-        ? relPath.slice(relPath.lastIndexOf("/") + 1)
-        : relPath;
-      const alreadyAdded =
-        existingAddresses.has(relPath) ||
-        (!!entry.gh && existingGhSlugs.has(entry.gh.toLowerCase()));
-      entries.push({
-        name,
-        relPath,
-        absPath: null,
-        hasManifest: false,
-        alreadyAdded,
-        source: "catalog",
-        gh: entry.gh,
-      });
-    }
-
-    return sendJson(res, entries);
+    // Local Dev calls this on load: concurrent loads share one walk.
+    return sendJson(
+      res,
+      await availableReposCache.get(
+        wsOverride ?? "",
+        () => buildAvailableRepos(wsOverride),
+        url.searchParams.get("refresh") === "1",
+      ),
+    );
   }
 
   // ── API: per-system auto-start toggle ─────────────────────────────────────
@@ -1411,6 +1471,10 @@ export async function handleWorkspaceRoutes(
     const startTime = Date.now();
     let overallExit = 0;
     let ranCount = 0;
+    // R16: each started service's resolved secret_env, by position in
+    // `ordered`. Resolved before a restart's down, so a missing key leaves
+    // the service running; the values only go into the runner's spawn env.
+    const preparedSecrets = new Map<number, Record<string, string>>();
 
     // Run services sequentially in dependency order
     const runNext = (i: number): void => {
@@ -1508,6 +1572,32 @@ export async function handleWorkspaceRoutes(
         }
       }
 
+      if ((cmd === "up" || cmd === "restart") && !preparedSecrets.has(i)) {
+        void (async () => {
+          // Phase 2 of a restart is a true `up`, so resolve for that.
+          const r = await prepareServiceSecretEnv(address, "up", ws.name);
+          if (cancelled) return;
+          if (!r.ok) {
+            ranCount++;
+            overallExit = 1;
+            emit({ type: "label", text: address });
+            emit({ type: "error", text: r.message });
+            emit({
+              type: "svc-done",
+              address,
+              exitCode: 1,
+              repoName: svcRepoName,
+              subsystem: svcSubsystem,
+            });
+            runNext(i + 1);
+            return;
+          }
+          preparedSecrets.set(i, r.secretEnv);
+          runNext(i);
+        })();
+        return;
+      }
+
       ranCount++;
 
       emit({ type: "label", text: address });
@@ -1540,6 +1630,7 @@ export async function handleWorkspaceRoutes(
           // the UI just hides the command they came here to watch.
           runtime: "daemon",
           env: getSudoEnv(),
+          secretEnv: preparedSecrets.get(i),
           onLine: (line) => emit({ type: "line", text: line }),
           onClose: (exitCode) => {
             // Release outer-scope refs so the disconnect handler no longer