@monster-orange/den 0.0.0 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/CHANGELOG.md +67 -0
  2. package/LICENSE +21 -0
  3. package/README.md +165 -1
  4. package/bin/den +61 -0
  5. package/docs/distributions.md +256 -0
  6. package/make/demo.mk +82 -0
  7. package/make/den.mk +80 -0
  8. package/make/journeys.mk +57 -0
  9. package/make/local_dev.mk +154 -0
  10. package/make/security.mk +59 -0
  11. package/make/test.mk +405 -0
  12. package/package.json +99 -4
  13. package/scripts/auth-configure.ts +415 -0
  14. package/scripts/heal-config.ts +92 -0
  15. package/scripts/jsonl-playwright-reporter.js +70 -0
  16. package/scripts/manifest-read.cjs +91 -0
  17. package/scripts/setup-1password.ts +570 -0
  18. package/scripts/setup-secrets.ts +149 -0
  19. package/scripts/view-results.js +156 -0
  20. package/service/install.sh +112 -0
  21. package/src/pillars/api/fixtures/index.ts +128 -0
  22. package/src/pillars/api/fixtures/openapi-validator.ts +111 -0
  23. package/src/pillars/api/global-setup.ts +103 -0
  24. package/src/pillars/api/playwright.config.ts +105 -0
  25. package/src/pillars/api/schemas/example-api.json +110 -0
  26. package/src/pillars/api/tests/health/health.spec.ts +19 -0
  27. package/src/pillars/api/tests/health/manifest.yml +6 -0
  28. package/src/pillars/browser/fixtures/index.ts +232 -0
  29. package/src/pillars/browser/fixtures/package.json +10 -0
  30. package/src/pillars/browser/global-setup.ts +140 -0
  31. package/src/pillars/browser/global-teardown.ts +48 -0
  32. package/src/pillars/browser/playwright.config.ts +212 -0
  33. package/src/pillars/browser/setup.ts +219 -0
  34. package/src/pillars/journeys/app-config.ts +237 -0
  35. package/src/pillars/journeys/discover.ts +160 -0
  36. package/src/pillars/journeys/hooks/deny-shell.cjs +44 -0
  37. package/src/pillars/journeys/observability/events.ts +288 -0
  38. package/src/pillars/journeys/observability/sdk-mapper.ts +306 -0
  39. package/src/pillars/journeys/report/friction.ts +304 -0
  40. package/src/pillars/journeys/report/junit.ts +105 -0
  41. package/src/pillars/journeys/report/render.ts +273 -0
  42. package/src/pillars/journeys/report/results.ts +323 -0
  43. package/src/pillars/journeys/run.ts +673 -0
  44. package/src/pillars/journeys/runner.ts +1025 -0
  45. package/src/pillars/journeys/sandbox.ts +406 -0
  46. package/src/pillars/journeys/spec/loader.ts +152 -0
  47. package/src/pillars/journeys/spec/schema.ts +208 -0
  48. package/src/pillars/journeys/teardown.ts +98 -0
  49. package/src/pillars/journeys/tools/browser-session.ts +219 -0
  50. package/src/pillars/journeys/tools/browser-tool.ts +914 -0
  51. package/src/pillars/journeys/tools/http-tool.ts +564 -0
  52. package/src/pillars/journeys/tools/ledger.ts +278 -0
  53. package/src/pillars/journeys/tools/snapshot-store.ts +117 -0
  54. package/src/pillars/journeys/verify/assertions.ts +502 -0
  55. package/src/pillars/journeys/verify/verdict.ts +71 -0
  56. package/src/pillars/perf/build-report.ts +36 -0
  57. package/src/pillars/perf/k6.config.js +112 -0
  58. package/src/pillars/perf/run.ts +345 -0
  59. package/src/pillars/perf/scenarios/load-profile.js +97 -0
  60. package/src/pillars/perf/staging.ts +73 -0
  61. package/src/pillars/security/discover.ts +108 -0
  62. package/src/pillars/security/findings.ts +151 -0
  63. package/src/pillars/security/plan.ts +355 -0
  64. package/src/pillars/security/run.ts +729 -0
  65. package/src/pillars/smoke/admin-probe.ts +68 -0
  66. package/src/pillars/smoke/base-url.ts +56 -0
  67. package/src/pillars/smoke/cleanup.ts +8 -0
  68. package/src/pillars/smoke/discover.ts +237 -0
  69. package/src/pillars/smoke/ensure-smoke-teardown.ts +40 -0
  70. package/src/pillars/smoke/fixture-manager.ts +217 -0
  71. package/src/pillars/smoke/fixtures.ts +468 -0
  72. package/src/pillars/smoke/global-setup-all.ts +262 -0
  73. package/src/pillars/smoke/global-setup-global.ts +262 -0
  74. package/src/pillars/smoke/global-setup.ts +273 -0
  75. package/src/pillars/smoke/global-teardown-all.ts +8 -0
  76. package/src/pillars/smoke/global-teardown-global.ts +8 -0
  77. package/src/pillars/smoke/global-teardown.ts +139 -0
  78. package/src/pillars/smoke/playwright-all.config.ts +147 -0
  79. package/src/pillars/smoke/playwright-global.config.ts +141 -0
  80. package/src/pillars/smoke/playwright.config.ts +162 -0
  81. package/src/pillars/smoke/setup-env.ts +132 -0
  82. package/src/pillars/smoke/setup-global.ts +126 -0
  83. package/src/pillars/smoke/setup.ts +115 -0
  84. package/src/pillars/smoke/smoke-repo.ts +16 -0
  85. package/src/pillars/smoke/staging.ts +49 -0
  86. package/src/pillars/suites/run.ts +147 -0
  87. package/src/scripts/configure-aws-profiles.ts +114 -0
  88. package/src/scripts/register-repos.ts +307 -0
  89. package/src/scripts/setup-default-workspace.ts +203 -0
  90. package/src/shared/applications.ts +138 -0
  91. package/src/shared/auth/api-token.ts +416 -0
  92. package/src/shared/auth/auth0-helper.ts +284 -0
  93. package/src/shared/auth/headless-login.ts +265 -0
  94. package/src/shared/auth/jwt-helper.ts +186 -0
  95. package/src/shared/auth/sudo-vault.ts +69 -0
  96. package/src/shared/auth/test-accounts.ts +98 -0
  97. package/src/shared/auth/token-injection.ts +184 -0
  98. package/src/shared/auth/token-providers/auth0.ts +80 -0
  99. package/src/shared/auth/token-providers/index.ts +182 -0
  100. package/src/shared/auth/token-refresh.ts +273 -0
  101. package/src/shared/banner.ts +15 -0
  102. package/src/shared/brand.ts +79 -0
  103. package/src/shared/cli-tabs.ts +131 -0
  104. package/src/shared/constants.ts +128 -0
  105. package/src/shared/den-config.ts +566 -0
  106. package/src/shared/dev-metrics/emit.ts +354 -0
  107. package/src/shared/edge/config.ts +229 -0
  108. package/src/shared/edge/render.ts +221 -0
  109. package/src/shared/edge/runtime.ts +268 -0
  110. package/src/shared/edge/types.ts +62 -0
  111. package/src/shared/engine.ts +67 -0
  112. package/src/shared/environment/aws-profiles.ts +49 -0
  113. package/src/shared/environment/env-selector.ts +161 -0
  114. package/src/shared/environment/playwright-env.ts +82 -0
  115. package/src/shared/environment-manager/catalog.ts +77 -0
  116. package/src/shared/environment-manager/clone-guard.ts +163 -0
  117. package/src/shared/environment-manager/dag.ts +286 -0
  118. package/src/shared/environment-manager/default-workspace.ts +182 -0
  119. package/src/shared/environment-manager/derivation/alias.ts +90 -0
  120. package/src/shared/environment-manager/derivation/cache.ts +97 -0
  121. package/src/shared/environment-manager/derivation/derive-graph.ts +421 -0
  122. package/src/shared/environment-manager/derivation/derived-up.ts +354 -0
  123. package/src/shared/environment-manager/derivation/load-alias.ts +11 -0
  124. package/src/shared/environment-manager/derivation/rules-compose.ts +42 -0
  125. package/src/shared/environment-manager/derivation/rules-events.ts +50 -0
  126. package/src/shared/environment-manager/derivation/rules-http.ts +247 -0
  127. package/src/shared/environment-manager/derivation/rules-infra.ts +64 -0
  128. package/src/shared/environment-manager/derivation/rules-readiness.ts +96 -0
  129. package/src/shared/environment-manager/derivation/scan.ts +160 -0
  130. package/src/shared/environment-manager/derivation/text.ts +40 -0
  131. package/src/shared/environment-manager/derivation/types.ts +69 -0
  132. package/src/shared/environment-manager/health.ts +589 -0
  133. package/src/shared/environment-manager/index.ts +6 -0
  134. package/src/shared/environment-manager/load-applications.ts +78 -0
  135. package/src/shared/environment-manager/log-tail.ts +212 -0
  136. package/src/shared/environment-manager/make-targets.ts +78 -0
  137. package/src/shared/environment-manager/opctl.ts +334 -0
  138. package/src/shared/environment-manager/readiness.ts +93 -0
  139. package/src/shared/environment-manager/register-local.ts +72 -0
  140. package/src/shared/environment-manager/secrets-check.ts +260 -0
  141. package/src/shared/environment-manager/service-env.ts +110 -0
  142. package/src/shared/environment-manager/service-runner.ts +1588 -0
  143. package/src/shared/environment-manager/up-plan.ts +302 -0
  144. package/src/shared/environment-manager/workspace-manager.ts +905 -0
  145. package/src/shared/environment-manager/workspaces.ts +860 -0
  146. package/src/shared/manifest/deps.ts +27 -0
  147. package/src/shared/manifest/migrate-manifest.ts +101 -0
  148. package/src/shared/manifest/read-manifest.ts +160 -0
  149. package/src/shared/manifest/schema.ts +299 -0
  150. package/src/shared/manifest/service-links.ts +93 -0
  151. package/src/shared/manifest/validate-manifest.ts +140 -0
  152. package/src/shared/manifest/workspace-manifests.ts +92 -0
  153. package/src/shared/open-url.ts +122 -0
  154. package/src/shared/org.ts +44 -0
  155. package/src/shared/paths.ts +178 -0
  156. package/src/shared/perf/optional-fields.ts +105 -0
  157. package/src/shared/perf/report-normalizer.ts +404 -0
  158. package/src/shared/perf/report-paths.ts +19 -0
  159. package/src/shared/perf/report-store.ts +110 -0
  160. package/src/shared/perf/report-types.ts +70 -0
  161. package/src/shared/perf/test-paths.ts +25 -0
  162. package/src/shared/process/log-rotation.ts +147 -0
  163. package/src/shared/process/spawn-streaming.ts +229 -0
  164. package/src/shared/product.ts +92 -0
  165. package/src/shared/profile.ts +141 -0
  166. package/src/shared/repo-kind.ts +32 -0
  167. package/src/shared/reporting/jsonl-reporter.ts +131 -0
  168. package/src/shared/reporting/slack-reporter.ts +340 -0
  169. package/src/shared/results/result-reader.ts +266 -0
  170. package/src/shared/results/result-writer.ts +168 -0
  171. package/src/shared/secrets/doppler.ts +126 -0
  172. package/src/shared/secrets/env.ts +48 -0
  173. package/src/shared/secrets/index.ts +197 -0
  174. package/src/shared/secrets/keys.ts +33 -0
  175. package/src/shared/secrets/onepassword.ts +75 -0
  176. package/src/shared/secrets/types.ts +52 -0
  177. package/src/shared/service-url.ts +119 -0
  178. package/src/shared/smoke/branch-resolver.ts +141 -0
  179. package/src/shared/smoke/git-remote.ts +70 -0
  180. package/src/shared/smoke/repo-cloner.ts +151 -0
  181. package/src/shared/stacks/config.ts +137 -0
  182. package/src/shared/stacks/manifest-commands.ts +88 -0
  183. package/src/shared/stacks/render.ts +253 -0
  184. package/src/shared/stacks/runtime.ts +334 -0
  185. package/src/shared/stacks/types.ts +53 -0
  186. package/src/shared/terminal/tabs.ts +212 -0
  187. package/src/shared/test-discovery/module-scanner.ts +569 -0
  188. package/src/shared/types/index.ts +678 -0
  189. package/src/shared/ui-marker.ts +19 -0
  190. package/src/ui/cli/applications-command.ts +116 -0
  191. package/src/ui/cli/config-command.ts +68 -0
  192. package/src/ui/cli/create-main.ts +9 -0
  193. package/src/ui/cli/create.ts +360 -0
  194. package/src/ui/cli/edge-command.ts +91 -0
  195. package/src/ui/cli/flag-parser.ts +246 -0
  196. package/src/ui/cli/graph-command.ts +524 -0
  197. package/src/ui/cli/health-display.ts +63 -0
  198. package/src/ui/cli/index.ts +3656 -0
  199. package/src/ui/cli/init.ts +855 -0
  200. package/src/ui/cli/invocation-target.ts +175 -0
  201. package/src/ui/cli/logs-command.ts +104 -0
  202. package/src/ui/cli/module-selection.ts +31 -0
  203. package/src/ui/cli/prompts.ts +767 -0
  204. package/src/ui/cli/results-viewer.ts +436 -0
  205. package/src/ui/cli/secrets-preflight.ts +91 -0
  206. package/src/ui/cli/stack-command.ts +144 -0
  207. package/src/ui/cli/stalled-setup-recovery.ts +129 -0
  208. package/src/ui/cli/test-runner.ts +1312 -0
  209. package/src/ui/cli/web-command.ts +1073 -0
  210. package/src/ui/server/changelog-parser.ts +94 -0
  211. package/src/ui/server/daemon-reaper.ts +129 -0
  212. package/src/ui/server/editor-manager.ts +301 -0
  213. package/src/ui/server/github-service.ts +223 -0
  214. package/src/ui/server/http-helpers.ts +98 -0
  215. package/src/ui/server/index.ts +3236 -0
  216. package/src/ui/server/log-history.ts +452 -0
  217. package/src/ui/server/perf-report.ts +42 -0
  218. package/src/ui/server/public/environments.html +494 -0
  219. package/src/ui/server/public/favicon.ico +0 -0
  220. package/src/ui/server/public/header.js +1371 -0
  221. package/src/ui/server/public/index.html +2595 -0
  222. package/src/ui/server/public/local-dev.html +7405 -0
  223. package/src/ui/server/public/logs.html +1489 -0
  224. package/src/ui/server/public/perf-report.js +359 -0
  225. package/src/ui/server/public/release.html +2111 -0
  226. package/src/ui/server/public/services.html +1098 -0
  227. package/src/ui/server/public/workspace-picker.js +69 -0
  228. package/src/ui/server/release-deploy-session.ts +246 -0
  229. package/src/ui/server/release-service.ts +1125 -0
  230. package/src/ui/server/sudo-store.ts +111 -0
  231. package/src/ui/server/workspace-routes.ts +2154 -0
@@ -0,0 +1,3236 @@
1
+ /**
2
+ * den Web UI Server
3
+ *
4
+ * Serves a browser-based test runner at http://localhost:7777
5
+ * Endpoints:
6
+ * GET / — UI (index.html)
7
+ * GET /report/* — Playwright HTML report (playwright-report/)
8
+ * GET /api/environments — Available environments
9
+ * GET /api/modules?pillar=X — Modules for a pillar (via moduleScanner)
10
+ * GET /api/run?pillar=X&... — SSE stream: runs tests, emits output line-by-line
11
+ */
12
+
13
+ import { composeHealthCmd } from "../../shared/stacks/manifest-commands.js";
14
+ import { hasRuntime, repoKind } from "../../shared/repo-kind.js";
15
+ import * as http from "http";
16
+ import * as fs from "fs";
17
+ import * as os from "os";
18
+ import * as path from "path";
19
+ import { execSync, spawn, spawnSync } from "child_process";
20
+ import { fileURLToPath } from "url";
21
+ import { envSelector } from "../../shared/environment/env-selector.js";
22
+ import { moduleScanner } from "../../shared/test-discovery/module-scanner.js";
23
+ import { resolveFieldEnv } from "../../shared/perf/optional-fields.js";
24
+ import { testRunner } from "../cli/test-runner.js";
25
+ import { discoverRepos } from "../../pillars/smoke/discover.js";
26
+ import {
27
+ awsProfileEnv,
28
+ loadAwsProfiles,
29
+ } from "../../shared/environment/aws-profiles.js";
30
+ import { loadCurrentPerfRunReport } from "../../shared/perf/report-store.js";
31
+ import type {
32
+ Pillar,
33
+ Environment,
34
+ LogSource,
35
+ ResolvedService,
36
+ WorkspaceConfig,
37
+ } from "../../shared/types/index.js";
38
+ import { getRunDonePayload } from "./perf-report.js";
39
+ import { readCatalog } from "../../shared/environment-manager/catalog.js";
40
+ import {
41
+ readWorkspacesFile,
42
+ getActiveWorkspace,
43
+ getActiveWorkspaces,
44
+ getFocusedWorkspaceName,
45
+ getWorkspaceByName,
46
+ isWorkspaceActive,
47
+ listWorkspaceServices,
48
+ resolveDepAddress,
49
+ parseAddress,
50
+ readManifestAnyBranch,
51
+ noAutostartRepos,
52
+ } from "../../shared/environment-manager/workspaces.js";
53
+ import { deriveGraph } from "../../shared/environment-manager/derivation/derive-graph.js";
54
+ import { orderForDown } from "../../shared/environment-manager/derivation/derived-up.js";
55
+ import { findConventionalContainer } from "../../shared/environment-manager/opctl.js";
56
+ import {
57
+ resolveHistorySource,
58
+ ensureSnapshot,
59
+ readPageBefore,
60
+ searchSystemHistory,
61
+ pruneDeadSnapshots,
62
+ isSafeServiceName,
63
+ } from "./log-history.js";
64
+ import {
65
+ pingHttp,
66
+ pingShell,
67
+ } from "../../shared/environment-manager/health.js";
68
+ import {
69
+ runServiceCommand,
70
+ resolveService,
71
+ listDaemons,
72
+ unregisterDaemon,
73
+ checkServiceHealth,
74
+ loadDaemonsFromFile,
75
+ pruneDeadDaemons,
76
+ isInteractiveServiceCommand,
77
+ rewriteLegacyOpctlKillCommandForContainer,
78
+ shouldDetachServiceCommand,
79
+ } from "../../shared/environment-manager/service-runner.js";
80
+ import { reapDeadDaemons } from "./daemon-reaper.js";
81
+ import { readManifest } from "../../shared/manifest/read-manifest.js";
82
+ import {
83
+ getServiceLogRoot,
84
+ spawnStreaming,
85
+ getServiceLogPath,
86
+ } from "../../shared/process/spawn-streaming.js";
87
+ import {
88
+ readDenConfig,
89
+ isLocalEnabled,
90
+ readUserConfig,
91
+ writeUserConfig,
92
+ readReleaseConfig,
93
+ resolveReleaseTarget,
94
+ writeUserReleaseConfig,
95
+ type ReleaseConfig,
96
+ } from "../../shared/den-config.js";
97
+ import { readOrg } from "../../shared/org.js";
98
+ import { getDistRoot } from "../../shared/paths.js";
99
+ import { enginePath } from "../../shared/engine.js";
100
+ import {
101
+ releaseService,
102
+ DEPLOYS_OFF_MESSAGE,
103
+ type DeployPlanEntry,
104
+ type ReleaseEvent,
105
+ } from "./release-service.js";
106
+ import { deploySessions } from "./release-deploy-session.js";
107
+ import { spawnDetachedWebRestart } from "../cli/web-command.js";
108
+ import { recordInvocation } from "../../shared/dev-metrics/emit.js";
109
+ import { validateManifest } from "../../shared/manifest/validate-manifest.js";
110
+ import { depService } from "../../shared/manifest/deps.js";
111
+ import {
112
+ isOpSignedIn,
113
+ hasSudoInVault,
114
+ readSudoFromVault,
115
+ saveSudoToVault,
116
+ } from "../../shared/auth/sudo-vault.js";
117
+ import { DEN_BANNER_LINES, DEN_SUBTITLE } from "../../shared/banner.js";
118
+
119
+ const __filename = fileURLToPath(import.meta.url);
120
+ const __dirname = path.dirname(__filename);
121
+
122
+ const ROOT = getDistRoot();
123
+ const DIST_PUBLIC_DIR = path.join(__dirname, "public");
124
+ const SOURCE_PUBLIC_DIR = enginePath("src/ui/server/public");
125
+ const PUBLIC_DIR = fs.existsSync(path.join(DIST_PUBLIC_DIR, "index.html"))
126
+ ? DIST_PUBLIC_DIR
127
+ : SOURCE_PUBLIC_DIR;
128
+ const REPORT_DIR = path.join(ROOT, "playwright-report");
129
+
130
+ /** One service's health, as returned by /api/health and its stream. */
131
+ interface HealthResult {
132
+ repo: string;
133
+ service: string;
134
+ label: string;
135
+ url: string | null;
136
+ repoPath: string;
137
+ mode: "dev-managed" | "managed";
138
+ isEntrypoint: boolean;
139
+ ok: boolean | null;
140
+ status?: number;
141
+ error?: string;
142
+ /** Where the app opens for this env (open-in-new-tab), or null. */
143
+ openUrl?: string | null;
144
+ /** Extra named links for this env. */
145
+ links?: OpenLink[];
146
+ }
147
+
148
+ /**
149
+ * Health is cached briefly so switching pages doesn't re-run every check. Short
150
+ * enough that a service going down still shows up within one poll interval.
151
+ */
152
+ /**
153
+ * Health results are cached host-agnostic; open links are rewritten per
154
+ * request so a viewer on an edge alias (e.g. *.internal) gets links that
155
+ * resolve from where they are.
156
+ */
157
+ function viewerRewrite(
158
+ host: string | undefined,
159
+ ): (r: HealthResult) => HealthResult {
160
+ let edge: { domain: string; aliases: string[] } | null = null;
161
+ try {
162
+ const cfg = readEdgeConfig();
163
+ edge = { domain: cfg.domain, aliases: cfg.aliases ?? [] };
164
+ } catch {
165
+ /* no edge configured */
166
+ }
167
+ if (!edge || !edge.aliases.length) return (r) => r;
168
+ const { domain, aliases } = edge;
169
+ const fix = (u: string) => rehost(u, domain, host, aliases);
170
+ return (r) => ({
171
+ ...r,
172
+ openUrl: r.openUrl ? fix(r.openUrl) : r.openUrl,
173
+ links: r.links?.map((l) => ({ ...l, url: fix(l.url) })),
174
+ });
175
+ }
176
+
177
+ const HEALTH_CACHE_TTL_MS = 8_000;
178
+ const healthCache = new Map<string, { results: HealthResult[]; at: number }>();
179
+ /** A workspace-scoped run covers a different service set than the default one. */
180
+ function healthCacheKey(env: Environment, workspace: string | null): string {
181
+ return `${env}|${workspace ?? "*"}`;
182
+ }
183
+ import {
184
+ LOG_SWEEP_INTERVAL_MS,
185
+ DEFAULT_PORT,
186
+ CLI_SPAWN_TIMEOUT_MS,
187
+ GIT_REMOTE_TIMEOUT_MS,
188
+ DAEMON_QUICK_EXIT_MS,
189
+ DAEMON_RETRY_DELAY_MS,
190
+ DAEMON_SHUTDOWN_GRACE_MS,
191
+ LIVE_TAIL_LINES,
192
+ HEALTH_CHECK_TIMEOUT_MS,
193
+ HOSTED_HEALTH_CHECK_TIMEOUT_MS,
194
+ } from "../../shared/constants.js";
195
+
196
+ const PORT = parseInt(process.env.PORT ?? String(readWebPortSafe()), 10);
197
+
198
+ // When this server process started. Used to detect a "stale" server — one whose
199
+ // loaded TypeScript is older than the on-disk source (the footgun where `make
200
+ // web`/`den web` keeps running old code after an edit). See computeStale().
201
+ const SERVER_STARTED_AT = Date.now();
202
+ const STALE_CACHE_TTL_MS = 2_000;
203
+ let _staleCache: { at: number; stale: boolean; scanned: number } | null = null;
204
+
205
+ /** Newest mtime (ms) among *.ts under `dir`, skipping node_modules + public/. */
206
+ function scanNewestTs(dir: string): { newest: number; count: number } {
207
+ let newest = 0;
208
+ let count = 0;
209
+ const stack = [dir];
210
+ while (stack.length) {
211
+ const d = stack.pop() as string;
212
+ let entries: fs.Dirent[];
213
+ try {
214
+ entries = fs.readdirSync(d, { withFileTypes: true });
215
+ } catch {
216
+ continue;
217
+ }
218
+ for (const e of entries) {
219
+ if (e.isDirectory()) {
220
+ // public/* is served fresh from disk per request (never stale); the
221
+ // server-side code that IS loaded at startup is the *.ts graph.
222
+ if (e.name === "node_modules" || e.name === "public") continue;
223
+ stack.push(path.join(d, e.name));
224
+ } else if (e.name.endsWith(".ts")) {
225
+ try {
226
+ const m = fs.statSync(path.join(d, e.name)).mtimeMs;
227
+ if (m > newest) newest = m;
228
+ count++;
229
+ } catch {
230
+ /* file vanished mid-scan — ignore */
231
+ }
232
+ }
233
+ }
234
+ }
235
+ return { newest, count };
236
+ }
237
+
238
+ /** True when any server-side *.ts is newer than this process's start. Cached. */
239
+ function computeStale(): { stale: boolean; scanned: number } {
240
+ const now = Date.now();
241
+ if (_staleCache && now - _staleCache.at < STALE_CACHE_TTL_MS) {
242
+ return { stale: _staleCache.stale, scanned: _staleCache.scanned };
243
+ }
244
+ const { newest, count } = scanNewestTs(enginePath("src"));
245
+ const stale = newest > SERVER_STARTED_AT;
246
+ _staleCache = { at: now, stale, scanned: count };
247
+ return { stale, scanned: count };
248
+ }
249
+ const SILENT = process.argv.includes("--silent");
250
+ /**
251
+ * Verbose mode — adds lifecycle detail to stdout + mirrors SSE `line` events
252
+ * from /api/run to stdout so the server terminal doubles as a live test
253
+ * console. Opt-in via `--verbose` flag or `DEN_LOG=verbose`. Default is
254
+ * unchanged: request lines only. The in-memory log ring populates either way
255
+ * so the /logs page's "den/server" stream works without the flag.
256
+ */
257
+ const VERBOSE =
258
+ process.argv.includes("--verbose") || process.env.DEN_LOG === "verbose";
259
+
260
+ // ── In-memory log ring ────────────────────────────────────────────────────────
261
+ // Every message written via serverLog() lands here in plain-text form (ANSI
262
+ // stripped) and fans out to any subscribers — primarily the /api/server-log
263
+ // SSE stream surfaced on the /logs page. Fixed-size ring so an idle server
264
+ // doesn't grow without bound; restart drops history (not persisted to disk).
265
+
266
+ interface LogEntry {
267
+ ts: string;
268
+ kind: string;
269
+ text: string;
270
+ }
271
+
272
+ const LOG_RING_MAX = 500;
273
+ const logRing: LogEntry[] = [];
274
+ const logSubscribers = new Set<(e: LogEntry) => void>();
275
+
276
+ // ANSI SGR escape sequences — stripping these requires a control-character
277
+ // literal (ESC = 0x1b), which triggers no-control-regex. Intentional.
278
+ // eslint-disable-next-line no-control-regex
279
+ const ANSI_RE = /\x1b\[[0-9;]*m/g;
280
+ function stripAnsi(s: string): string {
281
+ return s.replace(ANSI_RE, "");
282
+ }
283
+
284
+ function pushLog(kind: string, text: string): void {
285
+ const entry: LogEntry = {
286
+ ts: new Date().toISOString(),
287
+ kind,
288
+ text: stripAnsi(text),
289
+ };
290
+ logRing.push(entry);
291
+ if (logRing.length > LOG_RING_MAX) logRing.shift();
292
+ for (const sub of logSubscribers) {
293
+ try {
294
+ sub(entry);
295
+ } catch {
296
+ /* subscriber broke — don't let it take the server down */
297
+ }
298
+ }
299
+ }
300
+
301
+ /**
302
+ * Log to stdout unless --silent was passed, AND fan out to the log ring so the
303
+ * /logs page and any connected SSE subscribers see the same stream.
304
+ */
305
+ function serverLog(msg: string, kind: string = "info"): void {
306
+ pushLog(kind, msg);
307
+ if (!SILENT) process.stdout.write(msg + "\n");
308
+ }
309
+
310
+ /** ANSI helpers (no chalk dependency in the server) */
311
+ const dim = (s: string) => `\x1b[2m${s}\x1b[0m`;
312
+ const green = (s: string) => `\x1b[32m${s}\x1b[0m`;
313
+ const yellow = (s: string) => `\x1b[33m${s}\x1b[0m`;
314
+ const red = (s: string) => `\x1b[31m${s}\x1b[0m`;
315
+ const bold = (s: string) => `\x1b[1m${s}\x1b[0m`;
316
+
317
+ function logTimestamp(): string {
318
+ return dim(new Date().toLocaleTimeString("en-US", { hour12: false }));
319
+ }
320
+
321
+ function logStatus(code: number): string {
322
+ if (code < 300) return green(String(code));
323
+ if (code < 400) return yellow(String(code));
324
+ return red(String(code));
325
+ }
326
+
327
+ /** Static extensions we skip logging (too noisy). */
328
+ const STATIC_EXTS = new Set([
329
+ ".html",
330
+ ".js",
331
+ ".css",
332
+ ".ico",
333
+ ".map",
334
+ ".png",
335
+ ".svg",
336
+ ".woff",
337
+ ".woff2",
338
+ ]);
339
+
340
+ // ── Sudo password store ───────────────────────────────────────────────────────
341
+ import {
342
+ setSudoPassword,
343
+ clearSudoPassword,
344
+ getSudoEnv,
345
+ prewarmSudo,
346
+ hasSudoPassword,
347
+ hasPasswordlessSudo,
348
+ } from "./sudo-store.js";
349
+
350
+ // ── HTTP helpers ─────────────────────────────────────────────────────────────
351
+ import { sweepLogs } from "../../shared/process/log-rotation.js";
352
+ import {
353
+ readBody,
354
+ sendJson,
355
+ serveFile,
356
+ send400,
357
+ send404,
358
+ } from "./http-helpers.js";
359
+ import { orgOf, listOrgs } from "../../shared/manifest/service-links.js";
360
+ import {
361
+ readLatestSummary,
362
+ securityReportsDir,
363
+ } from "../../pillars/security/findings.js";
364
+
365
+ // ── Changelog parser ──────────────────────────────────────────────────────────
366
+ import { parseChangelog } from "./changelog-parser.js";
367
+
368
+ // ── Extracted services ───────────────────────────────────────────────────────
369
+ import { GitHubService } from "./github-service.js";
370
+ import { EditorManager, SUPPORTED_LANGUAGES } from "./editor-manager.js";
371
+ import { handleWorkspaceRoutes } from "./workspace-routes.js";
372
+ import { readEdgeConfig, readWebPort } from "../../shared/edge/config.js";
373
+ import {
374
+ linksFor,
375
+ openUrlFor,
376
+ rehost,
377
+ type OpenLink,
378
+ } from "../../shared/open-url.js";
379
+
380
+ import { product } from "../../shared/product.js";
381
+ /** config `web.port` (user > team), falling back to the built-in default. */
382
+ function readWebPortSafe(): number {
383
+ try {
384
+ return readWebPort();
385
+ } catch {
386
+ return DEFAULT_PORT;
387
+ }
388
+ }
389
+ const github = new GitHubService();
390
+ const editors = new EditorManager();
391
+
392
+ // ── Health check ─────────────────────────────────────────────────────────────
393
+ // pingHttp / pingShell are imported from shared health module above.
394
+
395
+ // ── Test source resolution ───────────────────────────────────────────────────
396
+ // Returns, for each repo contributing tests to a pillar, the resolved repo path
397
+ // plus its current git branch + short SHA. Used to surface the source of truth
398
+ // in the Tests UI and the run log so workspace/registry drift is visible.
399
+
400
+ interface TestSource {
401
+ repoName: string;
402
+ repoPath: string;
403
+ branch: string | null;
404
+ shortSha: string | null;
405
+ workspace: string | null;
406
+ }
407
+
408
+ function readGitField(gitArgs: string[], cwd: string): string | null {
409
+ const r = spawnSync("git", gitArgs, {
410
+ cwd,
411
+ encoding: "utf-8",
412
+ stdio: ["ignore", "pipe", "ignore"],
413
+ timeout: GIT_REMOTE_TIMEOUT_MS,
414
+ });
415
+ if (r.status !== 0 || !r.stdout) return null;
416
+ return r.stdout.trim() || null;
417
+ }
418
+
419
+ function collectTestSources(
420
+ pillar: "synthetic" | "smoke" | "journeys" | "security",
421
+ ): TestSource[] {
422
+ const workspace = getActiveWorkspace()?.name ?? null;
423
+ const sources: TestSource[] = [];
424
+ for (const repo of discoverRepos()) {
425
+ const manifest = repo.manifest as {
426
+ synthetics?: unknown;
427
+ smoke?: Record<string, unknown>;
428
+ journeys?: unknown;
429
+ security?: Record<string, unknown>;
430
+ } | null;
431
+ const matches =
432
+ pillar === "synthetic"
433
+ ? !!manifest?.synthetics
434
+ : pillar === "journeys"
435
+ ? !!manifest?.journeys
436
+ : pillar === "security"
437
+ ? Object.keys(manifest?.security ?? {}).length > 0
438
+ : Object.keys(manifest?.smoke ?? {}).length > 0;
439
+ if (!matches) continue;
440
+ sources.push({
441
+ repoName: repo.repoName,
442
+ repoPath: repo.repoPath,
443
+ branch: readGitField(
444
+ ["rev-parse", "--abbrev-ref", "HEAD"],
445
+ repo.repoPath,
446
+ ),
447
+ shortSha: readGitField(["rev-parse", "--short", "HEAD"], repo.repoPath),
448
+ workspace,
449
+ });
450
+ }
451
+ return sources;
452
+ }
453
+
454
+ function formatSourceLine(src: TestSource): string {
455
+ const wsTag = src.workspace ? ` @ ${src.workspace}` : "";
456
+ const branch = src.branch ?? "(no branch)";
457
+ const sha = src.shortSha ? ` (${src.shortSha})` : "";
458
+ return `${src.repoName}${wsTag} · ${branch}${sha} — ${src.repoPath}`;
459
+ }
460
+
461
+ // ── Lifecycle helpers ─────────────────────────────────────────────────────────
462
+
463
+ function formatDuration(ms: number): string {
464
+ if (ms < 1000) return `${ms}ms`;
465
+ const s = ms / 1000;
466
+ if (s < 60) return `${s.toFixed(1)}s`;
467
+ const m = Math.floor(s / 60);
468
+ const rs = Math.round(s - m * 60);
469
+ return `${m}m${rs}s`;
470
+ }
471
+
472
+ function shortRunId(): string {
473
+ return Math.random().toString(36).slice(2, 8);
474
+ }
475
+
476
+ /**
477
+ * When VERBOSE, mirror SSE `line` events to stdout prefixed with a run id so
478
+ * the server terminal doubles as a live test console. Off by default — this
479
+ * is the firehose mode, not for everyday use.
480
+ */
481
+ function mirrorLine(runId: string, text: string): void {
482
+ if (!VERBOSE) return;
483
+ process.stdout.write(` ${dim(`[${runId}]`)} ${stripAnsi(text)}\n`);
484
+ }
485
+
486
+ // ── Server ────────────────────────────────────────────────────────────────────
487
+
488
+ async function handleRequest(
489
+ req: http.IncomingMessage,
490
+ res: http.ServerResponse,
491
+ ): Promise<void> {
492
+ const url = new URL(req.url ?? "/", `http://localhost:${PORT}`);
493
+ const pathname = url.pathname;
494
+
495
+ // ── Request logging ────────────────────────────────────────────────────────
496
+ // /api/health fires on the UI's 5s polling interval forever — it dominates
497
+ // the request log and crowds out anything useful. Always filtered, from both
498
+ // stdout and the in-memory ring. Static assets are skipped for the same
499
+ // reason. Real errors on those paths still surface via send400/throw.
500
+ {
501
+ const ext = path.extname(pathname);
502
+ const isStatic = STATIC_EXTS.has(ext) || pathname === "/";
503
+ const isHealthPoll = pathname === "/api/health";
504
+ if (!isStatic && !isHealthPoll) {
505
+ const start = Date.now();
506
+ const method = bold((req.method ?? "GET").padEnd(4));
507
+ res.on("finish", () => {
508
+ const ms = dim(`${Date.now() - start}ms`);
509
+ serverLog(
510
+ ` ${logTimestamp()} ${method} ${pathname} → ${logStatus(res.statusCode)} ${ms}`,
511
+ "request",
512
+ );
513
+ });
514
+ }
515
+ }
516
+
517
+ // ── Static UI ──────────────────────────────────────────────────────────────
518
+ // Local Dev is the default page when local environment is enabled.
519
+ if (
520
+ pathname === "/" ||
521
+ pathname === "/local-dev" ||
522
+ pathname === "/local-dev.html"
523
+ ) {
524
+ if (!isLocalEnabled()) {
525
+ res.writeHead(302, { Location: "/tests" });
526
+ res.end();
527
+ return;
528
+ }
529
+ return serveFile(res, path.join(PUBLIC_DIR, "local-dev.html"));
530
+ }
531
+
532
+ if (pathname === "/tests" || pathname === "/index.html") {
533
+ return serveFile(res, path.join(PUBLIC_DIR, "index.html"));
534
+ }
535
+
536
+ if (pathname === "/services" || pathname === "/services.html") {
537
+ return serveFile(res, path.join(PUBLIC_DIR, "services.html"));
538
+ }
539
+ if (pathname === "/environments" || pathname === "/environments.html") {
540
+ return serveFile(res, path.join(PUBLIC_DIR, "environments.html"));
541
+ }
542
+
543
+ if (pathname === "/release" || pathname === "/release.html") {
544
+ return serveFile(res, path.join(PUBLIC_DIR, "release.html"));
545
+ }
546
+
547
+ if (pathname === "/logs") {
548
+ return serveFile(res, path.join(PUBLIC_DIR, "logs.html"));
549
+ }
550
+
551
+ if (pathname === "/header.js") {
552
+ return serveFile(res, path.join(PUBLIC_DIR, "header.js"));
553
+ }
554
+
555
+ if (pathname === "/perf-report.js") {
556
+ return serveFile(res, path.join(PUBLIC_DIR, "perf-report.js"));
557
+ }
558
+
559
+ // Imported by header.js (the workspace pill's picker models).
560
+ if (pathname === "/workspace-picker.js") {
561
+ return serveFile(res, path.join(PUBLIC_DIR, "workspace-picker.js"));
562
+ }
563
+
564
+ if (pathname === "/favicon.ico") {
565
+ return serveFile(res, path.join(PUBLIC_DIR, "favicon.ico"));
566
+ }
567
+
568
+ // ── Playwright report ──────────────────────────────────────────────────────
569
+ if (pathname.startsWith("/report")) {
570
+ const relPath = pathname.slice("/report".length) || "/index.html";
571
+ const filePath = path.resolve(path.join(REPORT_DIR, relPath));
572
+
573
+ // Path traversal guard — require a separator boundary so a sibling dir
574
+ // sharing the REPORT_DIR prefix (e.g. "playwright-report-evil") can't pass.
575
+ if (
576
+ filePath !== REPORT_DIR &&
577
+ !filePath.startsWith(REPORT_DIR + path.sep)
578
+ ) {
579
+ res.writeHead(403);
580
+ res.end("Forbidden");
581
+ return;
582
+ }
583
+
584
+ // Serve index.html for directories
585
+ const target =
586
+ fs.existsSync(filePath) && fs.statSync(filePath).isDirectory()
587
+ ? path.join(filePath, "index.html")
588
+ : filePath;
589
+
590
+ return serveFile(res, target);
591
+ }
592
+
593
+ // ── API: environments ──────────────────────────────────────────────────────
594
+ // Orgs the catalog's repos belong to, with config/den.yml display names.
595
+ if (pathname === "/api/orgs") {
596
+ const catalog = readCatalog();
597
+ const orgs = listOrgs(catalog, readDenConfig().orgs ?? {});
598
+ if (url.searchParams.get("repos") !== "1") return sendJson(res, orgs);
599
+ const repos = Object.fromEntries(
600
+ Object.keys(catalog).map((r) => [r, orgOf(r, catalog)]),
601
+ );
602
+ return sendJson(res, { orgs, repos });
603
+ }
604
+
605
+ if (pathname === "/api/environments") {
606
+ try {
607
+ const allEnvs = await envSelector.getAvailableEnvironments();
608
+ const { environments: enabledEnvs } = readDenConfig();
609
+ const envs = allEnvs.filter((e) => enabledEnvs.includes(e));
610
+ return sendJson(res, envs);
611
+ } catch (err) {
612
+ serverLog(` ⚠ /api/environments: ${(err as Error).message}`);
613
+ return sendJson(res, ["local"]);
614
+ }
615
+ }
616
+
617
+ // ── API: app URL for current env ──────────────────────────────────────────
618
+ if (pathname === "/api/app-url") {
619
+ const env = url.searchParams.get("env") ?? "local";
620
+ try {
621
+ const cfgPath = path.join(ROOT, "config", "environments", `${env}.json`);
622
+ const cfg = JSON.parse(fs.readFileSync(cfgPath, "utf-8")) as {
623
+ baseUrls?: { web?: string };
624
+ };
625
+ return sendJson(res, { url: cfg.baseUrls?.web ?? null });
626
+ } catch (err) {
627
+ serverLog(` ⚠ /api/app-url: ${(err as Error).message}`);
628
+ return sendJson(res, { url: null });
629
+ }
630
+ }
631
+
632
+ // ── API: modules ───────────────────────────────────────────────────────────
633
+ if (pathname === "/api/modules") {
634
+ const pillar = url.searchParams.get("pillar") as Pillar | null;
635
+ if (!pillar) return send400(res, "pillar required");
636
+ try {
637
+ const modules = await moduleScanner.scanModules(pillar);
638
+ return sendJson(
639
+ res,
640
+ modules.map((m) => ({
641
+ name: m.name,
642
+ displayName: m.displayName,
643
+ icon: m.icon,
644
+ testCount: m.testCount,
645
+ // Declarative launch-time options the web UI renders generically.
646
+ optionalFields: m.optionalFields ?? [],
647
+ })),
648
+ );
649
+ } catch (err) {
650
+ serverLog(` ⚠ /api/modules: ${(err as Error).message}`);
651
+ return sendJson(res, []);
652
+ }
653
+ }
654
+
655
+ // ── API: security scans ─────────────────────────────────────────────────────
656
+ // The newest ZAP run per env + mode (what the Tests page lists), and its
657
+ // HTML reports, served from reports/security/ only.
658
+ if (pathname === "/api/security/latest") {
659
+ const env = url.searchParams.get("env") ?? "local";
660
+ const modes = ["baseline", "api", "full"];
661
+ return sendJson(
662
+ res,
663
+ modes
664
+ .map((m) => readLatestSummary(env, m))
665
+ .filter((x): x is NonNullable<typeof x> => x !== null),
666
+ );
667
+ }
668
+ if (pathname.startsWith("/reports/security/")) {
669
+ const root = securityReportsDir();
670
+ const rel = decodeURIComponent(pathname.slice("/reports/security/".length));
671
+ const file = path.resolve(root, rel);
672
+ if (!file.startsWith(root + path.sep) || !/\.(html|json)$/.test(file))
673
+ return send404(res);
674
+ try {
675
+ const body = fs.readFileSync(file);
676
+ res.writeHead(200, {
677
+ "Content-Type": file.endsWith(".html")
678
+ ? "text/html; charset=utf-8"
679
+ : "application/json",
680
+ "Cache-Control": "no-store",
681
+ // ZAP's report pulls its own scripts; keep it away from Den's origin data.
682
+ "Content-Security-Policy": "sandbox allow-scripts",
683
+ });
684
+ return res.end(body);
685
+ } catch {
686
+ return send404(res);
687
+ }
688
+ }
689
+
690
+ // ── API: test sources ─────────────────────────────────────────────────────
691
+ // Returns the repos that contribute tests for a given pillar, with their
692
+ // current branch + short SHA. Lets the Tests UI show "you are testing
693
+ // against X" so workspace vs registered-repo drift is visible.
694
+ if (pathname === "/api/test-sources") {
695
+ const pillar = url.searchParams.get("pillar");
696
+ if (
697
+ pillar !== "synthetic" &&
698
+ pillar !== "smoke" &&
699
+ pillar !== "journeys" &&
700
+ pillar !== "security"
701
+ ) {
702
+ return sendJson(res, []);
703
+ }
704
+ try {
705
+ return sendJson(res, collectTestSources(pillar));
706
+ } catch (err) {
707
+ serverLog(` ⚠ /api/test-sources: ${(err as Error).message}`);
708
+ return sendJson(res, []);
709
+ }
710
+ }
711
+
712
+ // ── API: den server log (SSE) ──────────────────────────────────────────
713
+ // Tails this server's own stdout-equivalent log stream. The /logs page opens
714
+ // it as "den/server" alongside service log panes so engineers can watch
715
+ // den's lifecycle events without leaving the browser. The ring replays
716
+ // recent history on connect; new entries stream as they happen.
717
+ //
718
+ // Events emitted:
719
+ // { type: "history", entries: LogEntry[] } — once, on open
720
+ // { type: "log", entry: LogEntry } — per new entry
721
+ if (pathname === "/api/server-log") {
722
+ res.writeHead(200, {
723
+ "Content-Type": "text/event-stream",
724
+ "Cache-Control": "no-cache",
725
+ Connection: "keep-alive",
726
+ "X-Accel-Buffering": "no",
727
+ });
728
+ const emit = (obj: unknown): void => {
729
+ try {
730
+ res.write(`data: ${JSON.stringify(obj)}\n\n`);
731
+ } catch {
732
+ /* client disconnected mid-write; cleanup runs on `close` */
733
+ }
734
+ };
735
+ emit({ type: "history", entries: logRing.slice() });
736
+ const subscriber = (entry: LogEntry): void => {
737
+ emit({ type: "log", entry });
738
+ };
739
+ logSubscribers.add(subscriber);
740
+ req.on("close", () => {
741
+ logSubscribers.delete(subscriber);
742
+ });
743
+ return;
744
+ }
745
+
746
+ // ── API: latest perf report ────────────────────────────────────────────────
747
+ if (pathname === "/api/perf-report/current") {
748
+ const report = loadCurrentPerfRunReport(ROOT);
749
+ if (!report) {
750
+ res.writeHead(404, { "Content-Type": "application/json" });
751
+ res.end(JSON.stringify({ error: "No perf report available" }));
752
+ return;
753
+ }
754
+
755
+ return sendJson(res, report);
756
+ }
757
+
758
+ // ── API: health check ─────────────────────────────────────────────────────
759
+ /**
760
+ * Workspaces a health run covers: one named workspace, else EVERY active one
761
+ * (focus first). Health is machine-level — a service running under a
762
+ * non-focused active workspace is still up, and its card must say so.
763
+ */
764
+ function healthWorkspaces(
765
+ workspaceName: string | null,
766
+ ): Array<{ name: string; config: WorkspaceConfig }> {
767
+ if (workspaceName) {
768
+ const named = getWorkspaceByName(workspaceName);
769
+ return named ? [named] : [];
770
+ }
771
+ const focus = getFocusedWorkspaceName();
772
+ const names = [
773
+ ...(focus ? [focus] : []),
774
+ ...getActiveWorkspaces().filter((n) => n !== focus),
775
+ ];
776
+ return names.flatMap((n) => getWorkspaceByName(n) ?? []);
777
+ }
778
+
779
+ /**
780
+ * Run every health check for `env` (or one service via `addressFilter`).
781
+ * `onResult` fires per check as it lands, so a caller can stream instead of
782
+ * waiting on the slowest.
783
+ */
784
+ async function computeHealth(
785
+ env: Environment,
786
+ addressFilter: string | null,
787
+ onResult?: (r: HealthResult) => void,
788
+ workspaceName: string | null = null,
789
+ ): Promise<HealthResult[]> {
790
+ // Use workspace-aware service lists so repos added via the catalog/workspace
791
+ // (e.g. files-api) are included, not just .den/ symlinks.
792
+ const workspaces = healthWorkspaces(workspaceName);
793
+ if (workspaces.length === 0) return [];
794
+ const catalog = readCatalog();
795
+ // One entry per address across active workspaces; focus wins a duplicate,
796
+ // and only one checkout of a shared address can be running anyway. The
797
+ // owning workspace travels with the service — dep addresses resolve against
798
+ // the workspace the service came from.
799
+ const byAddress = new Map<
800
+ string,
801
+ { svc: ResolvedService; ws: { name: string; config: WorkspaceConfig } }
802
+ >();
803
+ for (const ws of workspaces)
804
+ for (const svc of listWorkspaceServices(ws.config, catalog, ws.name))
805
+ // Modules and libraries have no runtime, so no health dot.
806
+ if (!byAddress.has(svc.address) && hasRuntime(repoKind(svc.repoPath)))
807
+ byAddress.set(svc.address, { svc, ws });
808
+ let services = [...byAddress.values()];
809
+ if (addressFilter) {
810
+ services = services.filter((s) => s.svc.address === addressFilter);
811
+ }
812
+
813
+ const entries: Array<{
814
+ address: string;
815
+ deps: string[];
816
+ repo: string;
817
+ service: string;
818
+ label: string;
819
+ checkType: "http" | "shell" | "none";
820
+ target: string;
821
+ repoPath: string;
822
+ mode: "dev-managed" | "managed";
823
+ isEntrypoint: boolean;
824
+ /** Conventional http://<repo>/healthz probe for no-health-check primaries. */
825
+ conventional: string | null;
826
+ openUrl: string | null;
827
+ links: OpenLink[];
828
+ /** The GitHub org that owns the repo. */
829
+ org: string;
830
+ }> = [];
831
+ let edgeDomain: string | null = null;
832
+ try {
833
+ edgeDomain = readEdgeConfig().domain;
834
+ } catch {
835
+ /* no edge configured: local links come from manifests only */
836
+ }
837
+ for (const { svc: resolved, ws } of services) {
838
+ const manifest = readManifestAnyBranch(
839
+ resolved.repoName,
840
+ resolved.repoPath,
841
+ );
842
+ if (!manifest) {
843
+ // Keep the row (as "unknown"/muted) instead of dropping it: a service
844
+ // vanishing from the payload reads as "everything else is fine" and a
845
+ // merge-by-key client would freeze its last dot.
846
+ entries.push({
847
+ address: resolved.address,
848
+ deps: [],
849
+ repo: resolved.repoName,
850
+ service: resolved.subsystem,
851
+ label: resolved.subsystem,
852
+ checkType: "none",
853
+ target: "",
854
+ repoPath: resolved.repoPath,
855
+ mode: resolved.mode,
856
+ isEntrypoint: false,
857
+ conventional: null,
858
+ openUrl: null,
859
+ links: [],
860
+ org: orgOf(resolved.repoName, catalog),
861
+ });
862
+ continue;
863
+ }
864
+ const svcCfg = manifest.services?.[resolved.subsystem];
865
+ // A compose-run repo with no declared check gets the stack's own health
866
+ // (container state + Docker healthchecks), as `den status` does.
867
+ const composeHealth =
868
+ env === "local"
869
+ ? composeHealthCmd(
870
+ manifest,
871
+ resolved.subsystem,
872
+ resolved.repoName,
873
+ ws.name,
874
+ )
875
+ : null;
876
+ const healthCfg =
877
+ svcCfg?.health?.[env] ??
878
+ (composeHealth
879
+ ? { type: "shell" as const, cmd: composeHealth }
880
+ : undefined);
881
+ // Keep services with no health check (checkType "none") so systems like
882
+ // jobs-runtime still appear — as an "unknown" (muted) dot, matching
883
+ // `den status`, rather than vanishing from the health row.
884
+ const deps = (svcCfg?.depends_on ?? []).map((rawDep) => {
885
+ const d = depService(rawDep);
886
+ const { repoName } = parseAddress(
887
+ resolveDepAddress(d, catalog, ws.config, ws.name),
888
+ catalog,
889
+ );
890
+ return repoName === resolved.repoName
891
+ ? resolveDepAddress(d, catalog, ws.config, ws.name)
892
+ : repoName;
893
+ });
894
+ entries.push({
895
+ address: resolved.address,
896
+ deps,
897
+ repo: resolved.repoName,
898
+ service: resolved.subsystem,
899
+ label: healthCfg?.label ?? resolved.subsystem,
900
+ checkType: healthCfg
901
+ ? healthCfg.type === "shell"
902
+ ? "shell"
903
+ : "http"
904
+ : "none",
905
+ target: healthCfg
906
+ ? healthCfg.type === "shell"
907
+ ? healthCfg.cmd
908
+ : healthCfg.url
909
+ : "",
910
+ repoPath: resolved.repoPath,
911
+ mode: resolved.mode,
912
+ isEntrypoint: manifest.entrypoint === resolved.subsystem,
913
+ // Conventional local discovery (mirrors checkServiceHealth): a PRIMARY
914
+ // service with no declared local health check may still serve the stack
915
+ // convention at http://<repo>/healthz via opctl DNS.
916
+ conventional: (() => {
917
+ if (healthCfg || env !== "local") return null;
918
+ const keys = Object.keys(manifest.services ?? {});
919
+ const isPrimary = manifest.entrypoint
920
+ ? resolved.subsystem === manifest.entrypoint
921
+ : keys.length === 1;
922
+ return isPrimary ? `http://${resolved.repoName}/healthz` : null;
923
+ })(),
924
+ openUrl: openUrlFor({
925
+ manifest,
926
+ subsystem: resolved.subsystem,
927
+ repoName: resolved.repoName,
928
+ env,
929
+ healthUrl:
930
+ healthCfg && healthCfg.type !== "shell" ? healthCfg.url : null,
931
+ edgeDomain,
932
+ }),
933
+ links: linksFor(manifest, resolved.subsystem, env),
934
+ org: orgOf(resolved.repoName, catalog),
935
+ });
936
+ }
937
+
938
+ // Order systems top-down like the dependency graph — entry repo first
939
+ // (shop-web) … infra-stack last — so the health row reads the same
940
+ // direction as the graph (and `den status`). orderForDown is the reverse
941
+ // of the deps-first startup order.
942
+ const rank = new Map(
943
+ orderForDown(
944
+ deriveGraph({ scope: "all", workspace: workspaces[0]?.name }),
945
+ ).map((repo, i) => [repo, i]),
946
+ );
947
+ const sortedEntries = [...entries].sort((a, b) => {
948
+ const ra = rank.get(a.repo) ?? Number.MAX_SAFE_INTEGER;
949
+ const rb = rank.get(b.repo) ?? Number.MAX_SAFE_INTEGER;
950
+ return ra - rb || a.address.localeCompare(b.address);
951
+ });
952
+
953
+ const checkEntry = async ({
954
+ repo,
955
+ service,
956
+ label,
957
+ checkType,
958
+ target,
959
+ repoPath,
960
+ mode,
961
+ isEntrypoint,
962
+ conventional,
963
+ meta,
964
+ }) => {
965
+ if (checkType !== "none") {
966
+ const started = Date.now();
967
+ const ping =
968
+ checkType === "shell"
969
+ ? await pingShell(target, awsProfileEnv(env), repoPath)
970
+ : await pingHttp(
971
+ target,
972
+ env === "local"
973
+ ? HEALTH_CHECK_TIMEOUT_MS
974
+ : HOSTED_HEALTH_CHECK_TIMEOUT_MS,
975
+ );
976
+ return {
977
+ repo,
978
+ service,
979
+ label,
980
+ url: target,
981
+ repoPath,
982
+ mode,
983
+ isEntrypoint,
984
+ ...meta,
985
+ ms: Date.now() - started,
986
+ ...ping,
987
+ };
988
+ }
989
+ // No declared health check: try the conventional probe when eligible.
990
+ // Success → real "ok"; failure stays "unknown" (ok:null renders muted,
991
+ // not red — we can't distinguish "down" from "no endpoint").
992
+ if (conventional) {
993
+ const probe = await pingHttp(conventional);
994
+ if (probe.ok) {
995
+ return {
996
+ repo,
997
+ service,
998
+ label,
999
+ url: conventional,
1000
+ repoPath,
1001
+ mode,
1002
+ isEntrypoint,
1003
+ ...meta,
1004
+ ...probe,
1005
+ };
1006
+ }
1007
+ }
1008
+ return {
1009
+ repo,
1010
+ service,
1011
+ label,
1012
+ url: null,
1013
+ repoPath,
1014
+ mode,
1015
+ isEntrypoint,
1016
+ ...meta,
1017
+ ok: null,
1018
+ };
1019
+ };
1020
+ // Stream each result the moment it lands. The checks run concurrently but
1021
+ // the slowest (shell probes that shell out to the aws CLI, ~0.45s) used to
1022
+ // gate ALL of them, so every dot waited on the worst one.
1023
+ // `idx` carries the graph-ranked position above into each STREAMED result.
1024
+ // Streaming emits in completion order, which varies per poll (shell probes
1025
+ // finish at wildly different speeds), so without a stable key the client
1026
+ // repaints the dots in a different order every 5s.
1027
+ const results = await Promise.all(
1028
+ sortedEntries.map((e, idx) =>
1029
+ checkEntry(e).then((r) => {
1030
+ const ranked = { ...r, openUrl: e.openUrl, links: e.links, idx };
1031
+ onResult?.(ranked);
1032
+ return ranked;
1033
+ }),
1034
+ ),
1035
+ );
1036
+ return results;
1037
+ }
1038
+
1039
+ // Full-env health, cached. Every page navigation used to re-run all 30 checks
1040
+ // from scratch — and the slowest are shell probes that shell out to the aws
1041
+ // CLI (~0.45s each) — so the dots took ~0.6s to resolve on EVERY switch. A
1042
+ // single-service `?address=` probe always runs live: it exists to re-check one
1043
+ // node on demand, where staleness is the whole thing being avoided.
1044
+ if (pathname === "/api/health") {
1045
+ const forViewer = viewerRewrite(req.headers.host);
1046
+ const env = (url.searchParams.get("env") ?? "local") as Environment;
1047
+ const addressFilter = url.searchParams.get("address");
1048
+ const wsParam = url.searchParams.get("workspace");
1049
+ if (wsParam && !getWorkspaceByName(wsParam))
1050
+ return send400(res, `unknown workspace "${wsParam}"`);
1051
+ if (addressFilter) {
1052
+ return sendJson(
1053
+ res,
1054
+ (await computeHealth(env, addressFilter, undefined, wsParam)).map(
1055
+ forViewer,
1056
+ ),
1057
+ );
1058
+ }
1059
+ const refresh = url.searchParams.get("refresh") === "1";
1060
+ const cacheKey = healthCacheKey(env, wsParam);
1061
+ const hit = refresh ? null : healthCache.get(cacheKey);
1062
+ if (hit && Date.now() - hit.at < HEALTH_CACHE_TTL_MS) {
1063
+ return sendJson(res, hit.results.map(forViewer));
1064
+ }
1065
+ const results = await computeHealth(env, null, undefined, wsParam);
1066
+ healthCache.set(cacheKey, { results, at: Date.now() });
1067
+ return sendJson(res, results.map(forViewer));
1068
+ }
1069
+
1070
+ // Same checks, streamed: each result is sent the moment it lands so the fast
1071
+ // ones paint immediately instead of every dot waiting on the slowest.
1072
+ if (pathname === "/api/health/stream") {
1073
+ const forViewer = viewerRewrite(req.headers.host);
1074
+ const env = (url.searchParams.get("env") ?? "local") as Environment;
1075
+ const wsParam = url.searchParams.get("workspace");
1076
+ if (wsParam && !getWorkspaceByName(wsParam))
1077
+ return send400(res, `unknown workspace "${wsParam}"`);
1078
+ res.writeHead(200, {
1079
+ "Content-Type": "text/event-stream",
1080
+ "Cache-Control": "no-cache",
1081
+ Connection: "keep-alive",
1082
+ });
1083
+ const send = (o: unknown): void => {
1084
+ res.write(`data: ${JSON.stringify(o)}\n\n`);
1085
+ };
1086
+
1087
+ const refresh = url.searchParams.get("refresh") === "1";
1088
+ const cacheKey = healthCacheKey(env, wsParam);
1089
+ const hit = refresh ? null : healthCache.get(cacheKey);
1090
+ if (hit && Date.now() - hit.at < HEALTH_CACHE_TTL_MS) {
1091
+ // Cached: replay in one go. Nothing to stagger — it's already known.
1092
+ for (const r of hit.results)
1093
+ send({ type: "result", result: forViewer(r) });
1094
+ send({ type: "done", cached: true });
1095
+ return res.end();
1096
+ }
1097
+
1098
+ try {
1099
+ const results = await computeHealth(
1100
+ env,
1101
+ null,
1102
+ (r) => send({ type: "result", result: forViewer(r) }),
1103
+ wsParam,
1104
+ );
1105
+ healthCache.set(cacheKey, { results, at: Date.now() });
1106
+ send({ type: "done", cached: false });
1107
+ } catch (e) {
1108
+ send({ type: "error", error: (e as Error).message });
1109
+ }
1110
+ return res.end();
1111
+ }
1112
+
1113
+ // ── API: services list (catalog-aware) ───────────────────────────────────
1114
+ if (pathname === "/api/services") {
1115
+ const env = (url.searchParams.get("env") ?? "local") as Environment;
1116
+ // Optional: caller may request services for a specific (non-active) workspace.
1117
+ const wsOverride = url.searchParams.get("workspace") ?? null;
1118
+
1119
+ // Conventional lifecycle locally: `down` is opctl-conventional (stops
1120
+ // running containers with no declared command, no-ops gracefully), and
1121
+ // `restart` is orchestrated by the exec runner as down → verify → up — so
1122
+ // every service with an `up` can offer both even when the manifest doesn't
1123
+ // declare them.
1124
+ const withDown = (cmds: string[]): string[] => {
1125
+ if (env !== "local") return cmds;
1126
+ const out = [...cmds];
1127
+ if (!out.includes("down")) out.push("down");
1128
+ if (!out.includes("restart") && out.includes("up")) out.push("restart");
1129
+ return out;
1130
+ };
1131
+
1132
+ // Legacy: discovered repos from .den/ symlinks / registered-repos.json
1133
+ const legacyRepos = discoverRepos();
1134
+ const legacyEntries = legacyRepos.flatMap((repo) =>
1135
+ Object.entries(repo.manifest.services ?? {}).map(([svcName, svc]) => ({
1136
+ repo: repo.repoName,
1137
+ service: svcName,
1138
+ address: `${repo.repoName}/${svcName}`,
1139
+ repoPath: repo.repoPath,
1140
+ commands: withDown(Object.keys(svc.commands?.[env] ?? {})),
1141
+ logs: svc.logs?.[env] ?? null,
1142
+ mode: "dev-managed" as const,
1143
+ branch: null as string | null,
1144
+ fromCatalog: false,
1145
+ isEntrypoint: repo.manifest.entrypoint === svcName,
1146
+ autostart: true, // legacy discovery has no workspace entry to toggle
1147
+ deps: (svc.depends_on ?? []).map(depService),
1148
+ cloned: true, // legacy discovery only ever finds on-disk repos
1149
+ })),
1150
+ );
1151
+
1152
+ // Catalog entries (workspace-aware)
1153
+ const catalog = readCatalog();
1154
+ const wsFile = wsOverride ? readWorkspacesFile() : null;
1155
+ const ws = wsOverride
1156
+ ? wsFile?.workspaces[wsOverride]
1157
+ ? { name: wsOverride, config: wsFile!.workspaces[wsOverride] }
1158
+ : null
1159
+ : getActiveWorkspace();
1160
+ const getCurrentBranch = (repoPath: string): string | null => {
1161
+ try {
1162
+ return execSync("git rev-parse --abbrev-ref HEAD", {
1163
+ cwd: repoPath,
1164
+ encoding: "utf-8",
1165
+ stdio: ["ignore", "pipe", "ignore"],
1166
+ timeout: GIT_REMOTE_TIMEOUT_MS,
1167
+ }).trim();
1168
+ } catch {
1169
+ return null;
1170
+ }
1171
+ };
1172
+
1173
+ const noAutostart = ws
1174
+ ? noAutostartRepos(ws.config, catalog)
1175
+ : new Set<string>();
1176
+ const daemonOwners = new Map(
1177
+ listDaemons().map((d) => [d.address, d.workspaces ?? []] as const),
1178
+ );
1179
+ // Modules and libraries have no runtime: no card, health or commands.
1180
+ // They show as contributors on the Applications tab instead.
1181
+ const catalogEntries = ws
1182
+ ? listWorkspaceServices(ws.config, catalog, ws.name)
1183
+ .filter((r) => hasRuntime(repoKind(r.repoPath)))
1184
+ .flatMap((resolved) => {
1185
+ const manifest = readManifest(resolved.repoPath);
1186
+ const svc = manifest?.services?.[resolved.subsystem];
1187
+ if (!manifest || !svc) {
1188
+ // Workspace membership is config-driven: a managed entry
1189
+ // whose clone isn't on disk yet (or a checkout without a usable
1190
+ // manifest) must still render everywhere — flagged `cloned:false`
1191
+ // so the UI can offer Sync/clone instead of dropping the system.
1192
+ // `branch` comes from the workspace entry; never probe git on a
1193
+ // path that may not exist.
1194
+ return [
1195
+ {
1196
+ repo: resolved.repoName,
1197
+ service: resolved.subsystem,
1198
+ address: resolved.address,
1199
+ repoPath: resolved.repoPath,
1200
+ commands: [] as string[],
1201
+ logs: null,
1202
+ mode: resolved.mode,
1203
+ branch: resolved.branch,
1204
+ fromCatalog: true,
1205
+ isEntrypoint: false,
1206
+ autostart: !noAutostart.has(resolved.repoName),
1207
+ owners: daemonOwners.get(resolved.address) ?? [],
1208
+ deps: [] as string[],
1209
+ cloned: fs.existsSync(resolved.repoPath),
1210
+ },
1211
+ ];
1212
+ }
1213
+ return [
1214
+ {
1215
+ repo: resolved.repoName,
1216
+ service: resolved.subsystem,
1217
+ address: resolved.address,
1218
+ repoPath: resolved.repoPath,
1219
+ commands: withDown(Object.keys(svc.commands?.[env] ?? {})),
1220
+ logs: svc.logs?.[env] ?? null,
1221
+ mode: resolved.mode,
1222
+ branch: resolved.branch ?? getCurrentBranch(resolved.repoPath),
1223
+ fromCatalog: true,
1224
+ isEntrypoint: manifest.entrypoint === resolved.subsystem,
1225
+ autostart: !noAutostart.has(resolved.repoName),
1226
+ owners: daemonOwners.get(resolved.address) ?? [],
1227
+ deps: (svc.depends_on ?? []).map((d) =>
1228
+ resolveDepAddress(depService(d), catalog, ws.config, ws.name),
1229
+ ),
1230
+ cloned: true,
1231
+ },
1232
+ ];
1233
+ })
1234
+ : [];
1235
+
1236
+ // Merge: catalog entries take precedence over legacy for same repo.
1237
+ // When a workspace filter is active, also exclude legacy entries not in
1238
+ // the workspace (legacy discovery is not workspace-scoped).
1239
+ const catalogRepos = new Set(Object.keys(catalog));
1240
+ const wsServiceKeys = ws ? new Set(Object.keys(ws.config.services)) : null;
1241
+ const merged = [
1242
+ ...legacyEntries.filter((e) => {
1243
+ if (catalogRepos.has(e.repo)) return false;
1244
+ if (wsServiceKeys && !wsServiceKeys.has(e.repo)) return false;
1245
+ return true;
1246
+ }),
1247
+ ...catalogEntries,
1248
+ ];
1249
+
1250
+ // Attach the dependency-graph position so consumers can present systems
1251
+ // top-down (entry repo first, infra-stack last) — the same order the
1252
+ // health row and `den status` use. Added as a FIELD rather than by
1253
+ // re-sorting the array, because four pages consume this payload and some
1254
+ // rely on the current order.
1255
+ const graphRank = new Map(
1256
+ orderForDown(deriveGraph({ scope: "all", workspace: ws?.name })).map(
1257
+ (repo, i) => [repo, i],
1258
+ ),
1259
+ );
1260
+ const infraRepo = readDenConfig()?.graph?.infra?.repo ?? null;
1261
+ return sendJson(
1262
+ res,
1263
+ merged.map((e) => ({
1264
+ ...e,
1265
+ graphRank: graphRank.get(e.repo) ?? Number.MAX_SAFE_INTEGER,
1266
+ infra: infraRepo !== null && e.repo === infraRepo,
1267
+ })),
1268
+ );
1269
+ }
1270
+
1271
+ // ── API: service exec (SSE) — catalog-aware ────────────────────────────────
1272
+ if (pathname === "/api/service/exec") {
1273
+ const repoName = url.searchParams.get("repo");
1274
+ const svcName = url.searchParams.get("service");
1275
+ const cmdName = url.searchParams.get("cmd");
1276
+ const env = (url.searchParams.get("env") ?? "local") as Environment;
1277
+
1278
+ if (!repoName || !svcName || !cmdName)
1279
+ return send400(res, "repo, service, and cmd are required");
1280
+
1281
+ // Open the SSE stream immediately so any resolution errors are visible
1282
+ // in the output pane rather than triggering onerror ("Connection closed").
1283
+ res.writeHead(200, {
1284
+ "Content-Type": "text/event-stream",
1285
+ "Cache-Control": "no-cache",
1286
+ Connection: "keep-alive",
1287
+ "X-Accel-Buffering": "no",
1288
+ });
1289
+ const emit = (obj: unknown): void => {
1290
+ res.write(`data: ${JSON.stringify(obj)}\n\n`);
1291
+ };
1292
+ const fail = (msg: string): void => {
1293
+ emit({ type: "error", text: msg });
1294
+ emit({ type: "done", exitCode: 1, duration: 0 });
1295
+ res.end();
1296
+ };
1297
+
1298
+ const wsParam = url.searchParams.get("workspace");
1299
+ if (wsParam && !isWorkspaceActive(wsParam)) {
1300
+ return fail(`Workspace "${wsParam}" is not active — activate it first`);
1301
+ }
1302
+
1303
+ // Resolve repo path via resolveService (handles single-service repos where
1304
+ // the manifest key differs from the repo name, e.g. tasks-runner →
1305
+ // tool-runner). Try bare repo name first so single-service repos
1306
+ // don't need an explicit "repo/svc" address in the workspace config.
1307
+ let repoPath: string | undefined;
1308
+ let cmdDef:
1309
+ | string
1310
+ | { cmd: string; detachable?: boolean; interactive?: boolean }
1311
+ | undefined;
1312
+
1313
+ const resolved =
1314
+ resolveService(`${repoName}/${svcName}`, wsParam ?? undefined) ??
1315
+ resolveService(repoName, wsParam ?? undefined);
1316
+ if (resolved) {
1317
+ repoPath = resolved.repoPath;
1318
+ const manifest = readManifest(repoPath);
1319
+ if (manifest) {
1320
+ cmdDef =
1321
+ manifest.services?.[resolved.subsystem]?.commands?.[env]?.[cmdName];
1322
+ }
1323
+ }
1324
+
1325
+ if (!cmdDef) {
1326
+ // Legacy fallback: repos registered via .den/ symlinks / registered-repos.json
1327
+ const repos = discoverRepos();
1328
+ const repo = repos.find((r) => r.repoName === repoName);
1329
+ repoPath = repo?.repoPath;
1330
+ cmdDef = repo?.manifest.services?.[svcName]?.commands?.[env]?.[cmdName];
1331
+ }
1332
+
1333
+ if (!cmdDef || !repoPath) {
1334
+ // An un-cloned managed repo has no manifest to read commands from.
1335
+ // Report THAT, not "command not found" — the command may well exist; the
1336
+ // repo just isn't on disk yet.
1337
+ if (
1338
+ resolved &&
1339
+ resolved.mode === "managed" &&
1340
+ !fs.existsSync(resolved.repoPath)
1341
+ ) {
1342
+ return fail(
1343
+ `${repoName} is not cloned yet — run Sync (or ⋮ → clone now) to clone it into ` +
1344
+ `${product().servicesDir}/, then retry "${cmdName}".`,
1345
+ );
1346
+ }
1347
+ const ws = getActiveWorkspace();
1348
+ return fail(
1349
+ `Command "${cmdName}" not found for ${repoName}/${svcName} in env "${env}".` +
1350
+ (!ws ? " No active workspace." : ` Active workspace: ${ws.name}.`) +
1351
+ ` resolved=${JSON.stringify(resolved ?? null)}`,
1352
+ );
1353
+ }
1354
+
1355
+ const cmdString = rewriteLegacyOpctlKillCommandForContainer(
1356
+ cmdName,
1357
+ typeof cmdDef === "string" ? cmdDef : cmdDef.cmd,
1358
+ repoName,
1359
+ (line) => emit({ type: "line", text: line }),
1360
+ );
1361
+ const detached = shouldDetachServiceCommand(cmdName, cmdDef);
1362
+ const interactive = isInteractiveServiceCommand(cmdDef);
1363
+
1364
+ if (interactive) {
1365
+ return fail(
1366
+ `"${cmdName}" requires an interactive terminal (HITL).\n` +
1367
+ `Run in your terminal:\n cd ${repoPath}\n ${cmdString}`,
1368
+ );
1369
+ }
1370
+
1371
+ const startTime = Date.now();
1372
+ emit({ type: "cmd", text: `$ ${cmdString}`, detached });
1373
+
1374
+ const execId = shortRunId();
1375
+ const execDesc = `${repoName}/${svcName} ${cmdName} (${env})`;
1376
+ serverLog(`▶ exec ${execDesc} [${execId}]`, "lifecycle");
1377
+
1378
+ // For detachable commands (up/restart), the service is spawned as a daemon
1379
+ // writing to a log file so it survives den restarts. The tail proc
1380
+ // streams that log back to the SSE and keeps the connection open
1381
+ // indefinitely — the client stays attached until it closes the tab.
1382
+ const logPath = detached ? getServiceLogPath(repoName, svcName) : undefined;
1383
+ if (logPath) emit({ type: "line", text: `→ log: ${logPath}` });
1384
+
1385
+ let active = true; // false once client disconnects
1386
+
1387
+ prewarmSudo();
1388
+ const svcProc = spawnStreaming(
1389
+ cmdString,
1390
+ [],
1391
+ { ...process.env, E2E_ENVIRONMENT: env, ...getSudoEnv() },
1392
+ (line) => {
1393
+ mirrorLine(execId, line);
1394
+ emit({ type: "line", text: line });
1395
+ },
1396
+ (exitCode) => {
1397
+ if (!active) return; // client already closed — don't write to dead response
1398
+ const duration = Date.now() - startTime;
1399
+ serverLog(
1400
+ `■ exec ${execDesc} · exit=${exitCode} · ${formatDuration(duration)} [${execId}]`,
1401
+ "lifecycle",
1402
+ );
1403
+ // Dev-tooling telemetry: one invocation per service command run. The
1404
+ // action is whitelisted to common lifecycle verbs (low cardinality);
1405
+ // repo/service identifiers are internal and omitted.
1406
+ recordInvocation({
1407
+ command: "service.exec",
1408
+ surface: "ux",
1409
+ durationMs: duration,
1410
+ exit: exitCode,
1411
+ extra: {
1412
+ action: [
1413
+ "up",
1414
+ "down",
1415
+ "restart",
1416
+ "setup",
1417
+ "build",
1418
+ "logs",
1419
+ ].includes(cmdName)
1420
+ ? cmdName
1421
+ : "other",
1422
+ env,
1423
+ },
1424
+ });
1425
+ if (cmdName === "down" && exitCode === 0) {
1426
+ unregisterDaemon(resolved?.address ?? `${repoName}/${svcName}`);
1427
+ }
1428
+ if (exitCode !== 0)
1429
+ emit({ type: "error", text: `Command failed (exit ${exitCode})` });
1430
+ emit({ type: "done", exitCode, duration });
1431
+ res.end();
1432
+ },
1433
+ repoPath,
1434
+ detached,
1435
+ logPath,
1436
+ // For detachable commands (up): fire when the launch script itself exits.
1437
+ // This happens when `make up` hands off to a container runtime and exits.
1438
+ // The client uses this to transition from startup output to ongoing logs.
1439
+ detached
1440
+ ? (exitCode) => {
1441
+ if (!active) return;
1442
+ emit({
1443
+ type: "service-exited",
1444
+ exitCode,
1445
+ repoName,
1446
+ subsystem: svcName,
1447
+ });
1448
+ }
1449
+ : undefined,
1450
+ );
1451
+
1452
+ if (detached) {
1453
+ // Tell the client the daemon is up; keep the tail proc (svcProc) alive
1454
+ // so startup output keeps streaming until `make up` finishes.
1455
+ // Kill the tail proc only when the client disconnects (tab closed).
1456
+ emit({ type: "started", detached: true });
1457
+ res.on("close", () => {
1458
+ active = false;
1459
+ try {
1460
+ svcProc.kill();
1461
+ } catch {
1462
+ /* already exited */
1463
+ }
1464
+ });
1465
+ }
1466
+ return;
1467
+ }
1468
+
1469
+ // ── API: service logs (SSE) ───────────────────────────────────────────────
1470
+ if (pathname === "/api/service/logs") {
1471
+ const repoName = url.searchParams.get("repo");
1472
+ const svcName = url.searchParams.get("service");
1473
+ const env = (url.searchParams.get("env") ?? "local") as Environment;
1474
+
1475
+ if (!repoName || !svcName)
1476
+ return send400(res, "repo and service are required");
1477
+
1478
+ const repos = discoverRepos();
1479
+ const repo = repos.find((r) => r.repoName === repoName);
1480
+ const logSource = repo?.manifest.services?.[svcName]?.logs?.[env] as
1481
+ | LogSource
1482
+ | undefined;
1483
+
1484
+ if (!logSource) {
1485
+ res.writeHead(404, { "Content-Type": "text/plain" });
1486
+ res.end(
1487
+ `No log source configured for ${repoName}/${svcName} in env "${env}"`,
1488
+ );
1489
+ return;
1490
+ }
1491
+
1492
+ res.writeHead(200, {
1493
+ "Content-Type": "text/event-stream",
1494
+ "Cache-Control": "no-cache",
1495
+ Connection: "keep-alive",
1496
+ "X-Accel-Buffering": "no",
1497
+ });
1498
+
1499
+ const emit = (obj: unknown): void => {
1500
+ res.write(`data: ${JSON.stringify(obj)}\n\n`);
1501
+ };
1502
+
1503
+ if (logSource.type === "process") {
1504
+ emit({
1505
+ type: "error",
1506
+ text: "process logs: attach to a running process is not yet supported",
1507
+ });
1508
+ res.end();
1509
+ return;
1510
+ }
1511
+
1512
+ if (logSource.type === "shell") {
1513
+ emit({ type: "cmd", text: `$ ${logSource.cmd}` });
1514
+
1515
+ let active = true;
1516
+ const lineBuffer: string[] = [];
1517
+ let confirmed = false;
1518
+ let quickTimer: ReturnType<typeof setTimeout> | null = setTimeout(() => {
1519
+ quickTimer = null;
1520
+ confirmed = true;
1521
+ for (const line of lineBuffer) emit({ type: "line", text: line });
1522
+ lineBuffer.length = 0;
1523
+ }, DAEMON_QUICK_EXIT_MS);
1524
+
1525
+ const proc = testRunner.spawnStreaming(
1526
+ logSource.cmd,
1527
+ [],
1528
+ { ...process.env },
1529
+ (line) => {
1530
+ if (confirmed) {
1531
+ emit({ type: "line", text: line });
1532
+ } else {
1533
+ lineBuffer.push(line);
1534
+ }
1535
+ },
1536
+ (exitCode) => {
1537
+ if (!active) return;
1538
+ if (quickTimer !== null) {
1539
+ clearTimeout(quickTimer);
1540
+ quickTimer = null;
1541
+ lineBuffer.length = 0;
1542
+ emit({
1543
+ type: "line",
1544
+ text: `No running log stream for ${repoName}/${svcName}; the service may be down.`,
1545
+ });
1546
+ } else if (exitCode !== 0) {
1547
+ emit({
1548
+ type: "line",
1549
+ text: `── log stream exited (${exitCode}) ──`,
1550
+ });
1551
+ }
1552
+ emit({ type: "done", exitCode, duration: 0 });
1553
+ res.end();
1554
+ },
1555
+ repo?.repoPath,
1556
+ true, // detached — own process group
1557
+ );
1558
+
1559
+ res.on("close", () => {
1560
+ active = false;
1561
+ if (quickTimer !== null) {
1562
+ clearTimeout(quickTimer);
1563
+ quickTimer = null;
1564
+ }
1565
+ try {
1566
+ proc.kill("SIGTERM");
1567
+ } catch {
1568
+ /* already exited */
1569
+ }
1570
+ });
1571
+ return;
1572
+ }
1573
+
1574
+ if (logSource.type === "cloudwatch") {
1575
+ const region =
1576
+ logSource.region ?? process.env.AWS_DEFAULT_REGION ?? "us-east-1";
1577
+ const profile = loadAwsProfiles()[env];
1578
+ const args = [
1579
+ "logs",
1580
+ "tail",
1581
+ logSource.logGroup,
1582
+ "--follow",
1583
+ "--region",
1584
+ region,
1585
+ ];
1586
+ if (logSource.logStream)
1587
+ args.push("--log-stream-name-prefix", logSource.logStream);
1588
+ if (profile) args.push("--profile", profile);
1589
+
1590
+ emit({ type: "cmd", text: `$ aws ${args.join(" ")}` });
1591
+
1592
+ // Track output lines to detect AWS auth errors
1593
+ const AUTH_KEYWORDS = [
1594
+ "session has expired",
1595
+ "reauthenticate",
1596
+ "aws login",
1597
+ "aws sso login",
1598
+ "not authorized",
1599
+ "invalid token",
1600
+ "ExpiredToken",
1601
+ "NoCredentialsError",
1602
+ ];
1603
+ let authRequired = false;
1604
+
1605
+ // Strip static credential env vars so --profile takes effect.
1606
+ // AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY have higher precedence than
1607
+ // --profile and will silently override it if left in the environment.
1608
+ const awsEnv = { ...process.env };
1609
+ if (profile) {
1610
+ delete awsEnv.AWS_ACCESS_KEY_ID;
1611
+ delete awsEnv.AWS_SECRET_ACCESS_KEY;
1612
+ delete awsEnv.AWS_SESSION_TOKEN;
1613
+ delete awsEnv.AWS_SECURITY_TOKEN;
1614
+ delete awsEnv.AWS_PROFILE;
1615
+ delete awsEnv.AWS_DEFAULT_PROFILE;
1616
+ }
1617
+
1618
+ const proc = testRunner.spawnStreaming(
1619
+ "aws",
1620
+ args,
1621
+ awsEnv,
1622
+ (line) => {
1623
+ const lower = line.toLowerCase();
1624
+ if (AUTH_KEYWORDS.some((k) => lower.includes(k.toLowerCase())))
1625
+ authRequired = true;
1626
+ emit({ type: "line", text: line });
1627
+ },
1628
+ (exitCode) => {
1629
+ emit({
1630
+ type: "done",
1631
+ exitCode,
1632
+ duration: 0,
1633
+ authRequired: authRequired && exitCode !== 0,
1634
+ });
1635
+ res.end();
1636
+ },
1637
+ undefined,
1638
+ true, // detached — own process group
1639
+ );
1640
+
1641
+ res.on("close", () => {
1642
+ try {
1643
+ proc.kill("SIGTERM");
1644
+ } catch {
1645
+ /* already exited */
1646
+ }
1647
+ });
1648
+ return;
1649
+ }
1650
+
1651
+ emit({
1652
+ type: "error",
1653
+ text: `Unsupported log source type: ${(logSource as LogSource).type}`,
1654
+ });
1655
+ res.end();
1656
+ return;
1657
+ }
1658
+
1659
+ // ── API: running daemons list ──────────────────────────────────────────────
1660
+ if (pathname === "/api/service/daemons") {
1661
+ const daemons = listDaemons().map((d) => ({
1662
+ address: d.address,
1663
+ repoName: d.repoName,
1664
+ subsystem: d.subsystem,
1665
+ repoPath: d.repoPath,
1666
+ branch: d.branch ?? null,
1667
+ workspaces: d.workspaces ?? [],
1668
+ logPath: d.logPath,
1669
+ startedAt: d.startedAt.toISOString(),
1670
+ }));
1671
+ return sendJson(res, daemons);
1672
+ }
1673
+
1674
+ // ── API: service taillog (SSE) — live log stream for a running service ───────
1675
+ //
1676
+ // Source resolution, in order:
1677
+ // 1. The service's manifest `logs.local` shell command (e.g. `docker logs -f`)
1678
+ // 2. Conventional container derivation: a running `opctl_<base>_<hash>`
1679
+ // container matched from the repo's op manifest / repo name →
1680
+ // `docker logs -f` — real runtime logs with zero manifest config
1681
+ // 3. The den daemon startup log (~/.den/logs/<repo>/<service>.log) —
1682
+ // last resort, covers host-run processes with no container
1683
+ // A manifest command that exits within the quick-exit window (container not
1684
+ // found, stale filter) drops to tier 2/3, retrying tier 1 in between.
1685
+ // ── API: service log HISTORY — paged backwards to container startup ────────
1686
+ //
1687
+ // Docker retains the full log of the current container instance; this pages
1688
+ // it backwards with opaque cursors (byte offsets into an immutable snapshot
1689
+ // — see log-history.ts). env=local only in v1; other envs return
1690
+ // { unsupported: true } so the client renders a friendly notice.
1691
+ if (pathname === "/api/service/history") {
1692
+ // This handler sits ABOVE the blanket /api/service* local-dev guard
1693
+ // (later in this function), so it must enforce the guard itself.
1694
+ if (!isLocalEnabled()) {
1695
+ res.writeHead(403);
1696
+ res.end("local-dev is disabled");
1697
+ return;
1698
+ }
1699
+ const repoName = url.searchParams.get("repo");
1700
+ const svcName = url.searchParams.get("service");
1701
+ const env = url.searchParams.get("env") ?? "local";
1702
+ const cursor = url.searchParams.get("cursor") || null;
1703
+ const limit = Math.min(
1704
+ Math.max(
1705
+ parseInt(url.searchParams.get("limit") ?? "2000", 10) || 2000,
1706
+ 1,
1707
+ ),
1708
+ 10_000,
1709
+ );
1710
+ if (!repoName || !svcName)
1711
+ return send400(res, "repo and service are required");
1712
+ if (!isSafeServiceName(repoName) || !isSafeServiceName(svcName))
1713
+ return send400(res, "invalid repo/service name");
1714
+ try {
1715
+ const source = resolveHistorySource(repoName, svcName, env);
1716
+ if (source.kind === "unsupported")
1717
+ return sendJson(res, { unsupported: true, reason: source.reason });
1718
+ const snapshot = ensureSnapshot(source);
1719
+ if (!snapshot)
1720
+ return sendJson(res, {
1721
+ unsupported: true,
1722
+ reason: "could not snapshot the container log",
1723
+ });
1724
+ return sendJson(res, readPageBefore(snapshot, cursor, limit));
1725
+ } catch (e) {
1726
+ return send400(res, (e as Error).message);
1727
+ }
1728
+ }
1729
+
1730
+ // ── API: cross-system history search ───────────────────────────────────────
1731
+ //
1732
+ // Greps the FULL history of each requested system (targets = the /logs
1733
+ // toolbar's list, computed client-side so scoping lives in one place).
1734
+ if (pathname === "/api/logs/search") {
1735
+ // /api/logs is not covered by the downstream /api/service* local-dev
1736
+ // guard — enforce it here.
1737
+ if (!isLocalEnabled()) {
1738
+ res.writeHead(403);
1739
+ res.end("local-dev is disabled");
1740
+ return;
1741
+ }
1742
+ const q = url.searchParams.get("q") ?? "";
1743
+ const env = url.searchParams.get("env") ?? "local";
1744
+ const targets = (url.searchParams.get("targets") ?? "")
1745
+ .split(",")
1746
+ .map((t) => t.trim())
1747
+ .filter(Boolean)
1748
+ .slice(0, 40)
1749
+ .map((t) => {
1750
+ const slash = t.lastIndexOf("/");
1751
+ return { repo: t.slice(0, slash), service: t.slice(slash + 1) };
1752
+ })
1753
+ .filter(
1754
+ (t) =>
1755
+ t.repo &&
1756
+ t.service &&
1757
+ isSafeServiceName(t.repo) &&
1758
+ isSafeServiceName(t.service),
1759
+ );
1760
+ if (q.trim().length < 2) return send400(res, "q must be at least 2 chars");
1761
+ if (targets.length === 0) return send400(res, "targets required");
1762
+ void Promise.all(targets.map((t) => searchSystemHistory(t, q, env))).then(
1763
+ (results) => sendJson(res, { q, results }),
1764
+ );
1765
+ return;
1766
+ }
1767
+
1768
+ if (pathname === "/api/service/taillog") {
1769
+ const repoName = url.searchParams.get("repo");
1770
+ const svcName = url.searchParams.get("service");
1771
+ if (!repoName || !svcName)
1772
+ return send400(res, "repo and service are required");
1773
+
1774
+ // Resolve manifest log source. The client names an exact service, so
1775
+ // when the manifest has that key, use it directly — resolveService's
1776
+ // bare-repo correction rewrites the subsystem to the ENTRYPOINT (a
1777
+ // shop/db request would otherwise read api's log source).
1778
+ const tailResolved =
1779
+ resolveService(repoName) ?? resolveService(`${repoName}/${svcName}`);
1780
+ const tailManifest = tailResolved
1781
+ ? readManifest(tailResolved.repoPath)
1782
+ : null;
1783
+ const tailSubsystem = tailManifest?.services?.[svcName]
1784
+ ? svcName
1785
+ : (tailResolved?.subsystem ?? svcName);
1786
+ const logSource = tailManifest?.services?.[tailSubsystem]?.logs?.[
1787
+ "local"
1788
+ ] as LogSource | undefined;
1789
+
1790
+ const daemonLogPath = getServiceLogPath(repoName, svcName);
1791
+ const fallbackCmd = `tail -F -n ${LIVE_TAIL_LINES} "${daemonLogPath}"`;
1792
+
1793
+ const manifestCmd =
1794
+ logSource?.type === "shell" && logSource.cmd ? logSource.cmd : null;
1795
+ const manifestCwd = manifestCmd ? tailResolved?.repoPath : undefined;
1796
+
1797
+ res.writeHead(200, {
1798
+ "Content-Type": "text/event-stream",
1799
+ "Cache-Control": "no-cache",
1800
+ Connection: "keep-alive",
1801
+ "X-Accel-Buffering": "no",
1802
+ });
1803
+
1804
+ let active = true;
1805
+ const emit = (obj: unknown): void => {
1806
+ if (!active) return;
1807
+ res.write(`data: ${JSON.stringify(obj)}\n\n`);
1808
+ };
1809
+
1810
+ // If the manifest log command exits within QUICK_EXIT_MS, we consider it a
1811
+ // transient failure (e.g. container not yet running) and retry. After
1812
+ // MAX_RETRIES quick exits we give up and fall back to the daemon log.
1813
+ const QUICK_EXIT_MS = DAEMON_QUICK_EXIT_MS;
1814
+ const RETRY_DELAY_MS = DAEMON_RETRY_DELAY_MS;
1815
+ const MAX_RETRIES = 12; // up to ~60s of retries before giving up
1816
+
1817
+ let currentProc: ReturnType<typeof spawnStreaming> | null = null;
1818
+ let retryCount = 0;
1819
+ let retryTimer: ReturnType<typeof setTimeout> | null = null;
1820
+
1821
+ // Fallback tier 1: resolve the service's running container by opctl
1822
+ // naming convention (`opctl_<base>_<hash>`) and stream `docker logs -f`.
1823
+ // This gives REAL runtime logs for services with no (or broken)
1824
+ // manifest `logs.local` — no den daemon file needed. Returns false
1825
+ // when no conventional container is running.
1826
+ const startDockerFallback = (): boolean => {
1827
+ const container = findConventionalContainer(
1828
+ repoName,
1829
+ tailResolved?.repoPath,
1830
+ tailSubsystem,
1831
+ );
1832
+ if (!container) return false;
1833
+ emit({
1834
+ type: "line",
1835
+ text: `── streaming container logs: ${container} ──`,
1836
+ });
1837
+ currentProc = spawnStreaming(
1838
+ `docker logs -f --tail ${LIVE_TAIL_LINES} ${container}`,
1839
+ [],
1840
+ { ...process.env },
1841
+ (line) => emit({ type: "line", text: line }),
1842
+ (exitCode) => {
1843
+ if (!active) return;
1844
+ // Container stopped/removed — end the stream; the client's
1845
+ // auto-reattach re-resolves against whatever is running next.
1846
+ emit({
1847
+ type: "line",
1848
+ text: `── container log stream ended (${exitCode}) ──`,
1849
+ });
1850
+ res.end();
1851
+ },
1852
+ undefined,
1853
+ true,
1854
+ );
1855
+ return true;
1856
+ };
1857
+
1858
+ // Fallback tier 2 (last resort): tail the den daemon startup log —
1859
+ // covers host-run processes (no container to derive logs from).
1860
+ const startDaemonFileFallback = (): void => {
1861
+ emit({
1862
+ type: "line",
1863
+ text: "── container log unavailable — watching startup log ──",
1864
+ });
1865
+ currentProc = spawnStreaming(
1866
+ fallbackCmd,
1867
+ [],
1868
+ { ...process.env },
1869
+ (line) => emit({ type: "line", text: line }),
1870
+ () => {
1871
+ if (active) res.end();
1872
+ },
1873
+ undefined,
1874
+ true,
1875
+ );
1876
+ };
1877
+
1878
+ const startFallback = (): void => {
1879
+ if (startDockerFallback()) return;
1880
+ startDaemonFileFallback();
1881
+ };
1882
+
1883
+ const tryManifestCmd = (): void => {
1884
+ // Buffer lines during the quick-exit window — only flush them to the
1885
+ // client once we know the command has been alive long enough to be real.
1886
+ // This prevents docker error messages from cluttering the tab on retries.
1887
+ const lineBuffer: string[] = [];
1888
+ let confirmed = false; // true once QUICK_EXIT_MS has passed without exit
1889
+
1890
+ let quickTimer: ReturnType<typeof setTimeout> | null = setTimeout(() => {
1891
+ quickTimer = null;
1892
+ confirmed = true;
1893
+ // Flush any buffered lines now that we know the command is alive.
1894
+ for (const line of lineBuffer) emit({ type: "line", text: line });
1895
+ lineBuffer.length = 0;
1896
+ }, QUICK_EXIT_MS);
1897
+
1898
+ currentProc = spawnStreaming(
1899
+ manifestCmd!,
1900
+ [],
1901
+ { ...process.env },
1902
+ (line) => {
1903
+ if (confirmed) {
1904
+ emit({ type: "line", text: line });
1905
+ } else {
1906
+ lineBuffer.push(line);
1907
+ }
1908
+ },
1909
+ (exitCode) => {
1910
+ if (!active) return;
1911
+ if (quickTimer !== null) {
1912
+ // Exited before QUICK_EXIT_MS — the manifest log command is broken
1913
+ // or the container isn't ready. When den's own daemon log exists
1914
+ // (den spawned this service), fall back to it IMMEDIATELY instead
1915
+ // of retrying a dead command for ~60s of silent tab (the exact
1916
+ // failure mode of stale `docker ps --filter name=…` log commands).
1917
+ clearTimeout(quickTimer);
1918
+ quickTimer = null;
1919
+ lineBuffer.length = 0; // discard noisy error output
1920
+ if (startDockerFallback()) {
1921
+ // streaming the conventional container's logs
1922
+ } else if (fs.existsSync(daemonLogPath)) {
1923
+ startDaemonFileFallback();
1924
+ } else if (retryCount < MAX_RETRIES) {
1925
+ retryCount++;
1926
+ retryTimer = setTimeout(() => {
1927
+ retryTimer = null;
1928
+ if (active) tryManifestCmd();
1929
+ }, RETRY_DELAY_MS);
1930
+ } else {
1931
+ startFallback();
1932
+ }
1933
+ } else if (exitCode !== 0) {
1934
+ // Ran for a while then exited with error.
1935
+ emit({
1936
+ type: "line",
1937
+ text: `── log stream exited (${exitCode}) ──`,
1938
+ });
1939
+ res.end();
1940
+ } else {
1941
+ res.end();
1942
+ }
1943
+ },
1944
+ manifestCwd,
1945
+ true,
1946
+ );
1947
+ };
1948
+
1949
+ if (manifestCmd) {
1950
+ tryManifestCmd();
1951
+ } else {
1952
+ // No manifest log source (e.g. shop-web, control-plane): conventional
1953
+ // container logs first, daemon startup log as last resort.
1954
+ startFallback();
1955
+ }
1956
+
1957
+ res.on("close", () => {
1958
+ active = false;
1959
+ if (retryTimer) {
1960
+ clearTimeout(retryTimer);
1961
+ retryTimer = null;
1962
+ }
1963
+ try {
1964
+ currentProc?.kill("SIGTERM");
1965
+ } catch {
1966
+ /* already exited */
1967
+ }
1968
+ });
1969
+ return;
1970
+ }
1971
+
1972
+ // ── API: run tests (SSE) ───────────────────────────────────────────────────
1973
+ if (pathname === "/api/run") {
1974
+ const pillar = url.searchParams.get("pillar") as Pillar | null;
1975
+ const modules = url.searchParams.getAll("module").filter(Boolean);
1976
+ const env = (url.searchParams.get("env") ?? "local") as Environment;
1977
+ const baseApiUrl = url.searchParams.get("baseApiUrl") ?? undefined;
1978
+ const useLocalCheckout = url.searchParams.get("useLocalCheckout") === "1";
1979
+ const modeParam = url.searchParams.get("mode");
1980
+ const securityMode =
1981
+ modeParam === "api" || modeParam === "full" ? modeParam : "baseline";
1982
+ // Per-perf-test declarative optional-field values, as URL-encoded JSON:
1983
+ // { [moduleName]: { [fieldName]: value } }
1984
+ // Each module's declared `optionalFields` are resolved to env vars below and
1985
+ // routed ONLY to that module's k6 process (never globally).
1986
+ const optsRaw = url.searchParams.get("opts");
1987
+ let opts: Record<string, Record<string, unknown>> = {};
1988
+ if (optsRaw) {
1989
+ try {
1990
+ const parsed = JSON.parse(optsRaw);
1991
+ if (parsed && typeof parsed === "object") {
1992
+ opts = parsed as Record<string, Record<string, unknown>>;
1993
+ }
1994
+ } catch {
1995
+ // Malformed opts → ignore; fall back to per-field defaults.
1996
+ }
1997
+ }
1998
+
1999
+ if (!pillar) return send400(res, "pillar required");
2000
+
2001
+ // Resolve declarative optional fields → per-module env. Only the perf pillar
2002
+ // declares optionalFields; for every other pillar this is an empty map.
2003
+ let moduleEnv: Record<string, Record<string, string>> | undefined;
2004
+ if (pillar === "performance") {
2005
+ moduleEnv = {};
2006
+ try {
2007
+ const perfModules = await moduleScanner.scanModules("performance");
2008
+ // When "All modules" is selected the client sends no `module` params;
2009
+ // resolve fields for every perf module so its defaults still apply.
2010
+ const targetNames =
2011
+ modules.length > 0 ? modules : perfModules.map((m) => m.name);
2012
+ for (const m of perfModules) {
2013
+ if (!targetNames.includes(m.name)) continue;
2014
+ const fields = m.optionalFields ?? [];
2015
+ if (fields.length === 0) continue;
2016
+ const values = (opts[m.name] ?? {}) as Record<
2017
+ string,
2018
+ string | number | boolean | undefined
2019
+ >;
2020
+ const env = resolveFieldEnv(fields, values);
2021
+ if (Object.keys(env).length > 0) moduleEnv[m.name] = env;
2022
+ }
2023
+ } catch (err) {
2024
+ serverLog(` ⚠ /api/run optionalFields: ${(err as Error).message}`);
2025
+ }
2026
+ }
2027
+
2028
+ res.writeHead(200, {
2029
+ "Content-Type": "text/event-stream",
2030
+ "Cache-Control": "no-cache",
2031
+ Connection: "keep-alive",
2032
+ "X-Accel-Buffering": "no",
2033
+ });
2034
+
2035
+ const emit = (obj: unknown): void => {
2036
+ res.write(`data: ${JSON.stringify(obj)}\n\n`);
2037
+ };
2038
+
2039
+ const steps = testRunner.buildCommand({
2040
+ pillar,
2041
+ environment: env,
2042
+ modules,
2043
+ baseApiUrl,
2044
+ useLocalCheckout,
2045
+ moduleEnv,
2046
+ securityMode,
2047
+ });
2048
+
2049
+ if (steps.length === 0) {
2050
+ emit({
2051
+ type: "error",
2052
+ text: `No command configured for pillar: ${pillar}`,
2053
+ });
2054
+ res.end();
2055
+ return;
2056
+ }
2057
+
2058
+ // Log the source(s) of truth before the first step so run output captures
2059
+ // which repo/branch/SHA was actually tested. Silent for pillars that don't
2060
+ // read tests from repos (integration, perf are den-owned).
2061
+ if (
2062
+ pillar === "synthetic" ||
2063
+ pillar === "smoke" ||
2064
+ pillar === "journeys" ||
2065
+ pillar === "security"
2066
+ ) {
2067
+ try {
2068
+ const sources = collectTestSources(pillar);
2069
+ if (sources.length === 0) {
2070
+ emit({ type: "line", text: `→ source: (no repos discovered)` });
2071
+ } else {
2072
+ for (const s of sources) {
2073
+ emit({ type: "line", text: `→ source: ${formatSourceLine(s)}` });
2074
+ }
2075
+ }
2076
+ } catch (err) {
2077
+ emit({
2078
+ type: "line",
2079
+ text: `→ source: (resolution failed: ${(err as Error).message})`,
2080
+ });
2081
+ }
2082
+ }
2083
+
2084
+ const runId = shortRunId();
2085
+ const modDesc = modules.length ? modules.join(",") : "all";
2086
+ const runDesc = `${pillar}/${env} · modules=${modDesc}`;
2087
+ serverLog(`▶ run ${runDesc} [${runId}]`, "lifecycle");
2088
+ if (
2089
+ pillar === "synthetic" ||
2090
+ pillar === "smoke" ||
2091
+ pillar === "journeys" ||
2092
+ pillar === "security"
2093
+ ) {
2094
+ try {
2095
+ const sources = collectTestSources(pillar);
2096
+ for (const s of sources) {
2097
+ serverLog(` source: ${formatSourceLine(s)} [${runId}]`, "lifecycle");
2098
+ }
2099
+ } catch {
2100
+ /* already emitted above as an SSE line */
2101
+ }
2102
+ }
2103
+
2104
+ const startTime = Date.now();
2105
+ let overallExit = 0;
2106
+
2107
+ const finishLifecycle = (exitCode: number, note?: string): void => {
2108
+ const dur = formatDuration(Date.now() - startTime);
2109
+ const extra = note ? ` · ${note}` : "";
2110
+ serverLog(
2111
+ `■ run ${runDesc} · exit=${exitCode} · ${dur}${extra} [${runId}]`,
2112
+ "lifecycle",
2113
+ );
2114
+ // Dev-tooling telemetry: one invocation per test run (pillar/env are
2115
+ // low-cardinality enums; module count only — never module names/args).
2116
+ recordInvocation({
2117
+ command: "test.run",
2118
+ surface: "ux",
2119
+ durationMs: Date.now() - startTime,
2120
+ exit: exitCode,
2121
+ extra: {
2122
+ pillar: String(pillar),
2123
+ env: String(env),
2124
+ modules: modules.length,
2125
+ },
2126
+ });
2127
+ };
2128
+
2129
+ const runStep = (i: number): void => {
2130
+ if (i >= steps.length) {
2131
+ finishLifecycle(overallExit);
2132
+ emit(
2133
+ getRunDonePayload({
2134
+ pillar,
2135
+ exitCode: overallExit,
2136
+ duration: Date.now() - startTime,
2137
+ rootDir: ROOT,
2138
+ }),
2139
+ );
2140
+ res.end();
2141
+ return;
2142
+ }
2143
+
2144
+ const step = steps[i];
2145
+ emit({ type: "cmd", text: `$ ${step.command} ${step.args.join(" ")}` });
2146
+
2147
+ testRunner.spawnStreaming(
2148
+ step.command,
2149
+ step.args,
2150
+ step.env,
2151
+ (line) => {
2152
+ mirrorLine(runId, line);
2153
+ emit({ type: "line", text: line });
2154
+ },
2155
+ (exitCode) => {
2156
+ if (exitCode !== 0) {
2157
+ if (overallExit === 0) {
2158
+ overallExit = exitCode;
2159
+ }
2160
+ if (!step.continueOnFailure) {
2161
+ emit({
2162
+ type: "error",
2163
+ text: `Step "${step.label}" failed (exit ${exitCode}) — aborting`,
2164
+ });
2165
+ finishLifecycle(exitCode, `step "${step.label}" failed`);
2166
+ emit(
2167
+ getRunDonePayload({
2168
+ pillar,
2169
+ exitCode,
2170
+ duration: Date.now() - startTime,
2171
+ rootDir: ROOT,
2172
+ }),
2173
+ );
2174
+ res.end();
2175
+ return;
2176
+ }
2177
+ }
2178
+ runStep(i + 1);
2179
+ },
2180
+ );
2181
+ };
2182
+
2183
+ runStep(0);
2184
+ return;
2185
+ }
2186
+
2187
+ // ── API: AWS re-authenticate (SSE) ───────────────────────────────────────
2188
+ if (pathname === "/api/aws/login") {
2189
+ const env = (url.searchParams.get("env") ?? "local") as Environment;
2190
+ const profiles = loadAwsProfiles();
2191
+ const profile = profiles[env];
2192
+
2193
+ res.writeHead(200, {
2194
+ "Content-Type": "text/event-stream",
2195
+ "Cache-Control": "no-cache",
2196
+ Connection: "keep-alive",
2197
+ "X-Accel-Buffering": "no",
2198
+ });
2199
+
2200
+ const emit = (obj: unknown): void => {
2201
+ res.write(`data: ${JSON.stringify(obj)}\n\n`);
2202
+ };
2203
+
2204
+ if (!profile) {
2205
+ emit({
2206
+ type: "error",
2207
+ text: `No AWS profile configured for env "${env}". Run: make aws-configure`,
2208
+ });
2209
+ emit({ type: "done", exitCode: 1, duration: 0 });
2210
+ res.end();
2211
+ return;
2212
+ }
2213
+
2214
+ const args = ["sso", "login", "--profile", profile];
2215
+ emit({ type: "cmd", text: `$ aws ${args.join(" ")}` });
2216
+
2217
+ const proc = testRunner.spawnStreaming(
2218
+ "aws",
2219
+ args,
2220
+ { ...process.env },
2221
+ (line) => emit({ type: "line", text: line }),
2222
+ (exitCode) => {
2223
+ emit({ type: "done", exitCode, duration: 0 });
2224
+ res.end();
2225
+ },
2226
+ undefined,
2227
+ false, // not detached — we want signals to reach it
2228
+ );
2229
+
2230
+ res.on("close", () => {
2231
+ try {
2232
+ proc.kill("SIGTERM");
2233
+ } catch {
2234
+ /* already exited */
2235
+ }
2236
+ });
2237
+ return;
2238
+ }
2239
+
2240
+ // ── API: service catalog ──────────────────────────────────────────────────
2241
+ if (pathname === "/api/catalog") {
2242
+ return sendJson(res, readCatalog());
2243
+ }
2244
+
2245
+ // ── API: den config ────────────────────────────────────────────────────
2246
+ if (pathname === `/api/${product().cli}/config` && req.method === "GET") {
2247
+ // `org` resolved (id defaults to edge.org) for the header's org label.
2248
+ return sendJson(res, { ...readDenConfig(), org: readOrg() });
2249
+ }
2250
+
2251
+ // ── API: server info + staleness (drives the header "stale server" pill) ────
2252
+ if (pathname === "/api/server-info" && req.method === "GET") {
2253
+ const { stale, scanned } = computeStale();
2254
+ return sendJson(res, {
2255
+ pid: process.pid,
2256
+ port: PORT,
2257
+ startedAt: new Date(SERVER_STARTED_AT).toISOString(),
2258
+ uptimeMs: Date.now() - SERVER_STARTED_AT,
2259
+ stale,
2260
+ scanned,
2261
+ });
2262
+ }
2263
+
2264
+ // ── API: restart the web server (from the "stale server" UI) ────────────────
2265
+ // Spawns a detached `den web restart` that stops THIS process and starts a
2266
+ // fresh daemon on the same port. Respond first, then spawn on the next tick so
2267
+ // the response flushes before the driver child arrives to stop us.
2268
+ if (pathname === "/api/web/restart" && req.method === "POST") {
2269
+ serverLog("↻ web restart requested from the UI", "lifecycle");
2270
+ sendJson(res, { restarting: true, pid: process.pid });
2271
+ setImmediate(() => {
2272
+ try {
2273
+ spawnDetachedWebRestart(PORT);
2274
+ } catch (e) {
2275
+ serverLog(` ⚠ web restart failed to spawn: ${(e as Error).message}`);
2276
+ }
2277
+ });
2278
+ return;
2279
+ }
2280
+
2281
+ // ── API: release config (GET effective / PUT personal override) ────────────
2282
+ // Deliberately placed before the local-dev guard: the Release tab is for
2283
+ // hosted deploys and must work when `local` is disabled.
2284
+ if (pathname === "/api/release/config") {
2285
+ if (req.method === "GET") return sendJson(res, releaseService.config());
2286
+ if (req.method === "PUT") {
2287
+ try {
2288
+ const patch = JSON.parse(await readBody(req)) as Partial<ReleaseConfig>;
2289
+ if (typeof patch !== "object" || patch === null || Array.isArray(patch))
2290
+ return send400(res, "expected a release config object");
2291
+ if (patch.repos !== undefined) {
2292
+ if (!Array.isArray(patch.repos))
2293
+ return send400(res, "repos must be an array");
2294
+ for (const r of patch.repos) {
2295
+ if (!r || typeof r.name !== "string" || typeof r.wave !== "number")
2296
+ return send400(
2297
+ res,
2298
+ "each repo needs a string name and numeric wave",
2299
+ );
2300
+ }
2301
+ }
2302
+ // Whether the page may deploy is config/den.yml's call, never the page's.
2303
+ delete (patch as { deploy?: boolean }).deploy;
2304
+ writeUserReleaseConfig(patch);
2305
+ return sendJson(res, releaseService.config());
2306
+ } catch (e) {
2307
+ return send400(res, `Invalid request: ${(e as Error).message}`);
2308
+ }
2309
+ }
2310
+ res.writeHead(405);
2311
+ res.end("Method not allowed");
2312
+ return;
2313
+ }
2314
+
2315
+ // ── API: release status (verify table) ─────────────────────────────────────
2316
+ if (pathname === "/api/release/status" && req.method === "GET") {
2317
+ const env = url.searchParams.get("env") ?? "";
2318
+ const refresh = url.searchParams.get("refresh") === "1";
2319
+ const cfg = releaseService.config();
2320
+ if (!cfg.environments.includes(env))
2321
+ return send400(res, `Unknown env: ${env}`);
2322
+ try {
2323
+ const rows = await releaseService.verifyEnv(env, refresh);
2324
+ return sendJson(res, { env, rows, fetchedAt: new Date().toISOString() });
2325
+ } catch (e) {
2326
+ serverLog(` ⚠ /api/release/status: ${(e as Error).message}`);
2327
+ return sendJson(res, { env, rows: [], error: (e as Error).message });
2328
+ }
2329
+ }
2330
+
2331
+ // ── API: live CD runs (Release page poll) ──────────────────────────────────
2332
+ // Deliberately not env-scoped — see RepoRunState: a run object does not carry
2333
+ // the environment it targets, so a run in flight blocks that repo outright.
2334
+ if (pathname === "/api/release/runs" && req.method === "GET") {
2335
+ const refresh = url.searchParams.get("refresh") === "1";
2336
+ try {
2337
+ return sendJson(res, {
2338
+ runs: await releaseService.listRepoRuns(refresh),
2339
+ });
2340
+ } catch (e) {
2341
+ serverLog(` ⚠ /api/release/runs: ${(e as Error).message}`);
2342
+ return sendJson(res, { runs: [], error: (e as Error).message });
2343
+ }
2344
+ }
2345
+
2346
+ // ── API: release branches (deploy-time branch selector) ────────────────────
2347
+ if (pathname === "/api/release/branches" && req.method === "GET") {
2348
+ const repo = url.searchParams.get("repo") ?? "";
2349
+ const cfg = releaseService.config();
2350
+ if (!cfg.repos.some((r) => r.name === repo))
2351
+ return send400(res, `Unknown repo: ${repo}`);
2352
+ try {
2353
+ return sendJson(res, await releaseService.listBranches(repo));
2354
+ } catch {
2355
+ return sendJson(res, ["main"]);
2356
+ }
2357
+ }
2358
+
2359
+ // ── API: release deploy (session-backed) ───────────────────────────────────
2360
+ // Decoupled from the browser so the log survives page navigation and den
2361
+ // keeps monitoring even with no client attached: POST starts a background
2362
+ // deploy SESSION, GET ?session= attaches an SSE stream that REPLAYS the log so
2363
+ // far then streams live events, DELETE dismisses (closes) it.
2364
+ // Policy: prod requires confirm==="prod"; deploy-all→prod is enforced
2365
+ // main-only; a single non-main→prod deploy is an allowed "hotfix".
2366
+ if (pathname === "/api/release/deploy/active" && req.method === "GET") {
2367
+ return sendJson(res, deploySessions.active());
2368
+ }
2369
+
2370
+ if (pathname === "/api/release/deploy") {
2371
+ // ── Start (POST) → { id, label, status, ... } | 4xx ──
2372
+ if (req.method === "POST") {
2373
+ if (!readReleaseConfig().deploy) {
2374
+ res.writeHead(403, { "Content-Type": "application/json" });
2375
+ res.end(JSON.stringify({ error: DEPLOYS_OFF_MESSAGE }));
2376
+ return;
2377
+ }
2378
+ let body: {
2379
+ env?: string;
2380
+ mode?: string;
2381
+ repo?: string;
2382
+ ref?: string;
2383
+ confirm?: string;
2384
+ };
2385
+ try {
2386
+ body = JSON.parse((await readBody(req)) || "{}");
2387
+ } catch {
2388
+ return send400(res, "invalid JSON body");
2389
+ }
2390
+ const env = body.env ?? "";
2391
+ const mode = body.mode ?? "all";
2392
+ const repoParam = body.repo ?? null;
2393
+ const refParam = body.ref || "";
2394
+ const confirm = body.confirm ?? "";
2395
+
2396
+ const cfg = releaseService.config();
2397
+ if (!cfg.environments.includes(env))
2398
+ return send400(res, `Unknown env: ${env}`);
2399
+ const isProd = env === "prod";
2400
+ if (isProd && confirm !== "prod")
2401
+ return send400(res, "Production deploy requires typed confirmation.");
2402
+
2403
+ const notices: ReleaseEvent[] = [];
2404
+ let plan: DeployPlanEntry[];
2405
+ let label: string;
2406
+ if (mode === "single") {
2407
+ if (!repoParam)
2408
+ return send400(res, "repo required for a single deploy.");
2409
+ const r = cfg.repos.find((x) => x.name === repoParam);
2410
+ if (!r) return send400(res, `Unknown repo: ${repoParam}`);
2411
+ const target = resolveReleaseTarget(r, env, cfg);
2412
+ if (target.environments.length > 1)
2413
+ return send400(
2414
+ res,
2415
+ `${r.name} has ${target.environments.length} ${env} environments (${target.environments.join(", ")}); deploy each one from GitHub.`,
2416
+ );
2417
+ const ref = refParam || target.branch;
2418
+ if (ref !== target.branch) {
2419
+ const branches = await releaseService.listBranches(repoParam);
2420
+ if (!branches.includes(ref))
2421
+ return send400(res, `Unknown branch "${ref}" for ${repoParam}.`);
2422
+ }
2423
+ if (isProd && ref !== target.branch)
2424
+ notices.push({
2425
+ type: "log",
2426
+ level: "warn",
2427
+ text: `⚠ HOTFIX: dispatching "${ref}" (not ${target.branch}) to PRODUCTION.`,
2428
+ });
2429
+ plan = [
2430
+ {
2431
+ repo: r.name,
2432
+ workflow: target.workflows[0],
2433
+ ref,
2434
+ wave: r.wave,
2435
+ inputs: target.inputs,
2436
+ },
2437
+ ];
2438
+ label = `${r.name} @ ${ref} → ${env}`;
2439
+ } else {
2440
+ const enabled = cfg.repos.filter((r) => r.enabled !== false);
2441
+ if (!enabled.length)
2442
+ return send400(res, "No repos enabled for deploy.");
2443
+ const targets = new Map(
2444
+ enabled.map((r) => [r.name, resolveReleaseTarget(r, env, cfg)]),
2445
+ );
2446
+ // A repo with several environments here (one per tenant) is never
2447
+ // part of deploy-all: each tenant is its own decision.
2448
+ const multi = enabled.filter(
2449
+ (r) => targets.get(r.name)!.environments.length > 1,
2450
+ );
2451
+ for (const r of multi)
2452
+ notices.push({
2453
+ type: "log",
2454
+ level: "warn",
2455
+ text: `Skipping ${r.name}: ${targets.get(r.name)!.environments.length} ${env} environments; deploy each from GitHub.`,
2456
+ });
2457
+ plan = enabled
2458
+ .filter((r) => !multi.includes(r))
2459
+ .map((r) => {
2460
+ const t = targets.get(r.name)!;
2461
+ return {
2462
+ repo: r.name,
2463
+ workflow: t.workflows[0],
2464
+ ref: t.branch,
2465
+ wave: r.wave,
2466
+ inputs: t.inputs,
2467
+ };
2468
+ });
2469
+ if (isProd) {
2470
+ const nonMain = plan.filter((p) => p.ref !== "main");
2471
+ if (nonMain.length)
2472
+ return send400(
2473
+ res,
2474
+ `Deploy-all to prod is main-only. Off-main: ${nonMain
2475
+ .map((p) => `${p.repo}@${p.ref}`)
2476
+ .join(", ")}. Use a per-repo Hotfix instead.`,
2477
+ );
2478
+ }
2479
+ label = `all → ${env}`;
2480
+ }
2481
+
2482
+ notices.unshift({
2483
+ type: "log",
2484
+ text: `▶ deploy ${mode} → ${env} · ${plan.length} repo(s)`,
2485
+ });
2486
+
2487
+ // Concurrent deploys of different systems are fine — only the same
2488
+ // repo+env twice would race two runs against one GitHub concurrency
2489
+ // group.
2490
+ const clashing = deploySessions.conflicts(env, plan);
2491
+ if (clashing.length > 0) {
2492
+ res.writeHead(409, { "Content-Type": "application/json" });
2493
+ res.end(
2494
+ JSON.stringify({
2495
+ error: `Already deploying to ${env}: ${clashing.join(", ")}. Wait for it to finish or dismiss its log.`,
2496
+ repos: clashing,
2497
+ }),
2498
+ );
2499
+ return;
2500
+ }
2501
+ const session = deploySessions.start({ env, mode, label, plan, notices });
2502
+ serverLog(
2503
+ `▶ release deploy ${mode} → ${env} (${plan.length} repos) [${session.id}]`,
2504
+ "lifecycle",
2505
+ );
2506
+ return sendJson(res, session);
2507
+ }
2508
+
2509
+ // ── Attach (GET, SSE): replay buffered log, then stream live ──
2510
+ if (req.method === "GET") {
2511
+ const id = url.searchParams.get("session") ?? "";
2512
+ res.writeHead(200, {
2513
+ "Content-Type": "text/event-stream",
2514
+ "Cache-Control": "no-cache",
2515
+ Connection: "keep-alive",
2516
+ "X-Accel-Buffering": "no",
2517
+ });
2518
+ const emit = (ev: unknown): void => {
2519
+ try {
2520
+ res.write(`data: ${JSON.stringify(ev)}\n\n`);
2521
+ } catch {
2522
+ /* client gone */
2523
+ }
2524
+ };
2525
+ const unsub = deploySessions.attach(id, emit);
2526
+ if (!unsub) {
2527
+ emit({
2528
+ type: "log",
2529
+ level: "error",
2530
+ text: "Deploy session not found (it may have finished long ago or been closed).",
2531
+ });
2532
+ emit({ type: "done", ok: false, summary: "Session unavailable." });
2533
+ res.end();
2534
+ return;
2535
+ }
2536
+ req.on("close", unsub);
2537
+ return;
2538
+ }
2539
+
2540
+ // ── Dismiss (DELETE): user closed the log ──
2541
+ if (req.method === "DELETE") {
2542
+ const id = url.searchParams.get("session") ?? "";
2543
+ return sendJson(res, { ok: deploySessions.dismiss(id) });
2544
+ }
2545
+
2546
+ res.writeHead(405);
2547
+ res.end("Method not allowed");
2548
+ return;
2549
+ }
2550
+
2551
+ // ── Local-dev API guard ───────────────────────────────────────────────────
2552
+ // All workspace / service management APIs require local environment to be enabled.
2553
+ if (
2554
+ (pathname.startsWith("/api/workspaces") ||
2555
+ pathname.startsWith("/api/workspace/") ||
2556
+ pathname.startsWith("/api/service")) &&
2557
+ !isLocalEnabled()
2558
+ ) {
2559
+ res.writeHead(403);
2560
+ res.end("Local environment is not enabled for this deployment");
2561
+ return;
2562
+ }
2563
+
2564
+ // ── Workspace routes (extracted to workspace-routes.ts) ──────────────────
2565
+ await handleWorkspaceRoutes(req, res, pathname, url, editors, github);
2566
+ if (res.headersSent) return;
2567
+
2568
+ // ── API: manifest validation ──────────────────────────────────────────────
2569
+ // POST /api/manifest/validate body: { content: string } (raw YAML string)
2570
+ // GET /api/manifest/validate?path=<absolute-path>
2571
+ if (pathname === "/api/manifest/validate") {
2572
+ if (req.method === "POST") {
2573
+ try {
2574
+ const body = await readBody(req);
2575
+ const { content } = JSON.parse(body) as { content?: string };
2576
+ if (typeof content !== "string")
2577
+ return send400(res, '"content" string required');
2578
+ return sendJson(res, validateManifest(content));
2579
+ } catch (e) {
2580
+ return send400(res, `Invalid request: ${(e as Error).message}`);
2581
+ }
2582
+ }
2583
+ if (req.method === "GET") {
2584
+ const filePath = url.searchParams.get("path");
2585
+ if (!filePath) return send400(res, '"path" query param required');
2586
+ try {
2587
+ const denYml = fs.existsSync(filePath)
2588
+ ? fs.lstatSync(filePath).isDirectory()
2589
+ ? path.join(filePath, `${product().manifestYml}`)
2590
+ : filePath
2591
+ : filePath;
2592
+ const content = fs.readFileSync(denYml, "utf-8");
2593
+ return sendJson(res, validateManifest(content));
2594
+ } catch (e) {
2595
+ return sendJson(res, {
2596
+ valid: false,
2597
+ version: 1,
2598
+ errors: [{ path: "", message: (e as Error).message }],
2599
+ warnings: [],
2600
+ });
2601
+ }
2602
+ }
2603
+ }
2604
+
2605
+ // ── API: changelog ────────────────────────────────────────────────────────
2606
+ // GET /api/changelog — parses CHANGELOG.md and returns structured entries
2607
+ if (pathname === "/api/changelog" && req.method === "GET") {
2608
+ const changelogPath = enginePath("CHANGELOG.md");
2609
+ try {
2610
+ const raw = fs.readFileSync(changelogPath, "utf-8");
2611
+ const entries = parseChangelog(raw);
2612
+ return sendJson(res, entries);
2613
+ } catch {
2614
+ return sendJson(res, []);
2615
+ }
2616
+ }
2617
+
2618
+ // ── API: local-dev toggle ─────────────────────────────────────────────────
2619
+ // POST /api/local-dev body: { enabled: boolean }
2620
+ if (pathname === "/api/local-dev" && req.method === "POST") {
2621
+ try {
2622
+ const body = await readBody(req);
2623
+ const { enabled } = JSON.parse(body) as { enabled?: boolean };
2624
+ if (typeof enabled !== "boolean")
2625
+ return send400(res, '"enabled" boolean required');
2626
+ const user = readUserConfig();
2627
+ writeUserConfig({ ...user, local_dev: enabled });
2628
+ return sendJson(res, { ok: true, local_dev: enabled });
2629
+ } catch (e) {
2630
+ return send400(res, `Invalid request: ${(e as Error).message}`);
2631
+ }
2632
+ }
2633
+
2634
+ // ── API: sudo password ────────────────────────────────────────────────────
2635
+ // POST /api/sudo-password body: { password: string, saveToVault?: boolean }
2636
+ // GET /api/sudo-password/status → { set: boolean }
2637
+ // GET /api/sudo-password/vault-status → { opAvailable, hasStored }
2638
+ // POST /api/sudo-password/load-from-vault → { ok, reason? }
2639
+ if (pathname === "/api/sudo-password") {
2640
+ if (req.method !== "POST") {
2641
+ res.writeHead(405);
2642
+ res.end();
2643
+ return;
2644
+ }
2645
+ try {
2646
+ const body = await readBody(req);
2647
+ const { password, saveToVault } = JSON.parse(body) as {
2648
+ password?: string;
2649
+ saveToVault?: boolean;
2650
+ };
2651
+ if (typeof password !== "string" || !password)
2652
+ return send400(res, '"password" string required');
2653
+ // Validate the password before storing it.
2654
+ try {
2655
+ // -k: ignore cached credentials for this check (don't permanently invalidate)
2656
+ execSync("sudo -k -S -v", {
2657
+ input: password + "\n",
2658
+ stdio: ["pipe", "pipe", "pipe"],
2659
+ timeout: CLI_SPAWN_TIMEOUT_MS,
2660
+ });
2661
+ } catch {
2662
+ return sendJson(res, { ok: false, error: "Incorrect password" }, 401);
2663
+ }
2664
+ setSudoPassword(password);
2665
+ let savedToVault = false;
2666
+ let vaultError: string | undefined;
2667
+ if (saveToVault) {
2668
+ try {
2669
+ await saveSudoToVault(password);
2670
+ savedToVault = true;
2671
+ } catch (err) {
2672
+ vaultError = (err as Error).message;
2673
+ }
2674
+ }
2675
+ return sendJson(res, { ok: true, savedToVault, vaultError });
2676
+ } catch (e) {
2677
+ return send400(res, `Invalid request: ${(e as Error).message}`);
2678
+ }
2679
+ }
2680
+
2681
+ if (pathname === "/api/sudo-password/status") {
2682
+ // passwordless: NOPASSWD sudoer (or warm cache) — no admin login needed.
2683
+ return sendJson(res, {
2684
+ set: hasSudoPassword(),
2685
+ passwordless: hasPasswordlessSudo(),
2686
+ });
2687
+ }
2688
+
2689
+ // Advertises vault capability to the UI so it knows whether to show the
2690
+ // "Save to 1Password" checkbox and whether auto-load is worth trying.
2691
+ if (pathname === "/api/sudo-password/vault-status") {
2692
+ const opAvailable = await isOpSignedIn();
2693
+ const hasStored = opAvailable ? await hasSudoInVault() : false;
2694
+ return sendJson(res, { opAvailable, hasStored });
2695
+ }
2696
+
2697
+ // Auto-unlock path: UI calls this when status.set is false. We read from
2698
+ // vault, validate via sudo -k -S -v, and set the in-memory store on success.
2699
+ // Returns a reason code on failure so the UI can fall through to manual
2700
+ // entry without surfacing op CLI noise.
2701
+ if (pathname === "/api/sudo-password/load-from-vault") {
2702
+ if (req.method !== "POST") {
2703
+ res.writeHead(405);
2704
+ res.end();
2705
+ return;
2706
+ }
2707
+ const password = await readSudoFromVault();
2708
+ if (!password) {
2709
+ return sendJson(res, { ok: false, reason: "not-found" });
2710
+ }
2711
+ try {
2712
+ execSync("sudo -k -S -v", {
2713
+ input: password + "\n",
2714
+ stdio: ["pipe", "pipe", "pipe"],
2715
+ timeout: CLI_SPAWN_TIMEOUT_MS,
2716
+ });
2717
+ } catch {
2718
+ // Stored password no longer matches the system — likely the user
2719
+ // changed their admin password. Leave it in the vault; UI will prompt
2720
+ // and they can choose to overwrite on save.
2721
+ return sendJson(res, { ok: false, reason: "stale" });
2722
+ }
2723
+ setSudoPassword(password);
2724
+ return sendJson(res, { ok: true });
2725
+ }
2726
+
2727
+ // ── API: user settings ────────────────────────────────────────────────────
2728
+ // GET /api/settings → { editor, editors, terminal, terminals, editor_overrides, languages }
2729
+ // POST /api/settings body: { editor?, terminal?, editor_overrides? }
2730
+ if (pathname === "/api/settings") {
2731
+ if (req.method === "GET") {
2732
+ const user = readUserConfig();
2733
+ const editorList = editors.listEditors();
2734
+ const terminalList = editors.listTerminals();
2735
+ return sendJson(res, {
2736
+ editor: user.editor ?? null,
2737
+ editors: editorList,
2738
+ terminal: user.terminal ?? null,
2739
+ terminals: terminalList,
2740
+ editor_overrides: user.editor_overrides ?? {},
2741
+ languages: SUPPORTED_LANGUAGES,
2742
+ log_window_lines: user.log_window_lines ?? 20_000,
2743
+ });
2744
+ }
2745
+ if (req.method === "POST") {
2746
+ try {
2747
+ const body = JSON.parse(await readBody(req)) as {
2748
+ editor?: string | null;
2749
+ terminal?: string | null;
2750
+ editor_overrides?: Record<string, string> | null;
2751
+ };
2752
+ const user = readUserConfig();
2753
+ if (body.editor === null || body.editor === "") {
2754
+ delete user.editor;
2755
+ } else if (body.editor !== undefined) {
2756
+ if (!editors.isKnownEditor(body.editor))
2757
+ return send400(res, `Unknown editor "${body.editor}"`);
2758
+ user.editor = body.editor;
2759
+ }
2760
+ if (body.terminal === null || body.terminal === "") {
2761
+ delete user.terminal;
2762
+ } else if (body.terminal !== undefined) {
2763
+ if (!editors.isKnownTerminal(body.terminal))
2764
+ return send400(res, `Unknown terminal "${body.terminal}"`);
2765
+ user.terminal = body.terminal;
2766
+ }
2767
+ if (body.editor_overrides === null) {
2768
+ delete user.editor_overrides;
2769
+ } else if (body.editor_overrides !== undefined) {
2770
+ const langKeys = new Set<string>(
2771
+ SUPPORTED_LANGUAGES.map((l) => l.key),
2772
+ );
2773
+ const cleaned: Record<string, string> = {};
2774
+ for (const [lang, editorKey] of Object.entries(
2775
+ body.editor_overrides,
2776
+ )) {
2777
+ if (!langKeys.has(lang))
2778
+ return send400(res, `Unknown language "${lang}"`);
2779
+ if (!editorKey) continue; // empty = use default
2780
+ if (!editors.isKnownEditor(editorKey))
2781
+ return send400(res, `Unknown editor "${editorKey}"`);
2782
+ cleaned[lang] = editorKey;
2783
+ }
2784
+ if (Object.keys(cleaned).length > 0) {
2785
+ user.editor_overrides = cleaned;
2786
+ } else {
2787
+ delete user.editor_overrides;
2788
+ }
2789
+ }
2790
+ writeUserConfig(user);
2791
+ return sendJson(res, {
2792
+ ok: true,
2793
+ editor: user.editor ?? null,
2794
+ terminal: user.terminal ?? null,
2795
+ editor_overrides: user.editor_overrides ?? {},
2796
+ });
2797
+ } catch (e) {
2798
+ return send400(res, `Invalid request: ${(e as Error).message}`);
2799
+ }
2800
+ }
2801
+ }
2802
+
2803
+ // ── API: open repo in editor ──────────────────────────────────────────────
2804
+ // POST /api/editor/open body: { path: string }
2805
+ if (pathname === "/api/editor/open" && req.method === "POST") {
2806
+ try {
2807
+ const body = JSON.parse(await readBody(req)) as { path?: string };
2808
+ if (!body.path) return send400(res, '"path" is required');
2809
+
2810
+ // Walk up to the nearest existing ancestor if the path doesn't exist.
2811
+ let openPath = body.path;
2812
+ while (openPath && !fs.existsSync(openPath)) {
2813
+ const parent = path.dirname(openPath);
2814
+ if (parent === openPath) break; // reached filesystem root
2815
+ openPath = parent;
2816
+ }
2817
+
2818
+ const user = readUserConfig();
2819
+ // Resolve editor: per-language > workspace > global default
2820
+ const ws = getActiveWorkspace();
2821
+ const effectiveEditor = editors.resolveEditorForRepo(
2822
+ openPath,
2823
+ user.editor,
2824
+ user.editor_overrides,
2825
+ ws?.config.editor,
2826
+ );
2827
+ if (!effectiveEditor)
2828
+ return sendJson(
2829
+ res,
2830
+ { ok: false, error: "No editor configured. Set one in Settings." },
2831
+ 400,
2832
+ );
2833
+ const cmd = editors.resolveEditorCmd(effectiveEditor);
2834
+ if (!cmd)
2835
+ return sendJson(
2836
+ res,
2837
+ {
2838
+ ok: false,
2839
+ error: `Editor "${effectiveEditor}" not found on this machine.`,
2840
+ },
2841
+ 400,
2842
+ );
2843
+ const safePath = openPath.replace(/'/g, "'\\''");
2844
+ if (editors.isTerminalEditor(effectiveEditor)) {
2845
+ // Write a temporary .command file (Terminal.app executes these on open,
2846
+ // no Automation permission required — unlike osascript do script).
2847
+ const tmp = path.join(
2848
+ os.tmpdir(),
2849
+ `${product().cli}-open-${Date.now()}.command`,
2850
+ );
2851
+ fs.writeFileSync(tmp, `#!/bin/bash\ncd '${safePath}' && '${cmd}' .\n`, {
2852
+ mode: 0o755,
2853
+ });
2854
+ const result = spawnSync("open", [tmp], {
2855
+ timeout: CLI_SPAWN_TIMEOUT_MS,
2856
+ });
2857
+ // Give Terminal a moment to read the file before we remove it.
2858
+ setTimeout(() => {
2859
+ try {
2860
+ fs.unlinkSync(tmp);
2861
+ } catch {
2862
+ /* ignore */
2863
+ }
2864
+ }, 3_000);
2865
+ if (result.status !== 0) {
2866
+ const stderr = result.stderr?.toString().trim() ?? "";
2867
+ return sendJson(
2868
+ res,
2869
+ {
2870
+ ok: false,
2871
+ error: stderr || "Failed to open Terminal.app.",
2872
+ runCmd: `${cmd} ${openPath}`,
2873
+ },
2874
+ 500,
2875
+ );
2876
+ }
2877
+ } else {
2878
+ spawn(cmd, [openPath], { detached: true, stdio: "ignore" }).unref();
2879
+ }
2880
+ return sendJson(res, { ok: true });
2881
+ } catch (e) {
2882
+ return send400(res, `Invalid request: ${(e as Error).message}`);
2883
+ }
2884
+ }
2885
+
2886
+ // ── API: open directory in terminal app ──────────────────────────────────
2887
+ // POST /api/terminal/open body: { path: string }
2888
+ if (pathname === "/api/terminal/open" && req.method === "POST") {
2889
+ try {
2890
+ const body = JSON.parse(await readBody(req)) as { path?: string };
2891
+ if (!body.path) return send400(res, '"path" is required');
2892
+
2893
+ // Walk up to the nearest existing ancestor.
2894
+ let openPath = body.path;
2895
+ while (openPath && !fs.existsSync(openPath)) {
2896
+ const parent = path.dirname(openPath);
2897
+ if (parent === openPath) break;
2898
+ openPath = parent;
2899
+ }
2900
+
2901
+ const user = readUserConfig();
2902
+ if (!user.terminal)
2903
+ return sendJson(
2904
+ res,
2905
+ {
2906
+ ok: false,
2907
+ error: "No terminal app configured. Set one in Settings.",
2908
+ },
2909
+ 400,
2910
+ );
2911
+
2912
+ const appName = editors.resolveTerminalApp(user.terminal);
2913
+ if (!appName)
2914
+ return sendJson(
2915
+ res,
2916
+ {
2917
+ ok: false,
2918
+ error: `Terminal "${user.terminal}" not found on this machine.`,
2919
+ },
2920
+ 400,
2921
+ );
2922
+
2923
+ // Write a temporary .command script that cd's into the directory and
2924
+ // starts an interactive shell. Most macOS terminal apps execute .command
2925
+ // files when opened via `open -a`.
2926
+ const safePath = openPath.replace(/'/g, "'\\''");
2927
+ const tmp = path.join(
2928
+ os.tmpdir(),
2929
+ `${product().cli}-term-${Date.now()}.command`,
2930
+ );
2931
+ fs.writeFileSync(tmp, `#!/bin/bash\ncd '${safePath}' && exec $SHELL\n`, {
2932
+ mode: 0o755,
2933
+ });
2934
+ const result = spawnSync("open", ["-a", appName, tmp], {
2935
+ timeout: 5_000,
2936
+ });
2937
+ setTimeout(() => {
2938
+ try {
2939
+ fs.unlinkSync(tmp);
2940
+ } catch {
2941
+ /* ignore */
2942
+ }
2943
+ }, 3_000);
2944
+
2945
+ if (result.status !== 0) {
2946
+ const stderr = result.stderr?.toString().trim() ?? "";
2947
+ return sendJson(
2948
+ res,
2949
+ {
2950
+ ok: false,
2951
+ error: stderr || `Failed to open ${appName}.`,
2952
+ runCmd: `cd '${safePath}'`,
2953
+ },
2954
+ 500,
2955
+ );
2956
+ }
2957
+ return sendJson(res, { ok: true });
2958
+ } catch (e) {
2959
+ return send400(res, `Invalid request: ${(e as Error).message}`);
2960
+ }
2961
+ }
2962
+
2963
+ // ── 404 ────────────────────────────────────────────────────────────────────
2964
+ res.writeHead(404, { "Content-Type": "text/plain" });
2965
+ res.end("Not found");
2966
+ }
2967
+
2968
+ const server = http.createServer((req, res) => {
2969
+ // An async handler with no catch turns any route throw into an unhandled
2970
+ // rejection — which CRASHES the whole server on modern Node (freezing every
2971
+ // health dot) and leaves the request hanging. Fail the one request instead.
2972
+ handleRequest(req, res).catch((err) => {
2973
+ console.error(`[server] ${req.method} ${req.url} failed:`, err);
2974
+ try {
2975
+ if (!res.headersSent) {
2976
+ res.writeHead(500, { "Content-Type": "text/plain" });
2977
+ }
2978
+ res.end("internal server error");
2979
+ } catch {
2980
+ /* socket already gone */
2981
+ }
2982
+ });
2983
+ });
2984
+
2985
+ server.on("error", (err: NodeJS.ErrnoException) => {
2986
+ if (err.code === "EADDRINUSE") {
2987
+ console.error(
2988
+ `\n ✗ Port ${PORT} is already in use.\n` +
2989
+ `\n Another ${product().cli} instance may be running. To fix:\n` +
2990
+ `\n lsof -ti :${PORT} | xargs kill # kill whatever is on the port` +
2991
+ `\n make web # then restart\n`,
2992
+ );
2993
+ process.exit(1);
2994
+ }
2995
+ throw err;
2996
+ });
2997
+
2998
+ // Restore daemon registry from the previous session, then drop any whose opctl
2999
+ // containers are no longer running. Stale entries — from a prior session, or a
3000
+ // service stopped outside den — otherwise reopen as tabs in the web UI and
3001
+ // clutter the shutdown prompt. Container liveness (not health) is the signal: a
3002
+ // service's shared-infra health check stays green even when the service is down.
3003
+ // Runs synchronously at startup so /api/service/daemons never hands the UI a
3004
+ // stale entry on its first load.
3005
+ loadDaemonsFromFile();
3006
+ const prunedDaemons = pruneDeadDaemons();
3007
+ if (prunedDaemons.length > 0) {
3008
+ console.error(
3009
+ `Cleared ${prunedDaemons.length} stale daemon(s): ${prunedDaemons.join(", ")}`,
3010
+ );
3011
+ }
3012
+
3013
+ // Background recovery: after a short delay (long enough for services to be
3014
+ // responding), health-check each loaded daemon. Removal requires strong
3015
+ // evidence — a missing service, or two consecutive "error" health checks
3016
+ // (a single transient timeout/5xx never unregisters a possibly-running
3017
+ // daemon). See daemon-reaper.ts; behavior pinned by its unit tests.
3018
+ void (async () => {
3019
+ await new Promise<void>((resolve) =>
3020
+ setTimeout(resolve, DAEMON_SHUTDOWN_GRACE_MS),
3021
+ );
3022
+ await reapDeadDaemons({
3023
+ listDaemons,
3024
+ resolveService,
3025
+ getActiveWorkspaces,
3026
+ checkServiceHealth: (service) => checkServiceHealth(service),
3027
+ unregisterDaemon,
3028
+ delay: (ms) => new Promise<void>((resolve) => setTimeout(resolve, ms)),
3029
+ retryDelayMs: DAEMON_RETRY_DELAY_MS,
3030
+ });
3031
+ })();
3032
+
3033
+ const prunedSnapshots = pruneDeadSnapshots();
3034
+ if (prunedSnapshots > 0) {
3035
+ serverLog(`Pruned ${prunedSnapshots} orphaned log-history snapshot(s)`);
3036
+ }
3037
+
3038
+ // Cap service logs. Long-lived daemons never respawn (a localstack tail reached
3039
+ // 8.6 GB), so the spawn-time check alone is not enough — sweep on startup and
3040
+ // periodically. unref'd so it never holds the process open.
3041
+ function sweepServiceLogs(): void {
3042
+ try {
3043
+ const rotated = sweepLogs(getServiceLogRoot());
3044
+ if (rotated.length > 0) {
3045
+ serverLog(
3046
+ `Rotated ${rotated.length} oversized log(s): ${rotated
3047
+ .map((p) => path.basename(path.dirname(p)) + "/" + path.basename(p))
3048
+ .join(", ")}`,
3049
+ );
3050
+ }
3051
+ } catch (e) {
3052
+ serverLog(`Log sweep failed: ${(e as Error).message}`);
3053
+ }
3054
+ }
3055
+ sweepServiceLogs();
3056
+ setInterval(sweepServiceLogs, LOG_SWEEP_INTERVAL_MS).unref();
3057
+
3058
+ // Last-resort guards: an uncaught throw in an async callback (spawn handlers,
3059
+ // timers — code the per-request catch can't see) would kill the whole
3060
+ // dashboard; a TDZ bug in a down-flow callback did exactly that. For a local
3061
+ // dev server, staying alive degraded beats dying — log loudly and continue.
3062
+ process.on("uncaughtException", (err) => {
3063
+ console.error("[server] uncaught exception (continuing):", err);
3064
+ });
3065
+ process.on("unhandledRejection", (reason) => {
3066
+ console.error("[server] unhandled rejection (continuing):", reason);
3067
+ });
3068
+
3069
+ // Bind loopback only — this UI runs privileged local actions (sudo-backed
3070
+ // service commands), so it must not be reachable from the LAN. Opt out with
3071
+ // DEN_BIND_HOST=0.0.0.0 for deliberate exposure.
3072
+ const BIND_HOST = process.env.DEN_BIND_HOST ?? "127.0.0.1";
3073
+ server.listen(PORT, BIND_HOST, () => {
3074
+ const hosts = fs.existsSync("/etc/hosts")
3075
+ ? fs.readFileSync("/etc/hosts", "utf-8")
3076
+ : "";
3077
+ const host = hosts.includes(`127.0.0.1 ${product().cli}`)
3078
+ ? `${product().cli}`
3079
+ : "localhost";
3080
+
3081
+ let pkgVersion = "?";
3082
+ try {
3083
+ const pkg = JSON.parse(
3084
+ fs.readFileSync(enginePath("package.json"), "utf-8"),
3085
+ ) as { version?: string };
3086
+ if (pkg.version) pkgVersion = pkg.version;
3087
+ } catch (_e) {
3088
+ /* package.json unreadable — version stays "?" */
3089
+ }
3090
+
3091
+ console.log();
3092
+ for (const line of DEN_BANNER_LINES) {
3093
+ console.log(`\x1b[36m${line}\x1b[0m`); // cyan without chalk dependency
3094
+ }
3095
+ console.log();
3096
+ console.log(` ${DEN_SUBTITLE.trim()}`);
3097
+ console.log(` v${pkgVersion}`);
3098
+ console.log();
3099
+ console.log(` Web UI → http://${host}:${PORT}`);
3100
+ console.log();
3101
+ });
3102
+
3103
+ // ── Graceful shutdown ─────────────────────────────────────────────────────────
3104
+
3105
+ async function stopDaemons(): Promise<void> {
3106
+ const daemons = listDaemons();
3107
+ if (daemons.length === 0) return;
3108
+
3109
+ console.log("\nStopping services…");
3110
+
3111
+ const isTTY = process.stdout.isTTY;
3112
+ const total = daemons.length;
3113
+
3114
+ if (isTTY) {
3115
+ // Print every service line up front, then update in-place using ANSI cursor movement.
3116
+ for (const d of daemons) {
3117
+ process.stdout.write(` ⏳ ${d.address}\n`);
3118
+ }
3119
+ // Cursor is now one row below the last service line.
3120
+
3121
+ const updateLine = (idx: number, mark: string, suffix: string): void => {
3122
+ const movesUp = total - idx; // rows to climb to reach this service line
3123
+ process.stdout.write(
3124
+ `\x1b[${movesUp}A` + // move cursor up to the service's line
3125
+ `\r\x1b[2K` + // clear the line
3126
+ ` ${mark} ${daemons[idx].address}${suffix}` +
3127
+ `\x1b[${movesUp}E`, // move cursor back to beginning of original row
3128
+ );
3129
+ };
3130
+
3131
+ await Promise.allSettled(
3132
+ daemons.map(
3133
+ (d, i) =>
3134
+ new Promise<void>((resolve) => {
3135
+ runServiceCommand({
3136
+ address: d.address,
3137
+ command: "down",
3138
+ onLine: () => {},
3139
+ onClose: (exitCode) => {
3140
+ updateLine(
3141
+ i,
3142
+ exitCode === 0 ? "✓" : "✗",
3143
+ exitCode !== 0 ? ` (exit ${exitCode})` : "",
3144
+ );
3145
+ resolve();
3146
+ },
3147
+ });
3148
+ }),
3149
+ ),
3150
+ );
3151
+ process.stdout.write("\n");
3152
+ } else {
3153
+ // Non-TTY: simple line-per-service output, no cursor tricks.
3154
+ await Promise.allSettled(
3155
+ daemons.map(
3156
+ (d) =>
3157
+ new Promise<void>((resolve) => {
3158
+ process.stdout.write(` ↓ ${d.address}\n`);
3159
+ runServiceCommand({
3160
+ address: d.address,
3161
+ command: "down",
3162
+ onLine: () => {},
3163
+ onClose: (exitCode) => {
3164
+ process.stdout.write(
3165
+ ` ${exitCode === 0 ? "✓" : "✗"} ${d.address}\n`,
3166
+ );
3167
+ resolve();
3168
+ },
3169
+ });
3170
+ }),
3171
+ ),
3172
+ );
3173
+ }
3174
+
3175
+ console.log("Services stopped.");
3176
+ }
3177
+
3178
+ async function handleShutdown(signal: string): Promise<void> {
3179
+ clearSudoPassword();
3180
+ const daemons = listDaemons();
3181
+ if (daemons.length === 0) {
3182
+ process.exit(0);
3183
+ }
3184
+
3185
+ const isTTY = process.stdin.isTTY;
3186
+
3187
+ if (!isTTY) {
3188
+ // Non-interactive shutdown usually means `den web` is being restarted.
3189
+ // Do not tear down service daemons just because the UI process is cycling.
3190
+ console.log(
3191
+ `\n${product().title} web received ${signal}; leaving ${daemons.length} running service${daemons.length === 1 ? "" : "s"} alive.`,
3192
+ );
3193
+ process.exit(0);
3194
+ }
3195
+
3196
+ // Interactive terminal: ask the user.
3197
+ console.log(`\nRunning services (started by ${product().cli}):`);
3198
+ for (const d of daemons) {
3199
+ const age = Math.round((Date.now() - d.startedAt.getTime()) / 1000);
3200
+ console.log(` • ${d.address} (up ${age}s)`);
3201
+ }
3202
+
3203
+ process.stdout.write("\nStop these services before exiting? [Y/n] ");
3204
+
3205
+ // Restore normal terminal input for readline.
3206
+ if (process.stdin.setRawMode) process.stdin.setRawMode(false);
3207
+ process.stdin.resume();
3208
+
3209
+ await new Promise<void>((resolve) => {
3210
+ process.stdin.once("data", async (chunk) => {
3211
+ const answer = chunk.toString().trim().toLowerCase();
3212
+ if (answer === "" || answer === "y" || answer === "yes") {
3213
+ await stopDaemons();
3214
+ } else {
3215
+ console.log("Leaving services running.");
3216
+ }
3217
+ resolve();
3218
+ });
3219
+ });
3220
+
3221
+ process.exit(0);
3222
+ }
3223
+
3224
+ let _shuttingDown = false;
3225
+ process.on("SIGINT", () => {
3226
+ if (_shuttingDown) return;
3227
+ _shuttingDown = true;
3228
+ console.log(""); // newline after ^C
3229
+ void handleShutdown("SIGINT");
3230
+ });
3231
+
3232
+ process.on("SIGTERM", () => {
3233
+ if (_shuttingDown) return;
3234
+ _shuttingDown = true;
3235
+ void handleShutdown("SIGTERM");
3236
+ });