@monster-orange/den 0.2.4 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,63 +14,127 @@ import {
14
14
  writeWorkspacesFile,
15
15
  } from "./workspaces.js";
16
16
  import { readCatalog, writeCatalog } from "./catalog.js";
17
- import { readDenConfig, requireGithubOrg } from "../den-config.js";
18
- import type { WorkspaceServiceEntry } from "../types/index.js";
19
-
20
- const DEFAULT_BRANCH = "main";
17
+ import {
18
+ readWorkspaceBlueprints,
19
+ requireGithubOrg,
20
+ type ResolvedWorkspaceBlueprint,
21
+ } from "../den-config.js";
22
+ import type {
23
+ ServiceCatalog,
24
+ WorkspaceConfig,
25
+ WorkspaceServiceEntry,
26
+ WorkspacesFile,
27
+ } from "../types/index.js";
21
28
 
22
- export function defaultWorkspaceRepos(): string[] {
23
- return readDenConfig().defaults?.workspace?.repos ?? [];
29
+ /** The single-workspace blueprint (`defaults.workspace`), or null. */
30
+ function singleBlueprint(): ResolvedWorkspaceBlueprint | null {
31
+ const bps = readWorkspaceBlueprints();
32
+ return bps.shape === "single" ? bps.workspaces[0] : null;
24
33
  }
25
34
 
26
- function defaultBranch(): string {
27
- return readDenConfig().defaults?.workspace?.branch ?? DEFAULT_BRANCH;
35
+ /** True when the distribution uses `defaults.workspaces` (named workspaces). */
36
+ export function isMultiBlueprint(): boolean {
37
+ return readWorkspaceBlueprints().shape === "multi";
28
38
  }
29
39
 
30
- function org(): string {
31
- return requireGithubOrg();
40
+ /** The workspaces a fresh setup would seed, each with at least one repo. */
41
+ export function blueprintWorkspaces(): ResolvedWorkspaceBlueprint[] {
42
+ return readWorkspaceBlueprints().workspaces.filter((b) => b.repos.length > 0);
32
43
  }
33
44
 
34
- /** True on a fresh clone: a blueprint exists but the `default` workspace is empty. */
35
- export function defaultWorkspaceNeedsSeeding(): boolean {
36
- if (defaultWorkspaceRepos().length === 0) return false;
37
- const def = readWorkspacesFile().workspaces["default"];
38
- return !def || Object.keys(def.services ?? {}).length === 0;
45
+ /** `defaults.workspace.repos` — the single blueprint's repos (empty for the multi shape). */
46
+ export function defaultWorkspaceRepos(): string[] {
47
+ return singleBlueprint()?.repos ?? [];
39
48
  }
40
49
 
41
- /** Whether the `default` workspace already has services (so seeding would reset it). */
42
- export function defaultWorkspaceHasServices(): boolean {
43
- const def = readWorkspacesFile().workspaces["default"];
44
- return !!def && Object.keys(def.services ?? {}).length > 0;
50
+ function org(): string {
51
+ return requireGithubOrg();
45
52
  }
46
53
 
47
- /**
48
- * Seed (or reset) the `default` workspace from the blueprint — every repo as a
49
- * bare, first-class managed entry on `branch` — and ensure the catalog has
50
- * clone URLs (`<org>/<repo>`, only filled where missing). Returns the seeded repos.
51
- */
52
- export function seedDefaultWorkspace(): string[] {
53
- const repos = defaultWorkspaceRepos();
54
- const branch = defaultBranch();
54
+ function blueprintServices(
55
+ bp: ResolvedWorkspaceBlueprint,
56
+ repos: string[] = bp.repos,
57
+ ): Record<string, WorkspaceServiceEntry> {
55
58
  // Team default for systems a fresh workspace should NOT auto-start (e.g.
56
59
  // billing-api's `make up` opens an interactive Stripe tab). Personal after
57
60
  // seeding — toggled per system in the web ⋮ menu.
58
- const autostartOff = new Set(
59
- readDenConfig().defaults?.workspace?.autostart_off ?? [],
60
- );
61
+ const autostartOff = new Set(bp.autostartOff);
61
62
  const services: Record<string, WorkspaceServiceEntry> = {};
62
63
  for (const repo of repos) {
63
64
  services[repo] = {
64
65
  mode: "managed",
65
- branch,
66
+ branch: bp.branch,
66
67
  ...(autostartOff.has(repo) ? { autostart: false } : {}),
67
68
  };
68
69
  }
70
+ return services;
71
+ }
72
+
73
+ function fillCatalog(
74
+ repos: string[],
75
+ catalog: ServiceCatalog = readCatalog(),
76
+ ): void {
77
+ const o = org();
78
+ for (const repo of repos) {
79
+ if (!catalog[repo]?.gh) catalog[repo] = { gh: `${o}/${repo}` };
80
+ }
81
+ writeCatalog(catalog);
82
+ }
83
+
84
+ /**
85
+ * A genuine fresh setup: no workspace but `default` exists, `default` holds no
86
+ * services, and no blueprint was ever applied to it (`seeded_repos` unset —
87
+ * seeding, top-up and a declined offer all record it). An emptied `default`,
88
+ * or one next to other workspaces, is the user's deliberate state.
89
+ */
90
+ export function isFreshWorkspaceSetup(
91
+ file: WorkspacesFile = readWorkspacesFile(),
92
+ ): boolean {
93
+ const names = Object.keys(file.workspaces);
94
+ if (names.some((n) => n !== "default")) return false;
95
+ const def = file.workspaces["default"];
96
+ if (!def) return true;
97
+ return (
98
+ Object.keys(def.services ?? {}).length === 0 &&
99
+ def.seeded_repos === undefined
100
+ );
101
+ }
102
+
103
+ /** True on a genuine fresh setup with a non-empty blueprint (single or multi). */
104
+ export function defaultWorkspaceNeedsSeeding(): boolean {
105
+ if (blueprintWorkspaces().length === 0) return false;
106
+ return isFreshWorkspaceSetup();
107
+ }
108
+
109
+ /** Whether a workspace the blueprint seeds already has services (so seeding would reset it). */
110
+ export function defaultWorkspaceHasServices(): boolean {
111
+ const file = readWorkspacesFile();
112
+ const names = isMultiBlueprint()
113
+ ? blueprintWorkspaces().map((b) => b.name)
114
+ : ["default"];
115
+ return names.some(
116
+ (n) => Object.keys(file.workspaces[n]?.services ?? {}).length > 0,
117
+ );
118
+ }
119
+
120
+ /**
121
+ * Seed (or reset) the `default` workspace from the blueprint — every repo as a
122
+ * bare, first-class managed entry on `branch` — and ensure the catalog has
123
+ * clone URLs (`<org>/<repo>`, only filled where missing). Returns the seeded
124
+ * repos. With the multi blueprint (`defaults.workspaces`), seeds every named
125
+ * workspace instead (see seedBlueprintWorkspaces).
126
+ */
127
+ export function seedDefaultWorkspace(): string[] {
128
+ if (isMultiBlueprint())
129
+ return seedBlueprintWorkspaces().flatMap((w) => w.repos);
130
+ const bp = singleBlueprint();
131
+ if (!bp) return [];
132
+ const repos = bp.repos;
69
133
 
70
134
  const file = readWorkspacesFile();
71
135
  file.workspaces.default = {
72
136
  ...(file.workspaces.default ?? {}),
73
- services,
137
+ services: blueprintServices(bp),
74
138
  seeded_repos: [...repos],
75
139
  };
76
140
  if (!file.active?.length) {
@@ -78,27 +142,44 @@ export function seedDefaultWorkspace(): string[] {
78
142
  file.focus = "default";
79
143
  }
80
144
  writeWorkspacesFile(file);
81
-
82
- const catalog = readCatalog();
83
- const o = org();
84
- for (const repo of repos) {
85
- if (!catalog[repo]?.gh) catalog[repo] = { gh: `${o}/${repo}` };
86
- }
87
- writeCatalog(catalog);
88
-
145
+ fillCatalog(repos);
89
146
  return repos;
90
147
  }
91
148
 
92
149
  /**
93
- * Converge an EXISTING `default` workspace on the blueprint. Seeding is
94
- * one-shot, so a workspace seeded before a repo joined
95
- * `defaults.workspace.repos` (tasks-api, say) would never gain it on its own.
96
- * Adds each missing blueprint repo exactly as seeding would (bare
97
- * managed entry on `branch`, team autostart default, catalog clone URL
98
- * where absent) and returns the added names. No-op — and no file writes —
99
- * when already converged, or when the `default` workspace is empty/missing:
100
- * that state belongs to the seeding/declined flow, not top-up.
150
+ * Create (or reset) each workspace `defaults.workspaces` names with its repos,
151
+ * activate them all, and focus the one marked `focus: true` (else the first).
152
+ * Workspaces the blueprint does not name are left as they are. Returns what
153
+ * was seeded, in blueprint order.
101
154
  */
155
+ export function seedBlueprintWorkspaces(): { name: string; repos: string[] }[] {
156
+ const bps = blueprintWorkspaces();
157
+ if (bps.length === 0) return [];
158
+ const file = readWorkspacesFile();
159
+ for (const bp of bps) {
160
+ const prev: WorkspaceConfig | undefined = file.workspaces[bp.name];
161
+ file.workspaces[bp.name] = {
162
+ ...(prev ?? {}),
163
+ services: blueprintServices(bp),
164
+ seeded_repos: [...bp.repos],
165
+ };
166
+ }
167
+ const names = bps.map((b) => b.name);
168
+ // The untouched placeholder `default` of a fresh file is not the user's:
169
+ // only the blueprint's workspaces are active.
170
+ const kept = (file.active ?? []).filter(
171
+ (n) =>
172
+ names.includes(n) ||
173
+ n !== "default" ||
174
+ Object.keys(file.workspaces[n]?.services ?? {}).length > 0,
175
+ );
176
+ file.active = [...names, ...kept.filter((n) => !names.includes(n))];
177
+ file.focus = (bps.find((b) => b.focus) ?? bps[0]).name;
178
+ writeWorkspacesFile(file);
179
+ fillCatalog([...new Set(bps.flatMap((b) => b.repos))]);
180
+ return bps.map((b) => ({ name: b.name, repos: [...b.repos] }));
181
+ }
182
+
102
183
  /**
103
184
  * Converge an EXISTING `default` workspace on the blueprint. Seeding is
104
185
  * one-shot, so a workspace seeded before a repo joined
@@ -106,74 +187,88 @@ export function seedDefaultWorkspace(): string[] {
106
187
  *
107
188
  * Converges on blueprint DELTAS, not set difference: only repos that JOINED
108
189
  * the blueprint since it was last applied (`seeded_repos`) are candidates, so
109
- * a standard repo the user removed from the workspace stays removed. Presence
110
- * is checked at REPO level (workspace keys may be `repo/subsystem`
111
- * addresses), so a repo held as subsystem entries never gains a duplicate
112
- * bare entry. Added repos get exactly what seeding writes: bare
113
- * managed entry on `branch`, team autostart default, catalog clone
114
- * URL where absent. Returns the added names.
190
+ * a standard repo the user removed from the workspace stays removed. A
191
+ * candidate is skipped when any workspace already holds it, checked at REPO
192
+ * level (workspace keys may be `repo/subsystem` addresses). Added repos get
193
+ * exactly what seeding writes: bare managed entry on `branch`, team
194
+ * autostart default, catalog clone URL where absent. Returns the added names.
115
195
  *
116
- * No-op — and no file writes once converged — when the blueprint is empty or
117
- * the `default` workspace is empty/missing (that state belongs to the
118
- * seeding/declined flow). Workspaces from before `seeded_repos` existed
119
- * treat the whole blueprint as new once — the migration cannot distinguish
120
- * "removed" from "seeded before the repo joined"; afterwards removals stick.
196
+ * Never runs once the user has other workspaces: `default` is then one
197
+ * workspace among theirs, and repos they moved out must stay out. Also a
198
+ * no-op (no file writes once converged) for the multi blueprint, an empty
199
+ * blueprint, or an empty/missing `default` (that state belongs to the
200
+ * seeding flow). Workspaces from before `seeded_repos` existed treat the
201
+ * whole blueprint as new once; afterwards removals stick.
121
202
  */
122
203
  export function topUpDefaultWorkspace(): string[] {
123
- const repos = defaultWorkspaceRepos();
124
- if (repos.length === 0) return [];
204
+ const bp = singleBlueprint();
205
+ if (!bp || bp.repos.length === 0) return [];
206
+ const repos = bp.repos;
125
207
 
126
208
  const file = readWorkspacesFile();
209
+ if (Object.keys(file.workspaces).some((n) => n !== "default")) return [];
127
210
  const def = file.workspaces["default"];
128
211
  if (!def || Object.keys(def.services ?? {}).length === 0) return [];
129
212
 
130
213
  const applied = new Set(def.seeded_repos ?? []);
131
214
  const catalog = readCatalog();
132
215
  const present = new Set(
133
- Object.keys(def.services).map((k) => parseAddress(k, catalog).repoName),
134
- );
135
- const branch = defaultBranch();
136
- const autostartOff = new Set(
137
- readDenConfig().defaults?.workspace?.autostart_off ?? [],
216
+ Object.values(file.workspaces).flatMap((ws) =>
217
+ Object.keys(ws.services ?? {}).map(
218
+ (k) => parseAddress(k, catalog).repoName,
219
+ ),
220
+ ),
138
221
  );
139
222
 
140
- const added: string[] = [];
141
- for (const repo of repos) {
142
- if (applied.has(repo) || present.has(repo)) continue;
143
- def.services[repo] = {
144
- mode: "managed",
145
- branch,
146
- ...(autostartOff.has(repo) ? { autostart: false } : {}),
147
- };
148
- added.push(repo);
149
- }
223
+ const missing = repos.filter((r) => !applied.has(r) && !present.has(r));
224
+ Object.assign(def.services, blueprintServices(bp, missing));
150
225
 
151
226
  const blueprintChanged =
152
227
  def.seeded_repos?.length !== repos.length ||
153
228
  repos.some((r, i) => def.seeded_repos?.[i] !== r);
154
- if (added.length === 0 && !blueprintChanged) return [];
229
+ if (missing.length === 0 && !blueprintChanged) return [];
155
230
 
156
231
  def.seeded_repos = [...repos];
157
232
  writeWorkspacesFile(file);
233
+ if (missing.length > 0) fillCatalog(missing, catalog);
234
+ return missing;
235
+ }
158
236
 
159
- if (added.length > 0) {
160
- const o = org();
161
- for (const repo of added) {
162
- if (!catalog[repo]?.gh) catalog[repo] = { gh: `${o}/${repo}` };
163
- }
164
- writeCatalog(catalog);
237
+ /**
238
+ * Record that the user removed `repoName` from workspace `wsName`, so top-up
239
+ * never re-adds it. Only the `default` workspace is topped up, and only a
240
+ * blueprint repo could come back; `seeded_repos` already covers repos the
241
+ * blueprint applied. This closes the gap for a `default` from before
242
+ * `seeded_repos` existed, which top-up would otherwise migrate by re-adding
243
+ * the whole blueprint. Mutates `file` in place; the caller writes it.
244
+ */
245
+ export function recordRemovedFromWorkspace(
246
+ file: WorkspacesFile,
247
+ wsName: string,
248
+ repoName: string,
249
+ ): void {
250
+ if (wsName !== "default") return;
251
+ const def = file.workspaces["default"];
252
+ if (!def) return;
253
+ let blueprint: string[] = [];
254
+ try {
255
+ blueprint = defaultWorkspaceRepos();
256
+ } catch {
257
+ return; // a malformed blueprint is reported where seeding reads it
165
258
  }
166
-
167
- return added;
259
+ if (!blueprint.includes(repoName)) return;
260
+ const seeded = def.seeded_repos ?? [];
261
+ if (!seeded.includes(repoName)) def.seeded_repos = [...seeded, repoName];
168
262
  }
169
263
 
170
264
  /**
171
- * Persist an empty `default` workspace so the one-time auto-seed prompt isn't
172
- * asked again after the user declines it.
265
+ * Persist an empty `default` workspace, with an empty `seeded_repos`, so the
266
+ * one-time auto-seed prompt isn't asked again after the user declines it.
173
267
  */
174
268
  export function markDefaultWorkspaceDeclined(): void {
175
269
  const file = readWorkspacesFile();
176
270
  file.workspaces.default = file.workspaces.default ?? { services: {} };
271
+ file.workspaces.default.seeded_repos ??= [];
177
272
  if (!file.active?.length) {
178
273
  file.active = ["default"];
179
274
  file.focus = "default";
@@ -9,8 +9,13 @@ import { hasRuntime, repoKind } from "../repo-kind.js";
9
9
  import * as https from "https";
10
10
  import * as tls from "tls";
11
11
  import { readEdgeConfig } from "../edge/config.js";
12
- import { composeHealthCmd } from "../stacks/manifest-commands.js";
12
+ import {
13
+ composeHealth,
14
+ composeHealthTarget,
15
+ type ComposeHealthTarget,
16
+ } from "../stacks/compose-health.js";
13
17
  import * as fs from "fs";
18
+ import * as os from "os";
14
19
  import * as path from "path";
15
20
  import * as dns from "dns";
16
21
  import { spawn } from "child_process";
@@ -38,7 +43,15 @@ import type { Environment } from "../types/index.js";
38
43
 
39
44
  import { product } from "../product.js";
40
45
  export interface PingResult {
41
- ok: boolean;
46
+ /**
47
+ * true: the check answered healthy. false: it COMPLETED and failed. null:
48
+ * it did not complete or could not tell (no answer within its budget, its
49
+ * process could not start, or it reported its state unknown). Only false
50
+ * is down.
51
+ */
52
+ ok: boolean | null;
53
+ /** With ok null: the check gave no answer within its budget. */
54
+ pending?: boolean;
42
55
  status?: number;
43
56
  error?: string;
44
57
  }
@@ -50,7 +63,12 @@ export interface ServiceHealthResult {
50
63
  repoPath: string;
51
64
  mode: "dev-managed" | "managed";
52
65
  branch: string | null;
53
- health: "ok" | "error" | "unknown";
66
+ /**
67
+ * "pending": a check ran but gave no verdict (no answer in time, or it
68
+ * could not tell). Unlike "unknown" (no check configured) it is not a
69
+ * reason to proceed; unlike "error" it is not down.
70
+ */
71
+ health: "ok" | "error" | "unknown" | "pending";
54
72
  healthDetail?: string;
55
73
  /** Where the service runs for the env checked (R13). */
56
74
  runs?: Placement["runs"];
@@ -409,40 +427,185 @@ export async function pingShell(
409
427
  cmd: string,
410
428
  extraEnv: Record<string, string> = {},
411
429
  cwd?: string,
430
+ opts: {
431
+ /** The answer budget (default CLI_SPAWN_TIMEOUT_MS). */
432
+ timeoutMs?: number;
433
+ } = {},
412
434
  ): Promise<PingResult> {
435
+ // Bounded concurrency: a health cycle starts every check at once, and a
436
+ // distribution's shell checks are processes. Queued checks start their
437
+ // budget when they start, not while they wait.
438
+ const release = await shellSlots.acquire();
439
+ try {
440
+ return await runShellCheck(cmd, extraEnv, cwd, opts);
441
+ } finally {
442
+ release();
443
+ }
444
+ }
445
+
446
+ /** A counting semaphore. */
447
+ class Slots {
448
+ private free: number;
449
+ private waiting: (() => void)[] = [];
450
+ constructor(readonly size: number) {
451
+ this.free = size;
452
+ }
453
+ async acquire(): Promise<() => void> {
454
+ if (this.free > 0) this.free--;
455
+ else await new Promise<void>((r) => this.waiting.push(r));
456
+ let released = false;
457
+ return () => {
458
+ if (released) return;
459
+ released = true;
460
+ const next = this.waiting.shift();
461
+ if (next) next();
462
+ else this.free++;
463
+ };
464
+ }
465
+ }
466
+
467
+ /** Shell checks running at once: max(2, cpus/2). */
468
+ export const SHELL_CHECK_CONCURRENCY = Math.max(
469
+ 2,
470
+ Math.floor(os.cpus().length / 2),
471
+ );
472
+ const shellSlots = new Slots(SHELL_CHECK_CONCURRENCY);
473
+
474
+ function runShellCheck(
475
+ cmd: string,
476
+ extraEnv: Record<string, string>,
477
+ cwd: string | undefined,
478
+ opts: { timeoutMs?: number },
479
+ ): Promise<PingResult> {
480
+ const budget = opts.timeoutMs ?? CLI_SPAWN_TIMEOUT_MS;
413
481
  return new Promise((resolve) => {
414
482
  let settled = false;
415
483
  const settle = (r: PingResult): void => {
416
484
  if (settled) return;
417
485
  settled = true;
486
+ clearTimeout(timer);
418
487
  clearTimeout(guard);
419
488
  resolve(r);
420
489
  };
421
- const proc = spawn(cmd, [], {
422
- shell: true,
423
- timeout: CLI_SPAWN_TIMEOUT_MS,
424
- env: { ...process.env, ...extraEnv },
425
- cwd,
426
- // Output is never read: piped stdio would block a chatty script once the
427
- // 64KB buffer fills, and 'close' would wait on grandchildren holding the
428
- // pipes open. Nothing consumes it — don't create it.
429
- stdio: "ignore",
490
+ // A check that has not answered is not a failed check: past the budget
491
+ // it is pending (ok null), never down. Under load a CLI-based check
492
+ // (`<cli> stack health`, a full tsx start) can need longer than the
493
+ // budget on a cold poll, and calling that "down" painted healthy stacks
494
+ // red until the next poll.
495
+ const noAnswer = (): PingResult => ({
496
+ ok: null,
497
+ pending: true,
498
+ error: `no answer within ${Math.round(budget / 1000)}s`,
430
499
  });
500
+ let proc: ReturnType<typeof spawn>;
501
+ try {
502
+ proc = spawn(cmd, [], {
503
+ shell: true,
504
+ env: { ...process.env, ...extraEnv },
505
+ cwd,
506
+ // Output is never read: piped stdio would block a chatty script once
507
+ // the 64KB buffer fills, and 'close' would wait on grandchildren
508
+ // holding the pipes open. Nothing consumes it — don't create it.
509
+ stdio: "ignore",
510
+ });
511
+ } catch (err) {
512
+ resolve({ ok: null, error: (err as Error).message });
513
+ return;
514
+ }
515
+ const timer = setTimeout(() => {
516
+ settle(noAnswer());
517
+ proc.kill("SIGTERM");
518
+ }, budget);
431
519
  // 'exit', NOT 'close': close waits for stdio to drain, which a killed
432
520
  // shell's surviving grandchildren can hold open forever — one wedged
433
521
  // health check then hangs every /api/health poll for good. The guard
434
- // timer backstops even a spawn timeout that never fires.
435
- const guard = setTimeout(
436
- () => settle({ ok: false, error: "health check timed out" }),
437
- CLI_SPAWN_TIMEOUT_MS + 1_000,
438
- );
439
- proc.on("exit", (code) =>
440
- settle(code === 0 ? { ok: true } : { ok: false, error: `exit ${code}` }),
441
- );
442
- proc.on("error", (err) => settle({ ok: false, error: err.message }));
522
+ // timer backstops even a kill that never lands.
523
+ const guard = setTimeout(() => settle(noAnswer()), budget + 1_000);
524
+ proc.on("exit", (code, signal) => {
525
+ if (code === 0) return settle({ ok: true });
526
+ // Killed (by the budget above, or from outside): it never answered.
527
+ if (code === null)
528
+ return settle({ ...noAnswer(), error: `killed (${signal})` });
529
+ settle({ ok: false, error: `exit ${code}` });
530
+ });
531
+ // The check could not be started (e.g. EAGAIN under load): no verdict.
532
+ proc.on("error", (err) => settle({ ok: null, error: err.message }));
443
533
  });
444
534
  }
445
535
 
536
+ // ── Timeouts: pending first, down when repeated ─────────────────────────────
537
+ // A check that gets no answer within its budget (an HTTP timeout, a shell
538
+ // check past CLI_SPAWN_TIMEOUT_MS) has not completed: one such cycle is
539
+ // pending, not down. A cold first probe on a loaded machine (cold docker, a
540
+ // tsx-started `stack health`, cold TLS) can miss the budget while the service
541
+ // is fine, and reporting that as down painted healthy repos red until the
542
+ // next poll. Two consecutive timeouts are down: a hung service still shows.
543
+ // The web health row counts cycles (TimeoutStreaks); a one-shot caller
544
+ // (`<cli> status`, checkServiceHealth) re-probes once at once instead, so
545
+ // both say the same thing about the same service.
546
+
547
+ /** Whether a ping got no answer within its budget. */
548
+ export function timedOut(r: PingResult): boolean {
549
+ return r.pending === true || (r.ok === false && r.error === "timeout");
550
+ }
551
+
552
+ /** A timeout's verdict: pending on its first cycle, down on a repeat. */
553
+ function timeoutVerdict(r: PingResult, repeated: boolean): PingResult {
554
+ const error = r.error ?? "timeout";
555
+ return repeated
556
+ ? { ok: false, error: `${error} (twice in a row)` }
557
+ : { ok: null, pending: true, error };
558
+ }
559
+
560
+ /** Consecutive timed-out cycles per check, for a caller that polls. */
561
+ export class TimeoutStreaks {
562
+ private streak = new Map<string, number>();
563
+ /** `r` as this cycle's verdict for check `key`. */
564
+ judge(key: string, r: PingResult): PingResult {
565
+ if (!timedOut(r)) {
566
+ this.streak.delete(key);
567
+ return r;
568
+ }
569
+ const n = (this.streak.get(key) ?? 0) + 1;
570
+ this.streak.set(key, n);
571
+ return timeoutVerdict(r, n >= 2);
572
+ }
573
+ }
574
+
575
+ /** Probe once; on a timeout, probe again at once: down only if it times out twice. */
576
+ export async function pingTwiceOnTimeout(
577
+ probe: () => Promise<PingResult>,
578
+ ): Promise<PingResult> {
579
+ const first = await probe();
580
+ if (!timedOut(first)) return first;
581
+ const second = await probe();
582
+ return timedOut(second) ? timeoutVerdict(second, true) : second;
583
+ }
584
+
585
+ /**
586
+ * Run a shell health check. Den's own compose check (the `<cli> stack
587
+ * health` command it generates, `compose.cmd`) is answered in-process from
588
+ * the shared docker listing: no CLI process per check. Any other command
589
+ * (a distribution's own) runs as a shell command.
590
+ */
591
+ export async function pingShellCheck(
592
+ cmd: string,
593
+ extraEnv: Record<string, string>,
594
+ cwd: string | undefined,
595
+ compose: ComposeHealthTarget | null,
596
+ ): Promise<PingResult> {
597
+ if (!compose || cmd !== compose.cmd) return pingShell(cmd, extraEnv, cwd);
598
+ const v = await composeHealth(compose);
599
+ if (v.ok === true) return { ok: true };
600
+ if (v.ok === false) return { ok: false, error: v.detail };
601
+ return { ok: null, error: v.detail, ...(v.pending ? { pending: true } : {}) };
602
+ }
603
+
604
+ /** A ping result as a ServiceHealthResult health state. */
605
+ export function healthState(r: PingResult): ServiceHealthResult["health"] {
606
+ return r.ok === true ? "ok" : r.ok === false ? "error" : "pending";
607
+ }
608
+
446
609
  // ── Per-service health check ──────────────────────────────────────────────────
447
610
 
448
611
  /**
@@ -480,9 +643,9 @@ export async function checkServiceHealth(
480
643
  };
481
644
  }
482
645
 
483
- const composeHealth =
646
+ const compose =
484
647
  env === "local"
485
- ? composeHealthCmd(
648
+ ? composeHealthTarget(
486
649
  manifest,
487
650
  resolved.subsystem,
488
651
  resolved.repoName,
@@ -493,7 +656,7 @@ export async function checkServiceHealth(
493
656
  manifest,
494
657
  resolved.subsystem,
495
658
  env,
496
- composeHealth,
659
+ compose?.cmd ?? null,
497
660
  );
498
661
  const placed: Pick<ServiceHealthResult, "runs" | "target"> = {
499
662
  runs: plan.placement.runs,
@@ -544,21 +707,22 @@ export async function checkServiceHealth(
544
707
  };
545
708
  }
546
709
 
547
- const result =
710
+ const result = await pingTwiceOnTimeout(() =>
548
711
  healthCfg.type === "shell"
549
- ? await pingShell(healthCfg.cmd, {}, resolved.repoPath)
550
- : await pingHttp(
712
+ ? pingShellCheck(healthCfg.cmd, {}, resolved.repoPath, compose)
713
+ : pingHttp(
551
714
  healthCfg.url,
552
715
  env === "local" && plan.placement.runs !== "remote"
553
716
  ? HEALTH_CHECK_TIMEOUT_MS
554
717
  : HOSTED_HEALTH_CHECK_TIMEOUT_MS,
555
718
  plan.expect,
556
- );
719
+ ),
720
+ );
557
721
 
558
722
  return {
559
723
  ...base,
560
724
  ...placed,
561
- health: result.ok ? "ok" : "error",
725
+ health: healthState(result),
562
726
  healthDetail:
563
727
  result.error ??
564
728
  (!result.ok && result.status !== undefined