@cat-factory/server 0.179.0 → 0.181.0

package/dist/runtime/infraReachability.js

@@ -0,0 +1,188 @@
+import { INFRA_SETUP_PROBED_AREAS } from '@cat-factory/contracts';
+import { decideReachability, describeError, recordedUnreachableAreas, redactSecrets, runBestEffort, } from '@cat-factory/kernel';
+import { infraSetupAreaApplies } from '../modules/workspaces/infraSetup.js';
+/**
+ * Run one reachability pass across every workspace. Returns how many areas started failing and how
+ * many recovered — the two numbers worth a log line; an unchanged pass reports zeros and stays
+ * quiet. Enumerates boards from the workspace projection (`workspaceService.list(null)`), the same
+ * tenant-enumeration shape the platform-health and artifact-retention sweeps use.
+ *
+ * Time comes from the notification service's injected clock (`raise`/`clearByType` stamp `now`
+ * themselves) and from `Date.now()` for the pushed event, matching every other publisher.
+ */
+export async function sweepInfraReachability(container, logger) {
+    const cfg = container.config.infraReachability;
+    const notifications = container.notifications;
+    if (!cfg.enabled || !notifications)
+        return { raised: 0, cleared: 0 };
+    // NEVER from a mothership-mode node. This is a DEPLOYMENT-level sweep: every board such a node
+    // can see belongs to the org, so each laptop running it would probe every workspace's
+    // infrastructure and then race every other laptop on the same shared cards. The mothership runs
+    // it. This is also why the card read stays mothership-INTERNAL (`listOpenByType` is classified
+    // `sweeper`, not allow-listed on the persistence RPC) — without this guard the pass would fail
+    // with `unknown_method` on every interval instead of being cleanly out of scope.
+    if (container.config.localMode?.mothership)
+        return { raised: 0, cleared: 0 };
+    const sources = probeSources(container);
+    // Nothing on this deployment can be probed (neither integration is wired, or neither area applies
+    // here), so the pass would enumerate every board only to skip it. Bail before the workspace read.
+    if (!sources.ephemeralEnvironments && !sources.agentExecutor)
+        return { raised: 0, cleared: 0 };
+    const workspaces = await container.workspaceService.list(null);
+    const workspaceIds = workspaces.map((ws) => ws.id);
+    // Which boards already hold an open card — ONE batched read up front rather than a
+    // `findOpenByType` point-read per workspace inside the loop (that N+1 would run across the whole
+    // deployment every pass). It is also the state `decideReachability` compares against, so the
+    // healthy steady state costs exactly this one read plus the probes.
+    const cards = await notifications.service.listOpenByType(workspaceIds, 'infra_unreachable');
+    let raised = 0;
+    let cleared = 0;
+    for (const workspaceId of workspaceIds) {
+        try {
+            const previous = recordedUnreachableAreas(cards.get(workspaceId));
+            const observed = await probeWorkspace(sources, workspaceId, cfg.probeTimeoutMs);
+            const decision = decideReachability(previous, observed);
+            // Gate on the RECORD, not on the transitions: an area that is no longer configured drops out
+            // of the record while announcing nothing, and short-circuiting on `transitions` would leave
+            // its card open forever (nothing else ever clears it).
+            if (!decision.recordChanged && decision.transitions.length === 0)
+                continue;
+            // Persist FIRST, announce second. The card is the durable record the next pass compares
+            // against, so a push that fails after it commits costs one live banner update (the board's
+            // own snapshot still folds the card on next load); a push that succeeded against a record
+            // that never landed would re-announce the same outage on every subsequent pass.
+            if (decision.recordChanged) {
+                if (decision.unreachableAreas.length > 0) {
+                    await notifications.service.raise(workspaceId, {
+                        type: 'infra_unreachable',
+                        blockId: null,
+                        executionId: null,
+                        ...cardContent(decision.unreachableAreas),
+                        payload: { unreachableAreas: decision.unreachableAreas },
+                    });
+                }
+                else {
+                    await notifications.service.clearByType(workspaceId, 'infra_unreachable');
+                }
+            }
+            for (const change of decision.transitions) {
+                if (change.status === 'unreachable')
+                    raised += 1;
+                else
+                    cleared += 1;
+                await publish(container, workspaceId, change);
+            }
+        }
+        catch (err) {
+            logger?.warn('infra-reachability: failed to evaluate workspace', {
+                scope: 'infra-reachability',
+                workspaceId,
+                ...describeError(err),
+            });
+        }
+    }
+    return { raised, cleared };
+}
+/**
+ * Bind the per-area probes this deployment can actually run. An area that is absent here is one
+ * `decideReachability` never hears about, so it keeps whatever the card recorded rather than being
+ * reported as recovered by a deployment that never asked.
+ *
+ * Wiring is necessary but NOT sufficient: each area must also APPLY to this deployment
+ * (`infraSetupAreaApplies`, the same predicate the snapshot projection reports `not_applicable`
+ * from). Gating on the wired module alone was strictly looser than the projection — on Cloudflare
+ * and local mode the runner pool is an optional alternate target, so a dead one raised a card and
+ * paged Slack for an area whose banner the projection then refused to render. Probe only what this
+ * deployment would actually nag about.
+ */
+function probeSources(container) {
+    const environments = container.environments?.connectionService;
+    const runners = container.runners?.connectionService;
+    const applies = (area) => infraSetupAreaApplies(container, area);
+    return {
+        ...(environments && applies('ephemeralEnvironments')
+            ? { ephemeralEnvironments: (ws) => environments.probeSavedConnection(ws) }
+            : {}),
+        ...(runners && applies('agentExecutor')
+            ? { agentExecutor: (ws) => runners.probeSavedConnection(ws) }
+            : {}),
+    };
+}
+/**
+ * Probe every wired area for one workspace, concurrently — the areas are independent connections,
+ * so a slow apiserver must not serialise behind a slow runner pool.
+ */
+async function probeWorkspace(sources, workspaceId, timeoutMs) {
+    const wired = INFRA_SETUP_PROBED_AREAS.filter((area) => sources[area]);
+    return Promise.all(wired.map((area) => probeArea(() => sources[area](workspaceId), area, timeoutMs)));
+}
+/**
+ * Run one area's probe into a {@link ProbeOutcome}.
+ *
+ * FOUR results, deliberately distinct (degrade loudly), because they need four dispositions:
+ *  - a probe that ANSWERED `ok: false`, or didn't answer inside the budget ⇒ a real outage;
+ *  - a probe that THREW, or that could not be asked (`unprobeable`) ⇒ INDETERMINATE, leaving the
+ *    record untouched. A throw is a local fault (the connection wouldn't resolve, its secret bundle
+ *    wouldn't decrypt — classically a node with no access to the sealing key), and blaming the
+ *    operator's cluster for our own missing key is exactly the "never infer a cause from the
+ *    presence of an error" trap;
+ *  - nothing registered (`absent`) ⇒ NOT_CONFIGURED, which forgets a recorded outage without
+ *    announcing a recovery, so a card can't outlive the connection it reports on.
+ *
+ * NOTE the timeout only stops us WAITING: the underlying request keeps running to its own
+ * completion, because neither connection-test port takes an `AbortSignal` today. Threading one
+ * through both provider ports (and every backend that implements them) is the real fix and is out
+ * of this change's scope; the cost is a wasted in-flight request per timed-out probe, bounded by
+ * the interval.
+ */
+async function probeArea(probe, area, timeoutMs) {
+    let timer;
+    try {
+        const timeout = new Promise((resolve) => {
+            timer = setTimeout(() => resolve('timeout'), timeoutMs);
+        });
+        const probed = await Promise.race([probe(), timeout]);
+        if (probed === 'timeout') {
+            return { area, verdict: 'unreachable', detail: `No answer within ${timeoutMs}ms` };
+        }
+        if (probed.state === 'absent')
+            return { area, verdict: 'not_configured' };
+        if (probed.state === 'unprobeable') {
+            return { area, verdict: 'indeterminate', detail: probed.reason };
+        }
+        if (probed.result.ok)
+            return { area, verdict: 'reachable' };
+        // A provider's failure message routinely echoes the request URL it called, which can carry a
+        // token in the query — so it is scrubbed here, at the point it becomes operator-facing text.
+        const detail = probed.result.message ? redactSecrets(probed.result.message) : null;
+        return { area, verdict: 'unreachable', ...(detail ? { detail } : {}) };
+    }
+    catch {
+        return { area, verdict: 'indeterminate' };
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+/** Human-facing card text, derived from the failing AREAS only (see the payload's dedup note). */
+function cardContent(areas) {
+    const names = {
+        ephemeralEnvironments: 'the ephemeral-environment provider',
+        agentExecutor: 'the self-hosted runner pool',
+        binaryStorage: 'binary storage',
+    };
+    const listed = areas.map((area) => names[area]).join(' and ');
+    return {
+        title: 'Infrastructure is unreachable',
+        body: `A live probe could not reach ${listed}. It is configured, so this is an outage rather than a setup gap — the agents that depend on it cannot run until it answers again.`,
+    };
+}
+/** Push one transition, best-effort: a failed push must never abort the pass. */
+async function publish(container, workspaceId, change) {
+    const publisher = container.executionEventPublisher;
+    if (!publisher.infraSetupChanged)
+        return;
+    await runBestEffort(container.logger, 'infra-reachability publish', () => publisher.infraSetupChanged(workspaceId, change), { workspaceId, area: change.area, status: change.status });
+}
+//# sourceMappingURL=infraReachability.js.map

package/dist/runtime/infraReachability.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"infraReachability.js","sourceRoot":"","sources":["../../src/runtime/infraReachability.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAuB,MAAM,wBAAwB,CAAA;AACtF,OAAO,EACL,kBAAkB,EAClB,aAAa,EAIb,wBAAwB,EACxB,aAAa,EACb,aAAa,GAEd,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAA;AA2B3E;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,SAA0B,EAC1B,MAAe;IAEf,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAA;IAC9C,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,CAAA;IAC7C,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,aAAa;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAA;IACpE,+FAA+F;IAC/F,sFAAsF;IACtF,gGAAgG;IAChG,+FAA+F;IAC/F,+FAA+F;IAC/F,iFAAiF;IACjF,IAAI,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,UAAU;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAA;IAE5E,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,CAAA;IACvC,kGAAkG;IAClG,kGAAkG;IAClG,IAAI,CAAC,OAAO,CAAC,qBAAqB,IAAI,CAAC,OAAO,CAAC,aAAa;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAA;IAE9F,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC9D,MAAM,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAA;IAClD,mFAAmF;IACnF,iGAAiG;IACjG,6FAA6F;IAC7F,oEAAoE;IACpE,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAA;IAE3F,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,wBAAwB,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;YACjE,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,cAAc,CAAC,CAAA;YAC/E,MAAM,QAAQ,GAAG,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;YACvD,6FAA6F;YAC7F,4FAA4F;YAC5F,uDAAuD;YACvD,IAAI,CAAC,QAAQ,CAAC,aAAa,IAAI,QAAQ,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAQ;YAE1E,wFAAwF;YACxF,2FAA2F;YAC3F,0FAA0F;YAC1F,gFAAgF;YAChF,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;gBAC3B,IAAI,QAAQ,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzC,MAAM,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE;wBAC7C,IAAI,EAAE,mBAAmB;wBACzB,OAAO,EAAE,IAAI;wBACb,WAAW,EAAE,IAAI;wBACjB,GAAG,WAAW,CAAC,QAAQ,CAAC,gBAAgB,CAAC;wBACzC,OAAO,EAAE,EAAE,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB,EAAE;qBACzD,CAAC,CAAA;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,aAAa,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE,mBAAmB,CAAC,CAAA;gBAC3E,CAAC;YACH,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;gBAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,aAAa;oBAAE,MAAM,IAAI,CAAC,CAAA;;oBAC3C,OAAO,IAAI,CAAC,CAAA;gBACjB,MAAM,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,CAAA;YAC/C,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,EAAE,IAAI,CAAC,kDAAkD,EAAE;gBAC/D,KAAK,EAAE,oBAAoB;gBAC3B,WAAW;gBACX,GAAG,aAAa,CAAC,GAAG,CAAC;aACtB,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;AAC5B,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,YAAY,CAAC,SAA0B;IAC9C,MAAM,YAAY,GAAG,SAAS,CAAC,YAAY,EAAE,iBAAiB,CAAA;IAC9D,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACpD,MAAM,OAAO,GAAG,CAAC,IAAoB,EAAE,EAAE,CAAC,qBAAqB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IAChF,OAAO;QACL,GAAG,CAAC,YAAY,IAAI,OAAO,CAAC,uBAAuB,CAAC;YAClD,CAAC,CAAC,EAAE,qBAAqB,EAAE,CAAC,EAAU,EAAE,EAAE,CAAC,YAAY,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAAE;YAClF,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC,eAAe,CAAC;YACrC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,EAAU,EAAE,EAAE,CAAC,OAAO,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAAE;YACrE,CAAC,CAAC,EAAE,CAAC;KACR,CAAA;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,cAAc,CAC3B,OAAqB,EACrB,WAAmB,EACnB,SAAiB;IAEjB,MAAM,KAAK,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;IACtE,OAAO,OAAO,CAAC,GAAG,CAChB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CACnF,CAAA;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,KAAK,UAAU,SAAS,CACtB,KAA0C,EAC1C,IAAoB,EACpB,SAAiB;IAEjB,IAAI,KAAgD,CAAA;IACpD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;YACjD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,CAAA;QACzD,CAAC,CAAC,CAAA;QACF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC,CAAA;QACrD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,oBAAoB,SAAS,IAAI,EAAE,CAAA;QACpF,CAAC;QACD,IAAI,MAAM,CAAC,KAAK,KAAK,QAAQ;YAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAA;QACzE,IAAI,MAAM,CAAC,KAAK,KAAK,aAAa,EAAE,CAAC;YACnC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAA;QAClE,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE;YAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;QAC3D,6FAA6F;QAC7F,6FAA6F;QAC7F,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAClF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAA;IACxE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,CAAA;IAC3C,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAA;IAChC,CAAC;AACH,CAAC;AAED,kGAAkG;AAClG,SAAS,WAAW,CAAC,KAAgC;IACnD,MAAM,KAAK,GAAmC;QAC5C,qBAAqB,EAAE,oCAAoC;QAC3D,aAAa,EAAE,6BAA6B;QAC5C,aAAa,EAAE,gBAAgB;KAChC,CAAA;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAC7D,OAAO;QACL,KAAK,EAAE,+BAA+B;QACtC,IAAI,EAAE,gCAAgC,MAAM,oIAAoI;KACjL,CAAA;AACH,CAAC;AAED,iFAAiF;AACjF,KAAK,UAAU,OAAO,CACpB,SAA0B,EAC1B,WAAmB,EACnB,MAA4B;IAE5B,MAAM,SAAS,GAAG,SAAS,CAAC,uBAAuB,CAAA;IACnD,IAAI,CAAC,SAAS,CAAC,iBAAiB;QAAE,OAAM;IACxC,MAAM,aAAa,CACjB,SAAS,CAAC,MAAM,EAChB,4BAA4B,EAC5B,GAAG,EAAE,CAAC,SAAS,CAAC,iBAAkB,CAAC,WAAW,EAAE,MAAM,CAAC,EACvD,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAC1D,CAAA;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cat-factory/server",
-  "version": "0.179.0",
+  "version": "0.181.0",
   "description": "Runtime-neutral HTTP layer for the Agent Architecture Board: the Hono controllers, middleware (auth/authz/CORS/error), request helpers and the gateway seams shared by every deployment facade (Cloudflare Worker, Node service).",
   "repository": {
     "type": "git",
@@ -30,13 +30,13 @@
     "hono": "^4.12.32",
     "pino": "^10.3.1",
     "valibot": "^1.4.2",
-    "@cat-factory/agents": "0.88.0",
-    "@cat-factory/contracts": "0.197.0",
-    "@cat-factory/integrations": "0.111.1",
-    "@cat-factory/kernel": "0.194.0",
-    "@cat-factory/orchestration": "0.171.1",
-    "@cat-factory/prompt-fragments": "0.15.20",
-    "@cat-factory/spend": "0.12.124"
+    "@cat-factory/agents": "0.89.1",
+    "@cat-factory/contracts": "0.198.0",
+    "@cat-factory/integrations": "0.112.0",
+    "@cat-factory/kernel": "0.196.0",
+    "@cat-factory/orchestration": "0.172.1",
+    "@cat-factory/prompt-fragments": "0.15.21",
+    "@cat-factory/spend": "0.12.126"
   },
   "devDependencies": {
     "@types/node": "^26.1.1",