@teamscala/service-boot 0.23.425 → 0.23.427

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,18 +3,13 @@
3
3
  * @status handwritten
4
4
  * @edit edit directly
5
5
  *
6
- * Agreement lookups + the gate's resolution caches: the page→agreement
7
- * association cache, the owning/party org member-directory caches, and the
8
- * event-driven invalidation (LISTEN registry_config_changed) that clears them.
6
+ * Agreement lookup: typed first-match query over the agreements model via
7
+ * the page-server ORPC client.
9
8
  *
10
9
  * Split out of agreement-gate.ts (909 lines) — byte-for-byte move; import
11
10
  * paths adjusted only.
12
11
  */
13
12
 
14
- import { getAppLogger } from "@teamscala/logger/app-loggers";
15
- import { queryRows } from "@teamscala/db/query-rows";
16
- import { createCache } from "@teamscala/cache/create-cache";
17
- import { createNotifyListener, getDirectDatabaseUrl } from "@teamscala/db/notify-listener/index";
18
13
  import { getServerClient } from "../page-server-orpc/server-client.ts";
19
14
 
20
15
  export async function findFirstAgreement<T>(input: {
@@ -26,102 +21,3 @@ export async function findFirstAgreement<T>(input: {
26
21
  };
27
22
  return client.agreements.findFirst(input);
28
23
  }
29
-
30
- /**
31
- * Cache for the page→agreement association (new-flow pages that have no
32
- * pre-placed gate block). The association changes rarely (only when a new
33
- * agreement is created or a page is rebound), so a 30s TTL avoids a DB query
34
- * per request while staying fresh enough for just-created agreements to appear.
35
- * Negative results are cached too (id="") so non-agreement pages never re-query.
36
- */
37
- export const pageAgreementCache = createCache<{ id: string; name: string }>(
38
- "agreement-gate:page-association",
39
- { maxSize: 256, ttlMs: 30_000 },
40
- );
41
-
42
- /**
43
- * Cache for the owning org's member directory (org id → lowercased email → name),
44
- * used to grant VIEW-ONLY access to an agreement to any member of the org that
45
- * owns it (signing stays roster-bound). Membership changes on human timescales,
46
- * so a 30s TTL keeps the per-request authorize cheap (hot-path-resolutions-are-cached).
47
- */
48
- export const orgMemberCache = createCache<Map<string, string>>(
49
- "agreement-gate:org-members",
50
- { maxSize: 256, ttlMs: 30_000 },
51
- );
52
-
53
- /**
54
- * The owning org's member directory: lowercased email → display name, for every
55
- * member of `organisationId`. Empty when the org is unknown or has no members.
56
- *
57
- * Resolved via a parameterized raw join over better_auth_member + "user" rather
58
- * rather than a service-local Prisma `member` model: `member` is a shared
59
- * @@map'd Prisma model, and each
60
- * service's per-service client is generated by live DB introspection from only
61
- * the tables its service_schema declares — so that model is undefined
62
- * on a service (scala-sign, scala-proposals) that doesn't list better_auth_member.
63
- * A read-only raw query needs no model declaration and works on every client,
64
- * which is what makes org-member VIEW access actually resolve (the typed access
65
- * silently crashed → caught → empty directory → org members wrongly challenged).
66
- */
67
- export async function resolveOrgMemberDirectory(
68
- organisationId: string | null,
69
- ): Promise<Map<string, string>> {
70
- if (!organisationId) return new Map();
71
- const cached = orgMemberCache.get(organisationId);
72
- if (cached !== undefined) return cached;
73
- const directory = new Map<string, string>();
74
- try {
75
- const rows = await queryRows`
76
- SELECT u.email AS email, u.name AS name
77
- FROM better_auth_member m
78
- JOIN "user" u ON u.id = m.user_id
79
- WHERE m.organization_id = ${organisationId}
80
- ` as
81
- Array<{ email: string | null; name: string | null }>
82
- ;
83
- for (const r of rows) {
84
- const email = typeof r.email === "string" ? r.email.trim().toLowerCase() : "";
85
- if (email) directory.set(email, String(r.name ?? r.email));
86
- }
87
- orgMemberCache.set(organisationId, directory);
88
- return directory;
89
- } catch (err) {
90
- getAppLogger().error(
91
- `[agreement-gate] org-member resolve failed: ${err instanceof Error ? err.message : String(err)}`,
92
- );
93
- return directory;
94
- }
95
- }
96
-
97
- /**
98
- * The combined member directory for EVERY org an agreement's parties represent
99
- * (each party's `organisationId`) plus the agreement's owning org. Any of these
100
- * members may VIEW the agreement; signing stays restricted to the designated
101
- * signer (the party email). Usually 1 org (one of our orgs + an external party),
102
- * so the loop is over a 1-2 element set, each entry cached.
103
- */
104
- export async function resolveAgreementViewerDirectory(
105
- orgIds: Iterable<string | null | undefined>,
106
- ): Promise<Map<string, string>> {
107
- const out = new Map<string, string>();
108
- const unique = new Set(
109
- [...orgIds].filter((id): id is string => typeof id === "string" && id.length > 0),
110
- );
111
- for (const orgId of unique) {
112
- const dir = await resolveOrgMemberDirectory(orgId);
113
- for (const [email, name] of dir) if (!out.has(email)) out.set(email, name);
114
- }
115
- return out;
116
- }
117
-
118
- // Event-driven invalidation via LISTEN on registry_config_changed
119
- {
120
- const _url = getDirectDatabaseUrl();
121
- if (_url) {
122
- createNotifyListener(_url).listen("registry_config_changed", () => {
123
- pageAgreementCache.clear();
124
- orgMemberCache.clear();
125
- }).catch((e: unknown) => console.error("[cache-invalidation] subscribe failed: " + e));
126
- }
127
- }
@@ -12,7 +12,8 @@
12
12
 
13
13
  import { getAppLogger } from "@teamscala/logger/app-loggers";
14
14
  import { parseAgreementParties } from "@teamscala/document-signing/agreement-parties";
15
- import { findFirstAgreement, resolveAgreementViewerDirectory } from "./agreement-lookup.ts";
15
+ import { findFirstAgreement } from "./agreement-lookup.ts";
16
+ import { resolveAgreementViewerDirectory } from "./resolution-caches.ts";
16
17
 
17
18
  /** The agreement's authorized email set (its signer roster) for the access endpoint. */
18
19
  export async function resolveAgreementSignerEmails(
@@ -0,0 +1,108 @@
1
+ /**
2
+ * @system service-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * The agreement gate's resolution caches: the page→agreement association
7
+ * cache, the owning/party org member-directory caches, and the event-driven
8
+ * invalidation (LISTEN registry_config_changed) that clears them.
9
+ */
10
+
11
+ import { getAppLogger } from "@teamscala/logger/app-loggers";
12
+ import { queryRows } from "@teamscala/db/query-rows";
13
+ import { createCache } from "@teamscala/cache/create-cache";
14
+ import { registerCacheInvalidation } from "@teamscala/db/notify-listener/index";
15
+
16
+ /**
17
+ * Cache for the page→agreement association (new-flow pages that have no
18
+ * pre-placed gate block). The association changes rarely (only when a new
19
+ * agreement is created or a page is rebound), so a 30s TTL avoids a DB query
20
+ * per request while staying fresh enough for just-created agreements to appear.
21
+ * Negative results are cached too (id="") so non-agreement pages never re-query.
22
+ */
23
+ export const pageAgreementCache = createCache<{ id: string; name: string }>(
24
+ "agreement-gate:page-association",
25
+ { maxSize: 256, ttlMs: 30_000 },
26
+ );
27
+
28
+ /**
29
+ * Cache for the owning org's member directory (org id → lowercased email → name),
30
+ * used to grant VIEW-ONLY access to an agreement to any member of the org that
31
+ * owns it (signing stays roster-bound). Membership changes on human timescales,
32
+ * so a 30s TTL keeps the per-request authorize cheap (hot-path-resolutions-are-cached).
33
+ */
34
+ export const orgMemberCache = createCache<Map<string, string>>(
35
+ "agreement-gate:org-members",
36
+ { maxSize: 256, ttlMs: 30_000 },
37
+ );
38
+
39
+ /**
40
+ * The owning org's member directory: lowercased email → display name for
41
+ * every member of `organisationId`; empty when the org is unknown. Raw join,
42
+ * NOT the shared @@map'd Prisma `member` model — per-service clients are
43
+ * introspected from only the tables their service_schema declares, so that
44
+ * model is undefined on services that don't list better_auth_member. The
45
+ * typed access silently crashed → empty directory → org members wrongly
46
+ * challenged; the raw query works on every client.
47
+ */
48
+ export async function resolveOrgMemberDirectory(
49
+ organisationId: string | null,
50
+ ): Promise<Map<string, string>> {
51
+ if (!organisationId) return new Map();
52
+ const cached = orgMemberCache.get(organisationId);
53
+ if (cached !== undefined) return cached;
54
+ const directory = new Map<string, string>();
55
+ try {
56
+ const rows = await queryRows`
57
+ SELECT u.email AS email, u.name AS name
58
+ FROM better_auth_member m
59
+ JOIN "user" u ON u.id = m.user_id
60
+ WHERE m.organization_id = ${organisationId}
61
+ ` as
62
+ Array<{ email: string | null; name: string | null }>
63
+ ;
64
+ for (const r of rows) {
65
+ const email = typeof r.email === "string" ? r.email.trim().toLowerCase() : "";
66
+ if (email) directory.set(email, String(r.name ?? r.email));
67
+ }
68
+ orgMemberCache.set(organisationId, directory);
69
+ return directory;
70
+ } catch (err) {
71
+ getAppLogger().error(
72
+ `[agreement-gate] org-member resolve failed: ${err instanceof Error ? err.message : String(err)}`,
73
+ );
74
+ return directory;
75
+ }
76
+ }
77
+
78
+ /**
79
+ * The combined member directory for EVERY org an agreement's parties represent
80
+ * (each party's `organisationId`) plus the agreement's owning org. Any of these
81
+ * members may VIEW the agreement; signing stays restricted to the designated
82
+ * signer (the party email). Usually 1 org (one of our orgs + an external party),
83
+ * so the loop is over a 1-2 element set, each entry cached.
84
+ */
85
+ export async function resolveAgreementViewerDirectory(
86
+ orgIds: Iterable<string | null | undefined>,
87
+ ): Promise<Map<string, string>> {
88
+ const out = new Map<string, string>();
89
+ const unique = new Set(
90
+ [...orgIds].filter((id): id is string => typeof id === "string" && id.length > 0),
91
+ );
92
+ for (const orgId of unique) {
93
+ const dir = await resolveOrgMemberDirectory(orgId);
94
+ for (const [email, name] of dir) if (!out.has(email)) out.set(email, name);
95
+ }
96
+ return out;
97
+ }
98
+
99
+ // Event-driven invalidation via the process-wide shared LISTEN connection.
100
+ // `createNotifyListener` is a FACTORY — each call opens its own client
101
+ // (max:1, idle_timeout:0) that is never reused or stopped, so calling it at
102
+ // module scope costs one permanent connection per module. Measured
103
+ // 2026-08-28: twelve such sites gave every Puck UI service 8 LISTEN
104
+ // connections and zero query connections (scala-portal 18).
105
+ registerCacheInvalidation(() => {
106
+ pageAgreementCache.clear();
107
+ orgMemberCache.clear();
108
+ });
@@ -21,7 +21,8 @@ import {
21
21
  buildCertificateData,
22
22
  type CertificateData,
23
23
  } from "../fn-routers/agreement-certificate.ts";
24
- import { findFirstAgreement, resolveAgreementViewerDirectory } from "./agreement-lookup.ts";
24
+ import { findFirstAgreement } from "./agreement-lookup.ts";
25
+ import { resolveAgreementViewerDirectory } from "./resolution-caches.ts";
25
26
 
26
27
  export interface ContentBlock {
27
28
  type?: string;
@@ -31,7 +31,8 @@ import { injectResourceChatLauncher } from "./resource-chat-inject.ts";
31
31
  import { agTtlMs } from "./gate-ttls.ts";
32
32
  import { getAppLogger } from "@teamscala/logger/app-loggers";
33
33
  import { AG_COOKIE, AGREEMENT_GATE_DEFAULTS, loadAgreementGateSettings } from "./agreement-gate/settings.ts";
34
- import { findFirstAgreement, pageAgreementCache } from "./agreement-gate/agreement-lookup.ts";
34
+ import { findFirstAgreement } from "./agreement-gate/agreement-lookup.ts";
35
+ import { pageAgreementCache } from "./agreement-gate/resolution-caches.ts";
35
36
  import {
36
37
  brandOrgName,
37
38
  buildSigningSession,
@@ -49,6 +49,25 @@ export const FaultEscalationConfigSchema = v.object({
49
49
  }),
50
50
  ),
51
51
  ),
52
+ /** Per-kind CAPABILITY PROBES gating the recovery transition (operator
53
+ * ruling 2026-08-28: a layer emitting a degraded transition owes a
54
+ * recovery transition backed by a capability probe, never a timer over
55
+ * an empty event stream). A kind declared here resolves only when its
56
+ * probe answers HTTP OK; an unreachable probe HOLDS the episode open —
57
+ * once per episode-state, never an alarm. The volume floor above is
58
+ * still event-stream evidence and cannot answer alone: its window is
59
+ * cumulative, so hours of healthy history mask a just-died producer
60
+ * (measured 2026-08-28: llm.provider.call.failed resolved after 31
61
+ * quiet minutes on 12h-old volume while the pipeline was down). */
62
+ probe_sources: v.optional(
63
+ v.record(
64
+ v.string(),
65
+ v.object({
66
+ url: v.pipe(v.string(), v.minLength(1)),
67
+ timeout_seconds: v.optional(v.number()),
68
+ }),
69
+ ),
70
+ ),
52
71
  /** The orchestrator's work item — escalations are COMMENTS on it, never
53
72
  * task rows (the fleet-task-placement CHECK constraint refuses a system
54
73
  * task without session identity, and an escalation is a finding, not an
@@ -145,6 +145,26 @@ export async function fetchVolumeCount(pattern: string, windowMinutes: number):
145
145
  return Number(rows[0]?.n ?? 0);
146
146
  }
147
147
 
148
+ /** Whether a resolve-HOLD has already been raised for this episode state:
149
+ * the newest `platform.fault.resolve_held` naming the kind is newer than the
150
+ * episode's last transition. Durable, never process memory — a restart must
151
+ * not re-raise the hold, and a NEW episode state (a later last_updated_at)
152
+ * may raise it again. */
153
+ export async function resolveHoldRaisedSince(kind: string, since: Date | null): Promise<boolean> {
154
+ const prisma = getPrisma();
155
+ const rows = await prisma.$queryRawUnsafe<Array<{ occurred_at: Date }>>(
156
+ `SELECT occurred_at FROM event_log
157
+ WHERE kind = 'platform.fault.resolve_held'
158
+ AND payload_json->>'event_kind' = $1::text
159
+ ORDER BY occurred_at DESC LIMIT 1`,
160
+ kind,
161
+ );
162
+ const latest = rows[0]?.occurred_at;
163
+ if (!latest) return false;
164
+ if (!since) return true;
165
+ return new Date(latest) > since;
166
+ }
167
+
148
168
  /** Whether a zero-volume hold marker STANDS for a fault kind: derived from
149
169
  * the durable stream (latest flagged/cleared wins), never from process
150
170
  * memory — a restart must not re-raise or silently drop a standing hold.
@@ -0,0 +1,64 @@
1
+ /**
2
+ * @system server-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * Behavioural tests for the capability probe behind the resolve gate
7
+ * (operator ruling 2026-08-28): an episode whose subject is genuinely down
8
+ * must FAIL to close. Health surfaces encode capability in the STATUS CODE
9
+ * (llm-router /health answers 503 while degraded), so healthy means 2xx.
10
+ */
11
+ import { afterAll, beforeAll, describe, expect, test } from "bun:test";
12
+ import { probeHttpOk } from "./probe.ts";
13
+
14
+ describe("probeHttpOk — the resolve gate's instrument", () => {
15
+ let server: Bun.Server<undefined>;
16
+ let port = 0;
17
+
18
+ beforeAll(() => {
19
+ server = Bun.serve({
20
+ port: 0,
21
+ fetch(request) {
22
+ const url = new URL(request.url);
23
+ if (url.pathname === "/ok") return new Response('{"status":"ok"}');
24
+ if (url.pathname === "/degraded") return new Response('{"status":"degraded"}', { status: 503 });
25
+ return new Response("not found", { status: 404 });
26
+ },
27
+ });
28
+ port = server.port as number;
29
+ });
30
+
31
+ afterAll(() => {
32
+ server.stop(true);
33
+ });
34
+
35
+ test("an answering, healthy endpoint verifies", async () => {
36
+ expect(await probeHttpOk(`http://127.0.0.1:${port}/ok`)).toBe(true);
37
+ });
38
+
39
+ test("a subject answering DEGRADED must not read as healthy — 503 is not an OK", async () => {
40
+ // THE MEASURED SHAPE: the 2026-08-28 close-on-quiet fired while the
41
+ // pipeline was down. A probe that treated "any response" as health
42
+ // would resolve an episode whose subject is still answering
43
+ // "degraded" — the same verdict with no instrument behind it.
44
+ expect(await probeHttpOk(`http://127.0.0.1:${port}/degraded`)).toBe(false);
45
+ });
46
+
47
+ test("a subject that does not answer the probe's question must not read as healthy", async () => {
48
+ expect(await probeHttpOk(`http://127.0.0.1:${port}/missing`)).toBe(false);
49
+ });
50
+
51
+ test("a subject that cannot be reached at all must not read as healthy", async () => {
52
+ // Port 1 on loopback: nothing listens there; the probe must refuse
53
+ // promptly, not hang or throw.
54
+ expect(await probeHttpOk("http://127.0.0.1:1/health", 1)).toBe(false);
55
+ });
56
+
57
+ test("an unreachable subject must not read as healthy", async () => {
58
+ expect(await probeHttpOk("http://127.0.0.1:1/health", 1)).toBe(false);
59
+ });
60
+
61
+ test("an unreachable subject with a tiny timeout still refuses, never throws", async () => {
62
+ expect(await probeHttpOk("http://10.255.255.1:9/health", 1)).toBe(false);
63
+ });
64
+ });
@@ -0,0 +1,19 @@
1
+ /**
2
+ * @system server-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * The capability probe behind the resolve gate (operator ruling 2026-08-28):
7
+ * a recovery transition is backed by an ACTIVE question to the subject's
8
+ * capability surface, never a timer over an empty event stream. Healthy
9
+ * means HTTP OK (2xx) — llm-router /health answers 503 while degraded.
10
+ */
11
+
12
+ export async function probeHttpOk(url: string, timeoutSeconds = 5): Promise<boolean> {
13
+ try {
14
+ const response = await fetch(url, { signal: AbortSignal.timeout(timeoutSeconds * 1000) });
15
+ return response.ok;
16
+ } catch {
17
+ return false;
18
+ }
19
+ }
@@ -61,6 +61,17 @@ export interface SweepDecision {
61
61
  * model backend at all). The episode HOLDS OPEN and the hold raises
62
62
  * once; it never resolves from silence alone. */
63
63
  zeroVolumeHold: Array<{ episode: EscalationEpisode; quietMinutes: number; volumeCount: number }>;
64
+ /** Open episodes quiet past the resolve window whose kind declares a
65
+ * CAPABILITY PROBE: the close is withheld until the probe answers OK.
66
+ * The handler runs the probe and either resolves or holds — the decision
67
+ * core stays pure (it selects WHO must be probed, it never probes). */
68
+ resolveProbeGate: Array<{
69
+ episode: EscalationEpisode;
70
+ quietMinutes: number;
71
+ clearZeroVolumeFlag: boolean;
72
+ probeUrl: string;
73
+ probeTimeoutSeconds: number;
74
+ }>;
64
75
  }
65
76
 
66
77
  /** Per-kind config naming the kind whose VOLUME proves the producing
@@ -72,6 +83,15 @@ export interface ZeroVolumeSource {
72
83
  min_volume?: number;
73
84
  }
74
85
 
86
+ /** Per-kind config naming the capability probe that must answer OK before
87
+ * this fault kind's episode may close. The probe is an ACTIVE question to
88
+ * the subject's capability surface (HTTP OK), never a read of the event
89
+ * stream — absence of events is never evidence of state. */
90
+ export interface ProbeSource {
91
+ url: string;
92
+ timeout_seconds?: number;
93
+ }
94
+
75
95
  /** Everything decideSweep needs to judge quiet-vs-healthy, computed by the
76
96
  * caller from the valibot-validated config row + bounded queries. */
77
97
  export interface ZeroVolumeInputs {
@@ -125,10 +145,12 @@ export function decideSweep(
125
145
  now: Date,
126
146
  kindThresholds: Record<string, KindThresholdOverride> = {},
127
147
  zeroVolume?: ZeroVolumeInputs,
148
+ probeSources: Record<string, ProbeSource> = {},
128
149
  ): SweepDecision {
129
150
  const byKind = new Map(episodes.map((e) => [e.kind, e]));
130
151
  const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
131
- const decision: SweepDecision = { escalate: [], resolve: [], zeroVolumeHold: [] };
152
+ const decision: SweepDecision =
153
+ { escalate: [], resolve: [], resolveProbeGate: [], zeroVolumeHold: [] };
132
154
 
133
155
  for (const aggregate of aggregates) {
134
156
  const episode = byKind.get(aggregate.kind);
@@ -180,6 +202,22 @@ export function decideSweep(
180
202
  continue;
181
203
  }
182
204
  }
205
+ // PROBE-GATED RESOLUTION (operator ruling 2026-08-28): a kind with a
206
+ // declared capability probe does not close on quiet + volume alone —
207
+ // the handler must first ask the subject's capability surface and
208
+ // only an HTTP OK closes the episode. An unreachable probe holds the
209
+ // episode open (once per episode-state, never an alarm).
210
+ const probe = probeSources[episode.kind];
211
+ if (probe) {
212
+ decision.resolveProbeGate.push({
213
+ episode,
214
+ quietMinutes,
215
+ clearZeroVolumeFlag: Boolean(source && zeroVolume?.flagActiveByKind[episode.kind]),
216
+ probeUrl: probe.url,
217
+ probeTimeoutSeconds: probe.timeout_seconds ?? 5,
218
+ });
219
+ continue;
220
+ }
183
221
  decision.resolve.push({
184
222
  episode,
185
223
  quietMinutes,
@@ -115,7 +115,12 @@ describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
115
115
  const decision = decideSweep([sustained], [longOpen], POLICY, NOW);
116
116
  expect(decision.escalate).toHaveLength(0);
117
117
  expect(decision.resolve).toHaveLength(0);
118
- expect(Object.keys(decision).sort()).toEqual(["escalate", "resolve", "zeroVolumeHold"]);
118
+ expect(Object.keys(decision).sort()).toEqual([
119
+ "escalate",
120
+ "resolve",
121
+ "resolveProbeGate",
122
+ "zeroVolumeHold",
123
+ ]);
119
124
  });
120
125
 
121
126
  test("a claimed-but-undelivered episode is RETRIED, not counted as escalated", () => {
@@ -399,3 +404,43 @@ describe("composeZeroVolumeMessage — the quiet-is-not-healthy notice", () => {
399
404
  expect(message.split("\n").length).toBeLessThanOrEqual(4);
400
405
  });
401
406
  });
407
+
408
+ describe("decideSweep — probe-gated resolution (operator ruling 2026-08-28)", () => {
409
+ const PROBE_SOURCES = {
410
+ "llm.provider.call.failed": { url: "https://ai.scala.business/health", timeout_seconds: 5 },
411
+ };
412
+
413
+ test("a kind with a declared probe is gated, not resolved on quiet", () => {
414
+ const gone = episode({
415
+ kind: "llm.provider.call.failed",
416
+ lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
417
+ });
418
+ const decision = decideSweep([agg("llm.provider.call.failed", 10, 700, 60)], [gone], POLICY, NOW, {}, undefined, PROBE_SOURCES);
419
+ expect(decision.resolve).toHaveLength(0);
420
+ expect(decision.resolveProbeGate).toHaveLength(1);
421
+ expect(decision.resolveProbeGate[0]!.probeUrl).toBe("https://ai.scala.business/health");
422
+ expect(decision.resolveProbeGate[0]!.quietMinutes).toBeGreaterThanOrEqual(30);
423
+ });
424
+
425
+ test("a kind without a declared probe resolves on quiet exactly as before", () => {
426
+ const gone = episode({ kind: "wake.failed", lastSeenAt: new Date(NOW.getTime() - 60 * 60_000) });
427
+ const decision = decideSweep([agg("wake.failed", 10, 700, 60)], [gone], POLICY, NOW, {}, undefined, PROBE_SOURCES);
428
+ expect(decision.resolve).toHaveLength(1);
429
+ expect(decision.resolveProbeGate).toHaveLength(0);
430
+ });
431
+
432
+ test("the gate passes through the zero-volume flag so a healthy probe clears it", () => {
433
+ const gone = episode({
434
+ kind: "llm.provider.call.failed",
435
+ lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
436
+ });
437
+ const zeroVolume = {
438
+ sources: { "llm.provider.call.failed": { volume_kind_pattern: "^llm\.router\.routing_decision$", min_volume: 1 } },
439
+ countByKind: { "llm.provider.call.failed": 12 },
440
+ flagActiveByKind: { "llm.provider.call.failed": true },
441
+ };
442
+ const decision = decideSweep([agg("llm.provider.call.failed", 10, 700, 60)], [gone], POLICY, NOW, {}, zeroVolume, PROBE_SOURCES);
443
+ expect(decision.resolveProbeGate).toHaveLength(1);
444
+ expect(decision.resolveProbeGate[0]!.clearZeroVolumeFlag).toBe(true);
445
+ });
446
+ });
@@ -25,6 +25,8 @@ import {
25
25
  type ZeroVolumeInputs,
26
26
  } from "./platform-fault-escalation-logic.ts";
27
27
  import { loadFaultEscalationConfig } from "./platform-fault-escalation/config.ts";
28
+ import { probeHttpOk } from "./platform-fault-escalation/probe.ts";
29
+ import { resolveHoldRaisedSince } from "./platform-fault-escalation/event-log-queries.ts";
28
30
  import {
29
31
  fetchAggregates,
30
32
  fetchVolumeCount,
@@ -70,6 +72,7 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
70
72
  lastUpdatedAt: row.last_updated_at ? new Date(row.last_updated_at) : null,
71
73
  }));
72
74
 
75
+ const probeSources = config.probe_sources ?? {};
73
76
  const kindThresholds: Record<string, KindThresholdOverride> = {};
74
77
  for (const [kind, override] of Object.entries(config.kind_thresholds ?? {})) {
75
78
  kindThresholds[kind] = {
@@ -104,6 +107,7 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
104
107
  now,
105
108
  kindThresholds,
106
109
  Object.keys(volumeSources).length > 0 ? zeroVolume : undefined,
110
+ probeSources,
107
111
  );
108
112
 
109
113
  // Per-kind isolation: one fault's delivery failure must never block the
@@ -139,6 +143,39 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
139
143
  log.error(`[platform-fault-escalation] zero-volume hold for '${episode.kind}' failed (retried next sweep): ${err instanceof Error ? err.message : String(err)}`);
140
144
  }
141
145
  }
146
+ // PROBE-GATED RESOLUTION (operator ruling 2026-08-28): the recovery
147
+ // transition fires only when the subject's capability probe answers OK.
148
+ // An unreachable probe HOLDS the episode open — the close-refusal is
149
+ // raised ONCE per episode state as a durable event, never an alarm: the
150
+ // original verdict (failures went quiet) was correct; it lacked the
151
+ // probe behind the verdict.
152
+ let resolveProbeHeld = 0;
153
+ for (const { episode, quietMinutes, clearZeroVolumeFlag, probeUrl, probeTimeoutSeconds } of decision.resolveProbeGate) {
154
+ try {
155
+ if (await probeHttpOk(probeUrl, probeTimeoutSeconds)) {
156
+ await resolveEpisode(episode, quietMinutes);
157
+ if (clearZeroVolumeFlag) await clearZeroVolumeHold(episode, zeroVolume.countByKind[episode.kind] ?? 0);
158
+ log.info(`[platform-fault-escalation] RESOLVED '${episode.kind}' — capability probe ${probeUrl} answered OK after ${quietMinutes} quiet min`);
159
+ } else {
160
+ resolveProbeHeld += 1;
161
+ if (await resolveHoldRaisedSince(episode.kind, episode.lastUpdatedAt ?? episode.firstSeenAt)) continue;
162
+ recordEvent({
163
+ kind: "platform.fault.resolve_held",
164
+ payload: {
165
+ event_kind: episode.kind,
166
+ probe: probeUrl,
167
+ quiet_minutes: quietMinutes,
168
+ work_item_id: episode.workItemId,
169
+ },
170
+ });
171
+ log.warn(
172
+ `[platform-fault-escalation] resolve HELD for '${episode.kind}' — capability probe ${probeUrl} did not answer OK; episode stays open (raised once, not an alarm)`,
173
+ );
174
+ }
175
+ } catch (err) {
176
+ log.error(`[platform-fault-escalation] probe-gated resolve for '${episode.kind}' failed (episode stays open; retried next sweep): ${err instanceof Error ? err.message : String(err)}`);
177
+ }
178
+ }
142
179
  for (const { episode, quietMinutes, clearZeroVolumeFlag } of decision.resolve) {
143
180
  try {
144
181
  await resolveEpisode(episode, quietMinutes);
@@ -155,7 +192,8 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
155
192
  payload: {
156
193
  watched_kinds: aggregates.length,
157
194
  escalated: decision.escalate.length,
158
- resolved: decision.resolve.length,
195
+ resolved: decision.resolve.length + (decision.resolveProbeGate.length - resolveProbeHeld),
196
+ resolve_probe_held: resolveProbeHeld,
159
197
  zero_volume_holds: decision.zeroVolumeHold.length,
160
198
  },
161
199
  });
@@ -19,7 +19,6 @@
19
19
 
20
20
  import { ManagedCache } from "@teamscala/cache/cache";
21
21
  import { createCache } from "@teamscala/cache/create-cache";
22
- import { createNotifyListener, getDirectDatabaseUrl } from "@teamscala/db/notify-listener/index";
23
22
  import { createDiskCache } from "@teamscala/cache/disk-cache";
24
23
  import { join } from "node:path";
25
24
  import { userHome } from "@teamscala/os/host-paths";
@@ -181,5 +180,8 @@ function safeStringify(v: unknown): string {
181
180
  return String(v);
182
181
  }
183
182
  }
184
- // Guard predicate: event-driven invalidation subscription
185
- { const _u = getDirectDatabaseUrl(); if (_u) createNotifyListener(_u).listen("registry_config_changed", () => {}).catch((e: unknown) => console.error("[cache-invalidation] subscribe failed: " + e)); }
183
+ // No event-driven invalidation subscription here: this module caches nothing.
184
+ // The removed block opened a dedicated LISTEN connection whose handler was
185
+ // empty `() => {}` — a connection held for the process lifetime to run no
186
+ // code. Registering a no-op against the shared listener would be the same
187
+ // defect one layer down, so it is deleted rather than migrated.
@@ -58,10 +58,11 @@ export async function mountServerPackages(
58
58
  app: Hono,
59
59
  config: ServiceInstallConfig,
60
60
  serviceSlug: string,
61
+ getSharedDb?: () => unknown,
61
62
  ): Promise<MountRegistry> {
62
63
  const log = getAppLogger();
63
64
  const mountRegistry = createMountRegistry();
64
- const mountCtx = createMountContext(serviceSlug, config, app, mountRegistry);
65
+ const mountCtx = createMountContext(serviceSlug, config, app, mountRegistry, getSharedDb);
65
66
  const serverPackages = (config.server_packages ?? []) as string[];
66
67
 
67
68
  if (serverPackages.length === 0) {
@@ -30,7 +30,7 @@
30
30
 
31
31
  import { getClient } from "@teamscala/object-storage/create/client/get-client";
32
32
  import { createCache } from "@teamscala/cache/create-cache";
33
- import { createNotifyListener, getDirectDatabaseUrl } from "@teamscala/db/notify-listener/index";
33
+ import { registerCacheInvalidation } from "@teamscala/db/notify-listener/index";
34
34
  import { getServerClient } from "@teamscala/orpc/server-client-registry";
35
35
  import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
36
36
  import { getAppLogger } from "@teamscala/logger/app-loggers";
@@ -290,11 +290,9 @@ export function buildResourceNotFoundResponse(
290
290
  });
291
291
  }
292
292
 
293
- {
294
- const _url = getDirectDatabaseUrl();
295
- if (_url) {
296
- createNotifyListener(_url).listen("registry_config_changed", () => {
297
- logoCache.clear();
298
- }).catch((e: unknown) => console.error("[cache-invalidation] subscribe failed: " + e));
299
- }
300
- }
293
+ // Shares the one process-wide LISTEN connection rather than opening its own:
294
+ // `createNotifyListener` is a factory, so a module-scope call holds a
295
+ // connection for the process lifetime (see resolution-caches.ts).
296
+ registerCacheInvalidation(() => {
297
+ logoCache.clear();
298
+ });