@teamscala/service-boot 0.23.425 → 0.23.427
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +263 -207
- package/src/agreement-gate/agreement-lookup.ts +2 -106
- package/src/agreement-gate/authorized-emails.ts +2 -1
- package/src/agreement-gate/resolution-caches.ts +108 -0
- package/src/agreement-gate/signing-session.ts +2 -1
- package/src/agreement-gate.ts +2 -1
- package/src/handlers/platform-fault-escalation/config.ts +19 -0
- package/src/handlers/platform-fault-escalation/event-log-queries.ts +20 -0
- package/src/handlers/platform-fault-escalation/probe.test.ts +64 -0
- package/src/handlers/platform-fault-escalation/probe.ts +19 -0
- package/src/handlers/platform-fault-escalation-logic.ts +39 -1
- package/src/handlers/platform-fault-escalation.test.ts +46 -1
- package/src/handlers/platform-fault-escalation.ts +39 -1
- package/src/install-logging.ts +5 -3
- package/src/mount-server-packages.ts +2 -1
- package/src/resource-not-found.ts +7 -9
- package/src/two-phase-minimal-app.ts +2 -1
- package/src/ui-service.ts +3 -0
|
@@ -3,18 +3,13 @@
|
|
|
3
3
|
* @status handwritten
|
|
4
4
|
* @edit edit directly
|
|
5
5
|
*
|
|
6
|
-
* Agreement
|
|
7
|
-
*
|
|
8
|
-
* event-driven invalidation (LISTEN registry_config_changed) that clears them.
|
|
6
|
+
* Agreement lookup: typed first-match query over the agreements model via
|
|
7
|
+
* the page-server ORPC client.
|
|
9
8
|
*
|
|
10
9
|
* Split out of agreement-gate.ts (909 lines) — byte-for-byte move; import
|
|
11
10
|
* paths adjusted only.
|
|
12
11
|
*/
|
|
13
12
|
|
|
14
|
-
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
15
|
-
import { queryRows } from "@teamscala/db/query-rows";
|
|
16
|
-
import { createCache } from "@teamscala/cache/create-cache";
|
|
17
|
-
import { createNotifyListener, getDirectDatabaseUrl } from "@teamscala/db/notify-listener/index";
|
|
18
13
|
import { getServerClient } from "../page-server-orpc/server-client.ts";
|
|
19
14
|
|
|
20
15
|
export async function findFirstAgreement<T>(input: {
|
|
@@ -26,102 +21,3 @@ export async function findFirstAgreement<T>(input: {
|
|
|
26
21
|
};
|
|
27
22
|
return client.agreements.findFirst(input);
|
|
28
23
|
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Cache for the page→agreement association (new-flow pages that have no
|
|
32
|
-
* pre-placed gate block). The association changes rarely (only when a new
|
|
33
|
-
* agreement is created or a page is rebound), so a 30s TTL avoids a DB query
|
|
34
|
-
* per request while staying fresh enough for just-created agreements to appear.
|
|
35
|
-
* Negative results are cached too (id="") so non-agreement pages never re-query.
|
|
36
|
-
*/
|
|
37
|
-
export const pageAgreementCache = createCache<{ id: string; name: string }>(
|
|
38
|
-
"agreement-gate:page-association",
|
|
39
|
-
{ maxSize: 256, ttlMs: 30_000 },
|
|
40
|
-
);
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
* Cache for the owning org's member directory (org id → lowercased email → name),
|
|
44
|
-
* used to grant VIEW-ONLY access to an agreement to any member of the org that
|
|
45
|
-
* owns it (signing stays roster-bound). Membership changes on human timescales,
|
|
46
|
-
* so a 30s TTL keeps the per-request authorize cheap (hot-path-resolutions-are-cached).
|
|
47
|
-
*/
|
|
48
|
-
export const orgMemberCache = createCache<Map<string, string>>(
|
|
49
|
-
"agreement-gate:org-members",
|
|
50
|
-
{ maxSize: 256, ttlMs: 30_000 },
|
|
51
|
-
);
|
|
52
|
-
|
|
53
|
-
/**
|
|
54
|
-
* The owning org's member directory: lowercased email → display name, for every
|
|
55
|
-
* member of `organisationId`. Empty when the org is unknown or has no members.
|
|
56
|
-
*
|
|
57
|
-
* Resolved via a parameterized raw join over better_auth_member + "user" rather
|
|
58
|
-
* rather than a service-local Prisma `member` model: `member` is a shared
|
|
59
|
-
* @@map'd Prisma model, and each
|
|
60
|
-
* service's per-service client is generated by live DB introspection from only
|
|
61
|
-
* the tables its service_schema declares — so that model is undefined
|
|
62
|
-
* on a service (scala-sign, scala-proposals) that doesn't list better_auth_member.
|
|
63
|
-
* A read-only raw query needs no model declaration and works on every client,
|
|
64
|
-
* which is what makes org-member VIEW access actually resolve (the typed access
|
|
65
|
-
* silently crashed → caught → empty directory → org members wrongly challenged).
|
|
66
|
-
*/
|
|
67
|
-
export async function resolveOrgMemberDirectory(
|
|
68
|
-
organisationId: string | null,
|
|
69
|
-
): Promise<Map<string, string>> {
|
|
70
|
-
if (!organisationId) return new Map();
|
|
71
|
-
const cached = orgMemberCache.get(organisationId);
|
|
72
|
-
if (cached !== undefined) return cached;
|
|
73
|
-
const directory = new Map<string, string>();
|
|
74
|
-
try {
|
|
75
|
-
const rows = await queryRows`
|
|
76
|
-
SELECT u.email AS email, u.name AS name
|
|
77
|
-
FROM better_auth_member m
|
|
78
|
-
JOIN "user" u ON u.id = m.user_id
|
|
79
|
-
WHERE m.organization_id = ${organisationId}
|
|
80
|
-
` as
|
|
81
|
-
Array<{ email: string | null; name: string | null }>
|
|
82
|
-
;
|
|
83
|
-
for (const r of rows) {
|
|
84
|
-
const email = typeof r.email === "string" ? r.email.trim().toLowerCase() : "";
|
|
85
|
-
if (email) directory.set(email, String(r.name ?? r.email));
|
|
86
|
-
}
|
|
87
|
-
orgMemberCache.set(organisationId, directory);
|
|
88
|
-
return directory;
|
|
89
|
-
} catch (err) {
|
|
90
|
-
getAppLogger().error(
|
|
91
|
-
`[agreement-gate] org-member resolve failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
92
|
-
);
|
|
93
|
-
return directory;
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
/**
|
|
98
|
-
* The combined member directory for EVERY org an agreement's parties represent
|
|
99
|
-
* (each party's `organisationId`) plus the agreement's owning org. Any of these
|
|
100
|
-
* members may VIEW the agreement; signing stays restricted to the designated
|
|
101
|
-
* signer (the party email). Usually 1 org (one of our orgs + an external party),
|
|
102
|
-
* so the loop is over a 1-2 element set, each entry cached.
|
|
103
|
-
*/
|
|
104
|
-
export async function resolveAgreementViewerDirectory(
|
|
105
|
-
orgIds: Iterable<string | null | undefined>,
|
|
106
|
-
): Promise<Map<string, string>> {
|
|
107
|
-
const out = new Map<string, string>();
|
|
108
|
-
const unique = new Set(
|
|
109
|
-
[...orgIds].filter((id): id is string => typeof id === "string" && id.length > 0),
|
|
110
|
-
);
|
|
111
|
-
for (const orgId of unique) {
|
|
112
|
-
const dir = await resolveOrgMemberDirectory(orgId);
|
|
113
|
-
for (const [email, name] of dir) if (!out.has(email)) out.set(email, name);
|
|
114
|
-
}
|
|
115
|
-
return out;
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
// Event-driven invalidation via LISTEN on registry_config_changed
|
|
119
|
-
{
|
|
120
|
-
const _url = getDirectDatabaseUrl();
|
|
121
|
-
if (_url) {
|
|
122
|
-
createNotifyListener(_url).listen("registry_config_changed", () => {
|
|
123
|
-
pageAgreementCache.clear();
|
|
124
|
-
orgMemberCache.clear();
|
|
125
|
-
}).catch((e: unknown) => console.error("[cache-invalidation] subscribe failed: " + e));
|
|
126
|
-
}
|
|
127
|
-
}
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
|
|
13
13
|
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
14
14
|
import { parseAgreementParties } from "@teamscala/document-signing/agreement-parties";
|
|
15
|
-
import { findFirstAgreement
|
|
15
|
+
import { findFirstAgreement } from "./agreement-lookup.ts";
|
|
16
|
+
import { resolveAgreementViewerDirectory } from "./resolution-caches.ts";
|
|
16
17
|
|
|
17
18
|
/** The agreement's authorized email set (its signer roster) for the access endpoint. */
|
|
18
19
|
export async function resolveAgreementSignerEmails(
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system service-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* The agreement gate's resolution caches: the page→agreement association
|
|
7
|
+
* cache, the owning/party org member-directory caches, and the event-driven
|
|
8
|
+
* invalidation (LISTEN registry_config_changed) that clears them.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
12
|
+
import { queryRows } from "@teamscala/db/query-rows";
|
|
13
|
+
import { createCache } from "@teamscala/cache/create-cache";
|
|
14
|
+
import { registerCacheInvalidation } from "@teamscala/db/notify-listener/index";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Cache for the page→agreement association (new-flow pages that have no
|
|
18
|
+
* pre-placed gate block). The association changes rarely (only when a new
|
|
19
|
+
* agreement is created or a page is rebound), so a 30s TTL avoids a DB query
|
|
20
|
+
* per request while staying fresh enough for just-created agreements to appear.
|
|
21
|
+
* Negative results are cached too (id="") so non-agreement pages never re-query.
|
|
22
|
+
*/
|
|
23
|
+
export const pageAgreementCache = createCache<{ id: string; name: string }>(
|
|
24
|
+
"agreement-gate:page-association",
|
|
25
|
+
{ maxSize: 256, ttlMs: 30_000 },
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Cache for the owning org's member directory (org id → lowercased email → name),
|
|
30
|
+
* used to grant VIEW-ONLY access to an agreement to any member of the org that
|
|
31
|
+
* owns it (signing stays roster-bound). Membership changes on human timescales,
|
|
32
|
+
* so a 30s TTL keeps the per-request authorize cheap (hot-path-resolutions-are-cached).
|
|
33
|
+
*/
|
|
34
|
+
export const orgMemberCache = createCache<Map<string, string>>(
|
|
35
|
+
"agreement-gate:org-members",
|
|
36
|
+
{ maxSize: 256, ttlMs: 30_000 },
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The owning org's member directory: lowercased email → display name for
|
|
41
|
+
* every member of `organisationId`; empty when the org is unknown. Raw join,
|
|
42
|
+
* NOT the shared @@map'd Prisma `member` model — per-service clients are
|
|
43
|
+
* introspected from only the tables their service_schema declares, so that
|
|
44
|
+
* model is undefined on services that don't list better_auth_member. The
|
|
45
|
+
* typed access silently crashed → empty directory → org members wrongly
|
|
46
|
+
* challenged; the raw query works on every client.
|
|
47
|
+
*/
|
|
48
|
+
export async function resolveOrgMemberDirectory(
|
|
49
|
+
organisationId: string | null,
|
|
50
|
+
): Promise<Map<string, string>> {
|
|
51
|
+
if (!organisationId) return new Map();
|
|
52
|
+
const cached = orgMemberCache.get(organisationId);
|
|
53
|
+
if (cached !== undefined) return cached;
|
|
54
|
+
const directory = new Map<string, string>();
|
|
55
|
+
try {
|
|
56
|
+
const rows = await queryRows`
|
|
57
|
+
SELECT u.email AS email, u.name AS name
|
|
58
|
+
FROM better_auth_member m
|
|
59
|
+
JOIN "user" u ON u.id = m.user_id
|
|
60
|
+
WHERE m.organization_id = ${organisationId}
|
|
61
|
+
` as
|
|
62
|
+
Array<{ email: string | null; name: string | null }>
|
|
63
|
+
;
|
|
64
|
+
for (const r of rows) {
|
|
65
|
+
const email = typeof r.email === "string" ? r.email.trim().toLowerCase() : "";
|
|
66
|
+
if (email) directory.set(email, String(r.name ?? r.email));
|
|
67
|
+
}
|
|
68
|
+
orgMemberCache.set(organisationId, directory);
|
|
69
|
+
return directory;
|
|
70
|
+
} catch (err) {
|
|
71
|
+
getAppLogger().error(
|
|
72
|
+
`[agreement-gate] org-member resolve failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
73
|
+
);
|
|
74
|
+
return directory;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* The combined member directory for EVERY org an agreement's parties represent
|
|
80
|
+
* (each party's `organisationId`) plus the agreement's owning org. Any of these
|
|
81
|
+
* members may VIEW the agreement; signing stays restricted to the designated
|
|
82
|
+
* signer (the party email). Usually 1 org (one of our orgs + an external party),
|
|
83
|
+
* so the loop is over a 1-2 element set, each entry cached.
|
|
84
|
+
*/
|
|
85
|
+
export async function resolveAgreementViewerDirectory(
|
|
86
|
+
orgIds: Iterable<string | null | undefined>,
|
|
87
|
+
): Promise<Map<string, string>> {
|
|
88
|
+
const out = new Map<string, string>();
|
|
89
|
+
const unique = new Set(
|
|
90
|
+
[...orgIds].filter((id): id is string => typeof id === "string" && id.length > 0),
|
|
91
|
+
);
|
|
92
|
+
for (const orgId of unique) {
|
|
93
|
+
const dir = await resolveOrgMemberDirectory(orgId);
|
|
94
|
+
for (const [email, name] of dir) if (!out.has(email)) out.set(email, name);
|
|
95
|
+
}
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Event-driven invalidation via the process-wide shared LISTEN connection.
|
|
100
|
+
// `createNotifyListener` is a FACTORY — each call opens its own client
|
|
101
|
+
// (max:1, idle_timeout:0) that is never reused or stopped, so calling it at
|
|
102
|
+
// module scope costs one permanent connection per module. Measured
|
|
103
|
+
// 2026-08-28: twelve such sites gave every Puck UI service 8 LISTEN
|
|
104
|
+
// connections and zero query connections (scala-portal 18).
|
|
105
|
+
registerCacheInvalidation(() => {
|
|
106
|
+
pageAgreementCache.clear();
|
|
107
|
+
orgMemberCache.clear();
|
|
108
|
+
});
|
|
@@ -21,7 +21,8 @@ import {
|
|
|
21
21
|
buildCertificateData,
|
|
22
22
|
type CertificateData,
|
|
23
23
|
} from "../fn-routers/agreement-certificate.ts";
|
|
24
|
-
import { findFirstAgreement
|
|
24
|
+
import { findFirstAgreement } from "./agreement-lookup.ts";
|
|
25
|
+
import { resolveAgreementViewerDirectory } from "./resolution-caches.ts";
|
|
25
26
|
|
|
26
27
|
export interface ContentBlock {
|
|
27
28
|
type?: string;
|
package/src/agreement-gate.ts
CHANGED
|
@@ -31,7 +31,8 @@ import { injectResourceChatLauncher } from "./resource-chat-inject.ts";
|
|
|
31
31
|
import { agTtlMs } from "./gate-ttls.ts";
|
|
32
32
|
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
33
33
|
import { AG_COOKIE, AGREEMENT_GATE_DEFAULTS, loadAgreementGateSettings } from "./agreement-gate/settings.ts";
|
|
34
|
-
import { findFirstAgreement
|
|
34
|
+
import { findFirstAgreement } from "./agreement-gate/agreement-lookup.ts";
|
|
35
|
+
import { pageAgreementCache } from "./agreement-gate/resolution-caches.ts";
|
|
35
36
|
import {
|
|
36
37
|
brandOrgName,
|
|
37
38
|
buildSigningSession,
|
|
@@ -49,6 +49,25 @@ export const FaultEscalationConfigSchema = v.object({
|
|
|
49
49
|
}),
|
|
50
50
|
),
|
|
51
51
|
),
|
|
52
|
+
/** Per-kind CAPABILITY PROBES gating the recovery transition (operator
|
|
53
|
+
* ruling 2026-08-28: a layer emitting a degraded transition owes a
|
|
54
|
+
* recovery transition backed by a capability probe, never a timer over
|
|
55
|
+
* an empty event stream). A kind declared here resolves only when its
|
|
56
|
+
* probe answers HTTP OK; an unreachable probe HOLDS the episode open —
|
|
57
|
+
* once per episode-state, never an alarm. The volume floor above is
|
|
58
|
+
* still event-stream evidence and cannot answer alone: its window is
|
|
59
|
+
* cumulative, so hours of healthy history mask a just-died producer
|
|
60
|
+
* (measured 2026-08-28: llm.provider.call.failed resolved after 31
|
|
61
|
+
* quiet minutes on 12h-old volume while the pipeline was down). */
|
|
62
|
+
probe_sources: v.optional(
|
|
63
|
+
v.record(
|
|
64
|
+
v.string(),
|
|
65
|
+
v.object({
|
|
66
|
+
url: v.pipe(v.string(), v.minLength(1)),
|
|
67
|
+
timeout_seconds: v.optional(v.number()),
|
|
68
|
+
}),
|
|
69
|
+
),
|
|
70
|
+
),
|
|
52
71
|
/** The orchestrator's work item — escalations are COMMENTS on it, never
|
|
53
72
|
* task rows (the fleet-task-placement CHECK constraint refuses a system
|
|
54
73
|
* task without session identity, and an escalation is a finding, not an
|
|
@@ -145,6 +145,26 @@ export async function fetchVolumeCount(pattern: string, windowMinutes: number):
|
|
|
145
145
|
return Number(rows[0]?.n ?? 0);
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
+
/** Whether a resolve-HOLD has already been raised for this episode state:
|
|
149
|
+
* the newest `platform.fault.resolve_held` naming the kind is newer than the
|
|
150
|
+
* episode's last transition. Durable, never process memory — a restart must
|
|
151
|
+
* not re-raise the hold, and a NEW episode state (a later last_updated_at)
|
|
152
|
+
* may raise it again. */
|
|
153
|
+
export async function resolveHoldRaisedSince(kind: string, since: Date | null): Promise<boolean> {
|
|
154
|
+
const prisma = getPrisma();
|
|
155
|
+
const rows = await prisma.$queryRawUnsafe<Array<{ occurred_at: Date }>>(
|
|
156
|
+
`SELECT occurred_at FROM event_log
|
|
157
|
+
WHERE kind = 'platform.fault.resolve_held'
|
|
158
|
+
AND payload_json->>'event_kind' = $1::text
|
|
159
|
+
ORDER BY occurred_at DESC LIMIT 1`,
|
|
160
|
+
kind,
|
|
161
|
+
);
|
|
162
|
+
const latest = rows[0]?.occurred_at;
|
|
163
|
+
if (!latest) return false;
|
|
164
|
+
if (!since) return true;
|
|
165
|
+
return new Date(latest) > since;
|
|
166
|
+
}
|
|
167
|
+
|
|
148
168
|
/** Whether a zero-volume hold marker STANDS for a fault kind: derived from
|
|
149
169
|
* the durable stream (latest flagged/cleared wins), never from process
|
|
150
170
|
* memory — a restart must not re-raise or silently drop a standing hold.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* Behavioural tests for the capability probe behind the resolve gate
|
|
7
|
+
* (operator ruling 2026-08-28): an episode whose subject is genuinely down
|
|
8
|
+
* must FAIL to close. Health surfaces encode capability in the STATUS CODE
|
|
9
|
+
* (llm-router /health answers 503 while degraded), so healthy means 2xx.
|
|
10
|
+
*/
|
|
11
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
12
|
+
import { probeHttpOk } from "./probe.ts";
|
|
13
|
+
|
|
14
|
+
describe("probeHttpOk — the resolve gate's instrument", () => {
|
|
15
|
+
let server: Bun.Server<undefined>;
|
|
16
|
+
let port = 0;
|
|
17
|
+
|
|
18
|
+
beforeAll(() => {
|
|
19
|
+
server = Bun.serve({
|
|
20
|
+
port: 0,
|
|
21
|
+
fetch(request) {
|
|
22
|
+
const url = new URL(request.url);
|
|
23
|
+
if (url.pathname === "/ok") return new Response('{"status":"ok"}');
|
|
24
|
+
if (url.pathname === "/degraded") return new Response('{"status":"degraded"}', { status: 503 });
|
|
25
|
+
return new Response("not found", { status: 404 });
|
|
26
|
+
},
|
|
27
|
+
});
|
|
28
|
+
port = server.port as number;
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
afterAll(() => {
|
|
32
|
+
server.stop(true);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("an answering, healthy endpoint verifies", async () => {
|
|
36
|
+
expect(await probeHttpOk(`http://127.0.0.1:${port}/ok`)).toBe(true);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("a subject answering DEGRADED must not read as healthy — 503 is not an OK", async () => {
|
|
40
|
+
// THE MEASURED SHAPE: the 2026-08-28 close-on-quiet fired while the
|
|
41
|
+
// pipeline was down. A probe that treated "any response" as health
|
|
42
|
+
// would resolve an episode whose subject is still answering
|
|
43
|
+
// "degraded" — the same verdict with no instrument behind it.
|
|
44
|
+
expect(await probeHttpOk(`http://127.0.0.1:${port}/degraded`)).toBe(false);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test("a subject that does not answer the probe's question must not read as healthy", async () => {
|
|
48
|
+
expect(await probeHttpOk(`http://127.0.0.1:${port}/missing`)).toBe(false);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("a subject that cannot be reached at all must not read as healthy", async () => {
|
|
52
|
+
// Port 1 on loopback: nothing listens there; the probe must refuse
|
|
53
|
+
// promptly, not hang or throw.
|
|
54
|
+
expect(await probeHttpOk("http://127.0.0.1:1/health", 1)).toBe(false);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test("an unreachable subject must not read as healthy", async () => {
|
|
58
|
+
expect(await probeHttpOk("http://127.0.0.1:1/health", 1)).toBe(false);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("an unreachable subject with a tiny timeout still refuses, never throws", async () => {
|
|
62
|
+
expect(await probeHttpOk("http://10.255.255.1:9/health", 1)).toBe(false);
|
|
63
|
+
});
|
|
64
|
+
});
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* The capability probe behind the resolve gate (operator ruling 2026-08-28):
|
|
7
|
+
* a recovery transition is backed by an ACTIVE question to the subject's
|
|
8
|
+
* capability surface, never a timer over an empty event stream. Healthy
|
|
9
|
+
* means HTTP OK (2xx) — llm-router /health answers 503 while degraded.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export async function probeHttpOk(url: string, timeoutSeconds = 5): Promise<boolean> {
|
|
13
|
+
try {
|
|
14
|
+
const response = await fetch(url, { signal: AbortSignal.timeout(timeoutSeconds * 1000) });
|
|
15
|
+
return response.ok;
|
|
16
|
+
} catch {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -61,6 +61,17 @@ export interface SweepDecision {
|
|
|
61
61
|
* model backend at all). The episode HOLDS OPEN and the hold raises
|
|
62
62
|
* once; it never resolves from silence alone. */
|
|
63
63
|
zeroVolumeHold: Array<{ episode: EscalationEpisode; quietMinutes: number; volumeCount: number }>;
|
|
64
|
+
/** Open episodes quiet past the resolve window whose kind declares a
|
|
65
|
+
* CAPABILITY PROBE: the close is withheld until the probe answers OK.
|
|
66
|
+
* The handler runs the probe and either resolves or holds — the decision
|
|
67
|
+
* core stays pure (it selects WHO must be probed, it never probes). */
|
|
68
|
+
resolveProbeGate: Array<{
|
|
69
|
+
episode: EscalationEpisode;
|
|
70
|
+
quietMinutes: number;
|
|
71
|
+
clearZeroVolumeFlag: boolean;
|
|
72
|
+
probeUrl: string;
|
|
73
|
+
probeTimeoutSeconds: number;
|
|
74
|
+
}>;
|
|
64
75
|
}
|
|
65
76
|
|
|
66
77
|
/** Per-kind config naming the kind whose VOLUME proves the producing
|
|
@@ -72,6 +83,15 @@ export interface ZeroVolumeSource {
|
|
|
72
83
|
min_volume?: number;
|
|
73
84
|
}
|
|
74
85
|
|
|
86
|
+
/** Per-kind config naming the capability probe that must answer OK before
|
|
87
|
+
* this fault kind's episode may close. The probe is an ACTIVE question to
|
|
88
|
+
* the subject's capability surface (HTTP OK), never a read of the event
|
|
89
|
+
* stream — absence of events is never evidence of state. */
|
|
90
|
+
export interface ProbeSource {
|
|
91
|
+
url: string;
|
|
92
|
+
timeout_seconds?: number;
|
|
93
|
+
}
|
|
94
|
+
|
|
75
95
|
/** Everything decideSweep needs to judge quiet-vs-healthy, computed by the
|
|
76
96
|
* caller from the valibot-validated config row + bounded queries. */
|
|
77
97
|
export interface ZeroVolumeInputs {
|
|
@@ -125,10 +145,12 @@ export function decideSweep(
|
|
|
125
145
|
now: Date,
|
|
126
146
|
kindThresholds: Record<string, KindThresholdOverride> = {},
|
|
127
147
|
zeroVolume?: ZeroVolumeInputs,
|
|
148
|
+
probeSources: Record<string, ProbeSource> = {},
|
|
128
149
|
): SweepDecision {
|
|
129
150
|
const byKind = new Map(episodes.map((e) => [e.kind, e]));
|
|
130
151
|
const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
|
|
131
|
-
const decision: SweepDecision =
|
|
152
|
+
const decision: SweepDecision =
|
|
153
|
+
{ escalate: [], resolve: [], resolveProbeGate: [], zeroVolumeHold: [] };
|
|
132
154
|
|
|
133
155
|
for (const aggregate of aggregates) {
|
|
134
156
|
const episode = byKind.get(aggregate.kind);
|
|
@@ -180,6 +202,22 @@ export function decideSweep(
|
|
|
180
202
|
continue;
|
|
181
203
|
}
|
|
182
204
|
}
|
|
205
|
+
// PROBE-GATED RESOLUTION (operator ruling 2026-08-28): a kind with a
|
|
206
|
+
// declared capability probe does not close on quiet + volume alone —
|
|
207
|
+
// the handler must first ask the subject's capability surface and
|
|
208
|
+
// only an HTTP OK closes the episode. An unreachable probe holds the
|
|
209
|
+
// episode open (once per episode-state, never an alarm).
|
|
210
|
+
const probe = probeSources[episode.kind];
|
|
211
|
+
if (probe) {
|
|
212
|
+
decision.resolveProbeGate.push({
|
|
213
|
+
episode,
|
|
214
|
+
quietMinutes,
|
|
215
|
+
clearZeroVolumeFlag: Boolean(source && zeroVolume?.flagActiveByKind[episode.kind]),
|
|
216
|
+
probeUrl: probe.url,
|
|
217
|
+
probeTimeoutSeconds: probe.timeout_seconds ?? 5,
|
|
218
|
+
});
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
183
221
|
decision.resolve.push({
|
|
184
222
|
episode,
|
|
185
223
|
quietMinutes,
|
|
@@ -115,7 +115,12 @@ describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
|
|
|
115
115
|
const decision = decideSweep([sustained], [longOpen], POLICY, NOW);
|
|
116
116
|
expect(decision.escalate).toHaveLength(0);
|
|
117
117
|
expect(decision.resolve).toHaveLength(0);
|
|
118
|
-
expect(Object.keys(decision).sort()).toEqual([
|
|
118
|
+
expect(Object.keys(decision).sort()).toEqual([
|
|
119
|
+
"escalate",
|
|
120
|
+
"resolve",
|
|
121
|
+
"resolveProbeGate",
|
|
122
|
+
"zeroVolumeHold",
|
|
123
|
+
]);
|
|
119
124
|
});
|
|
120
125
|
|
|
121
126
|
test("a claimed-but-undelivered episode is RETRIED, not counted as escalated", () => {
|
|
@@ -399,3 +404,43 @@ describe("composeZeroVolumeMessage — the quiet-is-not-healthy notice", () => {
|
|
|
399
404
|
expect(message.split("\n").length).toBeLessThanOrEqual(4);
|
|
400
405
|
});
|
|
401
406
|
});
|
|
407
|
+
|
|
408
|
+
describe("decideSweep — probe-gated resolution (operator ruling 2026-08-28)", () => {
|
|
409
|
+
const PROBE_SOURCES = {
|
|
410
|
+
"llm.provider.call.failed": { url: "https://ai.scala.business/health", timeout_seconds: 5 },
|
|
411
|
+
};
|
|
412
|
+
|
|
413
|
+
test("a kind with a declared probe is gated, not resolved on quiet", () => {
|
|
414
|
+
const gone = episode({
|
|
415
|
+
kind: "llm.provider.call.failed",
|
|
416
|
+
lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
|
|
417
|
+
});
|
|
418
|
+
const decision = decideSweep([agg("llm.provider.call.failed", 10, 700, 60)], [gone], POLICY, NOW, {}, undefined, PROBE_SOURCES);
|
|
419
|
+
expect(decision.resolve).toHaveLength(0);
|
|
420
|
+
expect(decision.resolveProbeGate).toHaveLength(1);
|
|
421
|
+
expect(decision.resolveProbeGate[0]!.probeUrl).toBe("https://ai.scala.business/health");
|
|
422
|
+
expect(decision.resolveProbeGate[0]!.quietMinutes).toBeGreaterThanOrEqual(30);
|
|
423
|
+
});
|
|
424
|
+
|
|
425
|
+
test("a kind without a declared probe resolves on quiet exactly as before", () => {
|
|
426
|
+
const gone = episode({ kind: "wake.failed", lastSeenAt: new Date(NOW.getTime() - 60 * 60_000) });
|
|
427
|
+
const decision = decideSweep([agg("wake.failed", 10, 700, 60)], [gone], POLICY, NOW, {}, undefined, PROBE_SOURCES);
|
|
428
|
+
expect(decision.resolve).toHaveLength(1);
|
|
429
|
+
expect(decision.resolveProbeGate).toHaveLength(0);
|
|
430
|
+
});
|
|
431
|
+
|
|
432
|
+
test("the gate passes through the zero-volume flag so a healthy probe clears it", () => {
|
|
433
|
+
const gone = episode({
|
|
434
|
+
kind: "llm.provider.call.failed",
|
|
435
|
+
lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
|
|
436
|
+
});
|
|
437
|
+
const zeroVolume = {
|
|
438
|
+
sources: { "llm.provider.call.failed": { volume_kind_pattern: "^llm\.router\.routing_decision$", min_volume: 1 } },
|
|
439
|
+
countByKind: { "llm.provider.call.failed": 12 },
|
|
440
|
+
flagActiveByKind: { "llm.provider.call.failed": true },
|
|
441
|
+
};
|
|
442
|
+
const decision = decideSweep([agg("llm.provider.call.failed", 10, 700, 60)], [gone], POLICY, NOW, {}, zeroVolume, PROBE_SOURCES);
|
|
443
|
+
expect(decision.resolveProbeGate).toHaveLength(1);
|
|
444
|
+
expect(decision.resolveProbeGate[0]!.clearZeroVolumeFlag).toBe(true);
|
|
445
|
+
});
|
|
446
|
+
});
|
|
@@ -25,6 +25,8 @@ import {
|
|
|
25
25
|
type ZeroVolumeInputs,
|
|
26
26
|
} from "./platform-fault-escalation-logic.ts";
|
|
27
27
|
import { loadFaultEscalationConfig } from "./platform-fault-escalation/config.ts";
|
|
28
|
+
import { probeHttpOk } from "./platform-fault-escalation/probe.ts";
|
|
29
|
+
import { resolveHoldRaisedSince } from "./platform-fault-escalation/event-log-queries.ts";
|
|
28
30
|
import {
|
|
29
31
|
fetchAggregates,
|
|
30
32
|
fetchVolumeCount,
|
|
@@ -70,6 +72,7 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
70
72
|
lastUpdatedAt: row.last_updated_at ? new Date(row.last_updated_at) : null,
|
|
71
73
|
}));
|
|
72
74
|
|
|
75
|
+
const probeSources = config.probe_sources ?? {};
|
|
73
76
|
const kindThresholds: Record<string, KindThresholdOverride> = {};
|
|
74
77
|
for (const [kind, override] of Object.entries(config.kind_thresholds ?? {})) {
|
|
75
78
|
kindThresholds[kind] = {
|
|
@@ -104,6 +107,7 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
104
107
|
now,
|
|
105
108
|
kindThresholds,
|
|
106
109
|
Object.keys(volumeSources).length > 0 ? zeroVolume : undefined,
|
|
110
|
+
probeSources,
|
|
107
111
|
);
|
|
108
112
|
|
|
109
113
|
// Per-kind isolation: one fault's delivery failure must never block the
|
|
@@ -139,6 +143,39 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
139
143
|
log.error(`[platform-fault-escalation] zero-volume hold for '${episode.kind}' failed (retried next sweep): ${err instanceof Error ? err.message : String(err)}`);
|
|
140
144
|
}
|
|
141
145
|
}
|
|
146
|
+
// PROBE-GATED RESOLUTION (operator ruling 2026-08-28): the recovery
|
|
147
|
+
// transition fires only when the subject's capability probe answers OK.
|
|
148
|
+
// An unreachable probe HOLDS the episode open — the close-refusal is
|
|
149
|
+
// raised ONCE per episode state as a durable event, never an alarm: the
|
|
150
|
+
// original verdict (failures went quiet) was correct; it lacked the
|
|
151
|
+
// probe behind the verdict.
|
|
152
|
+
let resolveProbeHeld = 0;
|
|
153
|
+
for (const { episode, quietMinutes, clearZeroVolumeFlag, probeUrl, probeTimeoutSeconds } of decision.resolveProbeGate) {
|
|
154
|
+
try {
|
|
155
|
+
if (await probeHttpOk(probeUrl, probeTimeoutSeconds)) {
|
|
156
|
+
await resolveEpisode(episode, quietMinutes);
|
|
157
|
+
if (clearZeroVolumeFlag) await clearZeroVolumeHold(episode, zeroVolume.countByKind[episode.kind] ?? 0);
|
|
158
|
+
log.info(`[platform-fault-escalation] RESOLVED '${episode.kind}' — capability probe ${probeUrl} answered OK after ${quietMinutes} quiet min`);
|
|
159
|
+
} else {
|
|
160
|
+
resolveProbeHeld += 1;
|
|
161
|
+
if (await resolveHoldRaisedSince(episode.kind, episode.lastUpdatedAt ?? episode.firstSeenAt)) continue;
|
|
162
|
+
recordEvent({
|
|
163
|
+
kind: "platform.fault.resolve_held",
|
|
164
|
+
payload: {
|
|
165
|
+
event_kind: episode.kind,
|
|
166
|
+
probe: probeUrl,
|
|
167
|
+
quiet_minutes: quietMinutes,
|
|
168
|
+
work_item_id: episode.workItemId,
|
|
169
|
+
},
|
|
170
|
+
});
|
|
171
|
+
log.warn(
|
|
172
|
+
`[platform-fault-escalation] resolve HELD for '${episode.kind}' — capability probe ${probeUrl} did not answer OK; episode stays open (raised once, not an alarm)`,
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
} catch (err) {
|
|
176
|
+
log.error(`[platform-fault-escalation] probe-gated resolve for '${episode.kind}' failed (episode stays open; retried next sweep): ${err instanceof Error ? err.message : String(err)}`);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
142
179
|
for (const { episode, quietMinutes, clearZeroVolumeFlag } of decision.resolve) {
|
|
143
180
|
try {
|
|
144
181
|
await resolveEpisode(episode, quietMinutes);
|
|
@@ -155,7 +192,8 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
155
192
|
payload: {
|
|
156
193
|
watched_kinds: aggregates.length,
|
|
157
194
|
escalated: decision.escalate.length,
|
|
158
|
-
resolved: decision.resolve.length,
|
|
195
|
+
resolved: decision.resolve.length + (decision.resolveProbeGate.length - resolveProbeHeld),
|
|
196
|
+
resolve_probe_held: resolveProbeHeld,
|
|
159
197
|
zero_volume_holds: decision.zeroVolumeHold.length,
|
|
160
198
|
},
|
|
161
199
|
});
|
package/src/install-logging.ts
CHANGED
|
@@ -19,7 +19,6 @@
|
|
|
19
19
|
|
|
20
20
|
import { ManagedCache } from "@teamscala/cache/cache";
|
|
21
21
|
import { createCache } from "@teamscala/cache/create-cache";
|
|
22
|
-
import { createNotifyListener, getDirectDatabaseUrl } from "@teamscala/db/notify-listener/index";
|
|
23
22
|
import { createDiskCache } from "@teamscala/cache/disk-cache";
|
|
24
23
|
import { join } from "node:path";
|
|
25
24
|
import { userHome } from "@teamscala/os/host-paths";
|
|
@@ -181,5 +180,8 @@ function safeStringify(v: unknown): string {
|
|
|
181
180
|
return String(v);
|
|
182
181
|
}
|
|
183
182
|
}
|
|
184
|
-
//
|
|
185
|
-
|
|
183
|
+
// No event-driven invalidation subscription here: this module caches nothing.
|
|
184
|
+
// The removed block opened a dedicated LISTEN connection whose handler was
|
|
185
|
+
// empty `() => {}` — a connection held for the process lifetime to run no
|
|
186
|
+
// code. Registering a no-op against the shared listener would be the same
|
|
187
|
+
// defect one layer down, so it is deleted rather than migrated.
|
|
@@ -58,10 +58,11 @@ export async function mountServerPackages(
|
|
|
58
58
|
app: Hono,
|
|
59
59
|
config: ServiceInstallConfig,
|
|
60
60
|
serviceSlug: string,
|
|
61
|
+
getSharedDb?: () => unknown,
|
|
61
62
|
): Promise<MountRegistry> {
|
|
62
63
|
const log = getAppLogger();
|
|
63
64
|
const mountRegistry = createMountRegistry();
|
|
64
|
-
const mountCtx = createMountContext(serviceSlug, config, app, mountRegistry);
|
|
65
|
+
const mountCtx = createMountContext(serviceSlug, config, app, mountRegistry, getSharedDb);
|
|
65
66
|
const serverPackages = (config.server_packages ?? []) as string[];
|
|
66
67
|
|
|
67
68
|
if (serverPackages.length === 0) {
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
|
|
31
31
|
import { getClient } from "@teamscala/object-storage/create/client/get-client";
|
|
32
32
|
import { createCache } from "@teamscala/cache/create-cache";
|
|
33
|
-
import {
|
|
33
|
+
import { registerCacheInvalidation } from "@teamscala/db/notify-listener/index";
|
|
34
34
|
import { getServerClient } from "@teamscala/orpc/server-client-registry";
|
|
35
35
|
import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
|
|
36
36
|
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
@@ -290,11 +290,9 @@ export function buildResourceNotFoundResponse(
|
|
|
290
290
|
});
|
|
291
291
|
}
|
|
292
292
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
}
|
|
300
|
-
}
|
|
293
|
+
// Shares the one process-wide LISTEN connection rather than opening its own:
|
|
294
|
+
// `createNotifyListener` is a factory, so a module-scope call holds a
|
|
295
|
+
// connection for the process lifetime (see resolution-caches.ts).
|
|
296
|
+
registerCacheInvalidation(() => {
|
|
297
|
+
logoCache.clear();
|
|
298
|
+
});
|