@teamscala/service-boot 0.23.516 → 0.23.518
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +232 -1127
- package/src/agreement-gate/settings.ts +8 -8
- package/src/apply-logger-config.test.ts +46 -0
- package/src/apply-logger-config.ts +31 -0
- package/src/asset-proxy.test.ts +9 -5
- package/src/auth/sign-out.ts +122 -0
- package/src/build-client-assets/hydrate-bundle.ts +14 -4
- package/src/build-client-assets/manifest.ts +5 -18
- package/src/build-client-assets/styles-build.test.ts +11 -8
- package/src/build-client-assets/styles-build.ts +14 -63
- package/src/build-client-assets.test.ts +0 -161
- package/src/build-client-assets.ts +20 -67
- package/src/content-gate/token.ts +9 -2
- package/src/install-logging.ts +10 -26
- package/src/mount-orpc-router.ts +8 -2
- package/src/mutable-app-ref.ts +0 -7
- package/src/page-server-loader-graph.test.ts +31 -0
- package/src/page-server-loader-graph.ts +21 -0
- package/src/pg-boss-mount.ts +3 -16
- package/src/readiness-gate.ts +0 -4
- package/src/register-api-routes.ts +8 -17
- package/src/register-page-server-client.ts +6 -0
- package/src/register-registry-page-route/meta-title.test.ts +50 -0
- package/src/register-registry-page-route/meta-title.ts +1 -1
- package/src/register-registry-page-route/page-data-shared.ts +7 -11
- package/src/register-registry-page-route/resolve-request-graph.test.ts +1 -1
- package/src/register-registry-page-route/resolve-request-graph.ts +1 -2
- package/src/register-registry-page-route.ts +14 -16
- package/src/resolve-brand-favicon.test.ts +63 -0
- package/src/resolve-theme-mode.test.ts +51 -0
- package/src/resolvers/fetch-org-brand.test.ts +48 -0
- package/src/resolvers/website-by-host-map.test.ts +63 -0
- package/src/resolvers/website-by-host-map.ts +15 -3
- package/src/run-boot-hook-list.test.ts +47 -0
- package/src/run-boot-hooks.test.ts +56 -0
- package/src/scala-tools-currency-watchdog.test.ts +43 -0
- package/src/scala-tools-currency-watchdog.ts +0 -5
- package/src/transfers/presigned-upload.ts +165 -0
- package/src/two-phase-minimal-app.test.ts +24 -0
- package/src/two-phase-readiness.test.ts +102 -0
- package/src/types.test.ts +20 -0
- package/src/ui-service.test.ts +24 -0
- package/src/ws-boot-self-test.test.ts +43 -0
- package/src/build-client-assets/input-fingerprint.ts +0 -92
- package/src/build-client-assets/input-hashing.ts +0 -206
- package/src/handlers/bun-stable-watch.ts +0 -86
- package/src/handlers/messaging-send-reconcile-logic.test.ts +0 -79
- package/src/handlers/messaging-send-reconcile-logic.ts +0 -83
- package/src/handlers/messaging-send-recording-logic.test.ts +0 -86
- package/src/handlers/messaging-send-recording-logic.ts +0 -68
- package/src/handlers/messaging-send-recording-watch.test.ts +0 -91
- package/src/handlers/messaging-send-recording-watch.ts +0 -244
- package/src/handlers/messaging-send-sweep-selection.test.ts +0 -47
- package/src/handlers/messaging-send-sweep-selection.ts +0 -35
- package/src/handlers/platform-fault-escalation/config.ts +0 -95
- package/src/handlers/platform-fault-escalation/delivery.ts +0 -327
- package/src/handlers/platform-fault-escalation/event-log-queries.ts +0 -182
- package/src/handlers/platform-fault-escalation/orpc-client.ts +0 -114
- package/src/handlers/platform-fault-escalation/probe.test.ts +0 -63
- package/src/handlers/platform-fault-escalation/probe.ts +0 -19
- package/src/handlers/platform-fault-escalation-logic.ts +0 -382
- package/src/handlers/platform-fault-escalation-refresh.test.ts +0 -72
- package/src/handlers/platform-fault-escalation.test.ts +0 -497
- package/src/handlers/platform-fault-escalation.ts +0 -246
- package/src/handlers/projects-digest.ts +0 -198
- package/src/handlers/quiz-research.ts +0 -178
- package/src/handlers/registry-stale-inactive-sweep.ts +0 -82
- package/src/handlers/scala-tools-currency-watch.test.ts +0 -65
- package/src/handlers/scala-tools-currency-watch.ts +0 -162
- package/src/handlers/singleton-tools-currency-watch.test.ts +0 -51
- package/src/handlers/singleton-tools-currency-watch.ts +0 -380
- package/src/handlers/sweep-event-log.ts +0 -98
- package/src/handlers/upstream-issue-sync.ts +0 -220
- package/src/handlers/whatsapp-liveness-handler.test.ts +0 -56
- package/src/handlers/whatsapp-liveness-logic.ts +0 -53
- package/src/handlers/whatsapp-liveness.test.ts +0 -78
- package/src/handlers/whatsapp-liveness.ts +0 -138
- package/src/interpolate-meta.ts +0 -123
- package/src/load-service-prisma-client.ts +0 -53
- package/src/loader-errors.ts +0 -41
- package/src/read-viewport-from-request.ts +0 -36
- package/src/register-page-server-client.test.ts +0 -185
- package/src/resolve-app-base-url.ts +0 -35
- package/src/resolve-from-loader-config.test.ts +0 -429
- package/src/resolve-from-loader-config.ts +0 -316
- package/src/resolve-service-context.ts +0 -88
|
@@ -1,114 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* The system-authed in-process ORPC client of the platform-fault-escalation
|
|
7
|
-
* handler (lazy, built from the composed appRouter) and the system identity
|
|
8
|
-
* it falls back to.
|
|
9
|
-
*
|
|
10
|
-
* Split out of handlers/platform-fault-escalation.ts (500 lines) —
|
|
11
|
-
* byte-for-byte move; import paths adjusted only.
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* The system-authed in-process ORPC client, built LAZILY on first use from the
|
|
16
|
-
* SAME composed appRouter the /api/rpc mount uses (getAppRouter — never a raw
|
|
17
|
-
* re-import of the generated routers, per register-page-server-client's
|
|
18
|
-
* documented module-identity trap). Built here rather than read from
|
|
19
|
-
* page-server-orpc/server-client because a pg-boss handler can fire on a
|
|
20
|
-
* BACKEND service, where the page-server client is never registered — the
|
|
21
|
-
* empty-router fallback returns a client with no model namespaces (the
|
|
22
|
-
* TypeError this replaced). Per `no-direct-prisma-outside-orpc`, every WRITE
|
|
23
|
-
* in this handler routes through this client; the two raw reads (the
|
|
24
|
-
* event_log recurrence aggregate and the latest-payload sample, shapes no
|
|
25
|
-
* generated CRUD expresses) use
|
|
26
|
-
* the assigned-client form with $queryRawUnsafe. Timestamps are ISO strings,
|
|
27
|
-
* never Date instances (the generated valibot schemas require it).
|
|
28
|
-
*/
|
|
29
|
-
|
|
30
|
-
import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
|
|
31
|
-
import * as v from "valibot";
|
|
32
|
-
import { createCache } from "@teamscala/cache/create-cache";
|
|
33
|
-
|
|
34
|
-
const systemIdentityCache = createCache<{
|
|
35
|
-
userId: string | undefined;
|
|
36
|
-
organisationId: string | undefined;
|
|
37
|
-
}>("service-boot:fault-escalation-system-identity", {
|
|
38
|
-
ttlMs: Number.POSITIVE_INFINITY,
|
|
39
|
-
maxSize: 1,
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
export async function getSystemIdentity(): Promise<{ userId: string | undefined; organisationId: string | undefined }> {
|
|
43
|
-
const hit = systemIdentityCache.get("default");
|
|
44
|
-
if (hit) return hit;
|
|
45
|
-
const row = await loadRegistryConfig("config", "system-identity", v.looseObject({}));
|
|
46
|
-
const identity = {
|
|
47
|
-
userId: row?.system_user_id as string | undefined,
|
|
48
|
-
organisationId: row?.system_org_id as string | undefined,
|
|
49
|
-
};
|
|
50
|
-
systemIdentityCache.set("default", identity);
|
|
51
|
-
return identity;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
const orpcSurfaceCache = createCache<OrpcSurface>(
|
|
55
|
-
"service-boot:fault-escalation-orpc",
|
|
56
|
-
{ ttlMs: Number.POSITIVE_INFINITY, maxSize: 1 },
|
|
57
|
-
);
|
|
58
|
-
|
|
59
|
-
export async function orpc(): Promise<OrpcSurface> {
|
|
60
|
-
const hit = orpcSurfaceCache.get("default");
|
|
61
|
-
if (hit) return hit;
|
|
62
|
-
const { createRouterClient } = await import("@orpc/server");
|
|
63
|
-
const { getAppRouter } = await import("@teamscala/orpc/app-router");
|
|
64
|
-
const appRouter = (await getAppRouter()) as Record<string, unknown> | null;
|
|
65
|
-
if (!appRouter) {
|
|
66
|
-
throw new Error(
|
|
67
|
-
"platform-fault-escalation: no ORPC appRouter — the service has no generated routers to write through",
|
|
68
|
-
);
|
|
69
|
-
}
|
|
70
|
-
const prismaRegistry = await import("@teamscala/db/prisma-registry");
|
|
71
|
-
const registryLoader = await import("@teamscala/db/registry/load-config");
|
|
72
|
-
const { PlatformIdentitySchema } = await import(
|
|
73
|
-
"@teamscala/db-validation/registry-schemas/platform-identity"
|
|
74
|
-
);
|
|
75
|
-
const platformIdentity = await registryLoader.loadRegistryConfig(
|
|
76
|
-
"config",
|
|
77
|
-
"platform-identity",
|
|
78
|
-
PlatformIdentitySchema,
|
|
79
|
-
);
|
|
80
|
-
const surface = createRouterClient(
|
|
81
|
-
{ ...appRouter } as never,
|
|
82
|
-
{
|
|
83
|
-
context: {
|
|
84
|
-
prisma: prismaRegistry.getPrisma(),
|
|
85
|
-
user: {
|
|
86
|
-
id: "system",
|
|
87
|
-
isSuperAdmin: true,
|
|
88
|
-
organisationId: platformIdentity.platformOrganisationId,
|
|
89
|
-
},
|
|
90
|
-
},
|
|
91
|
-
} as never,
|
|
92
|
-
) as unknown as OrpcSurface;
|
|
93
|
-
orpcSurfaceCache.set("default", surface);
|
|
94
|
-
return surface;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
/** The router subset this handler uses — mirrors the generated procedure args. */
|
|
98
|
-
export interface OrpcSurface {
|
|
99
|
-
platform_fault_escalations: {
|
|
100
|
-
upsert(args: {
|
|
101
|
-
where: { kind: string };
|
|
102
|
-
create: Record<string, unknown>;
|
|
103
|
-
update: Record<string, unknown>;
|
|
104
|
-
}): Promise<unknown>;
|
|
105
|
-
update(args: { where: { kind: string }; data: Record<string, unknown> }): Promise<unknown>;
|
|
106
|
-
findMany(args?: { where?: Record<string, unknown> }): Promise<Array<Record<string, unknown>>>;
|
|
107
|
-
};
|
|
108
|
-
messaging_bots: {
|
|
109
|
-
findFirst(args: Record<string, unknown>): Promise<Record<string, unknown> | null>;
|
|
110
|
-
};
|
|
111
|
-
comments: {
|
|
112
|
-
create(args: { data: Record<string, unknown> }): Promise<{ id: string }>;
|
|
113
|
-
};
|
|
114
|
-
}
|
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
// @system codegen
|
|
2
|
-
// @status generated
|
|
3
|
-
// @edit change the suite in the owned-suites band, then re-run codegen. Hand-edits are overwritten.
|
|
4
|
-
//
|
|
5
|
-
// This suite's assertions are OWNED by the codegen band: the band module
|
|
6
|
-
// carries them verbatim, this file is the emission, and hand edits here are
|
|
7
|
-
// overwritten on the next run. The rationale each assertion carries moved
|
|
8
|
-
// with it into the band.
|
|
9
|
-
|
|
10
|
-
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
11
|
-
import { probeHttpOk } from "./probe.ts";
|
|
12
|
-
|
|
13
|
-
describe("probeHttpOk — the resolve gate's instrument", () => {
|
|
14
|
-
let server: Bun.Server<undefined>;
|
|
15
|
-
let port = 0;
|
|
16
|
-
|
|
17
|
-
beforeAll(() => {
|
|
18
|
-
server = Bun.serve({
|
|
19
|
-
port: 0,
|
|
20
|
-
fetch(request) {
|
|
21
|
-
const url = new URL(request.url);
|
|
22
|
-
if (url.pathname === "/ok") return new Response('{"status":"ok"}');
|
|
23
|
-
if (url.pathname === "/degraded") return new Response('{"status":"degraded"}', { status: 503 });
|
|
24
|
-
return new Response("not found", { status: 404 });
|
|
25
|
-
},
|
|
26
|
-
});
|
|
27
|
-
port = server.port as number;
|
|
28
|
-
});
|
|
29
|
-
|
|
30
|
-
afterAll(() => {
|
|
31
|
-
server.stop(true);
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
test("an answering, healthy endpoint verifies", async () => {
|
|
35
|
-
expect(await probeHttpOk(`http://127.0.0.1:${port}/ok`)).toBe(true);
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
test("a subject answering DEGRADED must not read as healthy — 503 is not an OK", async () => {
|
|
39
|
-
// THE MEASURED SHAPE: the 2026-08-28 close-on-quiet fired while the
|
|
40
|
-
// pipeline was down. A probe that treated "any response" as health
|
|
41
|
-
// would resolve an episode whose subject is still answering
|
|
42
|
-
// "degraded" — the same verdict with no instrument behind it.
|
|
43
|
-
expect(await probeHttpOk(`http://127.0.0.1:${port}/degraded`)).toBe(false);
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
test("a subject that does not answer the probe's question must not read as healthy", async () => {
|
|
47
|
-
expect(await probeHttpOk(`http://127.0.0.1:${port}/missing`)).toBe(false);
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
test("a subject that cannot be reached at all must not read as healthy", async () => {
|
|
51
|
-
// Port 1 on loopback: nothing listens there; the probe must refuse
|
|
52
|
-
// promptly, not hang or throw.
|
|
53
|
-
expect(await probeHttpOk("http://127.0.0.1:1/health", 1)).toBe(false);
|
|
54
|
-
});
|
|
55
|
-
|
|
56
|
-
test("an unreachable subject must not read as healthy", async () => {
|
|
57
|
-
expect(await probeHttpOk("http://127.0.0.1:1/health", 1)).toBe(false);
|
|
58
|
-
});
|
|
59
|
-
|
|
60
|
-
test("an unreachable subject with a tiny timeout still refuses, never throws", async () => {
|
|
61
|
-
expect(await probeHttpOk("http://10.255.255.1:9/health", 1)).toBe(false);
|
|
62
|
-
});
|
|
63
|
-
});
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* The capability probe behind the resolve gate (operator ruling 2026-08-28):
|
|
7
|
-
* a recovery transition is backed by an ACTIVE question to the subject's
|
|
8
|
-
* capability surface, never a timer over an empty event stream. Healthy
|
|
9
|
-
* means HTTP OK (2xx) — llm-router /health answers 503 while degraded.
|
|
10
|
-
*/
|
|
11
|
-
|
|
12
|
-
export async function probeHttpOk(url: string, timeoutSeconds = 5): Promise<boolean> {
|
|
13
|
-
try {
|
|
14
|
-
const response = await fetch(url, { signal: AbortSignal.timeout(timeoutSeconds * 1000) });
|
|
15
|
-
return response.ok;
|
|
16
|
-
} catch {
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
}
|
|
@@ -1,382 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* Pure decision logic for the platform-fault escalation sweep — full
|
|
7
|
-
* contract in curated `apps/escalation.md`. Episodes emit on TRANSITIONS
|
|
8
|
-
* ONLY; a future-dated `lastSeen` is never recurrence evidence.
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
export interface FaultAggregate {
|
|
12
|
-
kind: string;
|
|
13
|
-
count: number;
|
|
14
|
-
firstSeen: Date;
|
|
15
|
-
lastSeen: Date;
|
|
16
|
-
/** Id of the newest row backing `lastSeen`, when the caller resolved it —
|
|
17
|
-
* the resolvable handle the escalation cites so "last at T" is checkable
|
|
18
|
-
* against a real event_log row, not a computed or cached value. */
|
|
19
|
-
lastEventId?: string | null;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export interface EscalationEpisode {
|
|
23
|
-
kind: string;
|
|
24
|
-
firstSeenAt: Date;
|
|
25
|
-
lastSeenAt: Date;
|
|
26
|
-
eventCount: number;
|
|
27
|
-
escalatedAt: Date | null;
|
|
28
|
-
workItemId: string | null;
|
|
29
|
-
/** VERIFIED over — a capability probe answered OK, or a declared recovery
|
|
30
|
-
* transition arrived. Never written from silence. */
|
|
31
|
-
resolvedAt: Date | null;
|
|
32
|
-
/** The kind went quiet and the alarm is ARMED AGAIN. This is all that
|
|
33
|
-
* quiet ever honestly supported, and it is deliberately weaker than
|
|
34
|
-
* `resolvedAt`: it claims nothing about the condition. */
|
|
35
|
-
rearmedAt: Date | null;
|
|
36
|
-
lastUpdatedAt: Date | null;
|
|
37
|
-
deliveredCommentId?: string | null;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export interface FaultEscalationPolicy {
|
|
41
|
-
/** Recurrence floor: minimum event count inside the window. */
|
|
42
|
-
minCount: number;
|
|
43
|
-
/** Duration floor: minimum first→last spread inside the window, minutes. */
|
|
44
|
-
minSpanMinutes: number;
|
|
45
|
-
/** Quiet period that closes an episode and re-arms escalation, minutes. */
|
|
46
|
-
resolveAfterMinutes: number;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
/** Per-kind threshold overrides — a low-rate-but-persistent kind (guard.fail.open
|
|
50
|
-
* at ~1/hour over 12h) has different noise characteristics than a bursty one, so
|
|
51
|
-
* the recurrence/duration floors are per-kind policy in the config row. */
|
|
52
|
-
export interface KindThresholdOverride {
|
|
53
|
-
minCount?: number;
|
|
54
|
-
minSpanMinutes?: number;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export interface SweepDecision {
|
|
58
|
-
/** Sustained faults needing a FIRST delivery (new episode, or a claim whose send failed). */
|
|
59
|
-
escalate: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode | null }>;
|
|
60
|
-
/** Open episodes whose kind went quiet past the resolve window — close them.
|
|
61
|
-
* ONLY reachable from the probe gate: quiet alone never populates this. */
|
|
62
|
-
resolve: Array<{ episode: EscalationEpisode; quietMinutes: number; clearZeroVolumeFlag: boolean }>;
|
|
63
|
-
/** Open episodes quiet past the resolve window whose kind declares NO
|
|
64
|
-
* capability probe: the alarm is RE-ARMED (the next occurrence escalates
|
|
65
|
-
* again) and nothing is claimed about the condition. Quiet used to write
|
|
66
|
-
* `resolved_at` and post "Fault stopped", which is the unsound close
|
|
67
|
-
* `emit-transitions-not-conditions` names. It is re-armed rather than
|
|
68
|
-
* merely held open because `escalate` fires only for an absent, resolved
|
|
69
|
-
* or re-armed episode — holding it open would silence the alarm for good.
|
|
70
|
-
* Rationale + measurements: migration 811. */
|
|
71
|
-
rearm: Array<{ episode: EscalationEpisode; quietMinutes: number; clearZeroVolumeFlag: boolean }>;
|
|
72
|
-
/** Open episodes quiet past the resolve window whose producer ALSO went
|
|
73
|
-
* silent: zero failure rows AND zero call volume that would produce them.
|
|
74
|
-
* Quiet is NOT evidence of health here (measured 2026-08-27: llm-router
|
|
75
|
-
* died and produced zero llm.provider.call.failed events, which the
|
|
76
|
-
* sweep closed as "no failures for 30 min" while crush lanes had no
|
|
77
|
-
* model backend at all). The episode HOLDS OPEN and the hold raises
|
|
78
|
-
* once; it never resolves from silence alone. */
|
|
79
|
-
zeroVolumeHold: Array<{ episode: EscalationEpisode; quietMinutes: number; volumeCount: number }>;
|
|
80
|
-
/** Open episodes quiet past the resolve window whose kind declares a
|
|
81
|
-
* CAPABILITY PROBE: the close is withheld until the probe answers OK.
|
|
82
|
-
* The handler runs the probe and either resolves or holds — the decision
|
|
83
|
-
* core stays pure (it selects WHO must be probed, it never probes). */
|
|
84
|
-
resolveProbeGate: Array<{
|
|
85
|
-
episode: EscalationEpisode;
|
|
86
|
-
quietMinutes: number;
|
|
87
|
-
clearZeroVolumeFlag: boolean;
|
|
88
|
-
probeUrl: string;
|
|
89
|
-
probeTimeoutSeconds: number;
|
|
90
|
-
}>;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
/** Per-kind config naming the kind whose VOLUME proves the producing
|
|
94
|
-
* pipeline is alive — e.g. failures pair with completed calls. A fault kind
|
|
95
|
-
* without a source resolves on quiet exactly as before; with one, resolve
|
|
96
|
-
* additionally requires the volume floor to be met inside the window. */
|
|
97
|
-
export interface ZeroVolumeSource {
|
|
98
|
-
volume_kind_pattern: string;
|
|
99
|
-
min_volume?: number;
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/** Per-kind config naming the capability probe that must answer OK before
|
|
103
|
-
* this fault kind's episode may close. The probe is an ACTIVE question to
|
|
104
|
-
* the subject's capability surface (HTTP OK), never a read of the event
|
|
105
|
-
* stream — absence of events is never evidence of state. */
|
|
106
|
-
export interface ProbeSource {
|
|
107
|
-
url: string;
|
|
108
|
-
timeout_seconds?: number;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
/** Everything decideSweep needs to judge quiet-vs-healthy, computed by the
|
|
112
|
-
* caller from the valibot-validated config row + bounded queries. */
|
|
113
|
-
export interface ZeroVolumeInputs {
|
|
114
|
-
/** Configured sources keyed by watched kind. */
|
|
115
|
-
sources: Record<string, ZeroVolumeSource>;
|
|
116
|
-
/** Resolved volume count per watched kind over the window. */
|
|
117
|
-
countByKind: Record<string, number>;
|
|
118
|
-
/** Whether a flagged/cleared marker already stands for the kind (derived
|
|
119
|
-
* from event_log by the caller — never process memory). */
|
|
120
|
-
flagActiveByKind: Record<string, boolean>;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
function minutesBetween(a: Date, b: Date): number {
|
|
124
|
-
return (b.getTime() - a.getTime()) / 60_000;
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
function isSustained(
|
|
128
|
-
aggregate: FaultAggregate,
|
|
129
|
-
policy: FaultEscalationPolicy,
|
|
130
|
-
override: KindThresholdOverride | undefined,
|
|
131
|
-
now: Date,
|
|
132
|
-
): boolean {
|
|
133
|
-
const minCount = override?.minCount ?? policy.minCount;
|
|
134
|
-
const minSpan = override?.minSpanMinutes ?? policy.minSpanMinutes;
|
|
135
|
-
// Recency, not merely presence in the lookback window. Count and span
|
|
136
|
-
// answer "did this happen"; a fault qualifies only while it is STILL
|
|
137
|
-
// happening. Without this the 720-minute lookback keeps a dead fault
|
|
138
|
-
// qualifying for hours, and since a resolved episode reads as no episode
|
|
139
|
-
// it re-escalates on every sweep until its events age out (2026-08-26).
|
|
140
|
-
// A future `lastSeen` (producer clock skew / mocked test clock, measured
|
|
141
|
-
// 2026-08-26: "still recurring" citing a timestamp 44 min in its own
|
|
142
|
-
// future) makes minutesBetween go NEGATIVE — refuse it as evidence.
|
|
143
|
-
return (
|
|
144
|
-
aggregate.count >= minCount &&
|
|
145
|
-
minutesBetween(aggregate.firstSeen, aggregate.lastSeen) >= minSpan &&
|
|
146
|
-
aggregate.lastSeen.getTime() <= now.getTime() &&
|
|
147
|
-
minutesBetween(aggregate.lastSeen, now) < policy.resolveAfterMinutes
|
|
148
|
-
);
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
/**
|
|
152
|
-
* Classify the current sweep. `aggregates` are the per-kind error aggregates
|
|
153
|
-
* from the lookback window; `episodes` are the existing episode rows for those
|
|
154
|
-
* kinds PLUS every still-open episode (an open episode whose kind no longer
|
|
155
|
-
* appears in the window at all is the strongest resolve candidate).
|
|
156
|
-
*/
|
|
157
|
-
export function decideSweep(
|
|
158
|
-
aggregates: FaultAggregate[],
|
|
159
|
-
episodes: EscalationEpisode[],
|
|
160
|
-
policy: FaultEscalationPolicy,
|
|
161
|
-
now: Date,
|
|
162
|
-
kindThresholds: Record<string, KindThresholdOverride> = {},
|
|
163
|
-
zeroVolume?: ZeroVolumeInputs,
|
|
164
|
-
probeSources: Record<string, ProbeSource> = {},
|
|
165
|
-
): SweepDecision {
|
|
166
|
-
const byKind = new Map(episodes.map((e) => [e.kind, e]));
|
|
167
|
-
const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
|
|
168
|
-
const decision: SweepDecision =
|
|
169
|
-
{ escalate: [], resolve: [], rearm: [], resolveProbeGate: [], zeroVolumeHold: [] };
|
|
170
|
-
|
|
171
|
-
for (const aggregate of aggregates) {
|
|
172
|
-
const episode = byKind.get(aggregate.kind);
|
|
173
|
-
if (!isSustained(aggregate, policy, kindThresholds[aggregate.kind], now)) continue;
|
|
174
|
-
// No episode, or the previous episode closed (quiet past resolve window):
|
|
175
|
-
// a NEW episode begins and must be delivered.
|
|
176
|
-
// A RE-ARMED episode starts a new one exactly as a resolved one does.
|
|
177
|
-
// This is the half that keeps the alarm audible: without it, dropping
|
|
178
|
-
// the quiet-close would leave every probe-less kind permanently silent.
|
|
179
|
-
if (!episode || episode.resolvedAt || episode.rearmedAt) {
|
|
180
|
-
decision.escalate.push({ aggregate, episode: episode ?? null });
|
|
181
|
-
continue;
|
|
182
|
-
}
|
|
183
|
-
// Open episode with no verified delivery — the claim is retried, never
|
|
184
|
-
// counted as escalated (a component may not report a state it has not
|
|
185
|
-
// verified).
|
|
186
|
-
if (!episode.escalatedAt) {
|
|
187
|
-
decision.escalate.push({ aggregate, episode });
|
|
188
|
-
}
|
|
189
|
-
// An open, delivered episode emits NOTHING while it recurs: the open
|
|
190
|
-
// emission already happened and the close emission fires when the kind
|
|
191
|
-
// goes quiet. A cadence-driven "still recurring" repeat is exactly the
|
|
192
|
-
// silenced-alarm shape `emit-transitions-not-conditions` prohibits.
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
for (const episode of episodes) {
|
|
196
|
-
if (episode.resolvedAt || episode.rearmedAt) continue;
|
|
197
|
-
const aggregate = aggregateByKind.get(episode.kind);
|
|
198
|
-
// A future-dated aggregate lastSeen (producer clock skew) cannot serve
|
|
199
|
-
// as the quiet reference — it would freeze quietMinutes negative and
|
|
200
|
-
// hold the episode open forever while the kind is actually quiet. Fall
|
|
201
|
-
// back to the episode's own last_seen_at, which the bounded aggregate
|
|
202
|
-
// wrote and is therefore never in the future.
|
|
203
|
-
const quietReference =
|
|
204
|
-
aggregate && aggregate.lastSeen.getTime() <= now.getTime() ? aggregate.lastSeen : episode.lastSeenAt;
|
|
205
|
-
const quietMinutes = minutesBetween(quietReference, now);
|
|
206
|
-
if (quietMinutes < policy.resolveAfterMinutes) continue;
|
|
207
|
-
|
|
208
|
-
// ZERO-VOLUME GATE on resolution. When this fault kind declares a
|
|
209
|
-
// volume source and that volume is below its floor inside the window,
|
|
210
|
-
// "no failure rows" no longer separates healthy from dead-producer:
|
|
211
|
-
// hold the episode open instead of closing it. The hold raises once
|
|
212
|
-
// (when unflagged) through zeroVolumeHold; a flagged hold emits
|
|
213
|
-
// nothing while it persists — same transitions-only rule as an open
|
|
214
|
-
// escalation.
|
|
215
|
-
const source = zeroVolume?.sources[episode.kind];
|
|
216
|
-
const volumeCount = zeroVolume?.countByKind[episode.kind];
|
|
217
|
-
if (source) {
|
|
218
|
-
const floor = source.min_volume ?? 1;
|
|
219
|
-
if ((volumeCount ?? 0) < floor) {
|
|
220
|
-
decision.zeroVolumeHold.push({ episode, quietMinutes, volumeCount: volumeCount ?? 0 });
|
|
221
|
-
continue;
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
// PROBE-GATED RESOLUTION (operator ruling 2026-08-28): a kind with a
|
|
225
|
-
// declared capability probe does not close on quiet + volume alone —
|
|
226
|
-
// the handler must first ask the subject's capability surface and
|
|
227
|
-
// only an HTTP OK closes the episode. An unreachable probe holds the
|
|
228
|
-
// episode open (once per episode-state, never an alarm).
|
|
229
|
-
const probe = probeSources[episode.kind];
|
|
230
|
-
if (probe) {
|
|
231
|
-
decision.resolveProbeGate.push({
|
|
232
|
-
episode,
|
|
233
|
-
quietMinutes,
|
|
234
|
-
clearZeroVolumeFlag: Boolean(source && zeroVolume?.flagActiveByKind[episode.kind]),
|
|
235
|
-
probeUrl: probe.url,
|
|
236
|
-
probeTimeoutSeconds: probe.timeout_seconds ?? 5,
|
|
237
|
-
});
|
|
238
|
-
continue;
|
|
239
|
-
}
|
|
240
|
-
// NO PROBE DECLARED: re-arm, never resolve. Quiet re-arms the alarm and
|
|
241
|
-
// says nothing about the condition — see the `rearm` field's comment.
|
|
242
|
-
decision.rearm.push({
|
|
243
|
-
episode,
|
|
244
|
-
quietMinutes,
|
|
245
|
-
clearZeroVolumeFlag: Boolean(source && zeroVolume?.flagActiveByKind[episode.kind]),
|
|
246
|
-
});
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
return decision;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
/** Human "N min" span, rounded — for the phone-sized escalation message. */
|
|
253
|
-
/**
|
|
254
|
-
* Do this OPEN episode's stored window facts lag the live aggregate?
|
|
255
|
-
*
|
|
256
|
-
* Both NEGATIVE answers carry weight: a quiet kind keeps its last true
|
|
257
|
-
* sighting (the value resolve falls back to), and an aggregate no newer than
|
|
258
|
-
* the row is never written, so a steady-state sweep writes nothing.
|
|
259
|
-
*/
|
|
260
|
-
export function episodeNeedsRefresh(
|
|
261
|
-
episode: Pick<EscalationEpisode, "lastSeenAt">,
|
|
262
|
-
aggregate: { lastSeen: Date } | undefined,
|
|
263
|
-
): boolean {
|
|
264
|
-
if (!aggregate) return false;
|
|
265
|
-
return aggregate.lastSeen.getTime() > episode.lastSeenAt.getTime();
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
/**
|
|
269
|
-
* Does this fault kind page the operator, or land on the work item alone?
|
|
270
|
-
*
|
|
271
|
-
* DEFAULT IS TO PAGE. An unlisted kind reaches the phone, so forgetting to
|
|
272
|
-
* classify a new fault fails loudly rather than silently — the opposite
|
|
273
|
-
* default would let a kind be silenced by omission.
|
|
274
|
-
*/
|
|
275
|
-
export function pagesOperator(kind: string, agentHandledKinds?: string[]): boolean {
|
|
276
|
-
return !(agentHandledKinds ?? []).includes(kind);
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
export function formatSpanMinutes(minutes: number): string {
|
|
280
|
-
if (minutes < 60) return `${Math.round(minutes)} min`;
|
|
281
|
-
const hours = minutes / 60;
|
|
282
|
-
return `${hours >= 10 ? Math.round(hours) : Math.round(hours * 10) / 10} h`;
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
/**
|
|
286
|
-
* The WhatsApp message id from a messaging_whatsapp_send result. The
|
|
287
|
-
* intrinsic (scala-mcp / messaging-mcp-boot) returns
|
|
288
|
-
* `{ ok, message_id, error }` — the Rust scala-messaging-service owns the
|
|
289
|
-
* channel, picks the device, and records the send in messaging_logs itself.
|
|
290
|
-
* Delivery is only VERIFIED on a real id — a `"true"` ack, an error body, or
|
|
291
|
-
* an empty body is not a message id and must not be recorded as delivered.
|
|
292
|
-
*/
|
|
293
|
-
export function extractWhatsAppMessageId(sendResult: unknown): string | null {
|
|
294
|
-
if (sendResult && typeof sendResult === "object") {
|
|
295
|
-
const id = (sendResult as { message_id?: unknown }).message_id;
|
|
296
|
-
if (typeof id === "string" && id.length > 0) return id;
|
|
297
|
-
}
|
|
298
|
-
return null;
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
/**
|
|
302
|
-
* One sampled payload value rendered for the message's evidence line: a
|
|
303
|
-
* kind+count escalation is a pointer to evidence, not evidence (2026-08-26:
|
|
304
|
-
* the llm.provider.call.failed payloads held our own bounded-output contract
|
|
305
|
-
* violations while the escalation's impact line named healthy providers).
|
|
306
|
-
* JSON-encoded when non-string so structure survives, whitespace-collapsed
|
|
307
|
-
* to one line, credential shapes redacted before truncation (a hard cut can
|
|
308
|
-
* never expose the tail of a secret the length limit happened to spare),
|
|
309
|
-
* hard-truncated with an ellipsis so the reader knows the cut happened.
|
|
310
|
-
*/
|
|
311
|
-
export function formatEscalationSample(value: unknown): string {
|
|
312
|
-
const maxChars = 200;
|
|
313
|
-
const redactions: ReadonlyArray<[RegExp, string]> = [
|
|
314
|
-
[/\b(?:sk|gsk|ghp|gho|xox[baprs])[-_][A-Za-z0-9_-]{8,}/g, "[redacted]"],
|
|
315
|
-
[/\b(api[_-]?key|token|password|secret|authorization)\b["'=:\s]+(?:(?:bearer|basic)\s+)?[A-Za-z0-9._~+/=-]{4,}/gi, "$1=[redacted]"],
|
|
316
|
-
[/Bearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]"],
|
|
317
|
-
];
|
|
318
|
-
const raw = typeof value === "string" ? value : (JSON.stringify(value) ?? "");
|
|
319
|
-
if (raw.length === 0) return "";
|
|
320
|
-
const redacted = redactions.reduce((acc, [pattern, replacement]) => acc.replace(pattern, replacement), raw);
|
|
321
|
-
const collapsed = redacted.replace(/\s+/g, " ").trim();
|
|
322
|
-
return collapsed.length > maxChars ? `${collapsed.slice(0, maxChars)}…` : collapsed;
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
/**
|
|
326
|
-
* The escalation message — escalation-routing's three-field contract adapted
|
|
327
|
-
* to an automated detector: what was OBSERVED (evidence + sample), what it
|
|
328
|
-
* MEANS (impact), and the RECOMMENDATION. Short: the channel is a phone. The
|
|
329
|
-
* sample line is the triage evidence — `sample_fields` whitelisted per kind
|
|
330
|
-
* by the caller, formatted by formatEscalationSample; empty means the kind
|
|
331
|
-
* samples nothing or the latest payload carried none of its fields.
|
|
332
|
-
*/
|
|
333
|
-
export function composeEscalationMessage(input: {
|
|
334
|
-
kind: string;
|
|
335
|
-
aggregate: FaultAggregate;
|
|
336
|
-
impact: string;
|
|
337
|
-
recommendation: string;
|
|
338
|
-
sample: string;
|
|
339
|
-
/** ISO timestamp + event id of the newest backing row, when resolved —
|
|
340
|
-
* cited verbatim so the reader can resolve the claim against event_log. */
|
|
341
|
-
lastEvent?: { occurredAt: string; eventId: string } | null;
|
|
342
|
-
}): string {
|
|
343
|
-
const span = formatSpanMinutes(
|
|
344
|
-
(input.aggregate.lastSeen.getTime() - input.aggregate.firstSeen.getTime()) / 60_000,
|
|
345
|
-
);
|
|
346
|
-
const observed = input.lastEvent
|
|
347
|
-
? `Observed: ${input.aggregate.count} errors over ${span}, last ${input.lastEvent.occurredAt} (event ${input.lastEvent.eventId.slice(0, 8)}).`
|
|
348
|
-
: `Observed: ${input.aggregate.count} errors over ${span}, last ${input.aggregate.lastSeen.toISOString()}.`;
|
|
349
|
-
const lines = [`🔴 Platform fault: ${input.kind}`, observed];
|
|
350
|
-
if (input.sample) lines.push(`Sample: ${input.sample}`);
|
|
351
|
-
lines.push(`Impact: ${input.impact}`, `Recommendation: ${input.recommendation}`);
|
|
352
|
-
return lines.join("\n");
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
/**
|
|
356
|
-
* The zero-volume hold notice — the same three-field contract, for the shape
|
|
357
|
-
* where the watched failures went quiet AND the call volume that would have
|
|
358
|
-
* produced them went quiet too. That combination is NOT a resolution: it is
|
|
359
|
-
* either "fault ended" or "the producer itself died" (2026-08-27: the dead
|
|
360
|
-
* llm-router emitted zero llm.provider.call.failed rows and the sweep closed
|
|
361
|
-
* the episode as healthy while crush lanes had no model backend). Holds one
|
|
362
|
-
* episode open; raises once per episode-state, never on cadence.
|
|
363
|
-
*/
|
|
364
|
-
export function composeZeroVolumeMessage(input: {
|
|
365
|
-
kind: string;
|
|
366
|
-
volumeKindPattern: string;
|
|
367
|
-
minVolume: number;
|
|
368
|
-
quietMinutes: number;
|
|
369
|
-
volumeCount: number;
|
|
370
|
-
firstSeenAt: Date;
|
|
371
|
-
}): string {
|
|
372
|
-
const observed =
|
|
373
|
-
`Observed: '${input.kind}' open since ${input.firstSeenAt.toISOString()}, ` +
|
|
374
|
-
`quiet ${Math.round(input.quietMinutes)} min — but volume kind ~${input.volumeKindPattern} is ALSO silent ` +
|
|
375
|
-
`(${input.volumeCount} < floor ${input.minVolume} in window). Quiet is indistinguishable from dead.`;
|
|
376
|
-
return [
|
|
377
|
-
`🟡 Platform fault unresolved: ${input.kind}`,
|
|
378
|
-
observed,
|
|
379
|
-
"Impact: failures stopping means EITHER recovery OR its producer died entirely — this exact ambiguity hid llm-router's halt behind an all-clear.",
|
|
380
|
-
`Recommendation: probe/restart the producer of ${input.volumeKindPattern} directly; resolve the episode by hand only once completion is verified.`,
|
|
381
|
-
].join("\n");
|
|
382
|
-
}
|
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
// @system codegen
|
|
2
|
-
// @status generated
|
|
3
|
-
// @edit change the suite in the owned-suites band, then re-run codegen. Hand-edits are overwritten.
|
|
4
|
-
//
|
|
5
|
-
// This suite's assertions are OWNED by the codegen band: the band module
|
|
6
|
-
// carries them verbatim, this file is the emission, and hand edits here are
|
|
7
|
-
// overwritten on the next run. The rationale each assertion carries moved
|
|
8
|
-
// with it into the band.
|
|
9
|
-
|
|
10
|
-
import { describe, expect, test } from "bun:test";
|
|
11
|
-
import { episodeNeedsRefresh, pagesOperator } from "./platform-fault-escalation-logic.ts";
|
|
12
|
-
|
|
13
|
-
const at = (iso: string) => ({ lastSeenAt: new Date(iso) });
|
|
14
|
-
const seen = (iso: string) => ({ lastSeen: new Date(iso) });
|
|
15
|
-
|
|
16
|
-
describe("episodeNeedsRefresh", () => {
|
|
17
|
-
test("a still-firing fault whose row has fallen behind is refreshed", () => {
|
|
18
|
-
// The measured incident, in miniature: row frozen at escalation time,
|
|
19
|
-
// events still arriving fifteen hours later.
|
|
20
|
-
expect(episodeNeedsRefresh(at("2026-08-27T12:25:03Z"), seen("2026-08-28T03:00:41Z"))).toBe(
|
|
21
|
-
true,
|
|
22
|
-
);
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
test("a kind that has gone quiet keeps its last true sighting", () => {
|
|
26
|
-
// No aggregate means nothing landed in the window. The stored value must
|
|
27
|
-
// STAND rather than advance, because it is the reference the resolve
|
|
28
|
-
// decision falls back to — advancing it here would hold every quiet
|
|
29
|
-
// episode open forever.
|
|
30
|
-
expect(episodeNeedsRefresh(at("2026-08-27T12:25:03Z"), undefined)).toBe(false);
|
|
31
|
-
});
|
|
32
|
-
|
|
33
|
-
test("an unchanged fault is not written back", () => {
|
|
34
|
-
// Equal timestamps must not write: a sweep over steady state performs
|
|
35
|
-
// zero updates, so the refresh cannot become a per-sweep write storm.
|
|
36
|
-
const same = "2026-08-28T03:00:41Z";
|
|
37
|
-
expect(episodeNeedsRefresh(at(same), seen(same))).toBe(false);
|
|
38
|
-
});
|
|
39
|
-
|
|
40
|
-
test("an aggregate OLDER than the row never rewinds it", () => {
|
|
41
|
-
// A shrinking window can report an older newest-event than the row
|
|
42
|
-
// already holds. Rewinding would fabricate quiet time and resolve a
|
|
43
|
-
// live fault early.
|
|
44
|
-
expect(episodeNeedsRefresh(at("2026-08-28T03:00:41Z"), seen("2026-08-28T01:00:00Z"))).toBe(
|
|
45
|
-
false,
|
|
46
|
-
);
|
|
47
|
-
});
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
describe("pagesOperator", () => {
|
|
51
|
-
// Operator ruling 2026-08-28: "why is the WhatsApp PA messaging me about
|
|
52
|
-
// stuff that you should be doing". A kind a standing lane answers escalates
|
|
53
|
-
// to the work item; the phone is spared.
|
|
54
|
-
const AGENT_HANDLED = ["guard.fail.open", "host-command-bus.command.failed"];
|
|
55
|
-
|
|
56
|
-
test("a kind the maintenance lane answers does not page", () => {
|
|
57
|
-
expect(pagesOperator("guard.fail.open", AGENT_HANDLED)).toBe(false);
|
|
58
|
-
expect(pagesOperator("host-command-bus.command.failed", AGENT_HANDLED)).toBe(false);
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
test("an unlisted kind still pages", () => {
|
|
62
|
-
expect(pagesOperator("supervisor.halt.entered", AGENT_HANDLED)).toBe(true);
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
test("DEFAULT IS TO PAGE when nothing is configured", () => {
|
|
66
|
-
// The load-bearing direction: a kind is silenced only by being named.
|
|
67
|
-
// If the list is missing or empty, every fault reaches the operator, so
|
|
68
|
-
// a new kind can never be silenced by omission.
|
|
69
|
-
expect(pagesOperator("guard.fail.open", undefined)).toBe(true);
|
|
70
|
-
expect(pagesOperator("guard.fail.open", [])).toBe(true);
|
|
71
|
-
});
|
|
72
|
-
});
|