@teamscala/service-boot 0.23.407 → 0.23.409
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/fn-routers/index.ts +2 -0
- package/src/fn-routers/messaging-whatsapp-router.test.ts +119 -0
- package/src/fn-routers/messaging-whatsapp-router.ts +180 -0
- package/src/fn-routers/public-agreement-router.ts +1 -1
- package/src/handlers/platform-fault-escalation-logic.ts +30 -35
- package/src/handlers/platform-fault-escalation.test.ts +55 -19
- package/src/handlers/platform-fault-escalation.ts +65 -101
- package/src/mount-orpc-router.ts +8 -0
package/package.json
CHANGED
package/src/fn-routers/index.ts
CHANGED
|
@@ -30,6 +30,7 @@ import { publicGptSessionRouter } from "./public-gpt-session-router.ts";
|
|
|
30
30
|
import { signInFlowRouter } from "./sign-in-flow-router.ts";
|
|
31
31
|
import { userPreferencesRouter } from "./user-preferences-router.ts";
|
|
32
32
|
import { workItemsRouter } from "./work-items-router.ts";
|
|
33
|
+
import { messagingWhatsappRouter } from "./messaging-whatsapp-router.ts";
|
|
33
34
|
|
|
34
35
|
export function registerFnRouters(): void {
|
|
35
36
|
registerCustomRouter("fn", {
|
|
@@ -38,6 +39,7 @@ export function registerFnRouters(): void {
|
|
|
38
39
|
auth: authRouter,
|
|
39
40
|
cacheAdmin: cacheAdminRouter,
|
|
40
41
|
chatSession: chatSessionRouter,
|
|
42
|
+
messagingWhatsappSend: messagingWhatsappRouter.messagingWhatsappSend,
|
|
41
43
|
publicAgreement: publicAgreementRouter,
|
|
42
44
|
publicAssessment: publicAssessmentRouter,
|
|
43
45
|
publicGptSession: publicGptSessionRouter,
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system service-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* Proves fn.messagingWhatsappSend routes through scala-messaging-service: the
|
|
7
|
+
* URL is DERIVED from the service_install row (never spelled), the sealed-row
|
|
8
|
+
* credential is passed through the decrypt mock (identity, matching the
|
|
9
|
+
* public-assessment-router test pattern), the request carries the internal
|
|
10
|
+
* auth header, and the caller receives WhatsApp's message_id — the only thing
|
|
11
|
+
* that makes delivery provable. Failure modes (401, blank input) fail loudly
|
|
12
|
+
* with named causes, never a generic send error.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { createRouterClient } from "@orpc/server";
|
|
16
|
+
import { afterAll, describe, expect, mock, test } from "bun:test";
|
|
17
|
+
|
|
18
|
+
const SEND_CALLS: Array<{ url: string; headers: Record<string, string>; body: unknown }> = [];
|
|
19
|
+
|
|
20
|
+
mock.module("@teamscala/db/registry/load-config", () => ({
|
|
21
|
+
loadRegistryConfig: async (type: string, slug: string) => {
|
|
22
|
+
if (type === "service_install" && slug === "scala-messaging-service") {
|
|
23
|
+
return { deploy_target: "railway", port: 8080 };
|
|
24
|
+
}
|
|
25
|
+
if (type === "secret" && slug === "railway-scala-messaging-whatsapp-webhook-secret") {
|
|
26
|
+
return { token: "test-token" };
|
|
27
|
+
}
|
|
28
|
+
throw new Error(`unexpected loadRegistryConfig(${type}, ${slug})`);
|
|
29
|
+
},
|
|
30
|
+
}));
|
|
31
|
+
|
|
32
|
+
// Identity decrypt (the public-assessment-router test pattern).
|
|
33
|
+
mock.module("@teamscala/encryption/crypto/secret-config", () => ({
|
|
34
|
+
decryptSecretConfig: (raw: unknown) => raw,
|
|
35
|
+
}));
|
|
36
|
+
|
|
37
|
+
mock.module("@teamscala/logger/app-loggers", () => ({
|
|
38
|
+
getAppLogger: () => ({
|
|
39
|
+
info: () => {},
|
|
40
|
+
warn: () => {},
|
|
41
|
+
error: () => {},
|
|
42
|
+
debug: () => {},
|
|
43
|
+
}),
|
|
44
|
+
}));
|
|
45
|
+
|
|
46
|
+
const originalFetch = globalThis.fetch;
|
|
47
|
+
(globalThis as { fetch: unknown }).fetch = async (url: string, init: RequestInit) => {
|
|
48
|
+
SEND_CALLS.push({
|
|
49
|
+
url,
|
|
50
|
+
headers: init.headers as Record<string, string>,
|
|
51
|
+
body: JSON.parse(init.body as string),
|
|
52
|
+
});
|
|
53
|
+
const body = JSON.parse(init.body as string) as { phone?: string };
|
|
54
|
+
if (body.phone === "0-0-1-refuse") {
|
|
55
|
+
return new Response("no", { status: 401 });
|
|
56
|
+
}
|
|
57
|
+
return Response.json({ message_id: "3EB07EE8TESTID" });
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const { messagingWhatsappRouter } = await import("./messaging-whatsapp-router.ts");
|
|
61
|
+
|
|
62
|
+
// Invoke through a real ORPC client so the authed middleware runs too —
|
|
63
|
+
// calling the DecoratedProcedure directly is not a round trip.
|
|
64
|
+
const client = createRouterClient(
|
|
65
|
+
{ messagingWhatsappSend: messagingWhatsappRouter.messagingWhatsappSend } as never,
|
|
66
|
+
{
|
|
67
|
+
context: {
|
|
68
|
+
user: { id: "system", organisationId: null },
|
|
69
|
+
headers: new Headers(),
|
|
70
|
+
prisma: {},
|
|
71
|
+
} as never,
|
|
72
|
+
},
|
|
73
|
+
) as { messagingWhatsappSend: (a: unknown) => Promise<unknown> };
|
|
74
|
+
|
|
75
|
+
describe("fn.messagingWhatsappSend", () => {
|
|
76
|
+
test("derives the URL from service_install, sends with the decrypted header value, returns the message_id", async () => {
|
|
77
|
+
SEND_CALLS.length = 0;
|
|
78
|
+
const result = await client.messagingWhatsappSend({
|
|
79
|
+
phone: "61428193498",
|
|
80
|
+
message: "migration probe",
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
expect(SEND_CALLS).toHaveLength(1);
|
|
84
|
+
expect(SEND_CALLS[0]?.url).toBe(
|
|
85
|
+
"http://scala-messaging-service.railway.internal:8080/api/messaging/whatsapp/send",
|
|
86
|
+
);
|
|
87
|
+
expect(SEND_CALLS[0]?.headers["x-webhook-secret"]).toBe("test-token");
|
|
88
|
+
expect(SEND_CALLS[0]?.body).toEqual({ phone: "61428193498", message: "migration probe" });
|
|
89
|
+
expect(result).toEqual({ ok: true, message_id: "3EB07EE8TESTID" });
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
test("a 401 from the messaging service names the credential, not a generic failure", async () => {
|
|
93
|
+
SEND_CALLS.length = 0;
|
|
94
|
+
const result = await client.messagingWhatsappSend({
|
|
95
|
+
phone: "0-0-1-refuse",
|
|
96
|
+
message: "x",
|
|
97
|
+
});
|
|
98
|
+
expect(result).toEqual({
|
|
99
|
+
ok: false,
|
|
100
|
+
error: "messaging service refused the webhook secret (401)",
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("blank phone or message is refused before any request is made", async () => {
|
|
105
|
+
SEND_CALLS.length = 0;
|
|
106
|
+
const blank = await client.messagingWhatsappSend({
|
|
107
|
+
phone: " ",
|
|
108
|
+
message: "x",
|
|
109
|
+
});
|
|
110
|
+
expect(blank).toEqual({ ok: false, error: "phone and message are required" });
|
|
111
|
+
expect(SEND_CALLS).toHaveLength(0);
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
// Restore INSIDE afterAll — a top-level restore would run before the tests
|
|
116
|
+
// (all top-level code executes first) and hand the router back the real fetch.
|
|
117
|
+
afterAll(() => {
|
|
118
|
+
(globalThis as { fetch: unknown }).fetch = originalFetch;
|
|
119
|
+
});
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system service-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* fn.messagingWhatsappSend — the agent-facing WhatsApp send that routes through
|
|
7
|
+
* scala-messaging-service (the Rust owner of the channel), replacing the
|
|
8
|
+
* retired TS @teamscala/api-gowa SDK send path (operator ruling 2026-08-26:
|
|
9
|
+
* "make sure the migration is finished, i cant have half ts and half rust").
|
|
10
|
+
*
|
|
11
|
+
* The messaging service's POST /api/messaging/whatsapp/send is the single send
|
|
12
|
+
* surface: it resolves the sending device, returns WhatsApp's message_id for
|
|
13
|
+
* delivery correlation, and records the send in messaging_logs itself (Rust,
|
|
14
|
+
* `write_outbound_tracked`) — better than the deleted TS recorder, which only
|
|
15
|
+
* ran when the send happened to flow through tool-executor.
|
|
16
|
+
*
|
|
17
|
+
* Both facts this procedure needs are DERIVED at call time, never spelled:
|
|
18
|
+
* - the service URL comes from the service_install row (slug + port +
|
|
19
|
+
* deploy_target → Railway private DNS <slug>.railway.internal:<port>), the
|
|
20
|
+
* same derivation every *.railway.internal consumer uses;
|
|
21
|
+
* - the auth credential is the existing sealed secret row
|
|
22
|
+
* railway-scala-messaging-whatsapp-webhook-secret, decrypted per
|
|
23
|
+
* registry-via-helper + the agreement-send precedent.
|
|
24
|
+
*
|
|
25
|
+
* Registered flat under the `fn` namespace (two-segment procedure
|
|
26
|
+
* "fn.messagingWhatsappSend") because the tool-executor ORPC dispatch resolves
|
|
27
|
+
* "model.method" — nested routers would not be reachable as a tool procedure.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import { os } from "@orpc/server";
|
|
31
|
+
import * as v from "valibot";
|
|
32
|
+
import {
|
|
33
|
+
decryptSecretConfig,
|
|
34
|
+
type SecretConfigLike,
|
|
35
|
+
} from "@teamscala/encryption/crypto/secret-config";
|
|
36
|
+
import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
|
|
37
|
+
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
38
|
+
|
|
39
|
+
interface User {
|
|
40
|
+
id: string;
|
|
41
|
+
organisationId: string | null;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
interface Context {
|
|
45
|
+
user: User | null | undefined;
|
|
46
|
+
headers: Headers;
|
|
47
|
+
resolveUser?: (headers: Headers, prisma: any) => Promise<User | null>;
|
|
48
|
+
prisma: any;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const authed = os.$context<Context>().use(async (opt) => {
|
|
52
|
+
const { context } = opt;
|
|
53
|
+
let user = context.user;
|
|
54
|
+
if (!user && context.headers && context.resolveUser) {
|
|
55
|
+
user = await context.resolveUser(context.headers, context.prisma);
|
|
56
|
+
}
|
|
57
|
+
if (!user) {
|
|
58
|
+
const { ORPCError } = await import("@orpc/server");
|
|
59
|
+
throw new ORPCError("UNAUTHORIZED", { message: "Authentication required" });
|
|
60
|
+
}
|
|
61
|
+
return await opt.next({ context: { ...context, user } });
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
/** service_install shape for deriving the messaging service's internal URL. */
|
|
65
|
+
const MessagingInstallSchema = v.looseObject({
|
|
66
|
+
port: v.optional(v.number(), 8080),
|
|
67
|
+
deploy_target: v.string(),
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
/** The sealed webhook-secret row shape (value is decryptSecretConfig input). */
|
|
71
|
+
const MessagingSecretSchema = v.looseObject({});
|
|
72
|
+
|
|
73
|
+
/** Derive the messaging service base URL from its service_install row. */
|
|
74
|
+
async function resolveMessagingBaseUrl(): Promise<string> {
|
|
75
|
+
const install = await loadRegistryConfig(
|
|
76
|
+
"service_install",
|
|
77
|
+
"scala-messaging-service",
|
|
78
|
+
MessagingInstallSchema,
|
|
79
|
+
);
|
|
80
|
+
if (install.deploy_target !== "railway") {
|
|
81
|
+
throw new Error(
|
|
82
|
+
`scala-messaging-service deploy_target is "${install.deploy_target}", not "railway" — ` +
|
|
83
|
+
"the private-DNS URL derivation only holds on Railway. Fix the service_install row.",
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
return `http://scala-messaging-service.railway.internal:${install.port}`;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const sendSchema = v.object({
|
|
90
|
+
/** Destination phone number in E.164 (e.g. 61428193498). */
|
|
91
|
+
phone: v.string(),
|
|
92
|
+
/** Text body to send. */
|
|
93
|
+
message: v.string(),
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
export const messagingWhatsappRouter = {
|
|
97
|
+
messagingWhatsappSend: authed
|
|
98
|
+
.input(sendSchema)
|
|
99
|
+
.handler(async ({ input }) => {
|
|
100
|
+
const log = getAppLogger();
|
|
101
|
+
const phone = input.phone.trim();
|
|
102
|
+
const message = input.message;
|
|
103
|
+
if (!phone || !message) {
|
|
104
|
+
return { ok: false as const, error: "phone and message are required" };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
try {
|
|
108
|
+
const [baseUrl, rawSecret] = await Promise.all([
|
|
109
|
+
resolveMessagingBaseUrl(),
|
|
110
|
+
loadRegistryConfig(
|
|
111
|
+
"secret",
|
|
112
|
+
"railway-scala-messaging-whatsapp-webhook-secret",
|
|
113
|
+
MessagingSecretSchema,
|
|
114
|
+
).catch(() => {
|
|
115
|
+
throw new Error(
|
|
116
|
+
"secret row railway-scala-messaging-whatsapp-webhook-secret is missing — " +
|
|
117
|
+
"the messaging service send endpoint cannot be authenticated",
|
|
118
|
+
);
|
|
119
|
+
}),
|
|
120
|
+
]);
|
|
121
|
+
const decrypted = decryptSecretConfig(
|
|
122
|
+
rawSecret as SecretConfigLike,
|
|
123
|
+
) as { token?: string };
|
|
124
|
+
if (!decrypted.token) {
|
|
125
|
+
return {
|
|
126
|
+
ok: false as const,
|
|
127
|
+
error: "whatsapp webhook secret row has no token",
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const response = await fetch(
|
|
132
|
+
`${baseUrl}/api/messaging/whatsapp/send`,
|
|
133
|
+
{
|
|
134
|
+
method: "POST",
|
|
135
|
+
headers: {
|
|
136
|
+
"Content-Type": "application/json",
|
|
137
|
+
// The Rust handler accepts x-webhook-secret (or
|
|
138
|
+
// x-api-key) against WHATSAPP_WEBHOOK_SECRET.
|
|
139
|
+
"x-webhook-secret": decrypted.token,
|
|
140
|
+
},
|
|
141
|
+
body: JSON.stringify({ phone, message }),
|
|
142
|
+
signal: AbortSignal.timeout(30_000),
|
|
143
|
+
},
|
|
144
|
+
);
|
|
145
|
+
if (response.status === 401) {
|
|
146
|
+
return {
|
|
147
|
+
ok: false as const,
|
|
148
|
+
error: "messaging service refused the webhook secret (401)",
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
const body = (await response.json().catch(() => null)) as {
|
|
152
|
+
message_id?: string;
|
|
153
|
+
error?: string;
|
|
154
|
+
} | null;
|
|
155
|
+
if (!response.ok) {
|
|
156
|
+
return {
|
|
157
|
+
ok: false as const,
|
|
158
|
+
error:
|
|
159
|
+
body?.error ??
|
|
160
|
+
`messaging service send failed: ${response.status}`,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
// The message_id is the only thing that makes delivery provable —
|
|
164
|
+
// hand it back so the caller can correlate the ack (gowa.rs
|
|
165
|
+
// parse_message_id contract).
|
|
166
|
+
log.info("[fn.messagingWhatsappSend] sent", {
|
|
167
|
+
phone,
|
|
168
|
+
message_id: body?.message_id ?? null,
|
|
169
|
+
});
|
|
170
|
+
return {
|
|
171
|
+
ok: true as const,
|
|
172
|
+
message_id: body?.message_id ?? null,
|
|
173
|
+
};
|
|
174
|
+
} catch (err) {
|
|
175
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
176
|
+
log.error(`[fn.messagingWhatsappSend] failed: ${errorMessage}`);
|
|
177
|
+
return { ok: false as const, error: errorMessage };
|
|
178
|
+
}
|
|
179
|
+
}),
|
|
180
|
+
};
|
|
@@ -3,25 +3,9 @@
|
|
|
3
3
|
* @status handwritten
|
|
4
4
|
* @edit edit directly
|
|
5
5
|
*
|
|
6
|
-
* Pure decision logic for the platform-fault escalation sweep
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* dependencies (same shape as whatsapp-liveness-logic).
|
|
10
|
-
*
|
|
11
|
-
* The semantics this locks (from the 2026-08-15 "760 errors escalated to
|
|
12
|
-
* nobody" incident brief):
|
|
13
|
-
* - Threshold on DURATION and RECURRENCE, never a single event. A kind
|
|
14
|
-
* qualifies as a sustained fault only when BOTH hold inside the lookback
|
|
15
|
-
* window: at least `min_count` occurrences (recurrence) spread over at
|
|
16
|
-
* least `min_span_minutes` (duration). One failed push is noise; the same
|
|
17
|
-
* kind recurring for half an hour is an outage.
|
|
18
|
-
* - One escalation per distinct fault kind per episode. An episode is open
|
|
19
|
-
* from first qualification until `resolve_after_minutes` of quiet; while
|
|
20
|
-
* open, the fault never re-fires — it only appends throttled updates.
|
|
21
|
-
* - Delivery is verified, not queued: an episode row with `escalated_at`
|
|
22
|
-
* NULL is a CLAIM, not an escalation. The sweep re-attempts delivery for
|
|
23
|
-
* claimed-but-undelivered episodes until the WhatsApp send returns a
|
|
24
|
-
* message id.
|
|
6
|
+
* Pure decision logic for the platform-fault escalation sweep — full
|
|
7
|
+
* contract in curated `apps/escalation.md`. Episodes emit on TRANSITIONS
|
|
8
|
+
* ONLY; a future-dated `lastSeen` is never recurrence evidence.
|
|
25
9
|
*/
|
|
26
10
|
|
|
27
11
|
export interface FaultAggregate {
|
|
@@ -29,6 +13,10 @@ export interface FaultAggregate {
|
|
|
29
13
|
count: number;
|
|
30
14
|
firstSeen: Date;
|
|
31
15
|
lastSeen: Date;
|
|
16
|
+
/** Id of the newest row backing `lastSeen`, when the caller resolved it —
|
|
17
|
+
* the resolvable handle the escalation cites so "last at T" is checkable
|
|
18
|
+
* against a real event_log row, not a computed or cached value. */
|
|
19
|
+
lastEventId?: string | null;
|
|
32
20
|
}
|
|
33
21
|
|
|
34
22
|
export interface EscalationEpisode {
|
|
@@ -50,8 +38,6 @@ export interface FaultEscalationPolicy {
|
|
|
50
38
|
minSpanMinutes: number;
|
|
51
39
|
/** Quiet period that closes an episode and re-arms escalation, minutes. */
|
|
52
40
|
resolveAfterMinutes: number;
|
|
53
|
-
/** Minimum spacing between throttled updates on an open episode, minutes. */
|
|
54
|
-
updateAfterMinutes: number;
|
|
55
41
|
}
|
|
56
42
|
|
|
57
43
|
/** Per-kind threshold overrides — a low-rate-but-persistent kind (guard.fail.open
|
|
@@ -65,8 +51,6 @@ export interface KindThresholdOverride {
|
|
|
65
51
|
export interface SweepDecision {
|
|
66
52
|
/** Sustained faults needing a FIRST delivery (new episode, or a claim whose send failed). */
|
|
67
53
|
escalate: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode | null }>;
|
|
68
|
-
/** Open, delivered episodes still recurring — append evidence, never re-send. */
|
|
69
|
-
update: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode }>;
|
|
70
54
|
/** Open episodes whose kind went quiet past the resolve window — close them. */
|
|
71
55
|
resolve: Array<{ episode: EscalationEpisode; quietMinutes: number }>;
|
|
72
56
|
}
|
|
@@ -88,9 +72,13 @@ function isSustained(
|
|
|
88
72
|
// happening. Without this the 720-minute lookback keeps a dead fault
|
|
89
73
|
// qualifying for hours, and since a resolved episode reads as no episode
|
|
90
74
|
// it re-escalates on every sweep until its events age out (2026-08-26).
|
|
75
|
+
// A future `lastSeen` (producer clock skew / mocked test clock, measured
|
|
76
|
+
// 2026-08-26: "still recurring" citing a timestamp 44 min in its own
|
|
77
|
+
// future) makes minutesBetween go NEGATIVE — refuse it as evidence.
|
|
91
78
|
return (
|
|
92
79
|
aggregate.count >= minCount &&
|
|
93
80
|
minutesBetween(aggregate.firstSeen, aggregate.lastSeen) >= minSpan &&
|
|
81
|
+
aggregate.lastSeen.getTime() <= now.getTime() &&
|
|
94
82
|
minutesBetween(aggregate.lastSeen, now) < policy.resolveAfterMinutes
|
|
95
83
|
);
|
|
96
84
|
}
|
|
@@ -110,7 +98,7 @@ export function decideSweep(
|
|
|
110
98
|
): SweepDecision {
|
|
111
99
|
const byKind = new Map(episodes.map((e) => [e.kind, e]));
|
|
112
100
|
const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
|
|
113
|
-
const decision: SweepDecision = { escalate: [],
|
|
101
|
+
const decision: SweepDecision = { escalate: [], resolve: [] };
|
|
114
102
|
|
|
115
103
|
for (const aggregate of aggregates) {
|
|
116
104
|
const episode = byKind.get(aggregate.kind);
|
|
@@ -126,19 +114,23 @@ export function decideSweep(
|
|
|
126
114
|
// verified).
|
|
127
115
|
if (!episode.escalatedAt) {
|
|
128
116
|
decision.escalate.push({ aggregate, episode });
|
|
129
|
-
continue;
|
|
130
|
-
}
|
|
131
|
-
// Open, delivered, still recurring — throttled evidence update only.
|
|
132
|
-
const lastUpdate = episode.lastUpdatedAt ?? episode.escalatedAt;
|
|
133
|
-
if (minutesBetween(lastUpdate, now) >= policy.updateAfterMinutes) {
|
|
134
|
-
decision.update.push({ aggregate, episode });
|
|
135
117
|
}
|
|
118
|
+
// An open, delivered episode emits NOTHING while it recurs: the open
|
|
119
|
+
// emission already happened and the close emission fires when the kind
|
|
120
|
+
// goes quiet. A cadence-driven "still recurring" repeat is exactly the
|
|
121
|
+
// silenced-alarm shape `emit-transitions-not-conditions` prohibits.
|
|
136
122
|
}
|
|
137
123
|
|
|
138
124
|
for (const episode of episodes) {
|
|
139
125
|
if (episode.resolvedAt) continue;
|
|
140
126
|
const aggregate = aggregateByKind.get(episode.kind);
|
|
141
|
-
|
|
127
|
+
// A future-dated aggregate lastSeen (producer clock skew) cannot serve
|
|
128
|
+
// as the quiet reference — it would freeze quietMinutes negative and
|
|
129
|
+
// hold the episode open forever while the kind is actually quiet. Fall
|
|
130
|
+
// back to the episode's own last_seen_at, which the bounded aggregate
|
|
131
|
+
// wrote and is therefore never in the future.
|
|
132
|
+
const quietReference =
|
|
133
|
+
aggregate && aggregate.lastSeen.getTime() <= now.getTime() ? aggregate.lastSeen : episode.lastSeenAt;
|
|
142
134
|
const quietMinutes = minutesBetween(quietReference, now);
|
|
143
135
|
if (quietMinutes >= policy.resolveAfterMinutes) {
|
|
144
136
|
decision.resolve.push({ episode, quietMinutes });
|
|
@@ -214,14 +206,17 @@ export function composeEscalationMessage(input: {
|
|
|
214
206
|
impact: string;
|
|
215
207
|
recommendation: string;
|
|
216
208
|
sample: string;
|
|
209
|
+
/** ISO timestamp + event id of the newest backing row, when resolved —
|
|
210
|
+
* cited verbatim so the reader can resolve the claim against event_log. */
|
|
211
|
+
lastEvent?: { occurredAt: string; eventId: string } | null;
|
|
217
212
|
}): string {
|
|
218
213
|
const span = formatSpanMinutes(
|
|
219
214
|
(input.aggregate.lastSeen.getTime() - input.aggregate.firstSeen.getTime()) / 60_000,
|
|
220
215
|
);
|
|
221
|
-
const
|
|
222
|
-
|
|
223
|
-
`Observed: ${input.aggregate.count} errors over ${span},
|
|
224
|
-
];
|
|
216
|
+
const observed = input.lastEvent
|
|
217
|
+
? `Observed: ${input.aggregate.count} errors over ${span}, last ${input.lastEvent.occurredAt} (event ${input.lastEvent.eventId.slice(0, 8)}).`
|
|
218
|
+
: `Observed: ${input.aggregate.count} errors over ${span}, last ${input.aggregate.lastSeen.toISOString()}.`;
|
|
219
|
+
const lines = [`🔴 Platform fault: ${input.kind}`, observed];
|
|
225
220
|
if (input.sample) lines.push(`Sample: ${input.sample}`);
|
|
226
221
|
lines.push(`Impact: ${input.impact}`, `Recommendation: ${input.recommendation}`);
|
|
227
222
|
return lines.join("\n");
|
|
@@ -24,7 +24,6 @@ const POLICY: FaultEscalationPolicy = {
|
|
|
24
24
|
minCount: 10,
|
|
25
25
|
minSpanMinutes: 25,
|
|
26
26
|
resolveAfterMinutes: 30,
|
|
27
|
-
updateAfterMinutes: 30,
|
|
28
27
|
};
|
|
29
28
|
|
|
30
29
|
function agg(kind: string, count: number, firstSeenMinutesAgo: number, lastSeenMinutesAgo: number): FaultAggregate {
|
|
@@ -102,29 +101,20 @@ describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
|
|
|
102
101
|
expect(decision.escalate).toHaveLength(0);
|
|
103
102
|
});
|
|
104
103
|
|
|
105
|
-
test("open delivered episode
|
|
104
|
+
test("open delivered episode emits NOTHING while it recurs — transitions, not cadence", () => {
|
|
105
|
+
// 2026-08-26: the hourly "still recurring" comment made the alarm
|
|
106
|
+
// skippable noise and hid a real signal. An open, delivered episode now
|
|
107
|
+
// stays silent until the close transition fires (see the resolve suite).
|
|
106
108
|
const sustained = agg("llm.provider.call.failed", 300, 720, 0);
|
|
107
|
-
const
|
|
109
|
+
const longOpen = episode({
|
|
108
110
|
kind: "llm.provider.call.failed",
|
|
109
|
-
escalatedAt: new Date(NOW.getTime() - 90 * 60_000),
|
|
111
|
+
escalatedAt: new Date(NOW.getTime() - 90 * 60_000),
|
|
110
112
|
lastUpdatedAt: new Date(NOW.getTime() - 90 * 60_000),
|
|
111
113
|
});
|
|
112
|
-
const decision = decideSweep([sustained], [
|
|
113
|
-
expect(decision.escalate).toHaveLength(0);
|
|
114
|
-
expect(decision.update).toHaveLength(1);
|
|
115
|
-
expect(decision.update[0]!.episode.workItemId).toBe("wi-1");
|
|
116
|
-
});
|
|
117
|
-
|
|
118
|
-
test("open episode inside the update window gets NOTHING (dedup window)", () => {
|
|
119
|
-
const sustained = agg("llm.provider.call.failed", 300, 720, 0);
|
|
120
|
-
const open = episode({
|
|
121
|
-
kind: "llm.provider.call.failed",
|
|
122
|
-
escalatedAt: new Date(NOW.getTime() - 10 * 60_000),
|
|
123
|
-
lastUpdatedAt: new Date(NOW.getTime() - 10 * 60_000),
|
|
124
|
-
});
|
|
125
|
-
const decision = decideSweep([sustained], [open], POLICY, NOW);
|
|
114
|
+
const decision = decideSweep([sustained], [longOpen], POLICY, NOW);
|
|
126
115
|
expect(decision.escalate).toHaveLength(0);
|
|
127
|
-
expect(decision.
|
|
116
|
+
expect(decision.resolve).toHaveLength(0);
|
|
117
|
+
expect(Object.keys(decision).sort()).toEqual(["escalate", "resolve"]);
|
|
128
118
|
});
|
|
129
119
|
|
|
130
120
|
test("a claimed-but-undelivered episode is RETRIED, not counted as escalated", () => {
|
|
@@ -168,6 +158,34 @@ describe("decideSweep — resolution + re-arm", () => {
|
|
|
168
158
|
});
|
|
169
159
|
});
|
|
170
160
|
|
|
161
|
+
describe("decideSweep — a future-dated lastSeen is not recurrence evidence", () => {
|
|
162
|
+
// 2026-08-26: an escalation sent 08:20Z cited "last at 09:04:54Z" — 44
|
|
163
|
+
// minutes in its own future. The rows existed (producer envelopes whose
|
|
164
|
+
// clock ran 70 minutes ahead), and `minutesBetween(lastSeen, now)` went
|
|
165
|
+
// negative, passing every recency check while the kind was actually quiet.
|
|
166
|
+
// A timestamp that has not happened yet cannot evidence a fault happening
|
|
167
|
+
// now — no matter how the aggregate was built.
|
|
168
|
+
test("a future-dated aggregate does not escalate", () => {
|
|
169
|
+
const phantom = agg("guard.fail.open", 40, 720, -44);
|
|
170
|
+
const decision = decideSweep([phantom], [], POLICY, NOW);
|
|
171
|
+
expect(decision.escalate).toHaveLength(0);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
test("a future-dated aggregate does not hold an open episode open past the quiet window", () => {
|
|
175
|
+
// lastSeen 44 min in the future means the kind's real events are quiet:
|
|
176
|
+
// the episode's own last_seen_at (honest) drives the resolve decision,
|
|
177
|
+
// and the phantom aggregate must not veto it.
|
|
178
|
+
const quietEpisode = episode({
|
|
179
|
+
kind: "guard.fail.open",
|
|
180
|
+
lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
|
|
181
|
+
});
|
|
182
|
+
const phantom = agg("guard.fail.open", 40, 720, -44);
|
|
183
|
+
const decision = decideSweep([phantom], [quietEpisode], POLICY, NOW);
|
|
184
|
+
expect(decision.escalate).toHaveLength(0);
|
|
185
|
+
expect(decision.resolve).toHaveLength(1);
|
|
186
|
+
});
|
|
187
|
+
});
|
|
188
|
+
|
|
171
189
|
describe("extractGowaMessageId — delivery verification", () => {
|
|
172
190
|
test("the canonical GOWA SendResponse shape yields results.message_id", () => {
|
|
173
191
|
expect(extractGowaMessageId({ code: "SUCCESS", results: { message_id: "3EB089B9D6ADD58153C561", status: "sent" } })).toBe(
|
|
@@ -196,6 +214,7 @@ describe("composeEscalationMessage", () => {
|
|
|
196
214
|
impact: "scala-tools verbs are failing host-wide",
|
|
197
215
|
recommendation: "check oxmgr logs for the bus consumer",
|
|
198
216
|
sample: "",
|
|
217
|
+
lastEvent: null,
|
|
199
218
|
});
|
|
200
219
|
expect(message).toContain("Platform fault: host-command-bus.command.failed");
|
|
201
220
|
expect(message).toContain("Observed: 490 errors over 12 h");
|
|
@@ -204,6 +223,22 @@ describe("composeEscalationMessage", () => {
|
|
|
204
223
|
expect(message.split("\n")).toHaveLength(4);
|
|
205
224
|
});
|
|
206
225
|
|
|
226
|
+
// 2026-08-26: an escalation sent 08:20Z cited "last at 09:04:54Z", a
|
|
227
|
+
// timestamp 44 minutes in its own future matching no event the operator
|
|
228
|
+
// could find. The Observed line now cites the newest backing row's own
|
|
229
|
+
// occurred_at and id, so "last at T" resolves to a real event_log row.
|
|
230
|
+
test("cites the newest backing row's timestamp and event id — resolvable evidence", () => {
|
|
231
|
+
const message = composeEscalationMessage({
|
|
232
|
+
kind: "guard.fail.open",
|
|
233
|
+
aggregate: agg("guard.fail.open", 490, 720, 0),
|
|
234
|
+
impact: "guard enforcement is failing open",
|
|
235
|
+
recommendation: "check the guard daemon",
|
|
236
|
+
sample: "",
|
|
237
|
+
lastEvent: { occurredAt: "2026-08-26T08:20:29Z", eventId: "01a03b79-c124-748f-a79f-a9a436729544" },
|
|
238
|
+
});
|
|
239
|
+
expect(message).toContain("last 2026-08-26T08:20:29Z (event 01a03b79)");
|
|
240
|
+
});
|
|
241
|
+
|
|
207
242
|
// 2026-08-26: three escalations reached the orchestrator seat carrying
|
|
208
243
|
// kind, count and duration only — triaging each required a hand event_log
|
|
209
244
|
// query, and the llm.provider.call.failed impact line pointed AWAY from
|
|
@@ -217,6 +252,7 @@ describe("composeEscalationMessage", () => {
|
|
|
217
252
|
sample: 'status=429; message=Usage limit reached for 5 hour. Your limit will reset at 2026-08-26 01:19:03',
|
|
218
253
|
impact: "AI chat/completion calls are failing",
|
|
219
254
|
recommendation: "check provider quota",
|
|
255
|
+
lastEvent: null,
|
|
220
256
|
});
|
|
221
257
|
const lines = message.split("\n");
|
|
222
258
|
expect(lines).toHaveLength(5);
|
|
@@ -3,46 +3,10 @@
|
|
|
3
3
|
* @status handwritten @indivisible-unit
|
|
4
4
|
* @edit edit directly
|
|
5
5
|
*
|
|
6
|
-
* The `platform-fault-escalation` pg-boss handler
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* llm.provider.error) 231, guard.fail.open 16, scale-to-zero.wake_failed 12,
|
|
11
|
-
* listen.connect_failed 11). Emission works and observability-is-passive is satisfied; nothing
|
|
12
|
-
* acted. This sweep makes RECEIVING have CONSEQUENCE: sustained platform
|
|
13
|
-
* faults escalate to the OPERATOR over the existing PA path (WhatsApp, the
|
|
14
|
-
* scala PA operator-relay bot — escalation-routing's channel; no second
|
|
15
|
-
* channel invented), with the durable record as a work_items task the PA can
|
|
16
|
-
* read back and the operator can answer on.
|
|
17
|
-
*
|
|
18
|
-
* THIS IS NOT A DRIFT-DETECTOR PATROLLING INSTANCES. It reacts only to
|
|
19
|
-
* components reporting THEIR OWN failure — event_log kinds that already
|
|
20
|
-
* assert a fault about their emitter. Thresholds (duration + recurrence,
|
|
21
|
-
* never a single event), the watched-kind patterns, the resolve window, the
|
|
22
|
-
* per-kind impact/recommendation copy and the per-kind payload sample-field
|
|
23
|
-
* whitelists are all in the
|
|
24
|
-
* `config/platform-fault-escalation` registry row — changing policy is a row
|
|
25
|
-
* edit, not a redeploy. Episode dedup lives in the
|
|
26
|
-
* `platform_fault_escalations` table (migration 501): one escalation per
|
|
27
|
-
* distinct fault kind per episode; an already-escalated-and-unresolved fault
|
|
28
|
-
* only appends throttled work-item comments, never a second message.
|
|
29
|
-
*
|
|
30
|
-
* DELIVERY IS VERIFIED, NOT QUEUED. The WhatsApp send goes through the
|
|
31
|
-
* central `/mcp` loopback (`api_gowa_message` sendMessage — the single
|
|
32
|
-
* sealed GOWA-credential surface, same as whatsapp-liveness) and an episode
|
|
33
|
-
* only carries `escalated_at` once the send returned a WhatsApp message id;
|
|
34
|
-
* before that the row is a CLAIM the next sweep retries. Success is recorded
|
|
35
|
-
* only after readback of the delivery facts (message id + work item id).
|
|
36
|
-
*
|
|
37
|
-
* WHY A TICK EXISTS AT ALL, given observability-is-passive and
|
|
38
|
-
* no-polling-for-pushable-state: the trigger is DURATION + RECURRENCE of
|
|
39
|
-
* events already durably logged — a fact about the PATTERN of past events,
|
|
40
|
-
* not a state transition any single event can announce. The sweep reads one
|
|
41
|
-
* server-side aggregate (GROUP BY kind) per run against indexed event_log;
|
|
42
|
-
* while the platform is healthy it returns zero rows and costs nothing
|
|
43
|
-
* beyond the tick (idle-costs-nothing).
|
|
44
|
-
*
|
|
45
|
-
* Registered under slug `@teamscala/service-boot:platform-fault-escalation`.
|
|
6
|
+
* The `platform-fault-escalation` pg-boss handler. Full contract in curated
|
|
7
|
+
* `apps/escalation.md` — episodes EMIT ON TRANSITIONS ONLY; evidence bounded
|
|
8
|
+
* `occurred_at <= now()` and cited by resolvable event id; delivery verified
|
|
9
|
+
* by WhatsApp message id before `escalated_at` is recorded.
|
|
46
10
|
*/
|
|
47
11
|
import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
|
|
48
12
|
import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
|
|
@@ -82,7 +46,6 @@ const FaultEscalationConfigSchema = v.object({
|
|
|
82
46
|
),
|
|
83
47
|
),
|
|
84
48
|
resolve_after_minutes: v.number(),
|
|
85
|
-
update_after_minutes: v.number(),
|
|
86
49
|
/** The orchestrator's work item — escalations are COMMENTS on it, never
|
|
87
50
|
* task rows (the fleet-task-placement CHECK constraint refuses a system
|
|
88
51
|
* task without session identity, and an escalation is a finding, not an
|
|
@@ -204,6 +167,11 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
|
|
|
204
167
|
// Patterns come from the valibot-validated registry row and are regex-only
|
|
205
168
|
// by construction below; embedding them as one alternation keeps the query
|
|
206
169
|
// parameter-free on the pattern side while the window is a real parameter.
|
|
170
|
+
// The window is bounded on BOTH sides by the database's own statement
|
|
171
|
+
// clock: `occurred_at` is producer-claimed, and an unbounded top edge
|
|
172
|
+
// admits rows stamped in the future by a skewed or mocked producer clock —
|
|
173
|
+
// which then fabricate "still recurring" for a quiet fault (measured
|
|
174
|
+
// 2026-08-26). A future row is not evidence of anything happening now.
|
|
207
175
|
const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
|
|
208
176
|
for (const pattern of patterns) {
|
|
209
177
|
if (!regexClass.test(pattern)) {
|
|
@@ -213,10 +181,11 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
|
|
|
213
181
|
const combined = `(${patterns.join("|")})`;
|
|
214
182
|
const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
|
|
215
183
|
`SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
184
|
+
FROM event_log
|
|
185
|
+
WHERE occurred_at > now() - ($1::int * interval '1 minute')
|
|
186
|
+
AND occurred_at <= now()
|
|
187
|
+
AND kind ~ $2::text
|
|
188
|
+
GROUP BY kind`,
|
|
220
189
|
windowMinutes,
|
|
221
190
|
combined,
|
|
222
191
|
);
|
|
@@ -229,31 +198,47 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
|
|
|
229
198
|
}
|
|
230
199
|
|
|
231
200
|
/**
|
|
232
|
-
* The message's evidence
|
|
233
|
-
*
|
|
234
|
-
* config row and are constrained to bare
|
|
235
|
-
*
|
|
236
|
-
*
|
|
201
|
+
* The message's evidence: the whitelisted sample fields AND the row identity
|
|
202
|
+
* (id + occurred_at) of the newest payload for the kind. Field names come
|
|
203
|
+
* from the valibot-validated config row and are constrained to bare
|
|
204
|
+
* identifiers below, so the JSON path cannot break out of the string. The
|
|
205
|
+
* `occurred_at <= now()` bound matches the aggregate's top edge: the cited
|
|
206
|
+
* "last" row must be a real, non-future event whose id rides in the message
|
|
207
|
+
* so the claim resolves against event_log.
|
|
237
208
|
*/
|
|
238
|
-
async function
|
|
239
|
-
|
|
209
|
+
async function fetchLatestEvidence(
|
|
210
|
+
kind: string,
|
|
211
|
+
fields: string[],
|
|
212
|
+
): Promise<{ sample: string; lastEvent: { occurredAt: string; eventId: string } | null }> {
|
|
213
|
+
const prisma = getPrisma();
|
|
214
|
+
if (fields.length === 0) {
|
|
215
|
+
const rows = await prisma.$queryRawUnsafe<Array<{ id: unknown; occurred_at: Date }>>(
|
|
216
|
+
`SELECT id::text, occurred_at
|
|
217
|
+
FROM event_log
|
|
218
|
+
WHERE kind = $1::text AND occurred_at <= now()
|
|
219
|
+
ORDER BY occurred_at DESC
|
|
220
|
+
LIMIT 1`,
|
|
221
|
+
kind,
|
|
222
|
+
);
|
|
223
|
+
const top = rows[0];
|
|
224
|
+
return { sample: "", lastEvent: top ? { occurredAt: isoSeconds(top.occurred_at), eventId: String(top.id) } : null };
|
|
225
|
+
}
|
|
240
226
|
for (const field of fields) {
|
|
241
227
|
if (!/^[A-Za-z0-9_]+$/.test(field)) {
|
|
242
228
|
throw new Error(`config/${CONFIG_SLUG}: sample_fields entry is not a bare field name: ${field}`);
|
|
243
229
|
}
|
|
244
230
|
}
|
|
245
231
|
const selection = fields.map((field) => `payload_json->'${field}' AS "${field}"`).join(", ");
|
|
246
|
-
const prisma = getPrisma();
|
|
247
232
|
const rows = await prisma.$queryRawUnsafe<Array<Record<string, unknown>>>(
|
|
248
|
-
`SELECT ${selection}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
233
|
+
`SELECT id::text, occurred_at, ${selection}
|
|
234
|
+
FROM event_log
|
|
235
|
+
WHERE kind = $1::text AND occurred_at <= now()
|
|
236
|
+
ORDER BY occurred_at DESC
|
|
237
|
+
LIMIT 1`,
|
|
253
238
|
kind,
|
|
254
239
|
);
|
|
255
240
|
const row = rows[0];
|
|
256
|
-
if (!row) return "";
|
|
241
|
+
if (!row) return { sample: "", lastEvent: null };
|
|
257
242
|
const parts: string[] = [];
|
|
258
243
|
for (const field of fields) {
|
|
259
244
|
const value = row[field];
|
|
@@ -261,7 +246,19 @@ async function fetchLatestPayloadSample(kind: string, fields: string[]): Promise
|
|
|
261
246
|
const formatted = formatEscalationSample(value);
|
|
262
247
|
if (formatted) parts.push(`${field}=${formatted}`);
|
|
263
248
|
}
|
|
264
|
-
return
|
|
249
|
+
return {
|
|
250
|
+
sample: parts.join("; "),
|
|
251
|
+
lastEvent: {
|
|
252
|
+
occurredAt: isoSeconds(row.occurred_at as string | Date),
|
|
253
|
+
eventId: String(row.id),
|
|
254
|
+
},
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** ISO-8601 to whole seconds — the same precision event_log stores, so the
|
|
259
|
+
* cited string round-trips a `WHERE occurred_at = '<cited>'` query. */
|
|
260
|
+
function isoSeconds(value: Date | string): string {
|
|
261
|
+
return new Date(value).toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
265
262
|
}
|
|
266
263
|
|
|
267
264
|
async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
|
|
@@ -323,11 +320,12 @@ async function deliverEscalation(
|
|
|
323
320
|
const bot = await resolveOperatorRelay();
|
|
324
321
|
const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
|
|
325
322
|
const copy = config.copy[kind] ?? config.default_copy;
|
|
326
|
-
const sample = await
|
|
323
|
+
const { sample, lastEvent } = await fetchLatestEvidence(kind, config.sample_fields?.[kind] ?? []);
|
|
327
324
|
const message = composeEscalationMessage({
|
|
328
325
|
kind,
|
|
329
326
|
aggregate,
|
|
330
327
|
sample,
|
|
328
|
+
lastEvent,
|
|
331
329
|
impact: copy.impact,
|
|
332
330
|
recommendation: copy.recommendation,
|
|
333
331
|
});
|
|
@@ -390,6 +388,8 @@ async function deliverEscalation(
|
|
|
390
388
|
event_kind: kind,
|
|
391
389
|
count: aggregate.count,
|
|
392
390
|
sample,
|
|
391
|
+
last_event_id: lastEvent?.eventId ?? null,
|
|
392
|
+
last_event_occurred_at: lastEvent?.occurredAt ?? null,
|
|
393
393
|
work_item_id: config.orchestrator_work_item_id,
|
|
394
394
|
delivered_comment_id: comment.id,
|
|
395
395
|
delivered_message_id: messageId,
|
|
@@ -398,33 +398,6 @@ async function deliverEscalation(
|
|
|
398
398
|
return { messageId, workItemId: config.orchestrator_work_item_id };
|
|
399
399
|
}
|
|
400
400
|
|
|
401
|
-
async function updateEpisode(aggregate: FaultAggregate, episode: EscalationEpisode): Promise<void> {
|
|
402
|
-
const now = new Date();
|
|
403
|
-
await (await orpc()).platform_fault_escalations.update({
|
|
404
|
-
where: { kind: episode.kind },
|
|
405
|
-
data: {
|
|
406
|
-
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
407
|
-
event_count: aggregate.count,
|
|
408
|
-
last_updated_at: now.toISOString(),
|
|
409
|
-
updated_at: now.toISOString(),
|
|
410
|
-
},
|
|
411
|
-
});
|
|
412
|
-
if (episode.workItemId) {
|
|
413
|
-
await (await orpc()).comments.create({
|
|
414
|
-
data: {
|
|
415
|
-
work_item_id: episode.workItemId,
|
|
416
|
-
content: `Still recurring: ${aggregate.count} events, last at ${aggregate.lastSeen.toISOString()}. No new escalation (already escalated and unresolved).`,
|
|
417
|
-
sender_role: "system",
|
|
418
|
-
sender_name: "platform-fault-escalation",
|
|
419
|
-
},
|
|
420
|
-
});
|
|
421
|
-
}
|
|
422
|
-
recordEvent({
|
|
423
|
-
kind: "platform.fault.updated",
|
|
424
|
-
payload: { event_kind: episode.kind, count: aggregate.count, work_item_id: episode.workItemId },
|
|
425
|
-
});
|
|
426
|
-
}
|
|
427
|
-
|
|
428
401
|
async function resolveEpisode(episode: EscalationEpisode, quietMinutes: number): Promise<void> {
|
|
429
402
|
const now = new Date();
|
|
430
403
|
await (await orpc()).platform_fault_escalations.update({
|
|
@@ -493,7 +466,6 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
493
466
|
minCount: config.min_count,
|
|
494
467
|
minSpanMinutes: config.min_span_minutes,
|
|
495
468
|
resolveAfterMinutes: config.resolve_after_minutes,
|
|
496
|
-
updateAfterMinutes: config.update_after_minutes,
|
|
497
469
|
},
|
|
498
470
|
now,
|
|
499
471
|
kindThresholds,
|
|
@@ -514,13 +486,6 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
514
486
|
recordEvent({ kind: "platform.fault.escalation_failed", payload: { event_kind: aggregate.kind, reason } });
|
|
515
487
|
}
|
|
516
488
|
}
|
|
517
|
-
for (const { aggregate, episode } of decision.update) {
|
|
518
|
-
try {
|
|
519
|
-
await updateEpisode(aggregate, episode);
|
|
520
|
-
} catch (err) {
|
|
521
|
-
log.error(`[platform-fault-escalation] update for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
489
|
for (const { episode, quietMinutes } of decision.resolve) {
|
|
525
490
|
try {
|
|
526
491
|
await resolveEpisode(episode, quietMinutes);
|
|
@@ -536,18 +501,17 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
536
501
|
payload: {
|
|
537
502
|
watched_kinds: aggregates.length,
|
|
538
503
|
escalated: decision.escalate.length,
|
|
539
|
-
updated: decision.update.length,
|
|
540
504
|
resolved: decision.resolve.length,
|
|
541
505
|
},
|
|
542
506
|
});
|
|
543
|
-
if (decision.escalate.length > 0 || decision.
|
|
507
|
+
if (decision.escalate.length > 0 || decision.resolve.length > 0) {
|
|
544
508
|
log.info(
|
|
545
|
-
`[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.
|
|
509
|
+
`[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.resolve.length} resolved`,
|
|
546
510
|
);
|
|
547
511
|
}
|
|
548
|
-
}
|
|
512
|
+
}
|
|
549
513
|
|
|
550
|
-
pgBossHandlerRegistry.register(
|
|
514
|
+
pgBossHandlerRegistry.register(
|
|
551
515
|
"@teamscala/service-boot:platform-fault-escalation",
|
|
552
516
|
platformFaultEscalationSweep as (ctx: { config: ScheduledJobConfig }) => Promise<void>,
|
|
553
517
|
);
|
package/src/mount-orpc-router.ts
CHANGED
|
@@ -87,6 +87,14 @@ export async function mountOrpcRouter(
|
|
|
87
87
|
configureToolExecutor({
|
|
88
88
|
getAppRouter: () => getAppRouter(),
|
|
89
89
|
getPrisma: () => getConfiguredPrisma(),
|
|
90
|
+
// Central MCP gateway URL — sdk:* tool calls from any non-host service
|
|
91
|
+
// forward here (scala-mcp owns every @teamscala/api-* adapter per
|
|
92
|
+
// operator ruling 2026-08-26). The HTTP layer reads from the
|
|
93
|
+
// per-request config surface (the same place getAppRouter / getPrisma
|
|
94
|
+
// are resolved from), not the ambient environment, to honour
|
|
95
|
+
// `configured-primitives`.
|
|
96
|
+
getMcpGatewayUrl: () =>
|
|
97
|
+
(ctx as { config?: { MCP_GATEWAY_URL?: string } }).config?.MCP_GATEWAY_URL,
|
|
90
98
|
});
|
|
91
99
|
|
|
92
100
|
const resolveUser = async (headers: Headers) => {
|