@kb-labs/gateway-app 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2454 -0
- package/dist/index.js.map +1 -0
- package/package.json +7 -4
- package/.kb/database/kb.sqlite-shm +0 -0
- package/.kb/database/kb.sqlite-wal +0 -0
- package/src/__tests__/auth-routes.test.ts +0 -279
- package/src/__tests__/execute-routes.test.ts +0 -408
- package/src/__tests__/execution-registry.test.ts +0 -218
- package/src/__tests__/health.test.ts +0 -215
- package/src/__tests__/live-gateway.e2e.test.ts +0 -648
- package/src/__tests__/llm-gateway.test.ts +0 -361
- package/src/__tests__/observability-collector.test.ts +0 -59
- package/src/__tests__/platform-api.test.ts +0 -317
- package/src/__tests__/registry.test.ts +0 -546
- package/src/__tests__/retry-executor.test.ts +0 -244
- package/src/__tests__/server.integration.test.ts +0 -417
- package/src/__tests__/subscription-registry.test.ts +0 -308
- package/src/__tests__/telemetry-ingest.test.ts +0 -309
- package/src/__tests__/tokens.test.ts +0 -83
- package/src/__tests__/ws-client-connect.e2e.test.ts +0 -381
- package/src/__tests__/ws-handshake.e2e.test.ts +0 -288
- package/src/auth/middleware.ts +0 -50
- package/src/auth/routes.ts +0 -57
- package/src/auth/tokens.ts +0 -41
- package/src/bootstrap.ts +0 -98
- package/src/clients/subscription-registry.ts +0 -137
- package/src/clients/ws-handler.ts +0 -196
- package/src/config.ts +0 -20
- package/src/docs/routes.ts +0 -70
- package/src/execute/errors.ts +0 -21
- package/src/execute/execution-registry.ts +0 -84
- package/src/execute/retry-executor.ts +0 -159
- package/src/execute/routes.ts +0 -239
- package/src/hosts/dispatcher.ts +0 -2
- package/src/hosts/registry.ts +0 -305
- package/src/hosts/ws-handler.ts +0 -445
- package/src/index.ts +0 -7
- package/src/llm/routes.ts +0 -343
- package/src/manifest.ts +0 -21
- package/src/observability/collector.ts +0 -346
- package/src/platform/routes.ts +0 -195
- package/src/server.ts +0 -447
- package/src/telemetry/routes.ts +0 -89
- package/src/ws/gateway-ws.ts +0 -73
- package/tsconfig.build.json +0 -15
- package/tsconfig.json +0 -10
- package/tsup.config.ts +0 -8
- package/vitest.config.ts +0 -23
package/dist/index.js
ADDED
|
@@ -0,0 +1,2454 @@
|
|
|
1
|
+
import { logDiagnosticEvent } from '@kb-labs/core-platform';
|
|
2
|
+
import { createServiceBootstrap, platform } from '@kb-labs/core-runtime';
|
|
3
|
+
import { createCorrelatedLogger, registerOpenAPI, createServiceReadyResponse, OperationMetricsTracker, createServiceObservabilityDescribe, createServiceObservabilityHealth } from '@kb-labs/shared-http';
|
|
4
|
+
import { SqliteHostStore, globalDispatcher, AdaptiveBuffer } from '@kb-labs/gateway-core';
|
|
5
|
+
import { findNearestConfig, readJsonWithDiagnostics } from '@kb-labs/core-config';
|
|
6
|
+
import { GatewayConfigSchema, HostRegistrationSchema, RegisterRequestSchema, TokenRequestSchema, RefreshRequestSchema, ExecuteRequestSchema, ChatCompletionRequestSchema, TelemetryIngestRequestSchema, PlatformCallRequestSchema, HelloMessageSchema, SUPPORTED_PROTOCOL_VERSIONS, HostCapabilitySchema, AdapterCallMessageSchema, AdapterNameSchema, ClientHelloSchema, ClientCancelSchema, ClientUnsubscribeSchema, ClientSubscribeSchema, CLIENT_PROTOCOL_VERSION } from '@kb-labs/gateway-contracts';
|
|
7
|
+
import Fastify from 'fastify';
|
|
8
|
+
import fastifyCors from '@fastify/cors';
|
|
9
|
+
import fastifyHttpProxy from '@fastify/http-proxy';
|
|
10
|
+
import { AuthService, getClientByHostId } from '@kb-labs/gateway-auth';
|
|
11
|
+
import { randomUUID } from 'crypto';
|
|
12
|
+
import { mergeOpenAPISpecs } from '@kb-labs/core-registry';
|
|
13
|
+
import { WebSocketServer } from 'ws';
|
|
14
|
+
import { CANONICAL_OBSERVABILITY_METRICS, OBSERVABILITY_CONTRACT_VERSION, OBSERVABILITY_SCHEMA } from '@kb-labs/core-contracts';
|
|
15
|
+
import { hostname } from 'os';
|
|
16
|
+
import { monitorEventLoopDelay, performance } from 'perf_hooks';
|
|
17
|
+
|
|
18
|
+
// src/bootstrap.ts
|
|
19
|
+
async function loadGatewayConfig(repoRoot) {
|
|
20
|
+
const { path: configPath } = await findNearestConfig({
|
|
21
|
+
startDir: repoRoot,
|
|
22
|
+
filenames: [".kb/kb.config.json", "kb.config.json"]
|
|
23
|
+
});
|
|
24
|
+
if (!configPath) {
|
|
25
|
+
return GatewayConfigSchema.parse({});
|
|
26
|
+
}
|
|
27
|
+
const result = await readJsonWithDiagnostics(configPath);
|
|
28
|
+
if (!result.ok || !result.data.gateway) {
|
|
29
|
+
return GatewayConfigSchema.parse({});
|
|
30
|
+
}
|
|
31
|
+
return GatewayConfigSchema.parse(result.data.gateway);
|
|
32
|
+
}
|
|
33
|
+
async function resolveToken(token, cache, jwtConfig) {
|
|
34
|
+
const authService = new AuthService(cache, jwtConfig);
|
|
35
|
+
const jwtContext = await authService.verify(token);
|
|
36
|
+
if (jwtContext) {
|
|
37
|
+
return jwtContext;
|
|
38
|
+
}
|
|
39
|
+
const machineEntry = await cache.get(
|
|
40
|
+
`host:token:${token}`
|
|
41
|
+
);
|
|
42
|
+
if (machineEntry) {
|
|
43
|
+
return {
|
|
44
|
+
type: "machine",
|
|
45
|
+
userId: machineEntry.hostId,
|
|
46
|
+
namespaceId: machineEntry.namespaceId,
|
|
47
|
+
tier: "free",
|
|
48
|
+
permissions: ["host:connect"]
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
function extractBearerToken(authHeader) {
|
|
54
|
+
if (!authHeader) {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
const match = authHeader.match(/^Bearer\s+(.+)$/i);
|
|
58
|
+
return match ? match[1] ?? null : null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// src/auth/middleware.ts
|
|
62
|
+
var PUBLIC_ROUTES = /* @__PURE__ */ new Set([
|
|
63
|
+
"/health",
|
|
64
|
+
"/hosts/register",
|
|
65
|
+
// /hosts/connect and /clients/connect are handled at the HTTP upgrade level
|
|
66
|
+
// by gateway-ws.ts (raw ws) — they never reach Fastify routing.
|
|
67
|
+
"/auth/register",
|
|
68
|
+
"/auth/token",
|
|
69
|
+
"/auth/refresh",
|
|
70
|
+
"/internal/dispatch",
|
|
71
|
+
// has its own x-internal-secret auth
|
|
72
|
+
"/internal/resolve-host"
|
|
73
|
+
// has its own x-internal-secret auth
|
|
74
|
+
]);
|
|
75
|
+
function createAuthMiddleware(cache, jwtConfig) {
|
|
76
|
+
return async function authMiddleware(request, reply) {
|
|
77
|
+
const rawPath = new URL(request.url, "http://localhost").pathname;
|
|
78
|
+
const routePath = rawPath.replace(/\/+/g, "/").replace(/\/+$/, "") || "/";
|
|
79
|
+
if (PUBLIC_ROUTES.has(routePath)) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const queryToken = request.query["access_token"];
|
|
83
|
+
const token = extractBearerToken(request.headers.authorization) ?? queryToken ?? null;
|
|
84
|
+
if (!token) {
|
|
85
|
+
return reply.code(401).send({ error: "Unauthorized", message: "Missing Authorization header" });
|
|
86
|
+
}
|
|
87
|
+
const authContext = await resolveToken(token, cache, jwtConfig);
|
|
88
|
+
if (!authContext) {
|
|
89
|
+
return reply.code(401).send({ error: "Unauthorized", message: "Invalid token" });
|
|
90
|
+
}
|
|
91
|
+
request.authContext = authContext;
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
function registerAuthRoutes(app, authService) {
|
|
95
|
+
app.post("/auth/register", { schema: { tags: ["Auth"], summary: "Register new agent and get credentials" } }, async (request, reply) => {
|
|
96
|
+
const parsed = RegisterRequestSchema.safeParse(request.body);
|
|
97
|
+
if (!parsed.success) {
|
|
98
|
+
return reply.code(400).send({ error: "Bad Request", issues: parsed.error.issues });
|
|
99
|
+
}
|
|
100
|
+
const result = await authService.register(parsed.data);
|
|
101
|
+
return reply.code(201).send(result);
|
|
102
|
+
});
|
|
103
|
+
app.post("/auth/token", { schema: { tags: ["Auth"], summary: "Issue JWT token pair" } }, async (request, reply) => {
|
|
104
|
+
const parsed = TokenRequestSchema.safeParse(request.body);
|
|
105
|
+
if (!parsed.success) {
|
|
106
|
+
return reply.code(400).send({ error: "Bad Request", issues: parsed.error.issues });
|
|
107
|
+
}
|
|
108
|
+
const tokens = await authService.issueTokens(parsed.data.clientId, parsed.data.clientSecret);
|
|
109
|
+
if (!tokens) {
|
|
110
|
+
return reply.code(401).send({ error: "Unauthorized", message: "Invalid credentials" });
|
|
111
|
+
}
|
|
112
|
+
return reply.send(tokens);
|
|
113
|
+
});
|
|
114
|
+
app.post("/auth/refresh", { schema: { tags: ["Auth"], summary: "Refresh JWT token pair" } }, async (request, reply) => {
|
|
115
|
+
const parsed = RefreshRequestSchema.safeParse(request.body);
|
|
116
|
+
if (!parsed.success) {
|
|
117
|
+
return reply.code(400).send({ error: "Bad Request", issues: parsed.error.issues });
|
|
118
|
+
}
|
|
119
|
+
const tokens = await authService.refreshTokens(parsed.data.refreshToken);
|
|
120
|
+
if (!tokens) {
|
|
121
|
+
return reply.code(401).send({ error: "Unauthorized", message: "Invalid or expired refresh token" });
|
|
122
|
+
}
|
|
123
|
+
return reply.send(tokens);
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// src/execute/execution-registry.ts
|
|
128
|
+
var ExecutionRegistry = class {
|
|
129
|
+
executions = /* @__PURE__ */ new Map();
|
|
130
|
+
/** Register a new execution. Returns AbortSignal to wire into the dispatch. */
|
|
131
|
+
register(entry) {
|
|
132
|
+
const controller = new AbortController();
|
|
133
|
+
this.executions.set(entry.executionId, {
|
|
134
|
+
...entry,
|
|
135
|
+
controller,
|
|
136
|
+
startedAt: Date.now()
|
|
137
|
+
});
|
|
138
|
+
return controller.signal;
|
|
139
|
+
}
|
|
140
|
+
/** Cancel an execution by ID. Returns true if found and aborted. */
|
|
141
|
+
cancel(executionId, reason) {
|
|
142
|
+
const entry = this.executions.get(executionId);
|
|
143
|
+
if (!entry) {
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
if (entry.controller.signal.aborted) {
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
entry.cancelledReason = reason;
|
|
150
|
+
entry.controller.abort(reason);
|
|
151
|
+
return true;
|
|
152
|
+
}
|
|
153
|
+
/** Remove a completed/cancelled execution. */
|
|
154
|
+
remove(executionId) {
|
|
155
|
+
this.executions.delete(executionId);
|
|
156
|
+
}
|
|
157
|
+
/** Get an active execution. */
|
|
158
|
+
get(executionId) {
|
|
159
|
+
return this.executions.get(executionId);
|
|
160
|
+
}
|
|
161
|
+
/** Cancel all executions dispatched to a host (on host disconnect). */
|
|
162
|
+
cancelByHost(hostId, reason) {
|
|
163
|
+
const cancelled = [];
|
|
164
|
+
for (const entry of this.executions.values()) {
|
|
165
|
+
if (entry.hostId === hostId && !entry.controller.signal.aborted) {
|
|
166
|
+
entry.cancelledReason = reason;
|
|
167
|
+
entry.controller.abort(reason);
|
|
168
|
+
cancelled.push(entry.executionId);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
return cancelled;
|
|
172
|
+
}
|
|
173
|
+
/** Number of active executions. */
|
|
174
|
+
get size() {
|
|
175
|
+
return this.executions.size;
|
|
176
|
+
}
|
|
177
|
+
};
|
|
178
|
+
var executionRegistry = new ExecutionRegistry();
|
|
179
|
+
|
|
180
|
+
// src/clients/subscription-registry.ts
|
|
181
|
+
var SubscriptionRegistry = class {
|
|
182
|
+
/** executionId → Set<connectionId> */
|
|
183
|
+
byExecution = /* @__PURE__ */ new Map();
|
|
184
|
+
/** connectionId → Set<executionId> */
|
|
185
|
+
byConnection = /* @__PURE__ */ new Map();
|
|
186
|
+
/** connectionId → WebSocket (for sending events) */
|
|
187
|
+
sockets = /* @__PURE__ */ new Map();
|
|
188
|
+
/** Register a WebSocket for a connection. Must be called before subscribe(). */
|
|
189
|
+
registerSocket(connectionId, socket) {
|
|
190
|
+
this.sockets.set(connectionId, socket);
|
|
191
|
+
}
|
|
192
|
+
/** Remove a connection's socket on disconnect. */
|
|
193
|
+
removeSocket(connectionId) {
|
|
194
|
+
this.sockets.delete(connectionId);
|
|
195
|
+
}
|
|
196
|
+
subscribe(connectionId, executionId) {
|
|
197
|
+
let executionSubs = this.byExecution.get(executionId);
|
|
198
|
+
if (!executionSubs) {
|
|
199
|
+
executionSubs = /* @__PURE__ */ new Set();
|
|
200
|
+
this.byExecution.set(executionId, executionSubs);
|
|
201
|
+
}
|
|
202
|
+
executionSubs.add(connectionId);
|
|
203
|
+
let connectionSubs = this.byConnection.get(connectionId);
|
|
204
|
+
if (!connectionSubs) {
|
|
205
|
+
connectionSubs = /* @__PURE__ */ new Set();
|
|
206
|
+
this.byConnection.set(connectionId, connectionSubs);
|
|
207
|
+
}
|
|
208
|
+
connectionSubs.add(executionId);
|
|
209
|
+
}
|
|
210
|
+
unsubscribe(connectionId, executionId) {
|
|
211
|
+
this.byExecution.get(executionId)?.delete(connectionId);
|
|
212
|
+
this.byConnection.get(connectionId)?.delete(executionId);
|
|
213
|
+
if (this.byExecution.get(executionId)?.size === 0) {
|
|
214
|
+
this.byExecution.delete(executionId);
|
|
215
|
+
}
|
|
216
|
+
if (this.byConnection.get(connectionId)?.size === 0) {
|
|
217
|
+
this.byConnection.delete(connectionId);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
getSubscribers(executionId) {
|
|
221
|
+
return this.byExecution.get(executionId) ?? /* @__PURE__ */ new Set();
|
|
222
|
+
}
|
|
223
|
+
getSubscriptions(connectionId) {
|
|
224
|
+
return this.byConnection.get(connectionId) ?? /* @__PURE__ */ new Set();
|
|
225
|
+
}
|
|
226
|
+
removeConnection(connectionId) {
|
|
227
|
+
const subscriptions = this.byConnection.get(connectionId);
|
|
228
|
+
if (!subscriptions) {
|
|
229
|
+
this.sockets.delete(connectionId);
|
|
230
|
+
return [];
|
|
231
|
+
}
|
|
232
|
+
const orphaned = [];
|
|
233
|
+
for (const executionId of subscriptions) {
|
|
234
|
+
const subs = this.byExecution.get(executionId);
|
|
235
|
+
if (subs) {
|
|
236
|
+
subs.delete(connectionId);
|
|
237
|
+
if (subs.size === 0) {
|
|
238
|
+
this.byExecution.delete(executionId);
|
|
239
|
+
orphaned.push(executionId);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
this.byConnection.delete(connectionId);
|
|
244
|
+
this.sockets.delete(connectionId);
|
|
245
|
+
return orphaned;
|
|
246
|
+
}
|
|
247
|
+
broadcast(executionId, event) {
|
|
248
|
+
const subscribers = this.byExecution.get(executionId);
|
|
249
|
+
if (!subscribers || subscribers.size === 0) {
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
const payload = JSON.stringify(event);
|
|
253
|
+
for (const connectionId of subscribers) {
|
|
254
|
+
const socket = this.sockets.get(connectionId);
|
|
255
|
+
if (socket && socket.readyState === socket.OPEN) {
|
|
256
|
+
socket.send(payload);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
get connectionCount() {
|
|
261
|
+
return this.byConnection.size;
|
|
262
|
+
}
|
|
263
|
+
get subscriptionCount() {
|
|
264
|
+
let total = 0;
|
|
265
|
+
for (const subs of this.byExecution.values()) {
|
|
266
|
+
total += subs.size;
|
|
267
|
+
}
|
|
268
|
+
return total;
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
var subscriptionRegistry = new SubscriptionRegistry();
|
|
272
|
+
|
|
273
|
+
// src/execute/errors.ts
|
|
274
|
+
var CancelledError = class extends Error {
|
|
275
|
+
reason;
|
|
276
|
+
constructor(reason) {
|
|
277
|
+
super(`Execution cancelled: ${reason}`);
|
|
278
|
+
this.name = "CancelledError";
|
|
279
|
+
this.reason = reason;
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
|
|
283
|
+
// src/execute/retry-executor.ts
|
|
284
|
+
var DEFAULTS = {
|
|
285
|
+
maxAttempts: 1,
|
|
286
|
+
initialDelayMs: 1e3,
|
|
287
|
+
backoffMultiplier: 2,
|
|
288
|
+
maxDelayMs: 3e4,
|
|
289
|
+
onlyRetryable: true
|
|
290
|
+
};
|
|
291
|
+
async function executeWithRetry(ctx, dispatch) {
|
|
292
|
+
const cfg = { ...DEFAULTS, ...ctx.config };
|
|
293
|
+
const maxAttempts = Math.max(1, cfg.maxAttempts);
|
|
294
|
+
let lastError = null;
|
|
295
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
296
|
+
if (ctx.signal.aborted) {
|
|
297
|
+
throw new CancelledError(ctx.signal.reason);
|
|
298
|
+
}
|
|
299
|
+
try {
|
|
300
|
+
return await raceAbort(ctx.signal, dispatch());
|
|
301
|
+
} catch (err) {
|
|
302
|
+
if (err instanceof CancelledError) {
|
|
303
|
+
throw err;
|
|
304
|
+
}
|
|
305
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
306
|
+
if (attempt >= maxAttempts) {
|
|
307
|
+
break;
|
|
308
|
+
}
|
|
309
|
+
const classified = classifyError(err);
|
|
310
|
+
if (cfg.onlyRetryable && !classified.retryable) {
|
|
311
|
+
break;
|
|
312
|
+
}
|
|
313
|
+
const delay = Math.min(
|
|
314
|
+
cfg.initialDelayMs * Math.pow(cfg.backoffMultiplier, attempt - 1),
|
|
315
|
+
cfg.maxDelayMs
|
|
316
|
+
);
|
|
317
|
+
ctx.write({
|
|
318
|
+
type: "execution:retry",
|
|
319
|
+
requestId: ctx.requestId,
|
|
320
|
+
executionId: ctx.executionId,
|
|
321
|
+
attempt,
|
|
322
|
+
maxAttempts,
|
|
323
|
+
delayMs: delay,
|
|
324
|
+
error: classified.message
|
|
325
|
+
});
|
|
326
|
+
await interruptibleDelay(delay, ctx.signal);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
throw lastError ?? new Error("executeWithRetry: no attempts made");
|
|
330
|
+
}
|
|
331
|
+
function raceAbort(signal, promise) {
|
|
332
|
+
if (signal.aborted) {
|
|
333
|
+
return Promise.reject(new CancelledError(signal.reason));
|
|
334
|
+
}
|
|
335
|
+
return new Promise((resolve, reject) => {
|
|
336
|
+
const onAbort = () => reject(new CancelledError(signal.reason));
|
|
337
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
338
|
+
promise.then(
|
|
339
|
+
(v) => {
|
|
340
|
+
signal.removeEventListener("abort", onAbort);
|
|
341
|
+
resolve(v);
|
|
342
|
+
},
|
|
343
|
+
(e) => {
|
|
344
|
+
signal.removeEventListener("abort", onAbort);
|
|
345
|
+
reject(e);
|
|
346
|
+
}
|
|
347
|
+
);
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
function interruptibleDelay(ms, signal) {
|
|
351
|
+
if (signal.aborted) {
|
|
352
|
+
return Promise.reject(new CancelledError(signal.reason));
|
|
353
|
+
}
|
|
354
|
+
return new Promise((resolve, reject) => {
|
|
355
|
+
const timer = setTimeout(resolve, ms);
|
|
356
|
+
const onAbort = () => {
|
|
357
|
+
clearTimeout(timer);
|
|
358
|
+
reject(new CancelledError(signal.reason));
|
|
359
|
+
};
|
|
360
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
function classifyError(err) {
|
|
364
|
+
if (!(err instanceof Error)) {
|
|
365
|
+
return { code: "UNKNOWN", message: String(err), retryable: false };
|
|
366
|
+
}
|
|
367
|
+
const msg = err.message;
|
|
368
|
+
if (msg.includes("ECONNREFUSED") || msg.includes("ECONNRESET") || msg.includes("ETIMEDOUT") || msg.includes("timed out") || msg.includes("503") || msg.includes("Service Unavailable")) {
|
|
369
|
+
return { code: "TRANSPORT_ERROR", message: msg, retryable: true };
|
|
370
|
+
}
|
|
371
|
+
if (msg.includes("Host not connected")) {
|
|
372
|
+
return { code: "HOST_UNAVAILABLE", message: msg, retryable: true };
|
|
373
|
+
}
|
|
374
|
+
return { code: "HANDLER_ERROR", message: msg, retryable: false };
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
// src/execute/routes.ts
|
|
378
|
+
function registerExecuteRoutes(app, logger) {
|
|
379
|
+
app.post("/api/v1/execute", { schema: { tags: ["Execute"], summary: "Execute a plugin handler", hide: true } }, async (request, reply) => {
|
|
380
|
+
const auth = request.authContext;
|
|
381
|
+
if (!auth) {
|
|
382
|
+
return reply.code(401).send({ error: "Unauthorized" });
|
|
383
|
+
}
|
|
384
|
+
const parsed = ExecuteRequestSchema.safeParse(request.body);
|
|
385
|
+
if (!parsed.success) {
|
|
386
|
+
return reply.code(400).send({ error: "Bad Request", issues: parsed.error.issues });
|
|
387
|
+
}
|
|
388
|
+
const { pluginId, handlerRef, exportName, input, timeoutMs } = parsed.data;
|
|
389
|
+
const executionId = randomUUID();
|
|
390
|
+
const requestId = randomUUID();
|
|
391
|
+
const startTime = Date.now();
|
|
392
|
+
logger.info("Execute request received", {
|
|
393
|
+
executionId,
|
|
394
|
+
pluginId,
|
|
395
|
+
handlerRef,
|
|
396
|
+
namespaceId: auth.namespaceId
|
|
397
|
+
});
|
|
398
|
+
const hostId = globalDispatcher.firstHostWithCapability(auth.namespaceId, "execution");
|
|
399
|
+
if (!hostId) {
|
|
400
|
+
logDiagnosticEvent(logger, {
|
|
401
|
+
domain: "service",
|
|
402
|
+
event: "gateway.execution.dispatch",
|
|
403
|
+
level: "warn",
|
|
404
|
+
reasonCode: "execution_host_unavailable",
|
|
405
|
+
message: "No execution host connected for namespace",
|
|
406
|
+
outcome: "failed",
|
|
407
|
+
serviceId: "gateway",
|
|
408
|
+
evidence: {
|
|
409
|
+
namespaceId: auth.namespaceId,
|
|
410
|
+
pluginId,
|
|
411
|
+
handlerRef
|
|
412
|
+
}
|
|
413
|
+
});
|
|
414
|
+
return reply.code(503).send({
|
|
415
|
+
error: "No execution host connected",
|
|
416
|
+
hint: "Ensure a RuntimeServer is running and connected to Gateway",
|
|
417
|
+
namespaceId: auth.namespaceId
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
const signal = executionRegistry.register({
|
|
421
|
+
executionId,
|
|
422
|
+
requestId,
|
|
423
|
+
namespaceId: auth.namespaceId,
|
|
424
|
+
hostId,
|
|
425
|
+
pluginId,
|
|
426
|
+
handlerRef
|
|
427
|
+
});
|
|
428
|
+
reply.raw.on("close", () => {
|
|
429
|
+
if (!reply.raw.writableFinished && !signal.aborted) {
|
|
430
|
+
executionRegistry.cancel(executionId, "disconnect");
|
|
431
|
+
}
|
|
432
|
+
});
|
|
433
|
+
reply.raw.writeHead(200, {
|
|
434
|
+
"Content-Type": "application/x-ndjson",
|
|
435
|
+
"Transfer-Encoding": "chunked",
|
|
436
|
+
"Cache-Control": "no-cache",
|
|
437
|
+
"X-Execution-Id": executionId
|
|
438
|
+
});
|
|
439
|
+
reply.raw.flushHeaders();
|
|
440
|
+
const writeEvent = (event) => {
|
|
441
|
+
const payload = JSON.stringify(event) + "\n";
|
|
442
|
+
if (!reply.raw.writableEnded) {
|
|
443
|
+
reply.raw.write(payload);
|
|
444
|
+
}
|
|
445
|
+
subscriptionRegistry.broadcast(executionId, event);
|
|
446
|
+
};
|
|
447
|
+
try {
|
|
448
|
+
const result = await executeWithRetry(
|
|
449
|
+
{ executionId, requestId, signal, config: void 0, write: writeEvent },
|
|
450
|
+
() => globalDispatcher.call(
|
|
451
|
+
auth.namespaceId,
|
|
452
|
+
hostId,
|
|
453
|
+
"execution",
|
|
454
|
+
"execute",
|
|
455
|
+
[{ pluginId, handlerRef, exportName, input, executionId, requestId, timeoutMs }]
|
|
456
|
+
)
|
|
457
|
+
);
|
|
458
|
+
writeEvent({
|
|
459
|
+
type: "execution:done",
|
|
460
|
+
requestId,
|
|
461
|
+
executionId,
|
|
462
|
+
exitCode: 0,
|
|
463
|
+
durationMs: Date.now() - startTime,
|
|
464
|
+
metadata: { result }
|
|
465
|
+
});
|
|
466
|
+
} catch (err) {
|
|
467
|
+
if (err instanceof CancelledError) {
|
|
468
|
+
writeEvent({
|
|
469
|
+
type: "execution:cancelled",
|
|
470
|
+
requestId,
|
|
471
|
+
executionId,
|
|
472
|
+
reason: err.reason ?? "user",
|
|
473
|
+
durationMs: Date.now() - startTime
|
|
474
|
+
});
|
|
475
|
+
writeEvent({
|
|
476
|
+
type: "execution:done",
|
|
477
|
+
requestId,
|
|
478
|
+
executionId,
|
|
479
|
+
exitCode: 130,
|
|
480
|
+
durationMs: Date.now() - startTime
|
|
481
|
+
});
|
|
482
|
+
} else {
|
|
483
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
484
|
+
logDiagnosticEvent(logger, {
|
|
485
|
+
domain: "service",
|
|
486
|
+
event: "gateway.execution.dispatch",
|
|
487
|
+
level: "error",
|
|
488
|
+
reasonCode: "execution_dispatch_failed",
|
|
489
|
+
message: "Gateway execution dispatch failed",
|
|
490
|
+
outcome: "failed",
|
|
491
|
+
error: err instanceof Error ? err : new Error(String(err)),
|
|
492
|
+
serviceId: "gateway",
|
|
493
|
+
evidence: {
|
|
494
|
+
namespaceId: auth.namespaceId,
|
|
495
|
+
executionId,
|
|
496
|
+
requestId,
|
|
497
|
+
pluginId,
|
|
498
|
+
handlerRef,
|
|
499
|
+
hostId
|
|
500
|
+
}
|
|
501
|
+
});
|
|
502
|
+
writeEvent({
|
|
503
|
+
type: "execution:error",
|
|
504
|
+
requestId,
|
|
505
|
+
executionId,
|
|
506
|
+
code: "EXECUTION_FAILED",
|
|
507
|
+
message,
|
|
508
|
+
retryable: false
|
|
509
|
+
});
|
|
510
|
+
writeEvent({
|
|
511
|
+
type: "execution:done",
|
|
512
|
+
requestId,
|
|
513
|
+
executionId,
|
|
514
|
+
exitCode: 1,
|
|
515
|
+
durationMs: Date.now() - startTime
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
} finally {
|
|
519
|
+
executionRegistry.remove(executionId);
|
|
520
|
+
if (!reply.raw.writableEnded) {
|
|
521
|
+
reply.raw.end();
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
});
|
|
525
|
+
app.post("/api/v1/execute/:executionId/cancel", { schema: { tags: ["Execute"], summary: "Cancel an active execution" } }, async (request, reply) => {
|
|
526
|
+
const auth = request.authContext;
|
|
527
|
+
if (!auth) {
|
|
528
|
+
return reply.code(401).send({ error: "Unauthorized" });
|
|
529
|
+
}
|
|
530
|
+
const { executionId } = request.params;
|
|
531
|
+
const body = request.body;
|
|
532
|
+
const reason = body?.reason ?? "user";
|
|
533
|
+
const execution = executionRegistry.get(executionId);
|
|
534
|
+
if (!execution) {
|
|
535
|
+
return reply.code(404).send({ error: "Execution not found or already completed" });
|
|
536
|
+
}
|
|
537
|
+
if (execution.namespaceId !== auth.namespaceId) {
|
|
538
|
+
return reply.code(403).send({ error: "Forbidden \u2014 execution belongs to another namespace" });
|
|
539
|
+
}
|
|
540
|
+
const cancelled = executionRegistry.cancel(executionId, reason);
|
|
541
|
+
logger.info("Cancel request processed", {
|
|
542
|
+
executionId,
|
|
543
|
+
reason,
|
|
544
|
+
cancelled,
|
|
545
|
+
namespaceId: auth.namespaceId
|
|
546
|
+
});
|
|
547
|
+
return reply.code(cancelled ? 200 : 409).send({
|
|
548
|
+
executionId,
|
|
549
|
+
status: cancelled ? "cancelled" : "already_cancelled",
|
|
550
|
+
reason
|
|
551
|
+
});
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
function isLLMRouter(llm) {
|
|
555
|
+
return typeof llm.resolveAdapter === "function";
|
|
556
|
+
}
|
|
557
|
+
async function resolveLLMForTier(tier) {
|
|
558
|
+
const llm = platform.llm;
|
|
559
|
+
if (!llm) {
|
|
560
|
+
return void 0;
|
|
561
|
+
}
|
|
562
|
+
if (isLLMRouter(llm)) {
|
|
563
|
+
const binding = await llm.resolveAdapter({ tier });
|
|
564
|
+
return binding.adapter;
|
|
565
|
+
}
|
|
566
|
+
return llm;
|
|
567
|
+
}
|
|
568
|
+
function registerLLMGatewayRoutes(app, logger) {
|
|
569
|
+
app.post("/llm/v1/chat/completions", { schema: { tags: ["LLM"], summary: "OpenAI-compatible chat completions", hide: true } }, async (request, reply) => {
|
|
570
|
+
const auth = request.authContext;
|
|
571
|
+
if (!auth) {
|
|
572
|
+
return reply.code(401).send({ error: "Unauthorized" });
|
|
573
|
+
}
|
|
574
|
+
const parsed = ChatCompletionRequestSchema.safeParse(request.body);
|
|
575
|
+
if (!parsed.success) {
|
|
576
|
+
return reply.code(400).send({
|
|
577
|
+
error: {
|
|
578
|
+
message: "Bad Request",
|
|
579
|
+
type: "invalid_request_error",
|
|
580
|
+
code: null,
|
|
581
|
+
param: null
|
|
582
|
+
},
|
|
583
|
+
issues: parsed.error.issues
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
const req = parsed.data;
|
|
587
|
+
const requestId = `chatcmpl-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
588
|
+
const llm = await resolveLLMForTier(req.model);
|
|
589
|
+
if (!llm) {
|
|
590
|
+
return reply.code(503).send({
|
|
591
|
+
error: {
|
|
592
|
+
message: `LLM not available for tier "${req.model}"`,
|
|
593
|
+
type: "server_error",
|
|
594
|
+
code: null,
|
|
595
|
+
param: null
|
|
596
|
+
}
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
logger.info("AI Gateway request", {
|
|
600
|
+
requestId,
|
|
601
|
+
tier: req.model,
|
|
602
|
+
stream: req.stream,
|
|
603
|
+
messageCount: req.messages.length,
|
|
604
|
+
hasTools: !!req.tools?.length,
|
|
605
|
+
tenantId: auth.namespaceId
|
|
606
|
+
});
|
|
607
|
+
try {
|
|
608
|
+
if (req.stream) {
|
|
609
|
+
return await handleStreamingRequest(reply, llm, req, requestId, logger);
|
|
610
|
+
}
|
|
611
|
+
return await handleCompletionRequest(reply, llm, req, requestId);
|
|
612
|
+
} catch (err) {
|
|
613
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
614
|
+
logger.error("AI Gateway error", error, { requestId, tier: req.model });
|
|
615
|
+
return reply.code(500).send({
|
|
616
|
+
error: {
|
|
617
|
+
message: "Internal server error",
|
|
618
|
+
type: "server_error",
|
|
619
|
+
code: null,
|
|
620
|
+
param: null
|
|
621
|
+
}
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
async function handleCompletionRequest(reply, llm, req, requestId) {
|
|
627
|
+
const messages = toILLMMessages(req);
|
|
628
|
+
const startTime = Date.now();
|
|
629
|
+
const hasTools = req.tools && req.tools.length > 0;
|
|
630
|
+
if (hasTools && llm.chatWithTools) {
|
|
631
|
+
const tools = toILLMTools(req.tools);
|
|
632
|
+
const options = {
|
|
633
|
+
temperature: req.temperature,
|
|
634
|
+
maxTokens: req.max_tokens,
|
|
635
|
+
stop: normalizeStop(req.stop),
|
|
636
|
+
tools,
|
|
637
|
+
toolChoice: req.tool_choice
|
|
638
|
+
};
|
|
639
|
+
const result2 = await llm.chatWithTools(messages, options);
|
|
640
|
+
const toolCalls = result2.toolCalls?.map((tc) => ({
|
|
641
|
+
id: tc.id,
|
|
642
|
+
type: "function",
|
|
643
|
+
function: {
|
|
644
|
+
name: tc.name,
|
|
645
|
+
arguments: typeof tc.input === "string" ? tc.input : JSON.stringify(tc.input)
|
|
646
|
+
}
|
|
647
|
+
}));
|
|
648
|
+
const finishReason = result2.stopReason === "tool_use" ? "tool_calls" : result2.stopReason === "max_tokens" ? "length" : "stop";
|
|
649
|
+
const response2 = {
|
|
650
|
+
id: requestId,
|
|
651
|
+
object: "chat.completion",
|
|
652
|
+
created: Math.floor(startTime / 1e3),
|
|
653
|
+
model: req.model,
|
|
654
|
+
choices: [
|
|
655
|
+
{
|
|
656
|
+
index: 0,
|
|
657
|
+
message: {
|
|
658
|
+
role: "assistant",
|
|
659
|
+
content: result2.content || null,
|
|
660
|
+
...toolCalls?.length ? { tool_calls: toolCalls } : {}
|
|
661
|
+
},
|
|
662
|
+
finish_reason: finishReason
|
|
663
|
+
}
|
|
664
|
+
],
|
|
665
|
+
usage: {
|
|
666
|
+
prompt_tokens: result2.usage.promptTokens,
|
|
667
|
+
completion_tokens: result2.usage.completionTokens,
|
|
668
|
+
total_tokens: result2.usage.promptTokens + result2.usage.completionTokens
|
|
669
|
+
}
|
|
670
|
+
};
|
|
671
|
+
return reply.code(200).send(response2);
|
|
672
|
+
}
|
|
673
|
+
const systemPrompt = messages.find((m) => m.role === "system")?.content;
|
|
674
|
+
const userMessages = messages.filter((m) => m.role !== "system");
|
|
675
|
+
const prompt = userMessages.map((m) => m.content).join("\n\n");
|
|
676
|
+
const result = await llm.complete(prompt, {
|
|
677
|
+
systemPrompt,
|
|
678
|
+
temperature: req.temperature,
|
|
679
|
+
maxTokens: req.max_tokens,
|
|
680
|
+
stop: normalizeStop(req.stop)
|
|
681
|
+
});
|
|
682
|
+
const response = {
|
|
683
|
+
id: requestId,
|
|
684
|
+
object: "chat.completion",
|
|
685
|
+
created: Math.floor(startTime / 1e3),
|
|
686
|
+
model: req.model,
|
|
687
|
+
choices: [
|
|
688
|
+
{
|
|
689
|
+
index: 0,
|
|
690
|
+
message: { role: "assistant", content: result.content },
|
|
691
|
+
finish_reason: "stop"
|
|
692
|
+
}
|
|
693
|
+
],
|
|
694
|
+
usage: {
|
|
695
|
+
prompt_tokens: result.usage.promptTokens,
|
|
696
|
+
completion_tokens: result.usage.completionTokens,
|
|
697
|
+
total_tokens: result.usage.promptTokens + result.usage.completionTokens
|
|
698
|
+
}
|
|
699
|
+
};
|
|
700
|
+
return reply.code(200).send(response);
|
|
701
|
+
}
|
|
702
|
+
async function handleStreamingRequest(reply, llm, req, requestId, logger) {
|
|
703
|
+
const created = Math.floor(Date.now() / 1e3);
|
|
704
|
+
reply.raw.writeHead(200, {
|
|
705
|
+
"Content-Type": "text/event-stream",
|
|
706
|
+
"Cache-Control": "no-cache",
|
|
707
|
+
"Connection": "keep-alive",
|
|
708
|
+
"X-Request-Id": requestId
|
|
709
|
+
});
|
|
710
|
+
reply.raw.flushHeaders();
|
|
711
|
+
const writeChunk = (chunk) => {
|
|
712
|
+
if (!reply.raw.writableEnded) {
|
|
713
|
+
reply.raw.write(`data: ${JSON.stringify(chunk)}
|
|
714
|
+
|
|
715
|
+
`);
|
|
716
|
+
}
|
|
717
|
+
};
|
|
718
|
+
const writeDone = () => {
|
|
719
|
+
if (!reply.raw.writableEnded) {
|
|
720
|
+
reply.raw.write("data: [DONE]\n\n");
|
|
721
|
+
reply.raw.end();
|
|
722
|
+
}
|
|
723
|
+
};
|
|
724
|
+
try {
|
|
725
|
+
writeChunk({
|
|
726
|
+
id: requestId,
|
|
727
|
+
object: "chat.completion.chunk",
|
|
728
|
+
created,
|
|
729
|
+
model: req.model,
|
|
730
|
+
choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }]
|
|
731
|
+
});
|
|
732
|
+
const systemPrompt = req.messages.find((m) => m.role === "system")?.content;
|
|
733
|
+
const userMessages = req.messages.filter((m) => m.role !== "system");
|
|
734
|
+
const prompt = userMessages.map((m) => m.content).join("\n\n");
|
|
735
|
+
for await (const text of llm.stream(prompt, {
|
|
736
|
+
systemPrompt,
|
|
737
|
+
temperature: req.temperature,
|
|
738
|
+
maxTokens: req.max_tokens,
|
|
739
|
+
stop: normalizeStop(req.stop)
|
|
740
|
+
})) {
|
|
741
|
+
writeChunk({
|
|
742
|
+
id: requestId,
|
|
743
|
+
object: "chat.completion.chunk",
|
|
744
|
+
created,
|
|
745
|
+
model: req.model,
|
|
746
|
+
choices: [{ index: 0, delta: { content: text }, finish_reason: null }]
|
|
747
|
+
});
|
|
748
|
+
}
|
|
749
|
+
writeChunk({
|
|
750
|
+
id: requestId,
|
|
751
|
+
object: "chat.completion.chunk",
|
|
752
|
+
created,
|
|
753
|
+
model: req.model,
|
|
754
|
+
choices: [{ index: 0, delta: {}, finish_reason: "stop" }]
|
|
755
|
+
});
|
|
756
|
+
writeDone();
|
|
757
|
+
} catch (err) {
|
|
758
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
759
|
+
logger.error("AI Gateway stream error", error, { requestId });
|
|
760
|
+
if (!reply.raw.writableEnded) {
|
|
761
|
+
reply.raw.write(
|
|
762
|
+
`data: ${JSON.stringify({ error: { message: error.message, type: "server_error" } })}
|
|
763
|
+
|
|
764
|
+
`
|
|
765
|
+
);
|
|
766
|
+
reply.raw.end();
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
return reply;
|
|
770
|
+
}
|
|
771
|
+
function toILLMMessages(req) {
|
|
772
|
+
return req.messages.map((m) => {
|
|
773
|
+
const msg = { role: m.role, content: m.content };
|
|
774
|
+
if (m.tool_call_id) {
|
|
775
|
+
msg.toolCallId = m.tool_call_id;
|
|
776
|
+
}
|
|
777
|
+
if (m.tool_calls) {
|
|
778
|
+
msg.toolCalls = m.tool_calls.map((tc) => ({
|
|
779
|
+
id: tc.id,
|
|
780
|
+
name: tc.function.name,
|
|
781
|
+
input: safeJsonParse(tc.function.arguments)
|
|
782
|
+
}));
|
|
783
|
+
}
|
|
784
|
+
return msg;
|
|
785
|
+
});
|
|
786
|
+
}
|
|
787
|
+
function toILLMTools(tools) {
|
|
788
|
+
return tools.map((t) => ({
|
|
789
|
+
name: t.function.name,
|
|
790
|
+
description: t.function.description ?? "",
|
|
791
|
+
inputSchema: t.function.parameters ?? {}
|
|
792
|
+
}));
|
|
793
|
+
}
|
|
794
|
+
function normalizeStop(stop) {
|
|
795
|
+
if (!stop) {
|
|
796
|
+
return void 0;
|
|
797
|
+
}
|
|
798
|
+
return Array.isArray(stop) ? stop : [stop];
|
|
799
|
+
}
|
|
800
|
+
function safeJsonParse(str) {
|
|
801
|
+
try {
|
|
802
|
+
return JSON.parse(str);
|
|
803
|
+
} catch {
|
|
804
|
+
return str;
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
function registerTelemetryRoutes(app, logger) {
|
|
808
|
+
app.post("/telemetry/v1/ingest", { schema: { tags: ["Telemetry"], summary: "Ingest telemetry events" } }, async (request, reply) => {
|
|
809
|
+
const auth = request.authContext;
|
|
810
|
+
if (!auth) {
|
|
811
|
+
return reply.code(401).send({ error: "Unauthorized" });
|
|
812
|
+
}
|
|
813
|
+
const parsed = TelemetryIngestRequestSchema.safeParse(request.body);
|
|
814
|
+
if (!parsed.success) {
|
|
815
|
+
return reply.code(400).send({
|
|
816
|
+
error: "Bad Request",
|
|
817
|
+
issues: parsed.error.issues
|
|
818
|
+
});
|
|
819
|
+
}
|
|
820
|
+
const analytics = platform.analytics;
|
|
821
|
+
if (!analytics) {
|
|
822
|
+
return reply.code(503).send({
|
|
823
|
+
error: "Analytics adapter not configured"
|
|
824
|
+
});
|
|
825
|
+
}
|
|
826
|
+
const { events } = parsed.data;
|
|
827
|
+
let accepted = 0;
|
|
828
|
+
let rejected = 0;
|
|
829
|
+
const errors = [];
|
|
830
|
+
for (let i = 0; i < events.length; i++) {
|
|
831
|
+
const event = events[i];
|
|
832
|
+
try {
|
|
833
|
+
await analytics.track(event.type, {
|
|
834
|
+
// Source attribution — who sent this event
|
|
835
|
+
_source: event.source,
|
|
836
|
+
_tenantId: auth.namespaceId,
|
|
837
|
+
_ts: event.timestamp ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
838
|
+
// Tags as flat properties for indexing
|
|
839
|
+
...event.tags,
|
|
840
|
+
// Free-form payload
|
|
841
|
+
...event.payload
|
|
842
|
+
});
|
|
843
|
+
accepted++;
|
|
844
|
+
} catch (err) {
|
|
845
|
+
rejected++;
|
|
846
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
847
|
+
errors.push({ index: i, message });
|
|
848
|
+
logger.warn("Telemetry event rejected", {
|
|
849
|
+
index: i,
|
|
850
|
+
type: event.type,
|
|
851
|
+
source: event.source,
|
|
852
|
+
error: message
|
|
853
|
+
});
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
logger.info("Telemetry ingest", {
|
|
857
|
+
tenantId: auth.namespaceId,
|
|
858
|
+
accepted,
|
|
859
|
+
rejected,
|
|
860
|
+
totalEvents: events.length
|
|
861
|
+
});
|
|
862
|
+
const response = {
|
|
863
|
+
accepted,
|
|
864
|
+
rejected,
|
|
865
|
+
...errors.length > 0 ? { errors } : {}
|
|
866
|
+
};
|
|
867
|
+
return reply.code(accepted > 0 ? 200 : 422).send(response);
|
|
868
|
+
});
|
|
869
|
+
}
|
|
870
|
+
var ALLOWED_METHODS = {
|
|
871
|
+
llm: /* @__PURE__ */ new Set(["complete", "stream", "chatWithTools"]),
|
|
872
|
+
cache: /* @__PURE__ */ new Set(["get", "set", "delete", "clear"]),
|
|
873
|
+
vectorStore: /* @__PURE__ */ new Set(["search", "upsert", "delete", "count"]),
|
|
874
|
+
analytics: /* @__PURE__ */ new Set(["track", "identify", "flush", "getEvents", "getStats", "getDailyStats"]),
|
|
875
|
+
embeddings: /* @__PURE__ */ new Set(["embed"]),
|
|
876
|
+
storage: /* @__PURE__ */ new Set(["read", "write", "delete", "list", "exists"]),
|
|
877
|
+
eventBus: /* @__PURE__ */ new Set(["publish", "subscribe"]),
|
|
878
|
+
sqlDatabase: /* @__PURE__ */ new Set(["query", "execute"]),
|
|
879
|
+
documentDatabase: /* @__PURE__ */ new Set(["find", "findOne", "insert", "update", "delete"])
|
|
880
|
+
};
|
|
881
|
+
function resolveAdapter(name) {
|
|
882
|
+
const adapterMap = {
|
|
883
|
+
llm: () => platform.llm,
|
|
884
|
+
cache: () => platform.cache,
|
|
885
|
+
analytics: () => platform.analytics,
|
|
886
|
+
vectorStore: () => platform.vectorStore,
|
|
887
|
+
embeddings: () => platform.embeddings,
|
|
888
|
+
storage: () => platform.storage,
|
|
889
|
+
eventBus: () => platform.eventBus,
|
|
890
|
+
sqlDatabase: () => platform.sqlDatabase,
|
|
891
|
+
documentDatabase: () => platform.documentDatabase
|
|
892
|
+
};
|
|
893
|
+
const getter = adapterMap[name];
|
|
894
|
+
return getter ? getter() : void 0;
|
|
895
|
+
}
|
|
896
|
+
function isAsyncIterable(value) {
|
|
897
|
+
return value !== null && typeof value === "object" && Symbol.asyncIterator in value;
|
|
898
|
+
}
|
|
899
|
+
function registerPlatformRoutes(app, logger) {
|
|
900
|
+
app.post(
|
|
901
|
+
"/platform/v1/:adapter/:method",
|
|
902
|
+
{ schema: { tags: ["Platform"], summary: "Invoke a platform adapter method", hide: true } },
|
|
903
|
+
async (request, reply) => {
|
|
904
|
+
const auth = request.authContext;
|
|
905
|
+
if (!auth) {
|
|
906
|
+
return reply.code(401).send({ error: "Unauthorized" });
|
|
907
|
+
}
|
|
908
|
+
const { adapter: adapterName, method: methodName } = request.params;
|
|
909
|
+
const allowedMethods = ALLOWED_METHODS[adapterName];
|
|
910
|
+
if (!allowedMethods) {
|
|
911
|
+
return reply.code(404).send({
|
|
912
|
+
ok: false,
|
|
913
|
+
error: { message: `Unknown adapter: "${adapterName}"`, code: "ADAPTER_NOT_FOUND" },
|
|
914
|
+
durationMs: 0
|
|
915
|
+
});
|
|
916
|
+
}
|
|
917
|
+
if (!allowedMethods.has(methodName)) {
|
|
918
|
+
return reply.code(403).send({
|
|
919
|
+
ok: false,
|
|
920
|
+
error: { message: `Method "${methodName}" not allowed on adapter "${adapterName}"`, code: "METHOD_NOT_ALLOWED" },
|
|
921
|
+
durationMs: 0
|
|
922
|
+
});
|
|
923
|
+
}
|
|
924
|
+
const adapter = resolveAdapter(adapterName);
|
|
925
|
+
if (!adapter) {
|
|
926
|
+
return reply.code(503).send({
|
|
927
|
+
ok: false,
|
|
928
|
+
error: { message: `Adapter "${adapterName}" not configured`, code: "ADAPTER_UNAVAILABLE" },
|
|
929
|
+
durationMs: 0
|
|
930
|
+
});
|
|
931
|
+
}
|
|
932
|
+
const method = adapter[methodName];
|
|
933
|
+
if (typeof method !== "function") {
|
|
934
|
+
return reply.code(501).send({
|
|
935
|
+
ok: false,
|
|
936
|
+
error: { message: `Method "${methodName}" not implemented on adapter "${adapterName}"`, code: "METHOD_NOT_IMPLEMENTED" },
|
|
937
|
+
durationMs: 0
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
const parsed = PlatformCallRequestSchema.safeParse(request.body);
|
|
941
|
+
if (!parsed.success) {
|
|
942
|
+
return reply.code(400).send({
|
|
943
|
+
ok: false,
|
|
944
|
+
error: { message: "Invalid request body", code: "VALIDATION_ERROR" },
|
|
945
|
+
durationMs: 0
|
|
946
|
+
});
|
|
947
|
+
}
|
|
948
|
+
const { args } = parsed.data;
|
|
949
|
+
logger.info("Platform API call", {
|
|
950
|
+
adapter: adapterName,
|
|
951
|
+
method: methodName,
|
|
952
|
+
argCount: args.length,
|
|
953
|
+
tenantId: auth.namespaceId
|
|
954
|
+
});
|
|
955
|
+
const startTime = Date.now();
|
|
956
|
+
try {
|
|
957
|
+
const result = method.apply(adapter, args);
|
|
958
|
+
const resolved = result instanceof Promise ? await result : result;
|
|
959
|
+
if (isAsyncIterable(resolved)) {
|
|
960
|
+
reply.raw.writeHead(200, {
|
|
961
|
+
"Content-Type": "text/event-stream",
|
|
962
|
+
"Cache-Control": "no-cache",
|
|
963
|
+
"Connection": "keep-alive"
|
|
964
|
+
});
|
|
965
|
+
reply.raw.flushHeaders();
|
|
966
|
+
for await (const chunk of resolved) {
|
|
967
|
+
if (!reply.raw.writableEnded) {
|
|
968
|
+
const data = typeof chunk === "string" ? chunk : JSON.stringify(chunk);
|
|
969
|
+
reply.raw.write(`data: ${data}
|
|
970
|
+
|
|
971
|
+
`);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
if (!reply.raw.writableEnded) {
|
|
975
|
+
reply.raw.write("data: [DONE]\n\n");
|
|
976
|
+
reply.raw.end();
|
|
977
|
+
}
|
|
978
|
+
return reply;
|
|
979
|
+
}
|
|
980
|
+
const durationMs = Date.now() - startTime;
|
|
981
|
+
return reply.code(200).send({
|
|
982
|
+
ok: true,
|
|
983
|
+
result: resolved,
|
|
984
|
+
durationMs
|
|
985
|
+
});
|
|
986
|
+
} catch (err) {
|
|
987
|
+
const durationMs = Date.now() - startTime;
|
|
988
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
989
|
+
logger.error("Platform API error", error, {
|
|
990
|
+
adapter: adapterName,
|
|
991
|
+
method: methodName,
|
|
992
|
+
tenantId: auth.namespaceId
|
|
993
|
+
});
|
|
994
|
+
return reply.code(502).send({
|
|
995
|
+
ok: false,
|
|
996
|
+
error: { message: error.message, code: "ADAPTER_ERROR" },
|
|
997
|
+
durationMs
|
|
998
|
+
});
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
);
|
|
1002
|
+
}
|
|
1003
|
+
var MERGED_CACHE_KEY = "__gateway_merged_openapi";
|
|
1004
|
+
var MERGED_CACHE_TTL = 3e4;
|
|
1005
|
+
var UPSTREAM_SPEC_URLS = [
|
|
1006
|
+
"http://localhost:5050/openapi.json",
|
|
1007
|
+
"http://localhost:7778/openapi.json"
|
|
1008
|
+
];
|
|
1009
|
+
function registerAggregatedDocsRoutes(app, cache) {
|
|
1010
|
+
app.get("/openapi-merged.json", async (_req, reply) => {
|
|
1011
|
+
if (cache) {
|
|
1012
|
+
try {
|
|
1013
|
+
const hit = await cache.get(MERGED_CACHE_KEY);
|
|
1014
|
+
if (hit) {
|
|
1015
|
+
return reply.send(hit);
|
|
1016
|
+
}
|
|
1017
|
+
} catch {
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
const results = await Promise.allSettled(
|
|
1021
|
+
UPSTREAM_SPEC_URLS.map(
|
|
1022
|
+
(url) => fetch(url, { signal: AbortSignal.timeout(3e3) }).then((r) => r.json())
|
|
1023
|
+
)
|
|
1024
|
+
);
|
|
1025
|
+
const specs = results.filter((r) => r.status === "fulfilled").map((r) => r.value);
|
|
1026
|
+
const merged = mergeOpenAPISpecs(specs);
|
|
1027
|
+
if (cache) {
|
|
1028
|
+
try {
|
|
1029
|
+
await cache.set(MERGED_CACHE_KEY, merged, MERGED_CACHE_TTL);
|
|
1030
|
+
} catch {
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
return reply.send(merged);
|
|
1034
|
+
});
|
|
1035
|
+
app.register(async function docsAll(scope) {
|
|
1036
|
+
const swaggerUi = await import('@fastify/swagger-ui');
|
|
1037
|
+
await scope.register(swaggerUi.default ?? swaggerUi, {
|
|
1038
|
+
routePrefix: "/docs-all",
|
|
1039
|
+
uiConfig: {
|
|
1040
|
+
url: "/openapi-merged.json",
|
|
1041
|
+
docExpansion: "list",
|
|
1042
|
+
deepLinking: true
|
|
1043
|
+
}
|
|
1044
|
+
});
|
|
1045
|
+
});
|
|
1046
|
+
}
|
|
1047
|
+
var DEFAULT_RECONNECT_GRACE_MS = 1e4;
|
|
1048
|
+
var HostRegistry = class {
|
|
1049
|
+
constructor(cache, store, options) {
|
|
1050
|
+
this.cache = cache;
|
|
1051
|
+
this.store = store;
|
|
1052
|
+
this.reconnectGraceMs = options?.reconnectGraceMs ?? DEFAULT_RECONNECT_GRACE_MS;
|
|
1053
|
+
}
|
|
1054
|
+
graceTimers = /* @__PURE__ */ new Map();
|
|
1055
|
+
reconnectGraceMs;
|
|
1056
|
+
/**
|
|
1057
|
+
* Restore hosts from persistent store into cache on startup.
|
|
1058
|
+
* All restored hosts start as offline — live status comes from WS connections.
|
|
1059
|
+
*
|
|
1060
|
+
* Also resets any stale online/reconnecting hosts in cache to offline,
|
|
1061
|
+
* since no WebSocket connections survive a Gateway restart.
|
|
1062
|
+
*/
|
|
1063
|
+
async restore() {
|
|
1064
|
+
await this.resetStaleHosts();
|
|
1065
|
+
if (!this.store) {
|
|
1066
|
+
return 0;
|
|
1067
|
+
}
|
|
1068
|
+
const hosts = await this.store.listAll();
|
|
1069
|
+
for (const host of hosts) {
|
|
1070
|
+
const offline = { ...host, status: "offline", connections: [] };
|
|
1071
|
+
const cacheKey = this.hostKey(host.namespaceId, host.hostId);
|
|
1072
|
+
await this.cache.set(cacheKey, offline);
|
|
1073
|
+
await this.store.save(offline);
|
|
1074
|
+
await this.addToIndex(host.namespaceId, host.hostId);
|
|
1075
|
+
}
|
|
1076
|
+
return hosts.length;
|
|
1077
|
+
}
|
|
1078
|
+
/**
|
|
1079
|
+
* Reset all hosts in cache to offline.
|
|
1080
|
+
* Called on startup — no WS connections exist yet, so nothing should be online.
|
|
1081
|
+
* Uses namespace index maintained in cache to discover all namespaces.
|
|
1082
|
+
*/
|
|
1083
|
+
async resetStaleHosts() {
|
|
1084
|
+
const namespaces = await this.cache.get("host:namespaces") ?? ["default"];
|
|
1085
|
+
for (const ns of namespaces) {
|
|
1086
|
+
const hostIds = await this.cache.get(`host:index:${ns}`) ?? [];
|
|
1087
|
+
for (const hostId of hostIds) {
|
|
1088
|
+
const host = await this.cache.get(this.hostKey(ns, hostId));
|
|
1089
|
+
if (host && (host.status === "online" || host.status === "reconnecting")) {
|
|
1090
|
+
await this.cache.set(this.hostKey(ns, hostId), {
|
|
1091
|
+
...host,
|
|
1092
|
+
status: "offline",
|
|
1093
|
+
connections: []
|
|
1094
|
+
});
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
async register(reg) {
|
|
1100
|
+
const hostId = `host_${randomUUID().replace(/-/g, "").slice(0, 24)}`;
|
|
1101
|
+
const machineToken = randomUUID();
|
|
1102
|
+
const now = Date.now();
|
|
1103
|
+
const descriptor = {
|
|
1104
|
+
hostId,
|
|
1105
|
+
name: reg.name,
|
|
1106
|
+
namespaceId: reg.namespaceId,
|
|
1107
|
+
capabilities: reg.capabilities,
|
|
1108
|
+
status: "offline",
|
|
1109
|
+
lastSeen: now,
|
|
1110
|
+
connections: [],
|
|
1111
|
+
hostType: reg.hostType,
|
|
1112
|
+
createdAt: now,
|
|
1113
|
+
updatedAt: now
|
|
1114
|
+
};
|
|
1115
|
+
if (this.store) {
|
|
1116
|
+
await this.store.save(descriptor);
|
|
1117
|
+
await this.store.saveToken(machineToken, hostId, reg.namespaceId);
|
|
1118
|
+
}
|
|
1119
|
+
await this.cache.set(this.hostKey(reg.namespaceId, hostId), descriptor);
|
|
1120
|
+
await this.cache.set(this.tokenKey(machineToken), { hostId, namespaceId: reg.namespaceId });
|
|
1121
|
+
await this.addToIndex(reg.namespaceId, hostId);
|
|
1122
|
+
return { descriptor, machineToken };
|
|
1123
|
+
}
|
|
1124
|
+
async setOnline(hostId, namespaceId, connectionId) {
|
|
1125
|
+
const host = await this.getFromCache(hostId, namespaceId);
|
|
1126
|
+
if (!host) {
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
const graceKey = `${namespaceId}:${hostId}`;
|
|
1130
|
+
const graceTimer = this.graceTimers.get(graceKey);
|
|
1131
|
+
if (graceTimer) {
|
|
1132
|
+
clearTimeout(graceTimer);
|
|
1133
|
+
this.graceTimers.delete(graceKey);
|
|
1134
|
+
}
|
|
1135
|
+
const updated = {
|
|
1136
|
+
...host,
|
|
1137
|
+
status: "online",
|
|
1138
|
+
lastSeen: Date.now(),
|
|
1139
|
+
connections: [connectionId]
|
|
1140
|
+
};
|
|
1141
|
+
await this.cache.set(this.hostKey(namespaceId, hostId), updated);
|
|
1142
|
+
if (this.store) {
|
|
1143
|
+
await this.store.save(updated);
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
async setOffline(hostId, namespaceId, connectionId) {
|
|
1147
|
+
const host = await this.getFromCache(hostId, namespaceId);
|
|
1148
|
+
if (!host) {
|
|
1149
|
+
return;
|
|
1150
|
+
}
|
|
1151
|
+
const connections = host.connections.filter((c) => c !== connectionId);
|
|
1152
|
+
if (connections.length > 0) {
|
|
1153
|
+
await this.cache.set(this.hostKey(namespaceId, hostId), {
|
|
1154
|
+
...host,
|
|
1155
|
+
status: "online",
|
|
1156
|
+
lastSeen: Date.now(),
|
|
1157
|
+
connections
|
|
1158
|
+
});
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
const reconnecting = {
|
|
1162
|
+
...host,
|
|
1163
|
+
status: "reconnecting",
|
|
1164
|
+
lastSeen: Date.now(),
|
|
1165
|
+
connections: []
|
|
1166
|
+
};
|
|
1167
|
+
await this.cache.set(this.hostKey(namespaceId, hostId), reconnecting);
|
|
1168
|
+
if (this.store) {
|
|
1169
|
+
await this.store.save(reconnecting);
|
|
1170
|
+
}
|
|
1171
|
+
const graceKey = `${namespaceId}:${hostId}`;
|
|
1172
|
+
const existing = this.graceTimers.get(graceKey);
|
|
1173
|
+
if (existing) {
|
|
1174
|
+
clearTimeout(existing);
|
|
1175
|
+
}
|
|
1176
|
+
this.graceTimers.set(graceKey, setTimeout(async () => {
|
|
1177
|
+
this.graceTimers.delete(graceKey);
|
|
1178
|
+
const current = await this.getFromCache(hostId, namespaceId);
|
|
1179
|
+
if (current?.status === "reconnecting") {
|
|
1180
|
+
const offline = { ...current, status: "offline", lastSeen: Date.now() };
|
|
1181
|
+
await this.cache.set(this.hostKey(namespaceId, hostId), offline);
|
|
1182
|
+
if (this.store) {
|
|
1183
|
+
await this.store.save(offline);
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
}, this.reconnectGraceMs));
|
|
1187
|
+
}
|
|
1188
|
+
async heartbeat(hostId, namespaceId) {
|
|
1189
|
+
const host = await this.getFromCache(hostId, namespaceId);
|
|
1190
|
+
if (!host) {
|
|
1191
|
+
return;
|
|
1192
|
+
}
|
|
1193
|
+
await this.cache.set(this.hostKey(namespaceId, hostId), { ...host, lastSeen: Date.now() });
|
|
1194
|
+
}
|
|
1195
|
+
async get(hostId, namespaceId) {
|
|
1196
|
+
const cached = await this.cache.get(this.hostKey(namespaceId, hostId));
|
|
1197
|
+
if (cached) {
|
|
1198
|
+
return cached;
|
|
1199
|
+
}
|
|
1200
|
+
if (!this.store) {
|
|
1201
|
+
return null;
|
|
1202
|
+
}
|
|
1203
|
+
const stored = await this.store.get(hostId, namespaceId);
|
|
1204
|
+
if (!stored) {
|
|
1205
|
+
return null;
|
|
1206
|
+
}
|
|
1207
|
+
await this.cache.set(this.hostKey(namespaceId, hostId), { ...stored, status: "offline", connections: [] });
|
|
1208
|
+
await this.addToIndex(namespaceId, hostId);
|
|
1209
|
+
return { ...stored, status: "offline", connections: [] };
|
|
1210
|
+
}
|
|
1211
|
+
async resolveToken(token) {
|
|
1212
|
+
const cached = await this.cache.get(this.tokenKey(token));
|
|
1213
|
+
if (cached) {
|
|
1214
|
+
return cached;
|
|
1215
|
+
}
|
|
1216
|
+
if (!this.store) {
|
|
1217
|
+
return null;
|
|
1218
|
+
}
|
|
1219
|
+
const stored = await this.store.resolveToken(token);
|
|
1220
|
+
if (!stored) {
|
|
1221
|
+
return null;
|
|
1222
|
+
}
|
|
1223
|
+
await this.cache.set(this.tokenKey(token), stored);
|
|
1224
|
+
return stored;
|
|
1225
|
+
}
|
|
1226
|
+
async list(namespaceId) {
|
|
1227
|
+
if (this.store) {
|
|
1228
|
+
const persisted = await this.store.list(namespaceId);
|
|
1229
|
+
return Promise.all(
|
|
1230
|
+
persisted.map(async (host) => {
|
|
1231
|
+
const cached = await this.cache.get(this.hostKey(namespaceId, host.hostId));
|
|
1232
|
+
return cached ?? { ...host, status: "offline", connections: [] };
|
|
1233
|
+
})
|
|
1234
|
+
);
|
|
1235
|
+
}
|
|
1236
|
+
const indexKey = `host:index:${namespaceId}`;
|
|
1237
|
+
const hostIds = await this.cache.get(indexKey) ?? [];
|
|
1238
|
+
const results = await Promise.all(
|
|
1239
|
+
hostIds.map((id) => this.cache.get(this.hostKey(namespaceId, id)))
|
|
1240
|
+
);
|
|
1241
|
+
return results.filter((h) => h !== null);
|
|
1242
|
+
}
|
|
1243
|
+
async deregister(hostId, namespaceId) {
|
|
1244
|
+
const deleted = this.store ? await this.store.delete(hostId, namespaceId) : false;
|
|
1245
|
+
await this.cache.delete(this.hostKey(namespaceId, hostId));
|
|
1246
|
+
await this.removeFromIndex(namespaceId, hostId);
|
|
1247
|
+
return deleted;
|
|
1248
|
+
}
|
|
1249
|
+
async ensureRegistered(hostId, namespaceId, name, capabilities = []) {
|
|
1250
|
+
const existing = await this.get(hostId, namespaceId);
|
|
1251
|
+
if (existing) {
|
|
1252
|
+
if (capabilities.length > 0 && JSON.stringify(existing.capabilities) !== JSON.stringify(capabilities)) {
|
|
1253
|
+
const updated = { ...existing, capabilities, updatedAt: Date.now() };
|
|
1254
|
+
if (this.store) {
|
|
1255
|
+
await this.store.save(updated);
|
|
1256
|
+
}
|
|
1257
|
+
await this.cache.set(this.hostKey(namespaceId, hostId), updated);
|
|
1258
|
+
}
|
|
1259
|
+
return;
|
|
1260
|
+
}
|
|
1261
|
+
const now = Date.now();
|
|
1262
|
+
const descriptor = {
|
|
1263
|
+
hostId,
|
|
1264
|
+
name,
|
|
1265
|
+
namespaceId,
|
|
1266
|
+
capabilities,
|
|
1267
|
+
status: "offline",
|
|
1268
|
+
lastSeen: now,
|
|
1269
|
+
connections: [],
|
|
1270
|
+
createdAt: now,
|
|
1271
|
+
updatedAt: now
|
|
1272
|
+
};
|
|
1273
|
+
if (this.store) {
|
|
1274
|
+
await this.store.save(descriptor);
|
|
1275
|
+
}
|
|
1276
|
+
await this.cache.set(this.hostKey(namespaceId, hostId), descriptor);
|
|
1277
|
+
await this.addToIndex(namespaceId, hostId);
|
|
1278
|
+
}
|
|
1279
|
+
// ── Private helpers ──────────────────────────────────────────────
|
|
1280
|
+
hostKey(namespaceId, hostId) {
|
|
1281
|
+
return `host:registry:${namespaceId}:${hostId}`;
|
|
1282
|
+
}
|
|
1283
|
+
tokenKey(token) {
|
|
1284
|
+
return `host:token:${token}`;
|
|
1285
|
+
}
|
|
1286
|
+
async getFromCache(hostId, namespaceId) {
|
|
1287
|
+
return this.cache.get(this.hostKey(namespaceId, hostId));
|
|
1288
|
+
}
|
|
1289
|
+
async addToIndex(namespaceId, hostId) {
|
|
1290
|
+
const indexKey = `host:index:${namespaceId}`;
|
|
1291
|
+
const hostIds = await this.cache.get(indexKey) ?? [];
|
|
1292
|
+
if (!hostIds.includes(hostId)) {
|
|
1293
|
+
await this.cache.set(indexKey, [...hostIds, hostId]);
|
|
1294
|
+
}
|
|
1295
|
+
const nsKey = "host:namespaces";
|
|
1296
|
+
const namespaces = await this.cache.get(nsKey) ?? [];
|
|
1297
|
+
if (!namespaces.includes(namespaceId)) {
|
|
1298
|
+
await this.cache.set(nsKey, [...namespaces, namespaceId]);
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
async removeFromIndex(namespaceId, hostId) {
|
|
1302
|
+
const indexKey = `host:index:${namespaceId}`;
|
|
1303
|
+
const hostIds = await this.cache.get(indexKey) ?? [];
|
|
1304
|
+
await this.cache.set(indexKey, hostIds.filter((id) => id !== hostId));
|
|
1305
|
+
}
|
|
1306
|
+
};
|
|
1307
|
+
var HELLO_TIMEOUT_MS = 5e3;
|
|
1308
|
+
var HEARTBEAT_INTERVAL_MS = 3e4;
|
|
1309
|
+
var HEARTBEAT_GRACE_MS = 1e4;
|
|
1310
|
+
function send(ws, msg) {
|
|
1311
|
+
ws.send(JSON.stringify(msg));
|
|
1312
|
+
}
|
|
1313
|
+
function createWsHandler(cache, jwtConfig, logger, hostRegistry) {
|
|
1314
|
+
const registry = hostRegistry ?? new HostRegistry(cache);
|
|
1315
|
+
const buffer = new AdaptiveBuffer(cache);
|
|
1316
|
+
return async function wsHandler(socket, request) {
|
|
1317
|
+
const token = extractBearerToken(request.headers.authorization);
|
|
1318
|
+
if (!token) {
|
|
1319
|
+
logDiagnosticEvent(logger, {
|
|
1320
|
+
domain: "service",
|
|
1321
|
+
event: "gateway.hosts.ws.auth",
|
|
1322
|
+
level: "warn",
|
|
1323
|
+
reasonCode: "websocket_auth_failed",
|
|
1324
|
+
message: "Host WebSocket connection missing authorization token",
|
|
1325
|
+
outcome: "failed",
|
|
1326
|
+
serviceId: "gateway",
|
|
1327
|
+
route: "/hosts/connect"
|
|
1328
|
+
});
|
|
1329
|
+
socket.close(1008, "Missing Authorization header");
|
|
1330
|
+
return;
|
|
1331
|
+
}
|
|
1332
|
+
const tokenEntry = await resolveToken(token, cache, jwtConfig);
|
|
1333
|
+
if (!tokenEntry || tokenEntry.type !== "machine") {
|
|
1334
|
+
logDiagnosticEvent(logger, {
|
|
1335
|
+
domain: "service",
|
|
1336
|
+
event: "gateway.hosts.ws.auth",
|
|
1337
|
+
level: "warn",
|
|
1338
|
+
reasonCode: "websocket_auth_failed",
|
|
1339
|
+
message: "Host WebSocket machine token rejected",
|
|
1340
|
+
outcome: "failed",
|
|
1341
|
+
serviceId: "gateway",
|
|
1342
|
+
route: "/hosts/connect"
|
|
1343
|
+
});
|
|
1344
|
+
socket.close(1008, "Invalid machine token");
|
|
1345
|
+
return;
|
|
1346
|
+
}
|
|
1347
|
+
const { userId: hostId, namespaceId } = tokenEntry;
|
|
1348
|
+
const connectionId = randomUUID();
|
|
1349
|
+
const sessionId = randomUUID();
|
|
1350
|
+
let protocolVersion = null;
|
|
1351
|
+
let helloCaps = [];
|
|
1352
|
+
let helloDone = false;
|
|
1353
|
+
const protocolVersions = SUPPORTED_PROTOCOL_VERSIONS;
|
|
1354
|
+
await new Promise((resolve, reject) => {
|
|
1355
|
+
const helloTimeout = setTimeout(() => {
|
|
1356
|
+
if (!helloDone) {
|
|
1357
|
+
helloDone = true;
|
|
1358
|
+
logDiagnosticEvent(logger, {
|
|
1359
|
+
domain: "service",
|
|
1360
|
+
event: "gateway.hosts.ws.handshake",
|
|
1361
|
+
level: "warn",
|
|
1362
|
+
reasonCode: "websocket_hello_timeout",
|
|
1363
|
+
message: "Host WebSocket hello timed out",
|
|
1364
|
+
outcome: "failed",
|
|
1365
|
+
serviceId: "gateway",
|
|
1366
|
+
route: "/hosts/connect",
|
|
1367
|
+
evidence: {
|
|
1368
|
+
hostId,
|
|
1369
|
+
namespaceId
|
|
1370
|
+
}
|
|
1371
|
+
});
|
|
1372
|
+
socket.close(1008, "Hello timeout");
|
|
1373
|
+
reject(new Error("Hello timeout"));
|
|
1374
|
+
}
|
|
1375
|
+
}, HELLO_TIMEOUT_MS);
|
|
1376
|
+
socket.once("message", (raw) => {
|
|
1377
|
+
if (helloDone) {
|
|
1378
|
+
return;
|
|
1379
|
+
}
|
|
1380
|
+
helloDone = true;
|
|
1381
|
+
clearTimeout(helloTimeout);
|
|
1382
|
+
try {
|
|
1383
|
+
const msg = HelloMessageSchema.parse(JSON.parse(raw.toString()));
|
|
1384
|
+
if (!protocolVersions.includes(msg.protocolVersion)) {
|
|
1385
|
+
logDiagnosticEvent(logger, {
|
|
1386
|
+
domain: "service",
|
|
1387
|
+
event: "gateway.hosts.ws.handshake",
|
|
1388
|
+
level: "warn",
|
|
1389
|
+
reasonCode: "websocket_protocol_unsupported",
|
|
1390
|
+
message: "Host WebSocket protocol version is unsupported",
|
|
1391
|
+
outcome: "failed",
|
|
1392
|
+
serviceId: "gateway",
|
|
1393
|
+
route: "/hosts/connect",
|
|
1394
|
+
evidence: {
|
|
1395
|
+
hostId,
|
|
1396
|
+
namespaceId,
|
|
1397
|
+
protocolVersion: msg.protocolVersion,
|
|
1398
|
+
supportedVersions: [...SUPPORTED_PROTOCOL_VERSIONS]
|
|
1399
|
+
}
|
|
1400
|
+
});
|
|
1401
|
+
send(socket, {
|
|
1402
|
+
type: "negotiate",
|
|
1403
|
+
supportedVersions: [...SUPPORTED_PROTOCOL_VERSIONS]
|
|
1404
|
+
});
|
|
1405
|
+
socket.close(1008, "Unsupported protocol version");
|
|
1406
|
+
reject(new Error("Unsupported protocol version"));
|
|
1407
|
+
return;
|
|
1408
|
+
}
|
|
1409
|
+
protocolVersion = msg.protocolVersion;
|
|
1410
|
+
helloCaps = msg.capabilities ?? [];
|
|
1411
|
+
resolve();
|
|
1412
|
+
} catch (error) {
|
|
1413
|
+
logDiagnosticEvent(logger, {
|
|
1414
|
+
domain: "service",
|
|
1415
|
+
event: "gateway.hosts.ws.handshake",
|
|
1416
|
+
level: "warn",
|
|
1417
|
+
reasonCode: "websocket_handshake_invalid",
|
|
1418
|
+
message: "Host WebSocket hello message is invalid",
|
|
1419
|
+
outcome: "failed",
|
|
1420
|
+
error: error instanceof Error ? error : new Error(String(error)),
|
|
1421
|
+
serviceId: "gateway",
|
|
1422
|
+
route: "/hosts/connect",
|
|
1423
|
+
evidence: {
|
|
1424
|
+
hostId,
|
|
1425
|
+
namespaceId
|
|
1426
|
+
}
|
|
1427
|
+
});
|
|
1428
|
+
socket.close(1008, "Invalid hello message");
|
|
1429
|
+
reject(new Error("Invalid hello"));
|
|
1430
|
+
}
|
|
1431
|
+
});
|
|
1432
|
+
}).catch(() => {
|
|
1433
|
+
});
|
|
1434
|
+
if (!protocolVersion) {
|
|
1435
|
+
return;
|
|
1436
|
+
}
|
|
1437
|
+
const clientRecord = await getClientByHostId(cache, hostId);
|
|
1438
|
+
const registryCaps = (clientRecord?.capabilities ?? []).map((c) => HostCapabilitySchema.safeParse(c)).filter((r) => r.success).map((r) => r.data);
|
|
1439
|
+
const validatedHelloCaps = clientRecord ? [] : helloCaps.map((c) => HostCapabilitySchema.safeParse(c)).filter((r) => r.success).map((r) => r.data);
|
|
1440
|
+
const capabilities = clientRecord ? registryCaps : validatedHelloCaps;
|
|
1441
|
+
await registry.ensureRegistered(hostId, namespaceId, clientRecord?.name ?? hostId, capabilities);
|
|
1442
|
+
await registry.setOnline(hostId, namespaceId, connectionId);
|
|
1443
|
+
globalDispatcher.registerConnection(hostId, namespaceId, socket, capabilities);
|
|
1444
|
+
send(socket, {
|
|
1445
|
+
type: "connected",
|
|
1446
|
+
protocolVersion,
|
|
1447
|
+
hostId,
|
|
1448
|
+
sessionId
|
|
1449
|
+
});
|
|
1450
|
+
const buffered = await buffer.flush(hostId);
|
|
1451
|
+
for (const call of buffered) {
|
|
1452
|
+
send(socket, {
|
|
1453
|
+
type: "call",
|
|
1454
|
+
requestId: call.requestId,
|
|
1455
|
+
adapter: call.adapter,
|
|
1456
|
+
method: call.method,
|
|
1457
|
+
args: call.args,
|
|
1458
|
+
trace: { traceId: call.requestId, spanId: randomUUID() }
|
|
1459
|
+
});
|
|
1460
|
+
}
|
|
1461
|
+
let lastHeartbeat = Date.now();
|
|
1462
|
+
const heartbeatWatchdog = setInterval(async () => {
|
|
1463
|
+
const elapsed = Date.now() - lastHeartbeat;
|
|
1464
|
+
if (elapsed > HEARTBEAT_INTERVAL_MS + HEARTBEAT_GRACE_MS) {
|
|
1465
|
+
const host = await registry.get(hostId, namespaceId);
|
|
1466
|
+
if (host && host.status !== "degraded") {
|
|
1467
|
+
await cache.set(`host:registry:${namespaceId}:${hostId}`, {
|
|
1468
|
+
...host,
|
|
1469
|
+
status: "degraded"
|
|
1470
|
+
});
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
1474
|
+
socket.on("message", async (raw) => {
|
|
1475
|
+
try {
|
|
1476
|
+
const msg = JSON.parse(raw.toString());
|
|
1477
|
+
switch (msg.type) {
|
|
1478
|
+
case "heartbeat":
|
|
1479
|
+
lastHeartbeat = Date.now();
|
|
1480
|
+
await registry.heartbeat(hostId, namespaceId);
|
|
1481
|
+
send(socket, { type: "ack" });
|
|
1482
|
+
break;
|
|
1483
|
+
case "chunk":
|
|
1484
|
+
case "result":
|
|
1485
|
+
case "error":
|
|
1486
|
+
globalDispatcher.handleInbound(msg);
|
|
1487
|
+
break;
|
|
1488
|
+
case "adapter:call":
|
|
1489
|
+
void handleAdapterCall(msg, socket, hostId, namespaceId);
|
|
1490
|
+
break;
|
|
1491
|
+
}
|
|
1492
|
+
} catch (error) {
|
|
1493
|
+
logDiagnosticEvent(logger, {
|
|
1494
|
+
domain: "service",
|
|
1495
|
+
event: "gateway.hosts.ws.message",
|
|
1496
|
+
level: "warn",
|
|
1497
|
+
reasonCode: "websocket_message_invalid",
|
|
1498
|
+
message: "Host WebSocket message is malformed",
|
|
1499
|
+
outcome: "failed",
|
|
1500
|
+
error: error instanceof Error ? error : new Error(String(error)),
|
|
1501
|
+
serviceId: "gateway",
|
|
1502
|
+
route: "/hosts/connect",
|
|
1503
|
+
evidence: {
|
|
1504
|
+
hostId,
|
|
1505
|
+
namespaceId
|
|
1506
|
+
}
|
|
1507
|
+
});
|
|
1508
|
+
}
|
|
1509
|
+
});
|
|
1510
|
+
socket.on("close", async () => {
|
|
1511
|
+
clearInterval(heartbeatWatchdog);
|
|
1512
|
+
globalDispatcher.removeConnection(hostId, namespaceId);
|
|
1513
|
+
const cancelled = executionRegistry.cancelByHost(hostId, "disconnect");
|
|
1514
|
+
if (cancelled.length > 0) {
|
|
1515
|
+
logDiagnosticEvent(logger, {
|
|
1516
|
+
domain: "service",
|
|
1517
|
+
event: "gateway.hosts.ws.disconnect",
|
|
1518
|
+
level: "warn",
|
|
1519
|
+
reasonCode: "execution_dispatch_failed",
|
|
1520
|
+
message: "Host disconnected and active executions were cancelled",
|
|
1521
|
+
outcome: "failed",
|
|
1522
|
+
serviceId: "gateway",
|
|
1523
|
+
route: "/hosts/connect",
|
|
1524
|
+
evidence: {
|
|
1525
|
+
hostId,
|
|
1526
|
+
namespaceId,
|
|
1527
|
+
cancelledExecutions: cancelled.length
|
|
1528
|
+
}
|
|
1529
|
+
});
|
|
1530
|
+
}
|
|
1531
|
+
await registry.setOffline(hostId, namespaceId, connectionId);
|
|
1532
|
+
});
|
|
1533
|
+
};
|
|
1534
|
+
async function handleAdapterCall(msg, socket, hostId, namespaceId) {
|
|
1535
|
+
const requestId = msg["requestId"];
|
|
1536
|
+
const parsed = AdapterCallMessageSchema.safeParse(msg);
|
|
1537
|
+
if (!parsed.success) {
|
|
1538
|
+
logDiagnosticEvent(logger, {
|
|
1539
|
+
domain: "service",
|
|
1540
|
+
event: "gateway.hosts.adapter-call",
|
|
1541
|
+
level: "warn",
|
|
1542
|
+
reasonCode: "websocket_message_invalid",
|
|
1543
|
+
message: "Host adapter call message is invalid",
|
|
1544
|
+
outcome: "failed",
|
|
1545
|
+
serviceId: "gateway",
|
|
1546
|
+
route: "/hosts/connect",
|
|
1547
|
+
evidence: {
|
|
1548
|
+
hostId,
|
|
1549
|
+
namespaceId,
|
|
1550
|
+
requestId
|
|
1551
|
+
}
|
|
1552
|
+
});
|
|
1553
|
+
send(socket, {
|
|
1554
|
+
type: "adapter:error",
|
|
1555
|
+
requestId: requestId ?? "unknown",
|
|
1556
|
+
error: { code: "INVALID_MESSAGE", message: parsed.error.message, retryable: false }
|
|
1557
|
+
});
|
|
1558
|
+
return;
|
|
1559
|
+
}
|
|
1560
|
+
const call = parsed.data;
|
|
1561
|
+
const adapterCheck = AdapterNameSchema.safeParse(call.adapter);
|
|
1562
|
+
if (!adapterCheck.success) {
|
|
1563
|
+
logDiagnosticEvent(logger, {
|
|
1564
|
+
domain: "service",
|
|
1565
|
+
event: "gateway.hosts.adapter-call",
|
|
1566
|
+
level: "warn",
|
|
1567
|
+
reasonCode: "adapter_call_rejected",
|
|
1568
|
+
message: "Host adapter call rejected by gateway allowlist",
|
|
1569
|
+
outcome: "failed",
|
|
1570
|
+
serviceId: "gateway",
|
|
1571
|
+
route: "/hosts/connect",
|
|
1572
|
+
evidence: {
|
|
1573
|
+
hostId,
|
|
1574
|
+
namespaceId,
|
|
1575
|
+
requestId: call.requestId,
|
|
1576
|
+
adapter: call.adapter,
|
|
1577
|
+
method: call.method
|
|
1578
|
+
}
|
|
1579
|
+
});
|
|
1580
|
+
send(socket, {
|
|
1581
|
+
type: "adapter:error",
|
|
1582
|
+
requestId: call.requestId,
|
|
1583
|
+
error: { code: "ADAPTER_CALL_REJECTED", message: `Adapter not allowed: ${call.adapter}`, retryable: false }
|
|
1584
|
+
});
|
|
1585
|
+
return;
|
|
1586
|
+
}
|
|
1587
|
+
const restApiUrl = process.env.REST_API_URL ?? "http://localhost:5050";
|
|
1588
|
+
const internalSecret = process.env.GATEWAY_INTERNAL_SECRET ?? "";
|
|
1589
|
+
try {
|
|
1590
|
+
const response = await fetch(`${restApiUrl}/api/v1/internal/adapter-call`, {
|
|
1591
|
+
method: "POST",
|
|
1592
|
+
headers: {
|
|
1593
|
+
"Content-Type": "application/json",
|
|
1594
|
+
"x-internal-secret": internalSecret
|
|
1595
|
+
},
|
|
1596
|
+
body: JSON.stringify({
|
|
1597
|
+
requestId: call.requestId,
|
|
1598
|
+
adapter: call.adapter,
|
|
1599
|
+
method: call.method,
|
|
1600
|
+
args: call.args,
|
|
1601
|
+
context: {
|
|
1602
|
+
...call.context,
|
|
1603
|
+
namespaceId,
|
|
1604
|
+
hostId
|
|
1605
|
+
}
|
|
1606
|
+
})
|
|
1607
|
+
});
|
|
1608
|
+
const body = await response.json();
|
|
1609
|
+
if (body.ok) {
|
|
1610
|
+
send(socket, {
|
|
1611
|
+
type: "adapter:response",
|
|
1612
|
+
requestId: call.requestId,
|
|
1613
|
+
result: body.result
|
|
1614
|
+
});
|
|
1615
|
+
} else {
|
|
1616
|
+
send(socket, {
|
|
1617
|
+
type: "adapter:error",
|
|
1618
|
+
requestId: call.requestId,
|
|
1619
|
+
error: body.error ?? { code: "ADAPTER_ERROR", message: "Unknown error", retryable: false }
|
|
1620
|
+
});
|
|
1621
|
+
}
|
|
1622
|
+
} catch (err) {
|
|
1623
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1624
|
+
logDiagnosticEvent(logger, {
|
|
1625
|
+
domain: "service",
|
|
1626
|
+
event: "gateway.hosts.adapter-call",
|
|
1627
|
+
level: "error",
|
|
1628
|
+
reasonCode: "adapter_bridge_unavailable",
|
|
1629
|
+
message: "Gateway could not reach REST adapter bridge",
|
|
1630
|
+
outcome: "failed",
|
|
1631
|
+
error: err instanceof Error ? err : new Error(String(err)),
|
|
1632
|
+
serviceId: "gateway",
|
|
1633
|
+
route: "/hosts/connect",
|
|
1634
|
+
evidence: {
|
|
1635
|
+
hostId,
|
|
1636
|
+
namespaceId,
|
|
1637
|
+
requestId: call.requestId,
|
|
1638
|
+
adapter: call.adapter,
|
|
1639
|
+
method: call.method,
|
|
1640
|
+
restApiUrl
|
|
1641
|
+
}
|
|
1642
|
+
});
|
|
1643
|
+
send(socket, {
|
|
1644
|
+
type: "adapter:error",
|
|
1645
|
+
requestId: call.requestId,
|
|
1646
|
+
error: { code: "ADAPTER_CALL_TIMEOUT", message: `REST API unreachable: ${message}`, retryable: true }
|
|
1647
|
+
});
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1650
|
+
}
|
|
1651
|
+
var HELLO_TIMEOUT_MS2 = 5e3;
|
|
1652
|
+
function send2(ws, msg) {
|
|
1653
|
+
if (ws.readyState === ws.OPEN) {
|
|
1654
|
+
ws.send(JSON.stringify(msg));
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
function createClientWsHandler(cache, jwtConfig, logger) {
|
|
1658
|
+
return async function clientWsHandler(socket, request) {
|
|
1659
|
+
const queryToken = new URL(request.url ?? "/", "http://localhost").searchParams.get("access_token");
|
|
1660
|
+
const token = extractBearerToken(request.headers.authorization) ?? queryToken ?? null;
|
|
1661
|
+
if (!token) {
|
|
1662
|
+
socket.close(1008, "Missing Authorization");
|
|
1663
|
+
return;
|
|
1664
|
+
}
|
|
1665
|
+
const authContext = await resolveToken(token, cache, jwtConfig);
|
|
1666
|
+
if (!authContext) {
|
|
1667
|
+
socket.close(1008, "Invalid token");
|
|
1668
|
+
return;
|
|
1669
|
+
}
|
|
1670
|
+
const connectionId = randomUUID();
|
|
1671
|
+
let helloDone = false;
|
|
1672
|
+
await new Promise((resolve, reject) => {
|
|
1673
|
+
const helloTimeout = setTimeout(() => {
|
|
1674
|
+
if (!helloDone) {
|
|
1675
|
+
helloDone = true;
|
|
1676
|
+
socket.close(1008, "Hello timeout");
|
|
1677
|
+
reject(new Error("Hello timeout"));
|
|
1678
|
+
}
|
|
1679
|
+
}, HELLO_TIMEOUT_MS2);
|
|
1680
|
+
socket.once("message", (raw) => {
|
|
1681
|
+
if (helloDone) {
|
|
1682
|
+
return;
|
|
1683
|
+
}
|
|
1684
|
+
helloDone = true;
|
|
1685
|
+
clearTimeout(helloTimeout);
|
|
1686
|
+
try {
|
|
1687
|
+
const msg = ClientHelloSchema.parse(JSON.parse(raw.toString()));
|
|
1688
|
+
void msg;
|
|
1689
|
+
resolve();
|
|
1690
|
+
} catch {
|
|
1691
|
+
socket.close(1008, "Invalid hello message");
|
|
1692
|
+
reject(new Error("Invalid client:hello"));
|
|
1693
|
+
}
|
|
1694
|
+
});
|
|
1695
|
+
}).catch(() => {
|
|
1696
|
+
});
|
|
1697
|
+
if (!helloDone || socket.readyState !== socket.OPEN) {
|
|
1698
|
+
return;
|
|
1699
|
+
}
|
|
1700
|
+
subscriptionRegistry.registerSocket(connectionId, socket);
|
|
1701
|
+
send2(socket, {
|
|
1702
|
+
type: "client:connected",
|
|
1703
|
+
protocolVersion: CLIENT_PROTOCOL_VERSION,
|
|
1704
|
+
connectionId
|
|
1705
|
+
});
|
|
1706
|
+
logger.debug("Client connected", { connectionId, namespaceId: authContext.namespaceId });
|
|
1707
|
+
socket.on("message", (raw) => {
|
|
1708
|
+
let parsed;
|
|
1709
|
+
try {
|
|
1710
|
+
parsed = JSON.parse(raw.toString());
|
|
1711
|
+
} catch {
|
|
1712
|
+
send2(socket, {
|
|
1713
|
+
type: "client:error",
|
|
1714
|
+
code: "INVALID_MESSAGE",
|
|
1715
|
+
message: "Malformed JSON"
|
|
1716
|
+
});
|
|
1717
|
+
return;
|
|
1718
|
+
}
|
|
1719
|
+
switch (parsed.type) {
|
|
1720
|
+
case "client:subscribe": {
|
|
1721
|
+
const result = ClientSubscribeSchema.safeParse(parsed);
|
|
1722
|
+
if (!result.success) {
|
|
1723
|
+
send2(socket, { type: "client:error", code: "INVALID_MESSAGE", message: "Invalid subscribe message" });
|
|
1724
|
+
return;
|
|
1725
|
+
}
|
|
1726
|
+
const { executionId } = result.data;
|
|
1727
|
+
const execution = executionRegistry.get(executionId);
|
|
1728
|
+
if (!execution) {
|
|
1729
|
+
send2(socket, { type: "client:error", code: "EXECUTION_NOT_FOUND", message: `Execution ${executionId} not found`, executionId });
|
|
1730
|
+
return;
|
|
1731
|
+
}
|
|
1732
|
+
if (execution.namespaceId !== authContext.namespaceId) {
|
|
1733
|
+
send2(socket, { type: "client:error", code: "FORBIDDEN", message: "Execution belongs to another namespace", executionId });
|
|
1734
|
+
return;
|
|
1735
|
+
}
|
|
1736
|
+
subscriptionRegistry.subscribe(connectionId, executionId);
|
|
1737
|
+
break;
|
|
1738
|
+
}
|
|
1739
|
+
case "client:unsubscribe": {
|
|
1740
|
+
const result = ClientUnsubscribeSchema.safeParse(parsed);
|
|
1741
|
+
if (!result.success) {
|
|
1742
|
+
send2(socket, { type: "client:error", code: "INVALID_MESSAGE", message: "Invalid unsubscribe message" });
|
|
1743
|
+
return;
|
|
1744
|
+
}
|
|
1745
|
+
subscriptionRegistry.unsubscribe(connectionId, result.data.executionId);
|
|
1746
|
+
break;
|
|
1747
|
+
}
|
|
1748
|
+
case "client:cancel": {
|
|
1749
|
+
const result = ClientCancelSchema.safeParse(parsed);
|
|
1750
|
+
if (!result.success) {
|
|
1751
|
+
send2(socket, { type: "client:error", code: "INVALID_MESSAGE", message: "Invalid cancel message" });
|
|
1752
|
+
return;
|
|
1753
|
+
}
|
|
1754
|
+
const { executionId, reason } = result.data;
|
|
1755
|
+
const execution = executionRegistry.get(executionId);
|
|
1756
|
+
if (!execution) {
|
|
1757
|
+
send2(socket, { type: "client:error", code: "EXECUTION_NOT_FOUND", message: `Execution ${executionId} not found`, executionId });
|
|
1758
|
+
return;
|
|
1759
|
+
}
|
|
1760
|
+
if (execution.namespaceId !== authContext.namespaceId) {
|
|
1761
|
+
send2(socket, { type: "client:error", code: "FORBIDDEN", message: "Execution belongs to another namespace", executionId });
|
|
1762
|
+
return;
|
|
1763
|
+
}
|
|
1764
|
+
const cancelled = executionRegistry.cancel(executionId, reason ?? "user");
|
|
1765
|
+
if (!cancelled) {
|
|
1766
|
+
send2(socket, { type: "client:error", code: "CANCEL_FAILED", message: "Execution already completed or cancelled", executionId });
|
|
1767
|
+
}
|
|
1768
|
+
break;
|
|
1769
|
+
}
|
|
1770
|
+
default:
|
|
1771
|
+
send2(socket, { type: "client:error", code: "INVALID_MESSAGE", message: `Unknown message type: ${String(parsed.type)}` });
|
|
1772
|
+
}
|
|
1773
|
+
});
|
|
1774
|
+
socket.on("close", () => {
|
|
1775
|
+
subscriptionRegistry.removeConnection(connectionId);
|
|
1776
|
+
logger.debug("Client disconnected", { connectionId });
|
|
1777
|
+
});
|
|
1778
|
+
};
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
// src/ws/gateway-ws.ts
|
|
1782
|
+
var GATEWAY_WS_PATHS = /* @__PURE__ */ new Set(["/hosts/connect", "/clients/connect"]);
|
|
1783
|
+
function attachGatewayWs(server, cache, jwtConfig, logger, hostRegistry) {
|
|
1784
|
+
const wss = new WebSocketServer({ noServer: true });
|
|
1785
|
+
const hostsHandler = createWsHandler(cache, jwtConfig, logger, hostRegistry);
|
|
1786
|
+
const clientsHandler = createClientWsHandler(cache, jwtConfig, logger);
|
|
1787
|
+
const existingListeners = server.listeners("upgrade").slice();
|
|
1788
|
+
server.removeAllListeners("upgrade");
|
|
1789
|
+
server.on("upgrade", (req, socket, head) => {
|
|
1790
|
+
const pathname = new URL(req.url ?? "/", "http://localhost").pathname;
|
|
1791
|
+
if (GATEWAY_WS_PATHS.has(pathname)) {
|
|
1792
|
+
wss.handleUpgrade(req, socket, head, (ws) => {
|
|
1793
|
+
if (pathname === "/hosts/connect") {
|
|
1794
|
+
hostsHandler(ws, req);
|
|
1795
|
+
} else {
|
|
1796
|
+
clientsHandler(ws, req);
|
|
1797
|
+
}
|
|
1798
|
+
});
|
|
1799
|
+
} else {
|
|
1800
|
+
for (const listener of existingListeners) {
|
|
1801
|
+
listener.call(server, req, socket, head);
|
|
1802
|
+
}
|
|
1803
|
+
}
|
|
1804
|
+
});
|
|
1805
|
+
logger.info("Gateway WS endpoints attached", { paths: [...GATEWAY_WS_PATHS] });
|
|
1806
|
+
}
|
|
1807
|
+
function normalizeRoute(route) {
|
|
1808
|
+
if (!route) {
|
|
1809
|
+
return "unknown";
|
|
1810
|
+
}
|
|
1811
|
+
return route.split("?")[0].replace(/\/[0-9a-fA-F-]{6,}/g, "/:id");
|
|
1812
|
+
}
|
|
1813
|
+
function metricLine(name, value, labels) {
|
|
1814
|
+
if (!labels || Object.keys(labels).length === 0) {
|
|
1815
|
+
return `${name} ${value}`;
|
|
1816
|
+
}
|
|
1817
|
+
const pairs = Object.entries(labels).map(([key, labelValue]) => `${key}="${labelValue.replace(/"/g, '\\"')}"`);
|
|
1818
|
+
return `${name}{${pairs.join(",")}} ${value}`;
|
|
1819
|
+
}
|
|
1820
|
+
var GatewayObservabilityCollector = class {
|
|
1821
|
+
constructor(config) {
|
|
1822
|
+
this.config = config;
|
|
1823
|
+
this.dependencies = Object.keys(this.config.upstreams).map((serviceId) => ({
|
|
1824
|
+
serviceId,
|
|
1825
|
+
required: false,
|
|
1826
|
+
description: "Gateway upstream"
|
|
1827
|
+
}));
|
|
1828
|
+
}
|
|
1829
|
+
instanceId = `${hostname()}:${process.pid}`;
|
|
1830
|
+
eventLoop = monitorEventLoopDelay({ resolution: 20 });
|
|
1831
|
+
routeStats = /* @__PURE__ */ new Map();
|
|
1832
|
+
operationMetrics = new OperationMetricsTracker();
|
|
1833
|
+
startedAt = Date.now();
|
|
1834
|
+
dependencies;
|
|
1835
|
+
lastCpuUsage = process.cpuUsage();
|
|
1836
|
+
lastCpuTime = Date.now();
|
|
1837
|
+
intervalId = null;
|
|
1838
|
+
activeOperations = 0;
|
|
1839
|
+
requestsTotal = 0;
|
|
1840
|
+
errorsTotal = 0;
|
|
1841
|
+
lastSnapshot = {
|
|
1842
|
+
cpuPercent: 0,
|
|
1843
|
+
rssBytes: process.memoryUsage().rss,
|
|
1844
|
+
heapUsedBytes: process.memoryUsage().heapUsed,
|
|
1845
|
+
eventLoopLagMs: 0
|
|
1846
|
+
};
|
|
1847
|
+
register(server) {
|
|
1848
|
+
const hookServer = server;
|
|
1849
|
+
this.eventLoop.enable();
|
|
1850
|
+
this.intervalId = setInterval(() => this.captureRuntimeSnapshot(), 1e4);
|
|
1851
|
+
this.captureRuntimeSnapshot();
|
|
1852
|
+
hookServer.addHook("onRequest", (request, _reply, done) => {
|
|
1853
|
+
request.kbMetricsStart = performance.now();
|
|
1854
|
+
this.activeOperations += 1;
|
|
1855
|
+
done();
|
|
1856
|
+
});
|
|
1857
|
+
hookServer.addHook("onResponse", (request, reply, done) => {
|
|
1858
|
+
const started = request.kbMetricsStart ?? performance.now();
|
|
1859
|
+
const durationMs = Math.max(performance.now() - started, 0);
|
|
1860
|
+
const route = `${request.method.toUpperCase()} ${normalizeRoute(request.routeOptions?.url ?? request.url)}`;
|
|
1861
|
+
const stats = this.routeStats.get(route) ?? {
|
|
1862
|
+
count: 0,
|
|
1863
|
+
totalDurationMs: 0,
|
|
1864
|
+
maxDurationMs: 0,
|
|
1865
|
+
errorCount: 0
|
|
1866
|
+
};
|
|
1867
|
+
stats.count += 1;
|
|
1868
|
+
stats.totalDurationMs += durationMs;
|
|
1869
|
+
stats.maxDurationMs = Math.max(stats.maxDurationMs, durationMs);
|
|
1870
|
+
if (reply.statusCode >= 400) {
|
|
1871
|
+
stats.errorCount += 1;
|
|
1872
|
+
this.errorsTotal += 1;
|
|
1873
|
+
}
|
|
1874
|
+
this.routeStats.set(route, stats);
|
|
1875
|
+
this.requestsTotal += 1;
|
|
1876
|
+
this.activeOperations = Math.max(0, this.activeOperations - 1);
|
|
1877
|
+
done();
|
|
1878
|
+
});
|
|
1879
|
+
hookServer.addHook("onClose", (_instance, done) => {
|
|
1880
|
+
if (this.intervalId) {
|
|
1881
|
+
clearInterval(this.intervalId);
|
|
1882
|
+
this.intervalId = null;
|
|
1883
|
+
}
|
|
1884
|
+
this.eventLoop.disable();
|
|
1885
|
+
done();
|
|
1886
|
+
});
|
|
1887
|
+
}
|
|
1888
|
+
buildDescribe() {
|
|
1889
|
+
return createServiceObservabilityDescribe({
|
|
1890
|
+
schema: OBSERVABILITY_SCHEMA,
|
|
1891
|
+
contractVersion: OBSERVABILITY_CONTRACT_VERSION,
|
|
1892
|
+
serviceId: "gateway",
|
|
1893
|
+
instanceId: this.instanceId,
|
|
1894
|
+
serviceType: "gateway",
|
|
1895
|
+
version: "1.0.0",
|
|
1896
|
+
environment: process.env.NODE_ENV ?? "development",
|
|
1897
|
+
startedAt: new Date(this.startedAt).toISOString(),
|
|
1898
|
+
dependencies: this.dependencies,
|
|
1899
|
+
metricsEndpoint: "/metrics",
|
|
1900
|
+
healthEndpoint: "/observability/health",
|
|
1901
|
+
logsSource: "gateway",
|
|
1902
|
+
capabilities: ["httpMetrics", "eventLoopMetrics", "operationMetrics", "logCorrelation"],
|
|
1903
|
+
metricFamilies: [...CANONICAL_OBSERVABILITY_METRICS]
|
|
1904
|
+
});
|
|
1905
|
+
}
|
|
1906
|
+
buildHealth(input) {
|
|
1907
|
+
const checks = [
|
|
1908
|
+
...input.adapterChecks.map((entry) => ({
|
|
1909
|
+
id: `adapter:${entry.id}`,
|
|
1910
|
+
status: entry.available ? "ok" : "warn",
|
|
1911
|
+
latencyMs: entry.latencyMs,
|
|
1912
|
+
message: entry.available ? "Adapter available" : "Adapter unavailable"
|
|
1913
|
+
})),
|
|
1914
|
+
...input.upstreamChecks.map((entry) => ({
|
|
1915
|
+
id: `upstream:${entry.id}`,
|
|
1916
|
+
status: entry.status === "up" ? "ok" : "warn",
|
|
1917
|
+
latencyMs: entry.latencyMs,
|
|
1918
|
+
message: entry.status === "up" ? "Upstream healthy" : "Upstream unavailable"
|
|
1919
|
+
}))
|
|
1920
|
+
];
|
|
1921
|
+
const topOperations = mergeTopOperations(
|
|
1922
|
+
Array.from(this.routeStats.entries()).sort((a, b) => b[1].count - a[1].count || b[1].maxDurationMs - a[1].maxDurationMs).slice(0, 5).map(([operation, stats]) => ({
|
|
1923
|
+
operation: `http.${operation}`,
|
|
1924
|
+
count: stats.count,
|
|
1925
|
+
avgDurationMs: stats.count > 0 ? stats.totalDurationMs / stats.count : 0,
|
|
1926
|
+
maxDurationMs: stats.maxDurationMs,
|
|
1927
|
+
errorCount: stats.errorCount
|
|
1928
|
+
})),
|
|
1929
|
+
this.operationMetrics.getTopOperations()
|
|
1930
|
+
);
|
|
1931
|
+
return createServiceObservabilityHealth({
|
|
1932
|
+
schema: OBSERVABILITY_SCHEMA,
|
|
1933
|
+
contractVersion: OBSERVABILITY_CONTRACT_VERSION,
|
|
1934
|
+
serviceId: "gateway",
|
|
1935
|
+
instanceId: this.instanceId,
|
|
1936
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1937
|
+
status: input.status,
|
|
1938
|
+
uptimeSec: Math.floor((Date.now() - this.startedAt) / 1e3),
|
|
1939
|
+
metricsEndpoint: "/metrics",
|
|
1940
|
+
logsSource: "gateway",
|
|
1941
|
+
capabilities: ["httpMetrics", "eventLoopMetrics", "operationMetrics", "logCorrelation"],
|
|
1942
|
+
checks,
|
|
1943
|
+
snapshot: {
|
|
1944
|
+
cpuPercent: this.lastSnapshot.cpuPercent,
|
|
1945
|
+
rssBytes: this.lastSnapshot.rssBytes,
|
|
1946
|
+
heapUsedBytes: this.lastSnapshot.heapUsedBytes,
|
|
1947
|
+
eventLoopLagMs: this.lastSnapshot.eventLoopLagMs,
|
|
1948
|
+
activeOperations: this.activeOperations
|
|
1949
|
+
},
|
|
1950
|
+
topOperations,
|
|
1951
|
+
state: input.status === "healthy" ? "active" : input.status === "degraded" ? "partial_observability" : "insufficient_data",
|
|
1952
|
+
meta: {
|
|
1953
|
+
requestsTotal: this.requestsTotal,
|
|
1954
|
+
errorsTotal: this.errorsTotal
|
|
1955
|
+
}
|
|
1956
|
+
});
|
|
1957
|
+
}
|
|
1958
|
+
async renderPrometheusMetrics(healthStatus) {
|
|
1959
|
+
this.captureRuntimeSnapshot();
|
|
1960
|
+
const lines = [
|
|
1961
|
+
"# HELP process_cpu_percent Current process CPU usage percentage",
|
|
1962
|
+
"# TYPE process_cpu_percent gauge",
|
|
1963
|
+
metricLine("process_cpu_percent", this.lastSnapshot.cpuPercent),
|
|
1964
|
+
"# HELP process_rss_bytes Current process resident set size in bytes",
|
|
1965
|
+
"# TYPE process_rss_bytes gauge",
|
|
1966
|
+
metricLine("process_rss_bytes", this.lastSnapshot.rssBytes),
|
|
1967
|
+
"# HELP process_heap_used_bytes Current process heap used in bytes",
|
|
1968
|
+
"# TYPE process_heap_used_bytes gauge",
|
|
1969
|
+
metricLine("process_heap_used_bytes", this.lastSnapshot.heapUsedBytes),
|
|
1970
|
+
"# HELP process_event_loop_lag_ms Current event loop lag in milliseconds",
|
|
1971
|
+
"# TYPE process_event_loop_lag_ms gauge",
|
|
1972
|
+
metricLine("process_event_loop_lag_ms", this.lastSnapshot.eventLoopLagMs),
|
|
1973
|
+
"# HELP service_health_status Service health status (2=healthy, 1=degraded, 0=unhealthy)",
|
|
1974
|
+
"# TYPE service_health_status gauge",
|
|
1975
|
+
metricLine("service_health_status", healthStatus === "healthy" ? 2 : healthStatus === "degraded" ? 1 : 0),
|
|
1976
|
+
"# HELP service_restarts_total Service restart counter within current process lifetime",
|
|
1977
|
+
"# TYPE service_restarts_total gauge",
|
|
1978
|
+
metricLine("service_restarts_total", 0),
|
|
1979
|
+
"# HELP service_active_operations Current number of active operations",
|
|
1980
|
+
"# TYPE service_active_operations gauge",
|
|
1981
|
+
metricLine("service_active_operations", this.activeOperations),
|
|
1982
|
+
"# HELP http_requests_total Total number of HTTP requests",
|
|
1983
|
+
"# TYPE http_requests_total counter",
|
|
1984
|
+
"# HELP http_errors_total Total number of HTTP errors (4xx, 5xx)",
|
|
1985
|
+
"# TYPE http_errors_total counter",
|
|
1986
|
+
"# HELP http_request_duration_ms Total duration of HTTP requests grouped by route",
|
|
1987
|
+
"# TYPE http_request_duration_ms summary",
|
|
1988
|
+
"# HELP service_operation_total Total number of service operations",
|
|
1989
|
+
"# TYPE service_operation_total counter",
|
|
1990
|
+
"# HELP service_operation_duration_ms Total duration of service operations grouped by route",
|
|
1991
|
+
"# TYPE service_operation_duration_ms summary"
|
|
1992
|
+
];
|
|
1993
|
+
for (const [route, stats] of this.routeStats.entries()) {
|
|
1994
|
+
const status = stats.errorCount > 0 ? "error" : "ok";
|
|
1995
|
+
lines.push(metricLine("http_requests_total", stats.count, { route }));
|
|
1996
|
+
lines.push(metricLine("http_errors_total", stats.errorCount, { route }));
|
|
1997
|
+
lines.push(metricLine("http_request_duration_ms", Number(stats.totalDurationMs.toFixed(2)), { route }));
|
|
1998
|
+
lines.push(metricLine("service_operation_total", stats.count, { operation: `http.${route}`, status }));
|
|
1999
|
+
lines.push(metricLine("service_operation_duration_ms", Number(stats.totalDurationMs.toFixed(2)), { operation: `http.${route}`, status }));
|
|
2000
|
+
}
|
|
2001
|
+
lines.push(...this.operationMetrics.getMetricLines());
|
|
2002
|
+
return `${lines.join("\n")}
|
|
2003
|
+
`;
|
|
2004
|
+
}
|
|
2005
|
+
recordOperation(operation, durationMs = 0, status = "ok") {
|
|
2006
|
+
this.operationMetrics.recordOperation(operation, durationMs, status);
|
|
2007
|
+
}
|
|
2008
|
+
observeOperation(operation, work) {
|
|
2009
|
+
return this.operationMetrics.observeOperation(operation, work);
|
|
2010
|
+
}
|
|
2011
|
+
captureRuntimeSnapshot() {
|
|
2012
|
+
const currentUsage = process.cpuUsage(this.lastCpuUsage);
|
|
2013
|
+
const currentTime = Date.now();
|
|
2014
|
+
const deltaTime = Math.max(currentTime - this.lastCpuTime, 1);
|
|
2015
|
+
this.lastCpuUsage = process.cpuUsage();
|
|
2016
|
+
this.lastCpuTime = currentTime;
|
|
2017
|
+
const cpuTimeMs = (currentUsage.user + currentUsage.system) / 1e3;
|
|
2018
|
+
const memory = process.memoryUsage();
|
|
2019
|
+
const eventLoopLagMs = Number((this.eventLoop.mean / 1e6).toFixed(2));
|
|
2020
|
+
this.lastSnapshot = {
|
|
2021
|
+
cpuPercent: Number(Math.min(cpuTimeMs / deltaTime * 100, 100).toFixed(2)),
|
|
2022
|
+
rssBytes: memory.rss,
|
|
2023
|
+
heapUsedBytes: memory.heapUsed,
|
|
2024
|
+
eventLoopLagMs: Number.isFinite(eventLoopLagMs) ? eventLoopLagMs : 0
|
|
2025
|
+
};
|
|
2026
|
+
this.eventLoop.reset();
|
|
2027
|
+
}
|
|
2028
|
+
};
|
|
2029
|
+
function mergeTopOperations(httpOperations, domainOperations, limit = 5) {
|
|
2030
|
+
const merged = /* @__PURE__ */ new Map();
|
|
2031
|
+
for (const item of [...httpOperations, ...domainOperations]) {
|
|
2032
|
+
const existing = merged.get(item.operation);
|
|
2033
|
+
if (!existing) {
|
|
2034
|
+
merged.set(item.operation, { ...item });
|
|
2035
|
+
continue;
|
|
2036
|
+
}
|
|
2037
|
+
const count = (existing.count ?? 0) + (item.count ?? 0);
|
|
2038
|
+
const totalDurationMs = (existing.avgDurationMs ?? 0) * (existing.count ?? 0) + (item.avgDurationMs ?? 0) * (item.count ?? 0);
|
|
2039
|
+
merged.set(item.operation, {
|
|
2040
|
+
operation: item.operation,
|
|
2041
|
+
count,
|
|
2042
|
+
avgDurationMs: count > 0 ? totalDurationMs / count : 0,
|
|
2043
|
+
maxDurationMs: Math.max(existing.maxDurationMs ?? 0, item.maxDurationMs ?? 0),
|
|
2044
|
+
errorCount: (existing.errorCount ?? 0) + (item.errorCount ?? 0)
|
|
2045
|
+
});
|
|
2046
|
+
}
|
|
2047
|
+
const ranked = Array.from(merged.values()).sort((a, b) => (b.count ?? 0) - (a.count ?? 0) || (b.maxDurationMs ?? 0) - (a.maxDurationMs ?? 0));
|
|
2048
|
+
const sliced = ranked.slice(0, limit);
|
|
2049
|
+
if (domainOperations.length === 0 || sliced.some((item) => !item.operation.startsWith("http."))) {
|
|
2050
|
+
return sliced;
|
|
2051
|
+
}
|
|
2052
|
+
const firstDomainOperation = ranked.find((item) => !item.operation.startsWith("http."));
|
|
2053
|
+
if (!firstDomainOperation) {
|
|
2054
|
+
return sliced;
|
|
2055
|
+
}
|
|
2056
|
+
return [...sliced.slice(0, Math.max(0, limit - 1)), firstDomainOperation];
|
|
2057
|
+
}
|
|
2058
|
+
async function createServer(config, cache, logger, jwtConfig, registry) {
|
|
2059
|
+
const gatewayLogger = createCorrelatedLogger(logger, {
|
|
2060
|
+
serviceId: "gateway",
|
|
2061
|
+
logsSource: "gateway",
|
|
2062
|
+
layer: "gateway",
|
|
2063
|
+
service: "server",
|
|
2064
|
+
operation: "gateway.http"
|
|
2065
|
+
});
|
|
2066
|
+
const app = Fastify({
|
|
2067
|
+
logger: false
|
|
2068
|
+
});
|
|
2069
|
+
const isProduction = process.env.NODE_ENV === "production";
|
|
2070
|
+
await registerOpenAPI(app, {
|
|
2071
|
+
title: "KB Labs Gateway",
|
|
2072
|
+
description: "Central API gateway \u2014 auth, LLM, telemetry, platform dispatch",
|
|
2073
|
+
version: "1.0.0",
|
|
2074
|
+
servers: [{ url: "http://localhost:4000", description: "Local dev" }],
|
|
2075
|
+
ui: !isProduction
|
|
2076
|
+
});
|
|
2077
|
+
await app.register(fastifyCors, { origin: true });
|
|
2078
|
+
const observability = new GatewayObservabilityCollector(config);
|
|
2079
|
+
observability.register(app);
|
|
2080
|
+
app.addHook("onRequest", async (request, reply) => {
|
|
2081
|
+
const requestId = request.headers["x-request-id"] || request.id || randomUUID();
|
|
2082
|
+
const traceId = request.headers["x-trace-id"] || randomUUID();
|
|
2083
|
+
request.id = requestId;
|
|
2084
|
+
reply.header("X-Request-Id", requestId);
|
|
2085
|
+
reply.header("X-Trace-Id", traceId);
|
|
2086
|
+
request.kbLogger = createCorrelatedLogger(logger, {
|
|
2087
|
+
serviceId: "gateway",
|
|
2088
|
+
logsSource: "gateway",
|
|
2089
|
+
layer: "gateway",
|
|
2090
|
+
service: "request",
|
|
2091
|
+
requestId,
|
|
2092
|
+
traceId,
|
|
2093
|
+
method: request.method,
|
|
2094
|
+
url: request.url,
|
|
2095
|
+
operation: "http.request"
|
|
2096
|
+
});
|
|
2097
|
+
request.kbLogger.info(`\u2192 ${request.method.toUpperCase()} ${request.url}`);
|
|
2098
|
+
});
|
|
2099
|
+
app.addHook("onResponse", async (request, reply) => {
|
|
2100
|
+
const requestLogger = request.kbLogger;
|
|
2101
|
+
if (!requestLogger) {
|
|
2102
|
+
return;
|
|
2103
|
+
}
|
|
2104
|
+
requestLogger.info(`\u2713 ${request.method.toUpperCase()} ${request.url} ${reply.statusCode}`, {
|
|
2105
|
+
statusCode: reply.statusCode
|
|
2106
|
+
});
|
|
2107
|
+
});
|
|
2108
|
+
const PROXY_TIMEOUT_MS = 36e5;
|
|
2109
|
+
for (const [name, upstream] of Object.entries(config.upstreams)) {
|
|
2110
|
+
await app.register(fastifyHttpProxy, {
|
|
2111
|
+
upstream: upstream.url,
|
|
2112
|
+
prefix: upstream.prefix,
|
|
2113
|
+
rewritePrefix: upstream.rewritePrefix ?? upstream.prefix,
|
|
2114
|
+
disableCache: true,
|
|
2115
|
+
websocket: upstream.websocket ?? false,
|
|
2116
|
+
http: {
|
|
2117
|
+
requestOptions: {
|
|
2118
|
+
timeout: PROXY_TIMEOUT_MS
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
});
|
|
2122
|
+
gatewayLogger.info(`Upstream registered: ${name} \u2192 ${upstream.url} (${upstream.prefix}${upstream.websocket ? ", ws" : ""})`);
|
|
2123
|
+
}
|
|
2124
|
+
await app.register(async function gatewayRoutes(scope) {
|
|
2125
|
+
scope.addHook("onRequest", createAuthMiddleware(cache, jwtConfig));
|
|
2126
|
+
const authService = new AuthService(cache, jwtConfig);
|
|
2127
|
+
registerAuthRoutes(scope, authService);
|
|
2128
|
+
const HEALTH_CACHE_KEY = "__gateway_health";
|
|
2129
|
+
const HEALTH_CACHE_TTL = 15e3;
|
|
2130
|
+
const startupTime = Date.now();
|
|
2131
|
+
const collectHealthSnapshot = async () => {
|
|
2132
|
+
const cached = await cache.get(HEALTH_CACHE_KEY).catch(() => null);
|
|
2133
|
+
if (cached) {
|
|
2134
|
+
return cached;
|
|
2135
|
+
}
|
|
2136
|
+
const adapterNames = ["llm", "cache", "analytics", "vectorStore", "embeddings"];
|
|
2137
|
+
const adapters = {};
|
|
2138
|
+
for (const name of adapterNames) {
|
|
2139
|
+
await observability.observeOperation(`gateway.adapter.${name}`, async () => {
|
|
2140
|
+
const probeStart = Date.now();
|
|
2141
|
+
try {
|
|
2142
|
+
const adapter = platform[name];
|
|
2143
|
+
adapters[name] = { available: !!adapter, latencyMs: Date.now() - probeStart };
|
|
2144
|
+
} catch {
|
|
2145
|
+
adapters[name] = { available: false, latencyMs: Date.now() - probeStart };
|
|
2146
|
+
}
|
|
2147
|
+
});
|
|
2148
|
+
}
|
|
2149
|
+
const upstreams = {};
|
|
2150
|
+
for (const [name, upstream] of Object.entries(config.upstreams)) {
|
|
2151
|
+
await observability.observeOperation(`gateway.upstream.${name}.health`, async () => {
|
|
2152
|
+
const probeStart = Date.now();
|
|
2153
|
+
try {
|
|
2154
|
+
const res = await fetch(`${upstream.url}/health`, {
|
|
2155
|
+
signal: AbortSignal.timeout(2e3)
|
|
2156
|
+
});
|
|
2157
|
+
const latencyMs = Date.now() - probeStart;
|
|
2158
|
+
upstreams[name] = { status: res.ok ? "up" : "down", latencyMs };
|
|
2159
|
+
if (!res.ok) {
|
|
2160
|
+
logDiagnosticEvent(logger, {
|
|
2161
|
+
domain: "service",
|
|
2162
|
+
event: "gateway.upstream.health",
|
|
2163
|
+
level: "warn",
|
|
2164
|
+
reasonCode: "upstream_unavailable",
|
|
2165
|
+
message: "Gateway upstream health probe failed",
|
|
2166
|
+
outcome: "failed",
|
|
2167
|
+
serviceId: "gateway",
|
|
2168
|
+
route: `${upstream.prefix}/health`,
|
|
2169
|
+
evidence: {
|
|
2170
|
+
upstreamId: name,
|
|
2171
|
+
upstreamUrl: upstream.url,
|
|
2172
|
+
statusCode: res.status,
|
|
2173
|
+
latencyMs
|
|
2174
|
+
}
|
|
2175
|
+
});
|
|
2176
|
+
}
|
|
2177
|
+
} catch (error) {
|
|
2178
|
+
const latencyMs = Date.now() - probeStart;
|
|
2179
|
+
upstreams[name] = { status: "down", latencyMs };
|
|
2180
|
+
logDiagnosticEvent(logger, {
|
|
2181
|
+
domain: "service",
|
|
2182
|
+
event: "gateway.upstream.health",
|
|
2183
|
+
level: "warn",
|
|
2184
|
+
reasonCode: "upstream_unavailable",
|
|
2185
|
+
message: "Gateway upstream health probe failed",
|
|
2186
|
+
outcome: "failed",
|
|
2187
|
+
error: error instanceof Error ? error : new Error(String(error)),
|
|
2188
|
+
serviceId: "gateway",
|
|
2189
|
+
route: `${upstream.prefix}/health`,
|
|
2190
|
+
evidence: {
|
|
2191
|
+
upstreamId: name,
|
|
2192
|
+
upstreamUrl: upstream.url,
|
|
2193
|
+
latencyMs
|
|
2194
|
+
}
|
|
2195
|
+
});
|
|
2196
|
+
}
|
|
2197
|
+
});
|
|
2198
|
+
}
|
|
2199
|
+
const llmOk = adapters.llm?.available ?? false;
|
|
2200
|
+
const allOk = Object.values(adapters).every((a) => a.available);
|
|
2201
|
+
const snapshot = {
|
|
2202
|
+
status: llmOk ? allOk ? "healthy" : "degraded" : "unhealthy",
|
|
2203
|
+
version: "1.0",
|
|
2204
|
+
uptime: Math.floor((Date.now() - startupTime) / 1e3),
|
|
2205
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2206
|
+
adapters,
|
|
2207
|
+
upstreams
|
|
2208
|
+
};
|
|
2209
|
+
await cache.set(HEALTH_CACHE_KEY, snapshot, HEALTH_CACHE_TTL).catch(() => {
|
|
2210
|
+
});
|
|
2211
|
+
return snapshot;
|
|
2212
|
+
};
|
|
2213
|
+
scope.get("/health", { schema: { tags: ["System"], summary: "Gateway health check" } }, async () => {
|
|
2214
|
+
return collectHealthSnapshot();
|
|
2215
|
+
});
|
|
2216
|
+
scope.get("/ready", { schema: { tags: ["System"], summary: "Gateway readiness check" } }, async (_request, reply) => {
|
|
2217
|
+
const health = await collectHealthSnapshot();
|
|
2218
|
+
const upstreams = health.upstreams ?? {};
|
|
2219
|
+
const missingRequiredUpstreams = ["rest"].filter((id) => (upstreams[id]?.status ?? "down") !== "up");
|
|
2220
|
+
const ready = missingRequiredUpstreams.length === 0;
|
|
2221
|
+
return reply.code(ready ? 200 : 503).send(createServiceReadyResponse({
|
|
2222
|
+
ready,
|
|
2223
|
+
status: ready ? "ready" : "degraded",
|
|
2224
|
+
reason: ready ? "ready" : `upstream_unavailable:${missingRequiredUpstreams.join(",")}`,
|
|
2225
|
+
components: {
|
|
2226
|
+
gatewayAdapters: {
|
|
2227
|
+
ready: true
|
|
2228
|
+
},
|
|
2229
|
+
restUpstream: {
|
|
2230
|
+
ready: (upstreams.rest?.status ?? "down") === "up",
|
|
2231
|
+
status: upstreams.rest?.status ?? "down"
|
|
2232
|
+
},
|
|
2233
|
+
workflowUpstream: {
|
|
2234
|
+
ready: (upstreams.workflow?.status ?? "down") === "up",
|
|
2235
|
+
status: upstreams.workflow?.status ?? "down"
|
|
2236
|
+
},
|
|
2237
|
+
marketplaceUpstream: {
|
|
2238
|
+
ready: (upstreams.marketplace?.status ?? "down") === "up",
|
|
2239
|
+
status: upstreams.marketplace?.status ?? "down"
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
}));
|
|
2243
|
+
});
|
|
2244
|
+
scope.get("/metrics", { schema: { tags: ["Observability"], summary: "Gateway metrics in Prometheus format" } }, async (_request, reply) => {
|
|
2245
|
+
const health = await collectHealthSnapshot();
|
|
2246
|
+
const status = health.status ?? "healthy";
|
|
2247
|
+
reply.header("Content-Type", "text/plain; version=0.0.4; charset=utf-8");
|
|
2248
|
+
return observability.renderPrometheusMetrics(status);
|
|
2249
|
+
});
|
|
2250
|
+
scope.get("/observability/describe", {
|
|
2251
|
+
schema: { tags: ["Observability"], summary: "Gateway observability contract descriptor" }
|
|
2252
|
+
}, async () => observability.buildDescribe());
|
|
2253
|
+
scope.get("/observability/health", {
|
|
2254
|
+
schema: { tags: ["Observability"], summary: "Gateway observability health snapshot" }
|
|
2255
|
+
}, async () => {
|
|
2256
|
+
const health = await collectHealthSnapshot();
|
|
2257
|
+
const adapterChecks = Object.entries(health.adapters ?? {}).map(([id, value]) => ({ id, available: !!value?.available, latencyMs: value?.latencyMs }));
|
|
2258
|
+
const upstreamChecks = Object.entries(health.upstreams ?? {}).map(([id, value]) => ({ id, status: value?.status ?? "unknown", latencyMs: value?.latencyMs }));
|
|
2259
|
+
const status = health.status ?? "healthy";
|
|
2260
|
+
return observability.buildHealth({ status, adapterChecks, upstreamChecks });
|
|
2261
|
+
});
|
|
2262
|
+
if (!registry) {
|
|
2263
|
+
gatewayLogger.warn("No persistent HostRegistry injected \u2014 hosts will be lost on restart");
|
|
2264
|
+
}
|
|
2265
|
+
const hostRegistry = registry ?? new HostRegistry(cache);
|
|
2266
|
+
scope.post("/hosts/register", { schema: { tags: ["Hosts"], summary: "Register a host" } }, async (request, reply) => {
|
|
2267
|
+
const parsed = HostRegistrationSchema.safeParse(request.body);
|
|
2268
|
+
if (!parsed.success) {
|
|
2269
|
+
return reply.code(400).send({ error: "Bad Request", issues: parsed.error.issues });
|
|
2270
|
+
}
|
|
2271
|
+
const result = await hostRegistry.register(parsed.data);
|
|
2272
|
+
return reply.code(201).send({
|
|
2273
|
+
hostId: result.descriptor.hostId,
|
|
2274
|
+
machineToken: result.machineToken,
|
|
2275
|
+
status: result.descriptor.status
|
|
2276
|
+
});
|
|
2277
|
+
});
|
|
2278
|
+
scope.get("/hosts", { schema: { tags: ["Hosts"], summary: "List registered hosts" } }, async (request, reply) => {
|
|
2279
|
+
const auth = request.authContext;
|
|
2280
|
+
if (!auth) {
|
|
2281
|
+
return reply.code(401).send({ error: "Unauthorized" });
|
|
2282
|
+
}
|
|
2283
|
+
const hosts = await hostRegistry.list(auth.namespaceId);
|
|
2284
|
+
return { hosts };
|
|
2285
|
+
});
|
|
2286
|
+
scope.get("/hosts/:hostId", { schema: { tags: ["Hosts"], summary: "Get host by ID" } }, async (request, reply) => {
|
|
2287
|
+
const auth = request.authContext;
|
|
2288
|
+
if (!auth) {
|
|
2289
|
+
return reply.code(401).send({ error: "Unauthorized" });
|
|
2290
|
+
}
|
|
2291
|
+
const { hostId } = request.params;
|
|
2292
|
+
const host = await hostRegistry.get(hostId, auth.namespaceId);
|
|
2293
|
+
if (!host) {
|
|
2294
|
+
return reply.code(404).send({ error: "Host not found" });
|
|
2295
|
+
}
|
|
2296
|
+
return host;
|
|
2297
|
+
});
|
|
2298
|
+
scope.delete("/hosts/:hostId", { schema: { tags: ["Hosts"], summary: "Deregister a host" } }, async (request, reply) => {
|
|
2299
|
+
const auth = request.authContext;
|
|
2300
|
+
if (!auth) {
|
|
2301
|
+
return reply.code(401).send({ error: "Unauthorized" });
|
|
2302
|
+
}
|
|
2303
|
+
const { hostId } = request.params;
|
|
2304
|
+
const deleted = await hostRegistry.deregister(hostId, auth.namespaceId);
|
|
2305
|
+
if (!deleted) {
|
|
2306
|
+
return reply.code(404).send({ error: "Host not found" });
|
|
2307
|
+
}
|
|
2308
|
+
return reply.code(204).send();
|
|
2309
|
+
});
|
|
2310
|
+
registerExecuteRoutes(scope, logger);
|
|
2311
|
+
registerLLMGatewayRoutes(scope, logger);
|
|
2312
|
+
registerTelemetryRoutes(scope, logger);
|
|
2313
|
+
registerPlatformRoutes(scope, logger);
|
|
2314
|
+
registerAggregatedDocsRoutes(scope, cache);
|
|
2315
|
+
const internalSecret = process.env.GATEWAY_INTERNAL_SECRET;
|
|
2316
|
+
scope.post("/internal/dispatch", async (request, reply) => {
|
|
2317
|
+
const provided = request.headers["x-internal-secret"];
|
|
2318
|
+
if (!internalSecret || provided !== internalSecret) {
|
|
2319
|
+
return reply.code(403).send({ error: "Forbidden" });
|
|
2320
|
+
}
|
|
2321
|
+
const body = request.body;
|
|
2322
|
+
if (!body.namespaceId || !body.adapter || !body.method) {
|
|
2323
|
+
return reply.code(400).send({ error: "Missing required fields: namespaceId, adapter, method" });
|
|
2324
|
+
}
|
|
2325
|
+
const hostId = body.hostId ?? globalDispatcher.firstHostWithCapability(body.namespaceId, body.adapter) ?? globalDispatcher.firstHost(body.namespaceId);
|
|
2326
|
+
if (!hostId) {
|
|
2327
|
+
return reply.code(503).send({
|
|
2328
|
+
error: "No host connected",
|
|
2329
|
+
namespaceId: body.namespaceId
|
|
2330
|
+
});
|
|
2331
|
+
}
|
|
2332
|
+
try {
|
|
2333
|
+
const result = await globalDispatcher.call(
|
|
2334
|
+
body.namespaceId,
|
|
2335
|
+
hostId,
|
|
2336
|
+
body.adapter,
|
|
2337
|
+
body.method,
|
|
2338
|
+
body.args ?? []
|
|
2339
|
+
);
|
|
2340
|
+
return { result };
|
|
2341
|
+
} catch (err) {
|
|
2342
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2343
|
+
if (message.includes("Host not connected")) {
|
|
2344
|
+
return reply.code(503).send({ error: message });
|
|
2345
|
+
}
|
|
2346
|
+
return reply.code(502).send({ error: message });
|
|
2347
|
+
}
|
|
2348
|
+
});
|
|
2349
|
+
scope.post("/internal/resolve-host", async (request, reply) => {
|
|
2350
|
+
const provided = request.headers["x-internal-secret"];
|
|
2351
|
+
if (!internalSecret || provided !== internalSecret) {
|
|
2352
|
+
return reply.code(403).send({ error: "Forbidden" });
|
|
2353
|
+
}
|
|
2354
|
+
const body = request.body;
|
|
2355
|
+
const namespaceId = body.namespaceId ?? "default";
|
|
2356
|
+
const target = body.target ?? {};
|
|
2357
|
+
const strategy = target.hostSelection ?? "any-matching";
|
|
2358
|
+
let hostId;
|
|
2359
|
+
if (strategy === "pinned" && target.hostId) {
|
|
2360
|
+
const host = await hostRegistry.get(target.hostId, namespaceId);
|
|
2361
|
+
if (host?.status === "online" || host?.status === "reconnecting") {
|
|
2362
|
+
hostId = target.hostId;
|
|
2363
|
+
}
|
|
2364
|
+
} else {
|
|
2365
|
+
hostId = globalDispatcher.firstHostWithCapability(namespaceId, "execution");
|
|
2366
|
+
}
|
|
2367
|
+
if (!hostId) {
|
|
2368
|
+
return reply.code(404).send({ error: "No matching host found" });
|
|
2369
|
+
}
|
|
2370
|
+
return { hostId, strategy, namespaceId };
|
|
2371
|
+
});
|
|
2372
|
+
});
|
|
2373
|
+
await app.ready();
|
|
2374
|
+
attachGatewayWs(app.server, cache, jwtConfig, logger, registry);
|
|
2375
|
+
return app;
|
|
2376
|
+
}
|
|
2377
|
+
|
|
2378
|
+
// src/bootstrap.ts
|
|
2379
|
+
async function bootstrap(repoRoot = process.cwd()) {
|
|
2380
|
+
await createServiceBootstrap({ appId: "gateway", repoRoot });
|
|
2381
|
+
const logger = createCorrelatedLogger(platform.logger, {
|
|
2382
|
+
serviceId: "gateway",
|
|
2383
|
+
logsSource: "gateway",
|
|
2384
|
+
layer: "gateway",
|
|
2385
|
+
service: "bootstrap",
|
|
2386
|
+
operation: "gateway.bootstrap"
|
|
2387
|
+
});
|
|
2388
|
+
logger.info("Platform initialized", { repoRoot });
|
|
2389
|
+
const config = await loadGatewayConfig(repoRoot);
|
|
2390
|
+
logger.info("Gateway config loaded", {
|
|
2391
|
+
port: config.port,
|
|
2392
|
+
upstreams: Object.keys(config.upstreams)
|
|
2393
|
+
});
|
|
2394
|
+
let hostStore;
|
|
2395
|
+
const db = platform.getAdapter("sqlDatabase");
|
|
2396
|
+
if (db) {
|
|
2397
|
+
hostStore = new SqliteHostStore(db);
|
|
2398
|
+
logger.info("Host store: SQLite (persistent)");
|
|
2399
|
+
} else {
|
|
2400
|
+
logger.warn("Host store: none (cache-only, hosts will be lost on restart)");
|
|
2401
|
+
}
|
|
2402
|
+
const registry = new HostRegistry(platform.cache, hostStore);
|
|
2403
|
+
let restoredCount = 0;
|
|
2404
|
+
try {
|
|
2405
|
+
restoredCount = await registry.restore();
|
|
2406
|
+
} catch (error) {
|
|
2407
|
+
logDiagnosticEvent(platform.logger, {
|
|
2408
|
+
domain: "registry",
|
|
2409
|
+
event: "gateway.hosts.restore",
|
|
2410
|
+
level: "error",
|
|
2411
|
+
reasonCode: "registry_restore_failed",
|
|
2412
|
+
message: "Failed to restore gateway host registry",
|
|
2413
|
+
outcome: "failed",
|
|
2414
|
+
error: error instanceof Error ? error : new Error(String(error)),
|
|
2415
|
+
serviceId: "gateway",
|
|
2416
|
+
evidence: {
|
|
2417
|
+
persistentStore: !!hostStore
|
|
2418
|
+
}
|
|
2419
|
+
});
|
|
2420
|
+
throw error;
|
|
2421
|
+
}
|
|
2422
|
+
if (restoredCount > 0) {
|
|
2423
|
+
logger.info("Restored hosts from store", { count: restoredCount });
|
|
2424
|
+
}
|
|
2425
|
+
for (const [token, entry] of Object.entries(config.staticTokens)) {
|
|
2426
|
+
await platform.cache.set(`host:token:${token}`, entry);
|
|
2427
|
+
logger.info("Static token seeded", { hostId: entry.hostId, namespaceId: entry.namespaceId });
|
|
2428
|
+
}
|
|
2429
|
+
const jwtSecret = process.env.GATEWAY_JWT_SECRET;
|
|
2430
|
+
if (!jwtSecret) {
|
|
2431
|
+
logger.warn("GATEWAY_JWT_SECRET not set \u2014 using insecure default (dev only!)");
|
|
2432
|
+
}
|
|
2433
|
+
const jwtConfig = { secret: jwtSecret ?? "dev-insecure-secret-change-me" };
|
|
2434
|
+
const server = await createServer(config, platform.cache, platform.logger, jwtConfig, registry);
|
|
2435
|
+
const address = await server.listen({ port: config.port, host: "0.0.0.0" });
|
|
2436
|
+
logger.info("Gateway listening", { address });
|
|
2437
|
+
const shutdown = async (signal) => {
|
|
2438
|
+
logger.warn("Received shutdown signal", { signal });
|
|
2439
|
+
await platform.shutdown();
|
|
2440
|
+
await server.close();
|
|
2441
|
+
logger.info("Gateway shutdown complete");
|
|
2442
|
+
process.exit(0);
|
|
2443
|
+
};
|
|
2444
|
+
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
2445
|
+
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
2446
|
+
}
|
|
2447
|
+
|
|
2448
|
+
// src/index.ts
|
|
2449
|
+
bootstrap(process.cwd()).catch((error) => {
|
|
2450
|
+
console.error("Failed to start gateway:", error);
|
|
2451
|
+
process.exit(1);
|
|
2452
|
+
});
|
|
2453
|
+
//# sourceMappingURL=index.js.map
|
|
2454
|
+
//# sourceMappingURL=index.js.map
|