@syncular/server-hono 0.0.2-2 → 0.0.3-14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/console/gateway.d.ts +31 -0
- package/dist/console/gateway.d.ts.map +1 -0
- package/dist/console/gateway.js +2349 -0
- package/dist/console/gateway.js.map +1 -0
- package/dist/console/index.d.ts +3 -2
- package/dist/console/index.d.ts.map +1 -1
- package/dist/console/index.js +3 -1
- package/dist/console/index.js.map +1 -1
- package/dist/routes.d.ts.map +1 -1
- package/dist/routes.js +116 -94
- package/dist/routes.js.map +1 -1
- package/package.json +5 -5
- package/src/__tests__/console-gateway-live-routes.test.ts +246 -0
- package/src/__tests__/console-gateway-routes.test.ts +1364 -0
- package/src/__tests__/console-routes.test.ts +7 -3
- package/src/console/gateway.ts +3371 -0
- package/src/console/index.ts +3 -11
- package/src/routes.ts +122 -96
|
@@ -0,0 +1,1364 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test';
|
|
2
|
+
import { Hono } from 'hono';
|
|
3
|
+
import type {
|
|
4
|
+
ConsoleApiKey,
|
|
5
|
+
ConsoleClient,
|
|
6
|
+
ConsoleCommitListItem,
|
|
7
|
+
ConsoleHandler,
|
|
8
|
+
ConsoleOperationEvent,
|
|
9
|
+
ConsolePaginatedResponse,
|
|
10
|
+
ConsoleRequestEvent,
|
|
11
|
+
ConsoleRequestPayload,
|
|
12
|
+
ConsoleTimelineItem,
|
|
13
|
+
LatencyStatsResponse,
|
|
14
|
+
SyncStats,
|
|
15
|
+
TimeseriesStatsResponse,
|
|
16
|
+
} from '../console';
|
|
17
|
+
import { createConsoleGatewayRoutes } from '../console';
|
|
18
|
+
|
|
19
|
+
const CONSOLE_TOKEN = 'gateway-token';
|
|
20
|
+
|
|
21
|
+
interface MockInstanceData {
|
|
22
|
+
stats: SyncStats;
|
|
23
|
+
timeseries: TimeseriesStatsResponse;
|
|
24
|
+
latency: LatencyStatsResponse;
|
|
25
|
+
handlers: ConsoleHandler[];
|
|
26
|
+
apiKeys: ConsoleApiKey[];
|
|
27
|
+
commits: ConsoleCommitListItem[];
|
|
28
|
+
clients: ConsoleClient[];
|
|
29
|
+
timeline: ConsoleTimelineItem[];
|
|
30
|
+
operations: ConsoleOperationEvent[];
|
|
31
|
+
events: ConsoleRequestEvent[];
|
|
32
|
+
payloadsByEventId: Record<number, ConsoleRequestPayload>;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function readAuthorization(headers: HeadersInit | undefined): string | null {
|
|
36
|
+
if (!headers) return null;
|
|
37
|
+
if (headers instanceof Headers) {
|
|
38
|
+
return headers.get('Authorization');
|
|
39
|
+
}
|
|
40
|
+
if (Array.isArray(headers)) {
|
|
41
|
+
const header = headers.find(
|
|
42
|
+
([name]) => name.toLowerCase() === 'authorization'
|
|
43
|
+
);
|
|
44
|
+
return header?.[1] ?? null;
|
|
45
|
+
}
|
|
46
|
+
return headers.Authorization ?? headers.authorization ?? null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function createPaginatedResponse<T>(args: {
|
|
50
|
+
items: T[];
|
|
51
|
+
offset: number;
|
|
52
|
+
limit: number;
|
|
53
|
+
}): ConsolePaginatedResponse<T> {
|
|
54
|
+
const total = args.items.length;
|
|
55
|
+
return {
|
|
56
|
+
items: args.items.slice(args.offset, args.offset + args.limit),
|
|
57
|
+
total,
|
|
58
|
+
offset: args.offset,
|
|
59
|
+
limit: args.limit,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function parseJsonBody(init: RequestInit | undefined): unknown {
|
|
64
|
+
if (!init?.body || typeof init.body !== 'string') {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
try {
|
|
68
|
+
return JSON.parse(init.body) as unknown;
|
|
69
|
+
} catch {
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function createMockGatewayApp(args: {
|
|
75
|
+
instances: Record<string, MockInstanceData>;
|
|
76
|
+
failingInstances?: Set<string>;
|
|
77
|
+
}) {
|
|
78
|
+
const downstreamCalls: string[] = [];
|
|
79
|
+
const failingInstances = args.failingInstances ?? new Set<string>();
|
|
80
|
+
const instanceByHost = new Map<string, MockInstanceData>();
|
|
81
|
+
for (const [instanceId, data] of Object.entries(args.instances)) {
|
|
82
|
+
instanceByHost.set(`${instanceId}.example.test`, data);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const app = new Hono();
|
|
86
|
+
app.route(
|
|
87
|
+
'/console',
|
|
88
|
+
createConsoleGatewayRoutes({
|
|
89
|
+
instances: Object.keys(args.instances).map((instanceId) => ({
|
|
90
|
+
instanceId,
|
|
91
|
+
label: instanceId.toUpperCase(),
|
|
92
|
+
baseUrl: `https://${instanceId}.example.test/api/${instanceId}`,
|
|
93
|
+
})),
|
|
94
|
+
authenticate: async (c) => {
|
|
95
|
+
const authHeader = c.req.header('Authorization');
|
|
96
|
+
if (authHeader === `Bearer ${CONSOLE_TOKEN}`) {
|
|
97
|
+
return { consoleUserId: 'gateway-user' };
|
|
98
|
+
}
|
|
99
|
+
return null;
|
|
100
|
+
},
|
|
101
|
+
fetchImpl: async (input, init) => {
|
|
102
|
+
const url =
|
|
103
|
+
typeof input === 'string'
|
|
104
|
+
? new URL(input)
|
|
105
|
+
: input instanceof URL
|
|
106
|
+
? input
|
|
107
|
+
: new URL(input.url);
|
|
108
|
+
downstreamCalls.push(url.toString());
|
|
109
|
+
|
|
110
|
+
const instanceId = url.hostname.replace('.example.test', '');
|
|
111
|
+
if (!instanceId || failingInstances.has(instanceId)) {
|
|
112
|
+
return new Response(
|
|
113
|
+
JSON.stringify({ error: 'DOWNSTREAM_UNAVAILABLE' }),
|
|
114
|
+
{ status: 503 }
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const expectedAuthorization = `Bearer ${CONSOLE_TOKEN}`;
|
|
119
|
+
if (readAuthorization(init?.headers) !== expectedAuthorization) {
|
|
120
|
+
return new Response(JSON.stringify({ error: 'UNAUTHORIZED' }), {
|
|
121
|
+
status: 401,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const data = instanceByHost.get(url.hostname);
|
|
126
|
+
if (!data) {
|
|
127
|
+
return new Response(JSON.stringify({ error: 'NOT_FOUND' }), {
|
|
128
|
+
status: 404,
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const method = (init?.method ?? 'GET').toUpperCase();
|
|
133
|
+
const pathname = url.pathname;
|
|
134
|
+
if (pathname.endsWith('/console/stats')) {
|
|
135
|
+
return new Response(JSON.stringify(data.stats), { status: 200 });
|
|
136
|
+
}
|
|
137
|
+
if (pathname.endsWith('/console/stats/timeseries')) {
|
|
138
|
+
return new Response(JSON.stringify(data.timeseries), { status: 200 });
|
|
139
|
+
}
|
|
140
|
+
if (pathname.endsWith('/console/stats/latency')) {
|
|
141
|
+
return new Response(JSON.stringify(data.latency), { status: 200 });
|
|
142
|
+
}
|
|
143
|
+
if (pathname.endsWith('/console/handlers') && method === 'GET') {
|
|
144
|
+
return new Response(JSON.stringify({ items: data.handlers }), {
|
|
145
|
+
status: 200,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
if (pathname.endsWith('/console/prune/preview') && method === 'POST') {
|
|
149
|
+
return new Response(
|
|
150
|
+
JSON.stringify({
|
|
151
|
+
watermarkCommitSeq: data.stats.maxCommitSeq,
|
|
152
|
+
commitsToDelete: data.commits.length,
|
|
153
|
+
}),
|
|
154
|
+
{ status: 200 }
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
if (pathname.endsWith('/console/prune') && method === 'POST') {
|
|
158
|
+
return new Response(
|
|
159
|
+
JSON.stringify({ deletedCommits: data.commits.length }),
|
|
160
|
+
{ status: 200 }
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
if (pathname.endsWith('/console/compact') && method === 'POST') {
|
|
164
|
+
return new Response(
|
|
165
|
+
JSON.stringify({ deletedChanges: data.stats.changeCount }),
|
|
166
|
+
{ status: 200 }
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
if (
|
|
170
|
+
pathname.endsWith('/console/notify-data-change') &&
|
|
171
|
+
method === 'POST'
|
|
172
|
+
) {
|
|
173
|
+
const body = parseJsonBody(init) as {
|
|
174
|
+
tables?: string[];
|
|
175
|
+
} | null;
|
|
176
|
+
const tables = body?.tables ?? [];
|
|
177
|
+
return new Response(
|
|
178
|
+
JSON.stringify({
|
|
179
|
+
commitSeq: data.stats.maxCommitSeq + 1,
|
|
180
|
+
tables,
|
|
181
|
+
deletedChunks: 0,
|
|
182
|
+
}),
|
|
183
|
+
{ status: 200 }
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
if (pathname.endsWith('/console/events') && method === 'DELETE') {
|
|
187
|
+
return new Response(
|
|
188
|
+
JSON.stringify({ deletedCount: data.events.length }),
|
|
189
|
+
{ status: 200 }
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
if (pathname.endsWith('/console/events/prune') && method === 'POST') {
|
|
193
|
+
return new Response(JSON.stringify({ deletedCount: 1 }), {
|
|
194
|
+
status: 200,
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
if (pathname.endsWith('/console/api-keys') && method === 'GET') {
|
|
198
|
+
return new Response(
|
|
199
|
+
JSON.stringify(
|
|
200
|
+
createPaginatedResponse({
|
|
201
|
+
items: data.apiKeys,
|
|
202
|
+
offset: Number(url.searchParams.get('offset') ?? '0'),
|
|
203
|
+
limit: Number(url.searchParams.get('limit') ?? '50'),
|
|
204
|
+
})
|
|
205
|
+
),
|
|
206
|
+
{ status: 200 }
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
if (pathname.endsWith('/console/api-keys') && method === 'POST') {
|
|
210
|
+
const createdAt = '2026-02-17T10:10:00.000Z';
|
|
211
|
+
return new Response(
|
|
212
|
+
JSON.stringify({
|
|
213
|
+
key: {
|
|
214
|
+
keyId: `${instanceId}-new-key`,
|
|
215
|
+
keyPrefix: `${instanceId}-new`,
|
|
216
|
+
name: `${instanceId} created key`,
|
|
217
|
+
keyType: 'admin',
|
|
218
|
+
scopeKeys: ['*'],
|
|
219
|
+
actorId: null,
|
|
220
|
+
createdAt,
|
|
221
|
+
expiresAt: null,
|
|
222
|
+
lastUsedAt: null,
|
|
223
|
+
revokedAt: null,
|
|
224
|
+
},
|
|
225
|
+
secretKey: `${instanceId}_secret_key`,
|
|
226
|
+
}),
|
|
227
|
+
{ status: 201 }
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
const limit = Number(url.searchParams.get('limit') ?? '50');
|
|
232
|
+
const offset = Number(url.searchParams.get('offset') ?? '0');
|
|
233
|
+
if (pathname.endsWith('/console/commits')) {
|
|
234
|
+
return new Response(
|
|
235
|
+
JSON.stringify(
|
|
236
|
+
createPaginatedResponse({
|
|
237
|
+
items: data.commits,
|
|
238
|
+
offset,
|
|
239
|
+
limit,
|
|
240
|
+
})
|
|
241
|
+
),
|
|
242
|
+
{ status: 200 }
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
const commitDetailMatch = pathname.match(/\/console\/commits\/(.+)$/);
|
|
246
|
+
if (commitDetailMatch) {
|
|
247
|
+
const commitSeq = Number(commitDetailMatch[1]);
|
|
248
|
+
const commit = data.commits.find(
|
|
249
|
+
(item) => item.commitSeq === commitSeq
|
|
250
|
+
);
|
|
251
|
+
if (!commit) {
|
|
252
|
+
return new Response(JSON.stringify({ error: 'NOT_FOUND' }), {
|
|
253
|
+
status: 404,
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
return new Response(
|
|
257
|
+
JSON.stringify({
|
|
258
|
+
...commit,
|
|
259
|
+
changes: [],
|
|
260
|
+
}),
|
|
261
|
+
{ status: 200 }
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
if (pathname.endsWith('/console/clients')) {
|
|
265
|
+
return new Response(
|
|
266
|
+
JSON.stringify(
|
|
267
|
+
createPaginatedResponse({
|
|
268
|
+
items: data.clients,
|
|
269
|
+
offset,
|
|
270
|
+
limit,
|
|
271
|
+
})
|
|
272
|
+
),
|
|
273
|
+
{ status: 200 }
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
if (pathname.endsWith('/console/timeline')) {
|
|
277
|
+
return new Response(
|
|
278
|
+
JSON.stringify(
|
|
279
|
+
createPaginatedResponse({
|
|
280
|
+
items: data.timeline,
|
|
281
|
+
offset,
|
|
282
|
+
limit,
|
|
283
|
+
})
|
|
284
|
+
),
|
|
285
|
+
{ status: 200 }
|
|
286
|
+
);
|
|
287
|
+
}
|
|
288
|
+
if (pathname.endsWith('/console/operations')) {
|
|
289
|
+
return new Response(
|
|
290
|
+
JSON.stringify(
|
|
291
|
+
createPaginatedResponse({
|
|
292
|
+
items: data.operations,
|
|
293
|
+
offset,
|
|
294
|
+
limit,
|
|
295
|
+
})
|
|
296
|
+
),
|
|
297
|
+
{ status: 200 }
|
|
298
|
+
);
|
|
299
|
+
}
|
|
300
|
+
if (pathname.endsWith('/console/events')) {
|
|
301
|
+
return new Response(
|
|
302
|
+
JSON.stringify(
|
|
303
|
+
createPaginatedResponse({
|
|
304
|
+
items: data.events,
|
|
305
|
+
offset,
|
|
306
|
+
limit,
|
|
307
|
+
})
|
|
308
|
+
),
|
|
309
|
+
{ status: 200 }
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
const evictClientMatch = pathname.match(/\/console\/clients\/(.+)$/);
|
|
313
|
+
if (evictClientMatch && method === 'DELETE') {
|
|
314
|
+
return new Response(JSON.stringify({ evicted: true }), {
|
|
315
|
+
status: 200,
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
const eventDetailMatch = pathname.match(/\/console\/events\/(\d+)$/);
|
|
320
|
+
if (eventDetailMatch) {
|
|
321
|
+
const eventId = Number(eventDetailMatch[1]);
|
|
322
|
+
const event = data.events.find((item) => item.eventId === eventId);
|
|
323
|
+
if (!event) {
|
|
324
|
+
return new Response(JSON.stringify({ error: 'NOT_FOUND' }), {
|
|
325
|
+
status: 404,
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
return new Response(JSON.stringify(event), { status: 200 });
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
const eventPayloadMatch = pathname.match(
|
|
332
|
+
/\/console\/events\/(\d+)\/payload$/
|
|
333
|
+
);
|
|
334
|
+
if (eventPayloadMatch) {
|
|
335
|
+
const eventId = Number(eventPayloadMatch[1]);
|
|
336
|
+
const payload = data.payloadsByEventId[eventId];
|
|
337
|
+
if (!payload) {
|
|
338
|
+
return new Response(JSON.stringify({ error: 'NOT_FOUND' }), {
|
|
339
|
+
status: 404,
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
return new Response(JSON.stringify(payload), { status: 200 });
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
const apiKeyStageRotateMatch = pathname.match(
|
|
346
|
+
/\/console\/api-keys\/([^/]+)\/rotate\/stage$/
|
|
347
|
+
);
|
|
348
|
+
if (apiKeyStageRotateMatch && method === 'POST') {
|
|
349
|
+
const keyId = apiKeyStageRotateMatch[1] ?? 'unknown';
|
|
350
|
+
return new Response(
|
|
351
|
+
JSON.stringify({
|
|
352
|
+
key: {
|
|
353
|
+
keyId: `${keyId}-staged`,
|
|
354
|
+
keyPrefix: `${keyId}-stg`,
|
|
355
|
+
name: `${instanceId} staged key`,
|
|
356
|
+
keyType: 'admin',
|
|
357
|
+
scopeKeys: ['*'],
|
|
358
|
+
actorId: null,
|
|
359
|
+
createdAt: '2026-02-17T10:11:00.000Z',
|
|
360
|
+
expiresAt: null,
|
|
361
|
+
lastUsedAt: null,
|
|
362
|
+
revokedAt: null,
|
|
363
|
+
},
|
|
364
|
+
secretKey: `${instanceId}_staged_secret`,
|
|
365
|
+
}),
|
|
366
|
+
{ status: 200 }
|
|
367
|
+
);
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
const apiKeyRotateMatch = pathname.match(
|
|
371
|
+
/\/console\/api-keys\/([^/]+)\/rotate$/
|
|
372
|
+
);
|
|
373
|
+
if (apiKeyRotateMatch && method === 'POST') {
|
|
374
|
+
const keyId = apiKeyRotateMatch[1] ?? 'unknown';
|
|
375
|
+
return new Response(
|
|
376
|
+
JSON.stringify({
|
|
377
|
+
key: {
|
|
378
|
+
keyId: `${keyId}-rotated`,
|
|
379
|
+
keyPrefix: `${keyId}-rot`,
|
|
380
|
+
name: `${instanceId} rotated key`,
|
|
381
|
+
keyType: 'admin',
|
|
382
|
+
scopeKeys: ['*'],
|
|
383
|
+
actorId: null,
|
|
384
|
+
createdAt: '2026-02-17T10:12:00.000Z',
|
|
385
|
+
expiresAt: null,
|
|
386
|
+
lastUsedAt: null,
|
|
387
|
+
revokedAt: null,
|
|
388
|
+
},
|
|
389
|
+
secretKey: `${instanceId}_rotated_secret`,
|
|
390
|
+
}),
|
|
391
|
+
{ status: 200 }
|
|
392
|
+
);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const apiKeyMatch = pathname.match(/\/console\/api-keys\/([^/]+)$/);
|
|
396
|
+
if (apiKeyMatch && method === 'GET') {
|
|
397
|
+
const keyId = apiKeyMatch[1] ?? '';
|
|
398
|
+
const key = data.apiKeys.find((item) => item.keyId === keyId);
|
|
399
|
+
if (!key) {
|
|
400
|
+
return new Response(JSON.stringify({ error: 'NOT_FOUND' }), {
|
|
401
|
+
status: 404,
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
return new Response(JSON.stringify(key), { status: 200 });
|
|
405
|
+
}
|
|
406
|
+
if (apiKeyMatch && method === 'DELETE') {
|
|
407
|
+
return new Response(JSON.stringify({ revoked: true }), {
|
|
408
|
+
status: 200,
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
if (pathname.endsWith('/console/api-keys/bulk-revoke')) {
|
|
412
|
+
const body = parseJsonBody(init) as { keyIds?: string[] } | null;
|
|
413
|
+
const keyIds = body?.keyIds ?? [];
|
|
414
|
+
return new Response(
|
|
415
|
+
JSON.stringify({
|
|
416
|
+
requestedCount: keyIds.length,
|
|
417
|
+
revokedCount: keyIds.length,
|
|
418
|
+
alreadyRevokedCount: 0,
|
|
419
|
+
notFoundCount: 0,
|
|
420
|
+
revokedKeyIds: keyIds,
|
|
421
|
+
alreadyRevokedKeyIds: [],
|
|
422
|
+
notFoundKeyIds: [],
|
|
423
|
+
}),
|
|
424
|
+
{ status: 200 }
|
|
425
|
+
);
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
return new Response(JSON.stringify({ error: 'NOT_IMPLEMENTED' }), {
|
|
429
|
+
status: 404,
|
|
430
|
+
});
|
|
431
|
+
},
|
|
432
|
+
})
|
|
433
|
+
);
|
|
434
|
+
|
|
435
|
+
return { app, downstreamCalls };
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
describe('createConsoleGatewayRoutes', () => {
|
|
439
|
+
const alphaStats: SyncStats = {
|
|
440
|
+
commitCount: 10,
|
|
441
|
+
changeCount: 30,
|
|
442
|
+
minCommitSeq: 1,
|
|
443
|
+
maxCommitSeq: 40,
|
|
444
|
+
clientCount: 2,
|
|
445
|
+
activeClientCount: 1,
|
|
446
|
+
minActiveClientCursor: 35,
|
|
447
|
+
maxActiveClientCursor: 35,
|
|
448
|
+
};
|
|
449
|
+
|
|
450
|
+
const betaStats: SyncStats = {
|
|
451
|
+
commitCount: 4,
|
|
452
|
+
changeCount: 9,
|
|
453
|
+
minCommitSeq: 3,
|
|
454
|
+
maxCommitSeq: 18,
|
|
455
|
+
clientCount: 1,
|
|
456
|
+
activeClientCount: 1,
|
|
457
|
+
minActiveClientCursor: 18,
|
|
458
|
+
maxActiveClientCursor: 18,
|
|
459
|
+
};
|
|
460
|
+
|
|
461
|
+
const alphaTimeseries: TimeseriesStatsResponse = {
|
|
462
|
+
interval: 'hour',
|
|
463
|
+
range: '24h',
|
|
464
|
+
buckets: [
|
|
465
|
+
{
|
|
466
|
+
timestamp: '2026-02-17T09:00:00.000Z',
|
|
467
|
+
pushCount: 1,
|
|
468
|
+
pullCount: 1,
|
|
469
|
+
errorCount: 0,
|
|
470
|
+
avgLatencyMs: 20,
|
|
471
|
+
},
|
|
472
|
+
{
|
|
473
|
+
timestamp: '2026-02-17T10:00:00.000Z',
|
|
474
|
+
pushCount: 0,
|
|
475
|
+
pullCount: 1,
|
|
476
|
+
errorCount: 1,
|
|
477
|
+
avgLatencyMs: 40,
|
|
478
|
+
},
|
|
479
|
+
],
|
|
480
|
+
};
|
|
481
|
+
|
|
482
|
+
const betaTimeseries: TimeseriesStatsResponse = {
|
|
483
|
+
interval: 'hour',
|
|
484
|
+
range: '24h',
|
|
485
|
+
buckets: [
|
|
486
|
+
{
|
|
487
|
+
timestamp: '2026-02-17T09:00:00.000Z',
|
|
488
|
+
pushCount: 2,
|
|
489
|
+
pullCount: 0,
|
|
490
|
+
errorCount: 1,
|
|
491
|
+
avgLatencyMs: 50,
|
|
492
|
+
},
|
|
493
|
+
{
|
|
494
|
+
timestamp: '2026-02-17T10:00:00.000Z',
|
|
495
|
+
pushCount: 1,
|
|
496
|
+
pullCount: 1,
|
|
497
|
+
errorCount: 0,
|
|
498
|
+
avgLatencyMs: 10,
|
|
499
|
+
},
|
|
500
|
+
],
|
|
501
|
+
};
|
|
502
|
+
|
|
503
|
+
const alphaLatency: LatencyStatsResponse = {
|
|
504
|
+
push: { p50: 10, p90: 20, p99: 30 },
|
|
505
|
+
pull: { p50: 12, p90: 22, p99: 32 },
|
|
506
|
+
range: '24h',
|
|
507
|
+
};
|
|
508
|
+
|
|
509
|
+
const betaLatency: LatencyStatsResponse = {
|
|
510
|
+
push: { p50: 20, p90: 40, p99: 60 },
|
|
511
|
+
pull: { p50: 14, p90: 24, p99: 34 },
|
|
512
|
+
range: '24h',
|
|
513
|
+
};
|
|
514
|
+
|
|
515
|
+
const alphaHandlers: ConsoleHandler[] = [
|
|
516
|
+
{ table: 'tasks', dependsOn: ['projects'], snapshotChunkTtlMs: 60000 },
|
|
517
|
+
];
|
|
518
|
+
const betaHandlers: ConsoleHandler[] = [
|
|
519
|
+
{ table: 'orders', dependsOn: ['customers'], snapshotChunkTtlMs: 60000 },
|
|
520
|
+
];
|
|
521
|
+
|
|
522
|
+
const alphaApiKeys: ConsoleApiKey[] = [
|
|
523
|
+
{
|
|
524
|
+
keyId: 'alpha-key-1',
|
|
525
|
+
keyPrefix: 'alpha-key',
|
|
526
|
+
name: 'Alpha admin key',
|
|
527
|
+
keyType: 'admin',
|
|
528
|
+
scopeKeys: ['*'],
|
|
529
|
+
actorId: null,
|
|
530
|
+
createdAt: '2026-02-17T10:00:00.000Z',
|
|
531
|
+
expiresAt: null,
|
|
532
|
+
lastUsedAt: null,
|
|
533
|
+
revokedAt: null,
|
|
534
|
+
},
|
|
535
|
+
];
|
|
536
|
+
const betaApiKeys: ConsoleApiKey[] = [
|
|
537
|
+
{
|
|
538
|
+
keyId: 'beta-key-1',
|
|
539
|
+
keyPrefix: 'beta-key',
|
|
540
|
+
name: 'Beta relay key',
|
|
541
|
+
keyType: 'relay',
|
|
542
|
+
scopeKeys: ['tenant-b'],
|
|
543
|
+
actorId: null,
|
|
544
|
+
createdAt: '2026-02-17T10:00:00.000Z',
|
|
545
|
+
expiresAt: null,
|
|
546
|
+
lastUsedAt: null,
|
|
547
|
+
revokedAt: null,
|
|
548
|
+
},
|
|
549
|
+
];
|
|
550
|
+
|
|
551
|
+
const alphaCommits: ConsoleCommitListItem[] = [
|
|
552
|
+
{
|
|
553
|
+
commitSeq: 40,
|
|
554
|
+
actorId: 'a1',
|
|
555
|
+
clientId: 'client-shared',
|
|
556
|
+
clientCommitId: 'alpha-40',
|
|
557
|
+
createdAt: '2026-02-17T10:04:00.000Z',
|
|
558
|
+
changeCount: 2,
|
|
559
|
+
affectedTables: ['tasks'],
|
|
560
|
+
},
|
|
561
|
+
{
|
|
562
|
+
commitSeq: 39,
|
|
563
|
+
actorId: 'a2',
|
|
564
|
+
clientId: 'client-shared',
|
|
565
|
+
clientCommitId: 'alpha-39',
|
|
566
|
+
createdAt: '2026-02-17T10:03:00.000Z',
|
|
567
|
+
changeCount: 1,
|
|
568
|
+
affectedTables: ['tasks'],
|
|
569
|
+
},
|
|
570
|
+
];
|
|
571
|
+
|
|
572
|
+
const betaCommits: ConsoleCommitListItem[] = [
|
|
573
|
+
{
|
|
574
|
+
commitSeq: 18,
|
|
575
|
+
actorId: 'b1',
|
|
576
|
+
clientId: 'client-shared',
|
|
577
|
+
clientCommitId: 'beta-18',
|
|
578
|
+
createdAt: '2026-02-17T10:05:00.000Z',
|
|
579
|
+
changeCount: 3,
|
|
580
|
+
affectedTables: ['orders'],
|
|
581
|
+
},
|
|
582
|
+
];
|
|
583
|
+
|
|
584
|
+
const alphaClients: ConsoleClient[] = [
|
|
585
|
+
{
|
|
586
|
+
clientId: 'client-shared',
|
|
587
|
+
actorId: 'a1',
|
|
588
|
+
cursor: 35,
|
|
589
|
+
lagCommitCount: 5,
|
|
590
|
+
connectionPath: 'direct',
|
|
591
|
+
connectionMode: 'realtime',
|
|
592
|
+
realtimeConnectionCount: 1,
|
|
593
|
+
isRealtimeConnected: true,
|
|
594
|
+
activityState: 'active',
|
|
595
|
+
lastRequestAt: '2026-02-17T10:06:00.000Z',
|
|
596
|
+
lastRequestType: 'pull',
|
|
597
|
+
lastRequestOutcome: 'ok',
|
|
598
|
+
effectiveScopes: { org_id: 'alpha' },
|
|
599
|
+
updatedAt: '2026-02-17T10:06:00.000Z',
|
|
600
|
+
},
|
|
601
|
+
];
|
|
602
|
+
|
|
603
|
+
const betaClients: ConsoleClient[] = [
|
|
604
|
+
{
|
|
605
|
+
clientId: 'client-shared',
|
|
606
|
+
actorId: 'b1',
|
|
607
|
+
cursor: 18,
|
|
608
|
+
lagCommitCount: 0,
|
|
609
|
+
connectionPath: 'relay',
|
|
610
|
+
connectionMode: 'polling',
|
|
611
|
+
realtimeConnectionCount: 0,
|
|
612
|
+
isRealtimeConnected: false,
|
|
613
|
+
activityState: 'idle',
|
|
614
|
+
lastRequestAt: '2026-02-17T10:05:30.000Z',
|
|
615
|
+
lastRequestType: 'pull',
|
|
616
|
+
lastRequestOutcome: 'ok',
|
|
617
|
+
effectiveScopes: { org_id: 'beta' },
|
|
618
|
+
updatedAt: '2026-02-17T10:05:30.000Z',
|
|
619
|
+
},
|
|
620
|
+
];
|
|
621
|
+
|
|
622
|
+
const alphaEvents: ConsoleRequestEvent[] = [
|
|
623
|
+
{
|
|
624
|
+
eventId: 1001,
|
|
625
|
+
partitionId: 'tenant-a',
|
|
626
|
+
requestId: 'alpha-req-1',
|
|
627
|
+
traceId: null,
|
|
628
|
+
spanId: null,
|
|
629
|
+
eventType: 'pull',
|
|
630
|
+
syncPath: 'http-combined',
|
|
631
|
+
transportPath: 'direct',
|
|
632
|
+
actorId: 'a1',
|
|
633
|
+
clientId: 'client-shared',
|
|
634
|
+
statusCode: 200,
|
|
635
|
+
outcome: 'success',
|
|
636
|
+
responseStatus: 'ok',
|
|
637
|
+
errorCode: null,
|
|
638
|
+
durationMs: 23,
|
|
639
|
+
commitSeq: 40,
|
|
640
|
+
operationCount: 1,
|
|
641
|
+
rowCount: 2,
|
|
642
|
+
subscriptionCount: 0,
|
|
643
|
+
scopesSummary: null,
|
|
644
|
+
tables: ['tasks'],
|
|
645
|
+
errorMessage: null,
|
|
646
|
+
payloadRef: 'payload-alpha-1001',
|
|
647
|
+
createdAt: '2026-02-17T10:03:30.000Z',
|
|
648
|
+
},
|
|
649
|
+
];
|
|
650
|
+
|
|
651
|
+
const betaEvents: ConsoleRequestEvent[] = [
|
|
652
|
+
{
|
|
653
|
+
eventId: 2001,
|
|
654
|
+
partitionId: 'tenant-b',
|
|
655
|
+
requestId: 'beta-req-1',
|
|
656
|
+
traceId: null,
|
|
657
|
+
spanId: null,
|
|
658
|
+
eventType: 'push',
|
|
659
|
+
syncPath: 'http-combined',
|
|
660
|
+
transportPath: 'relay',
|
|
661
|
+
actorId: 'b1',
|
|
662
|
+
clientId: 'client-shared',
|
|
663
|
+
statusCode: 200,
|
|
664
|
+
outcome: 'success',
|
|
665
|
+
responseStatus: 'ok',
|
|
666
|
+
errorCode: null,
|
|
667
|
+
durationMs: 30,
|
|
668
|
+
commitSeq: 18,
|
|
669
|
+
operationCount: 1,
|
|
670
|
+
rowCount: 3,
|
|
671
|
+
subscriptionCount: 0,
|
|
672
|
+
scopesSummary: null,
|
|
673
|
+
tables: ['orders'],
|
|
674
|
+
errorMessage: null,
|
|
675
|
+
payloadRef: 'payload-beta-2001',
|
|
676
|
+
createdAt: '2026-02-17T10:05:00.000Z',
|
|
677
|
+
},
|
|
678
|
+
];
|
|
679
|
+
|
|
680
|
+
const alphaOperations: ConsoleOperationEvent[] = [
|
|
681
|
+
{
|
|
682
|
+
operationId: 101,
|
|
683
|
+
operationType: 'notify_data_change',
|
|
684
|
+
consoleUserId: 'console-a',
|
|
685
|
+
partitionId: 'tenant-a',
|
|
686
|
+
targetClientId: null,
|
|
687
|
+
requestPayload: { tables: ['tasks'] },
|
|
688
|
+
resultPayload: { commitSeq: 40 },
|
|
689
|
+
createdAt: '2026-02-17T10:06:30.000Z',
|
|
690
|
+
},
|
|
691
|
+
];
|
|
692
|
+
|
|
693
|
+
const betaOperations: ConsoleOperationEvent[] = [
|
|
694
|
+
{
|
|
695
|
+
operationId: 201,
|
|
696
|
+
operationType: 'evict_client',
|
|
697
|
+
consoleUserId: 'console-b',
|
|
698
|
+
partitionId: 'tenant-b',
|
|
699
|
+
targetClientId: 'client-shared',
|
|
700
|
+
requestPayload: { clientId: 'client-shared' },
|
|
701
|
+
resultPayload: { evicted: true },
|
|
702
|
+
createdAt: '2026-02-17T10:06:00.000Z',
|
|
703
|
+
},
|
|
704
|
+
];
|
|
705
|
+
|
|
706
|
+
const alphaTimeline: ConsoleTimelineItem[] = [
|
|
707
|
+
{
|
|
708
|
+
type: 'commit',
|
|
709
|
+
timestamp: '2026-02-17T10:04:00.000Z',
|
|
710
|
+
commit: alphaCommits[0],
|
|
711
|
+
event: null,
|
|
712
|
+
},
|
|
713
|
+
{
|
|
714
|
+
type: 'event',
|
|
715
|
+
timestamp: '2026-02-17T10:03:30.000Z',
|
|
716
|
+
commit: null,
|
|
717
|
+
event: alphaEvents[0] ?? null,
|
|
718
|
+
},
|
|
719
|
+
];
|
|
720
|
+
|
|
721
|
+
const betaTimeline: ConsoleTimelineItem[] = [
|
|
722
|
+
{
|
|
723
|
+
type: 'event',
|
|
724
|
+
timestamp: '2026-02-17T10:05:00.000Z',
|
|
725
|
+
commit: null,
|
|
726
|
+
event: betaEvents[0] ?? null,
|
|
727
|
+
},
|
|
728
|
+
];
|
|
729
|
+
|
|
730
|
+
const instanceData: Record<string, MockInstanceData> = {
|
|
731
|
+
alpha: {
|
|
732
|
+
stats: alphaStats,
|
|
733
|
+
timeseries: alphaTimeseries,
|
|
734
|
+
latency: alphaLatency,
|
|
735
|
+
handlers: alphaHandlers,
|
|
736
|
+
apiKeys: alphaApiKeys,
|
|
737
|
+
commits: alphaCommits,
|
|
738
|
+
clients: alphaClients,
|
|
739
|
+
timeline: alphaTimeline,
|
|
740
|
+
operations: alphaOperations,
|
|
741
|
+
events: alphaEvents,
|
|
742
|
+
payloadsByEventId: {
|
|
743
|
+
1001: {
|
|
744
|
+
payloadRef: 'payload-alpha-1001',
|
|
745
|
+
partitionId: 'tenant-a',
|
|
746
|
+
requestPayload: {
|
|
747
|
+
clientId: 'client-shared',
|
|
748
|
+
pull: { cursor: 39 },
|
|
749
|
+
},
|
|
750
|
+
responsePayload: {
|
|
751
|
+
pull: { commitSeq: 40 },
|
|
752
|
+
},
|
|
753
|
+
createdAt: '2026-02-17T10:03:30.000Z',
|
|
754
|
+
},
|
|
755
|
+
},
|
|
756
|
+
},
|
|
757
|
+
beta: {
|
|
758
|
+
stats: betaStats,
|
|
759
|
+
timeseries: betaTimeseries,
|
|
760
|
+
latency: betaLatency,
|
|
761
|
+
handlers: betaHandlers,
|
|
762
|
+
apiKeys: betaApiKeys,
|
|
763
|
+
commits: betaCommits,
|
|
764
|
+
clients: betaClients,
|
|
765
|
+
timeline: betaTimeline,
|
|
766
|
+
operations: betaOperations,
|
|
767
|
+
events: betaEvents,
|
|
768
|
+
payloadsByEventId: {
|
|
769
|
+
2001: {
|
|
770
|
+
payloadRef: 'payload-beta-2001',
|
|
771
|
+
partitionId: 'tenant-b',
|
|
772
|
+
requestPayload: {
|
|
773
|
+
clientId: 'client-shared',
|
|
774
|
+
push: { clientCommitId: 'beta-18' },
|
|
775
|
+
},
|
|
776
|
+
responsePayload: {
|
|
777
|
+
push: { commitSeq: 18 },
|
|
778
|
+
},
|
|
779
|
+
createdAt: '2026-02-17T10:05:00.000Z',
|
|
780
|
+
},
|
|
781
|
+
},
|
|
782
|
+
},
|
|
783
|
+
};
|
|
784
|
+
|
|
785
|
+
it('requires auth and lists configured instances', async () => {
|
|
786
|
+
const { app } = createMockGatewayApp({ instances: instanceData });
|
|
787
|
+
|
|
788
|
+
const unauthorized = await app.request(
|
|
789
|
+
'http://localhost/console/instances'
|
|
790
|
+
);
|
|
791
|
+
expect(unauthorized.status).toBe(401);
|
|
792
|
+
|
|
793
|
+
const response = await app.request('http://localhost/console/instances', {
|
|
794
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
795
|
+
});
|
|
796
|
+
expect(response.status).toBe(200);
|
|
797
|
+
const body = (await response.json()) as {
|
|
798
|
+
items: Array<{ instanceId: string; enabled: boolean }>;
|
|
799
|
+
};
|
|
800
|
+
|
|
801
|
+
expect(body.items.map((item) => item.instanceId).sort()).toEqual([
|
|
802
|
+
'alpha',
|
|
803
|
+
'beta',
|
|
804
|
+
]);
|
|
805
|
+
expect(body.items.every((item) => item.enabled)).toBe(true);
|
|
806
|
+
});
|
|
807
|
+
|
|
808
|
+
it('reports downstream instance health and supports instance filters', async () => {
|
|
809
|
+
const { app } = createMockGatewayApp({
|
|
810
|
+
instances: instanceData,
|
|
811
|
+
failingInstances: new Set(['beta']),
|
|
812
|
+
});
|
|
813
|
+
|
|
814
|
+
const response = await app.request(
|
|
815
|
+
'http://localhost/console/instances/health',
|
|
816
|
+
{
|
|
817
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
818
|
+
}
|
|
819
|
+
);
|
|
820
|
+
expect(response.status).toBe(200);
|
|
821
|
+
const body = (await response.json()) as {
|
|
822
|
+
items: Array<{
|
|
823
|
+
instanceId: string;
|
|
824
|
+
healthy: boolean;
|
|
825
|
+
status?: number;
|
|
826
|
+
reason?: string;
|
|
827
|
+
responseTimeMs: number;
|
|
828
|
+
checkedAt: string;
|
|
829
|
+
}>;
|
|
830
|
+
partial: boolean;
|
|
831
|
+
failedInstances: Array<{
|
|
832
|
+
instanceId: string;
|
|
833
|
+
reason: string;
|
|
834
|
+
status?: number;
|
|
835
|
+
}>;
|
|
836
|
+
};
|
|
837
|
+
|
|
838
|
+
expect(body.items).toHaveLength(2);
|
|
839
|
+
const alpha = body.items.find((item) => item.instanceId === 'alpha');
|
|
840
|
+
const beta = body.items.find((item) => item.instanceId === 'beta');
|
|
841
|
+
|
|
842
|
+
expect(alpha?.healthy).toBe(true);
|
|
843
|
+
expect(alpha?.status).toBe(200);
|
|
844
|
+
expect(typeof alpha?.responseTimeMs).toBe('number');
|
|
845
|
+
expect(typeof alpha?.checkedAt).toBe('string');
|
|
846
|
+
|
|
847
|
+
expect(beta?.healthy).toBe(false);
|
|
848
|
+
expect(beta?.status).toBe(503);
|
|
849
|
+
expect(beta?.reason).toBe('HTTP 503');
|
|
850
|
+
expect(body.partial).toBe(true);
|
|
851
|
+
expect(body.failedInstances).toEqual([
|
|
852
|
+
{ instanceId: 'beta', reason: 'HTTP 503', status: 503 },
|
|
853
|
+
]);
|
|
854
|
+
|
|
855
|
+
const filteredResponse = await app.request(
|
|
856
|
+
'http://localhost/console/instances/health?instanceId=alpha',
|
|
857
|
+
{
|
|
858
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
859
|
+
}
|
|
860
|
+
);
|
|
861
|
+
expect(filteredResponse.status).toBe(200);
|
|
862
|
+
const filteredBody = (await filteredResponse.json()) as {
|
|
863
|
+
items: Array<{ instanceId: string; healthy: boolean }>;
|
|
864
|
+
partial: boolean;
|
|
865
|
+
};
|
|
866
|
+
|
|
867
|
+
expect(filteredBody.items).toHaveLength(1);
|
|
868
|
+
expect(filteredBody.items[0]?.instanceId).toBe('alpha');
|
|
869
|
+
expect(filteredBody.items[0]?.healthy).toBe(true);
|
|
870
|
+
expect(filteredBody.partial).toBe(false);
|
|
871
|
+
|
|
872
|
+
const noMatchResponse = await app.request(
|
|
873
|
+
'http://localhost/console/instances/health?instanceId=missing',
|
|
874
|
+
{
|
|
875
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
876
|
+
}
|
|
877
|
+
);
|
|
878
|
+
expect(noMatchResponse.status).toBe(400);
|
|
879
|
+
const noMatchBody = (await noMatchResponse.json()) as {
|
|
880
|
+
error: string;
|
|
881
|
+
};
|
|
882
|
+
expect(noMatchBody.error).toBe('NO_INSTANCES_SELECTED');
|
|
883
|
+
});
|
|
884
|
+
|
|
885
|
+
it('merges stats and reports partial failures', async () => {
|
|
886
|
+
const { app } = createMockGatewayApp({
|
|
887
|
+
instances: instanceData,
|
|
888
|
+
failingInstances: new Set(['beta']),
|
|
889
|
+
});
|
|
890
|
+
|
|
891
|
+
const response = await app.request('http://localhost/console/stats', {
|
|
892
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
893
|
+
});
|
|
894
|
+
expect(response.status).toBe(200);
|
|
895
|
+
const body = (await response.json()) as SyncStats & {
|
|
896
|
+
partial: boolean;
|
|
897
|
+
failedInstances: Array<{ instanceId: string; reason: string }>;
|
|
898
|
+
maxCommitSeqByInstance: Record<string, number>;
|
|
899
|
+
minCommitSeqByInstance: Record<string, number>;
|
|
900
|
+
};
|
|
901
|
+
|
|
902
|
+
expect(body.commitCount).toBe(alphaStats.commitCount);
|
|
903
|
+
expect(body.changeCount).toBe(alphaStats.changeCount);
|
|
904
|
+
expect(body.maxCommitSeqByInstance.alpha).toBe(alphaStats.maxCommitSeq);
|
|
905
|
+
expect(body.minCommitSeqByInstance.alpha).toBe(alphaStats.minCommitSeq);
|
|
906
|
+
expect(body.partial).toBe(true);
|
|
907
|
+
expect(body.failedInstances).toEqual([
|
|
908
|
+
{ instanceId: 'beta', reason: 'HTTP 503', status: 503 },
|
|
909
|
+
]);
|
|
910
|
+
});
|
|
911
|
+
|
|
912
|
+
it('merges timeseries and latency stats across instances', async () => {
|
|
913
|
+
const { app } = createMockGatewayApp({ instances: instanceData });
|
|
914
|
+
|
|
915
|
+
const timeseriesResponse = await app.request(
|
|
916
|
+
'http://localhost/console/stats/timeseries?interval=hour&range=24h',
|
|
917
|
+
{
|
|
918
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
919
|
+
}
|
|
920
|
+
);
|
|
921
|
+
expect(timeseriesResponse.status).toBe(200);
|
|
922
|
+
const timeseriesBody = (await timeseriesResponse.json()) as {
|
|
923
|
+
buckets: TimeseriesStatsResponse['buckets'];
|
|
924
|
+
interval: 'hour' | 'minute' | 'day';
|
|
925
|
+
range: '1h' | '6h' | '24h' | '7d' | '30d';
|
|
926
|
+
partial: boolean;
|
|
927
|
+
};
|
|
928
|
+
|
|
929
|
+
expect(timeseriesBody.interval).toBe('hour');
|
|
930
|
+
expect(timeseriesBody.range).toBe('24h');
|
|
931
|
+
expect(timeseriesBody.partial).toBe(false);
|
|
932
|
+
expect(timeseriesBody.buckets).toHaveLength(2);
|
|
933
|
+
expect(timeseriesBody.buckets[0]).toMatchObject({
|
|
934
|
+
timestamp: '2026-02-17T09:00:00.000Z',
|
|
935
|
+
pushCount: 3,
|
|
936
|
+
pullCount: 1,
|
|
937
|
+
errorCount: 1,
|
|
938
|
+
});
|
|
939
|
+
expect(timeseriesBody.buckets[1]).toMatchObject({
|
|
940
|
+
timestamp: '2026-02-17T10:00:00.000Z',
|
|
941
|
+
pushCount: 1,
|
|
942
|
+
pullCount: 2,
|
|
943
|
+
errorCount: 1,
|
|
944
|
+
});
|
|
945
|
+
expect(timeseriesBody.buckets[0]?.avgLatencyMs).toBeCloseTo(35, 5);
|
|
946
|
+
expect(timeseriesBody.buckets[1]?.avgLatencyMs).toBeCloseTo(20, 5);
|
|
947
|
+
|
|
948
|
+
const latencyResponse = await app.request(
|
|
949
|
+
'http://localhost/console/stats/latency?range=24h',
|
|
950
|
+
{
|
|
951
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
952
|
+
}
|
|
953
|
+
);
|
|
954
|
+
expect(latencyResponse.status).toBe(200);
|
|
955
|
+
const latencyBody =
|
|
956
|
+
(await latencyResponse.json()) as LatencyStatsResponse & {
|
|
957
|
+
partial: boolean;
|
|
958
|
+
};
|
|
959
|
+
|
|
960
|
+
expect(latencyBody.range).toBe('24h');
|
|
961
|
+
expect(latencyBody.partial).toBe(false);
|
|
962
|
+
expect(latencyBody.push.p50).toBe(15);
|
|
963
|
+
expect(latencyBody.push.p90).toBe(30);
|
|
964
|
+
expect(latencyBody.push.p99).toBe(45);
|
|
965
|
+
expect(latencyBody.pull.p50).toBe(13);
|
|
966
|
+
expect(latencyBody.pull.p90).toBe(23);
|
|
967
|
+
expect(latencyBody.pull.p99).toBe(33);
|
|
968
|
+
});
|
|
969
|
+
|
|
970
|
+
it('merges timeline globally and supports instance filters', async () => {
|
|
971
|
+
const { app, downstreamCalls } = createMockGatewayApp({
|
|
972
|
+
instances: instanceData,
|
|
973
|
+
});
|
|
974
|
+
|
|
975
|
+
const response = await app.request(
|
|
976
|
+
'http://localhost/console/timeline?offset=0&limit=2',
|
|
977
|
+
{
|
|
978
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
979
|
+
}
|
|
980
|
+
);
|
|
981
|
+
expect(response.status).toBe(200);
|
|
982
|
+
const body = (await response.json()) as {
|
|
983
|
+
items: Array<{
|
|
984
|
+
instanceId: string;
|
|
985
|
+
timestamp: string;
|
|
986
|
+
federatedTimelineId: string;
|
|
987
|
+
}>;
|
|
988
|
+
total: number;
|
|
989
|
+
partial: boolean;
|
|
990
|
+
};
|
|
991
|
+
|
|
992
|
+
expect(body.total).toBe(3);
|
|
993
|
+
expect(body.partial).toBe(false);
|
|
994
|
+
expect(body.items.map((item) => item.instanceId)).toEqual([
|
|
995
|
+
'beta',
|
|
996
|
+
'alpha',
|
|
997
|
+
]);
|
|
998
|
+
expect(body.items[0]?.federatedTimelineId).toBe('beta:event:2001');
|
|
999
|
+
expect(body.items[1]?.federatedTimelineId).toBe('alpha:commit:40');
|
|
1000
|
+
|
|
1001
|
+
const filtered = await app.request(
|
|
1002
|
+
'http://localhost/console/timeline?instanceId=alpha&offset=0&limit=10',
|
|
1003
|
+
{
|
|
1004
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1005
|
+
}
|
|
1006
|
+
);
|
|
1007
|
+
expect(filtered.status).toBe(200);
|
|
1008
|
+
const filteredBody = (await filtered.json()) as {
|
|
1009
|
+
items: Array<{ instanceId: string }>;
|
|
1010
|
+
total: number;
|
|
1011
|
+
};
|
|
1012
|
+
expect(
|
|
1013
|
+
filteredBody.items.every((item) => item.instanceId === 'alpha')
|
|
1014
|
+
).toBe(true);
|
|
1015
|
+
expect(filteredBody.total).toBe(2);
|
|
1016
|
+
|
|
1017
|
+
expect(
|
|
1018
|
+
downstreamCalls.some((url) => url.includes('alpha.example.test'))
|
|
1019
|
+
).toBe(true);
|
|
1020
|
+
expect(
|
|
1021
|
+
downstreamCalls.some((url) => url.includes('beta.example.test'))
|
|
1022
|
+
).toBe(true);
|
|
1023
|
+
});
|
|
1024
|
+
|
|
1025
|
+
it('returns 502 when all selected instances fail', async () => {
|
|
1026
|
+
const { app } = createMockGatewayApp({
|
|
1027
|
+
instances: instanceData,
|
|
1028
|
+
failingInstances: new Set(['alpha', 'beta']),
|
|
1029
|
+
});
|
|
1030
|
+
|
|
1031
|
+
const response = await app.request('http://localhost/console/stats', {
|
|
1032
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1033
|
+
});
|
|
1034
|
+
expect(response.status).toBe(502);
|
|
1035
|
+
const body = (await response.json()) as {
|
|
1036
|
+
error: string;
|
|
1037
|
+
failedInstances: Array<{ instanceId: string; reason: string }>;
|
|
1038
|
+
};
|
|
1039
|
+
expect(body.error).toBe('DOWNSTREAM_UNAVAILABLE');
|
|
1040
|
+
expect(body.failedInstances).toEqual([
|
|
1041
|
+
{ instanceId: 'alpha', reason: 'HTTP 503', status: 503 },
|
|
1042
|
+
{ instanceId: 'beta', reason: 'HTTP 503', status: 503 },
|
|
1043
|
+
]);
|
|
1044
|
+
});
|
|
1045
|
+
|
|
1046
|
+
it('merges commits and clients with federated ids', async () => {
|
|
1047
|
+
const { app } = createMockGatewayApp({ instances: instanceData });
|
|
1048
|
+
|
|
1049
|
+
const commitsResponse = await app.request(
|
|
1050
|
+
'http://localhost/console/commits?offset=0&limit=10',
|
|
1051
|
+
{
|
|
1052
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1053
|
+
}
|
|
1054
|
+
);
|
|
1055
|
+
expect(commitsResponse.status).toBe(200);
|
|
1056
|
+
const commitsBody = (await commitsResponse.json()) as {
|
|
1057
|
+
items: Array<{ federatedCommitId: string; instanceId: string }>;
|
|
1058
|
+
total: number;
|
|
1059
|
+
};
|
|
1060
|
+
expect(commitsBody.total).toBe(3);
|
|
1061
|
+
expect(commitsBody.items[0]?.federatedCommitId).toBe('beta:18');
|
|
1062
|
+
expect(commitsBody.items[1]?.federatedCommitId).toBe('alpha:40');
|
|
1063
|
+
|
|
1064
|
+
const clientsResponse = await app.request(
|
|
1065
|
+
'http://localhost/console/clients?offset=0&limit=10',
|
|
1066
|
+
{
|
|
1067
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1068
|
+
}
|
|
1069
|
+
);
|
|
1070
|
+
expect(clientsResponse.status).toBe(200);
|
|
1071
|
+
const clientsBody = (await clientsResponse.json()) as {
|
|
1072
|
+
items: Array<{ federatedClientId: string; instanceId: string }>;
|
|
1073
|
+
total: number;
|
|
1074
|
+
};
|
|
1075
|
+
expect(clientsBody.total).toBe(2);
|
|
1076
|
+
expect(clientsBody.items[0]?.federatedClientId).toBe('alpha:client-shared');
|
|
1077
|
+
expect(clientsBody.items[1]?.federatedClientId).toBe('beta:client-shared');
|
|
1078
|
+
});
|
|
1079
|
+
|
|
1080
|
+
it('resolves commit detail by federated id and local id with instance filter', async () => {
|
|
1081
|
+
const { app } = createMockGatewayApp({ instances: instanceData });
|
|
1082
|
+
|
|
1083
|
+
const federatedResponse = await app.request(
|
|
1084
|
+
'http://localhost/console/commits/alpha:40',
|
|
1085
|
+
{
|
|
1086
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1087
|
+
}
|
|
1088
|
+
);
|
|
1089
|
+
expect(federatedResponse.status).toBe(200);
|
|
1090
|
+
const federatedBody = (await federatedResponse.json()) as {
|
|
1091
|
+
instanceId: string;
|
|
1092
|
+
localCommitSeq: number;
|
|
1093
|
+
federatedCommitId: string;
|
|
1094
|
+
commitSeq: number;
|
|
1095
|
+
};
|
|
1096
|
+
expect(federatedBody.instanceId).toBe('alpha');
|
|
1097
|
+
expect(federatedBody.localCommitSeq).toBe(40);
|
|
1098
|
+
expect(federatedBody.federatedCommitId).toBe('alpha:40');
|
|
1099
|
+
expect(federatedBody.commitSeq).toBe(40);
|
|
1100
|
+
|
|
1101
|
+
const localResponse = await app.request(
|
|
1102
|
+
'http://localhost/console/commits/40?instanceId=alpha',
|
|
1103
|
+
{
|
|
1104
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1105
|
+
}
|
|
1106
|
+
);
|
|
1107
|
+
expect(localResponse.status).toBe(200);
|
|
1108
|
+
|
|
1109
|
+
const ambiguousResponse = await app.request(
|
|
1110
|
+
'http://localhost/console/commits/40',
|
|
1111
|
+
{
|
|
1112
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1113
|
+
}
|
|
1114
|
+
);
|
|
1115
|
+
expect(ambiguousResponse.status).toBe(400);
|
|
1116
|
+
});
|
|
1117
|
+
|
|
1118
|
+
it('merges operations and events with federated ids', async () => {
|
|
1119
|
+
const { app } = createMockGatewayApp({ instances: instanceData });
|
|
1120
|
+
|
|
1121
|
+
const operationsResponse = await app.request(
|
|
1122
|
+
'http://localhost/console/operations?offset=0&limit=10',
|
|
1123
|
+
{
|
|
1124
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1125
|
+
}
|
|
1126
|
+
);
|
|
1127
|
+
expect(operationsResponse.status).toBe(200);
|
|
1128
|
+
const operationsBody = (await operationsResponse.json()) as {
|
|
1129
|
+
items: Array<{ federatedOperationId: string; instanceId: string }>;
|
|
1130
|
+
total: number;
|
|
1131
|
+
partial: boolean;
|
|
1132
|
+
};
|
|
1133
|
+
expect(operationsBody.total).toBe(2);
|
|
1134
|
+
expect(operationsBody.partial).toBe(false);
|
|
1135
|
+
expect(operationsBody.items[0]?.federatedOperationId).toBe('alpha:101');
|
|
1136
|
+
expect(operationsBody.items[1]?.federatedOperationId).toBe('beta:201');
|
|
1137
|
+
|
|
1138
|
+
const eventsResponse = await app.request(
|
|
1139
|
+
'http://localhost/console/events?offset=0&limit=10',
|
|
1140
|
+
{
|
|
1141
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1142
|
+
}
|
|
1143
|
+
);
|
|
1144
|
+
expect(eventsResponse.status).toBe(200);
|
|
1145
|
+
const eventsBody = (await eventsResponse.json()) as {
|
|
1146
|
+
items: Array<{ federatedEventId: string; instanceId: string }>;
|
|
1147
|
+
total: number;
|
|
1148
|
+
partial: boolean;
|
|
1149
|
+
};
|
|
1150
|
+
expect(eventsBody.total).toBe(2);
|
|
1151
|
+
expect(eventsBody.partial).toBe(false);
|
|
1152
|
+
expect(eventsBody.items[0]?.federatedEventId).toBe('beta:2001');
|
|
1153
|
+
expect(eventsBody.items[1]?.federatedEventId).toBe('alpha:1001');
|
|
1154
|
+
});
|
|
1155
|
+
|
|
1156
|
+
it('resolves event detail and payload by federated id', async () => {
|
|
1157
|
+
const { app } = createMockGatewayApp({ instances: instanceData });
|
|
1158
|
+
|
|
1159
|
+
const eventResponse = await app.request(
|
|
1160
|
+
'http://localhost/console/events/alpha:1001',
|
|
1161
|
+
{
|
|
1162
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1163
|
+
}
|
|
1164
|
+
);
|
|
1165
|
+
expect(eventResponse.status).toBe(200);
|
|
1166
|
+
const eventBody = (await eventResponse.json()) as {
|
|
1167
|
+
instanceId: string;
|
|
1168
|
+
localEventId: number;
|
|
1169
|
+
federatedEventId: string;
|
|
1170
|
+
eventId: number;
|
|
1171
|
+
payloadRef: string | null;
|
|
1172
|
+
};
|
|
1173
|
+
expect(eventBody.instanceId).toBe('alpha');
|
|
1174
|
+
expect(eventBody.localEventId).toBe(1001);
|
|
1175
|
+
expect(eventBody.federatedEventId).toBe('alpha:1001');
|
|
1176
|
+
expect(eventBody.eventId).toBe(1001);
|
|
1177
|
+
expect(eventBody.payloadRef).toBe('payload-alpha-1001');
|
|
1178
|
+
|
|
1179
|
+
const payloadResponse = await app.request(
|
|
1180
|
+
'http://localhost/console/events/alpha:1001/payload',
|
|
1181
|
+
{
|
|
1182
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1183
|
+
}
|
|
1184
|
+
);
|
|
1185
|
+
expect(payloadResponse.status).toBe(200);
|
|
1186
|
+
const payloadBody = (await payloadResponse.json()) as {
|
|
1187
|
+
instanceId: string;
|
|
1188
|
+
localEventId: number;
|
|
1189
|
+
federatedEventId: string;
|
|
1190
|
+
payloadRef: string;
|
|
1191
|
+
partitionId: string;
|
|
1192
|
+
};
|
|
1193
|
+
expect(payloadBody.instanceId).toBe('alpha');
|
|
1194
|
+
expect(payloadBody.localEventId).toBe(1001);
|
|
1195
|
+
expect(payloadBody.federatedEventId).toBe('alpha:1001');
|
|
1196
|
+
expect(payloadBody.payloadRef).toBe('payload-alpha-1001');
|
|
1197
|
+
expect(payloadBody.partitionId).toBe('tenant-a');
|
|
1198
|
+
});
|
|
1199
|
+
|
|
1200
|
+
it('validates federated event id and instance', async () => {
|
|
1201
|
+
const { app } = createMockGatewayApp({ instances: instanceData });
|
|
1202
|
+
|
|
1203
|
+
const invalidFormat = await app.request(
|
|
1204
|
+
'http://localhost/console/events/not-a-federated-id',
|
|
1205
|
+
{
|
|
1206
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1207
|
+
}
|
|
1208
|
+
);
|
|
1209
|
+
expect(invalidFormat.status).toBe(400);
|
|
1210
|
+
|
|
1211
|
+
const missingInstance = await app.request(
|
|
1212
|
+
'http://localhost/console/events/unknown:1001',
|
|
1213
|
+
{
|
|
1214
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1215
|
+
}
|
|
1216
|
+
);
|
|
1217
|
+
expect(missingInstance.status).toBe(404);
|
|
1218
|
+
|
|
1219
|
+
const ambiguousLocalId = await app.request(
|
|
1220
|
+
'http://localhost/console/events/1001',
|
|
1221
|
+
{
|
|
1222
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1223
|
+
}
|
|
1224
|
+
);
|
|
1225
|
+
expect(ambiguousLocalId.status).toBe(400);
|
|
1226
|
+
|
|
1227
|
+
const resolvedLocalId = await app.request(
|
|
1228
|
+
'http://localhost/console/events/1001?instanceId=alpha',
|
|
1229
|
+
{
|
|
1230
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1231
|
+
}
|
|
1232
|
+
);
|
|
1233
|
+
expect(resolvedLocalId.status).toBe(200);
|
|
1234
|
+
});
|
|
1235
|
+
|
|
1236
|
+
it('requires explicit single instance for non-federated gateway endpoints', async () => {
|
|
1237
|
+
const { app } = createMockGatewayApp({ instances: instanceData });
|
|
1238
|
+
|
|
1239
|
+
const pruneWithoutInstance = await app.request(
|
|
1240
|
+
'http://localhost/console/prune',
|
|
1241
|
+
{
|
|
1242
|
+
method: 'POST',
|
|
1243
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1244
|
+
}
|
|
1245
|
+
);
|
|
1246
|
+
expect(pruneWithoutInstance.status).toBe(400);
|
|
1247
|
+
const pruneBody = (await pruneWithoutInstance.json()) as {
|
|
1248
|
+
error: string;
|
|
1249
|
+
message: string;
|
|
1250
|
+
};
|
|
1251
|
+
expect(pruneBody.error).toBe('INSTANCE_REQUIRED');
|
|
1252
|
+
|
|
1253
|
+
const handlersWithoutInstance = await app.request(
|
|
1254
|
+
'http://localhost/console/handlers',
|
|
1255
|
+
{
|
|
1256
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1257
|
+
}
|
|
1258
|
+
);
|
|
1259
|
+
expect(handlersWithoutInstance.status).toBe(400);
|
|
1260
|
+
const handlersBody = (await handlersWithoutInstance.json()) as {
|
|
1261
|
+
error: string;
|
|
1262
|
+
};
|
|
1263
|
+
expect(handlersBody.error).toBe('INSTANCE_REQUIRED');
|
|
1264
|
+
});
|
|
1265
|
+
|
|
1266
|
+
it('proxies single-instance mutation and config endpoints', async () => {
|
|
1267
|
+
const { app } = createMockGatewayApp({ instances: instanceData });
|
|
1268
|
+
|
|
1269
|
+
const handlersResponse = await app.request(
|
|
1270
|
+
'http://localhost/console/handlers?instanceId=alpha',
|
|
1271
|
+
{
|
|
1272
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1273
|
+
}
|
|
1274
|
+
);
|
|
1275
|
+
expect(handlersResponse.status).toBe(200);
|
|
1276
|
+
const handlersBody = (await handlersResponse.json()) as {
|
|
1277
|
+
items: ConsoleHandler[];
|
|
1278
|
+
};
|
|
1279
|
+
expect(handlersBody.items[0]?.table).toBe('tasks');
|
|
1280
|
+
|
|
1281
|
+
const pruneResponse = await app.request(
|
|
1282
|
+
'http://localhost/console/prune?instanceId=alpha',
|
|
1283
|
+
{
|
|
1284
|
+
method: 'POST',
|
|
1285
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1286
|
+
}
|
|
1287
|
+
);
|
|
1288
|
+
expect(pruneResponse.status).toBe(200);
|
|
1289
|
+
const pruneBody = (await pruneResponse.json()) as {
|
|
1290
|
+
deletedCommits: number;
|
|
1291
|
+
};
|
|
1292
|
+
expect(pruneBody.deletedCommits).toBe(alphaCommits.length);
|
|
1293
|
+
|
|
1294
|
+
const notifyResponse = await app.request(
|
|
1295
|
+
'http://localhost/console/notify-data-change?instanceId=alpha',
|
|
1296
|
+
{
|
|
1297
|
+
method: 'POST',
|
|
1298
|
+
headers: {
|
|
1299
|
+
Authorization: `Bearer ${CONSOLE_TOKEN}`,
|
|
1300
|
+
'Content-Type': 'application/json',
|
|
1301
|
+
},
|
|
1302
|
+
body: JSON.stringify({ tables: ['tasks'], partitionId: 'tenant-a' }),
|
|
1303
|
+
}
|
|
1304
|
+
);
|
|
1305
|
+
expect(notifyResponse.status).toBe(200);
|
|
1306
|
+
const notifyBody = (await notifyResponse.json()) as {
|
|
1307
|
+
commitSeq: number;
|
|
1308
|
+
tables: string[];
|
|
1309
|
+
deletedChunks: number;
|
|
1310
|
+
};
|
|
1311
|
+
expect(notifyBody.tables).toEqual(['tasks']);
|
|
1312
|
+
expect(notifyBody.deletedChunks).toBe(0);
|
|
1313
|
+
|
|
1314
|
+
const clearEventsResponse = await app.request(
|
|
1315
|
+
'http://localhost/console/events?instanceId=alpha',
|
|
1316
|
+
{
|
|
1317
|
+
method: 'DELETE',
|
|
1318
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1319
|
+
}
|
|
1320
|
+
);
|
|
1321
|
+
expect(clearEventsResponse.status).toBe(200);
|
|
1322
|
+
const clearEventsBody = (await clearEventsResponse.json()) as {
|
|
1323
|
+
deletedCount: number;
|
|
1324
|
+
};
|
|
1325
|
+
expect(clearEventsBody.deletedCount).toBe(alphaEvents.length);
|
|
1326
|
+
|
|
1327
|
+
const apiKeysResponse = await app.request(
|
|
1328
|
+
'http://localhost/console/api-keys?instanceId=alpha&offset=0&limit=10',
|
|
1329
|
+
{
|
|
1330
|
+
headers: { Authorization: `Bearer ${CONSOLE_TOKEN}` },
|
|
1331
|
+
}
|
|
1332
|
+
);
|
|
1333
|
+
expect(apiKeysResponse.status).toBe(200);
|
|
1334
|
+
const apiKeysBody = (await apiKeysResponse.json()) as {
|
|
1335
|
+
total: number;
|
|
1336
|
+
items: ConsoleApiKey[];
|
|
1337
|
+
};
|
|
1338
|
+
expect(apiKeysBody.total).toBe(alphaApiKeys.length);
|
|
1339
|
+
expect(apiKeysBody.items[0]?.keyId).toBe('alpha-key-1');
|
|
1340
|
+
|
|
1341
|
+
const createApiKeyResponse = await app.request(
|
|
1342
|
+
'http://localhost/console/api-keys?instanceId=alpha',
|
|
1343
|
+
{
|
|
1344
|
+
method: 'POST',
|
|
1345
|
+
headers: {
|
|
1346
|
+
Authorization: `Bearer ${CONSOLE_TOKEN}`,
|
|
1347
|
+
'Content-Type': 'application/json',
|
|
1348
|
+
},
|
|
1349
|
+
body: JSON.stringify({
|
|
1350
|
+
name: 'Created key',
|
|
1351
|
+
keyType: 'admin',
|
|
1352
|
+
scopeKeys: ['*'],
|
|
1353
|
+
}),
|
|
1354
|
+
}
|
|
1355
|
+
);
|
|
1356
|
+
expect(createApiKeyResponse.status).toBe(201);
|
|
1357
|
+
const createApiKeyBody = (await createApiKeyResponse.json()) as {
|
|
1358
|
+
key: ConsoleApiKey;
|
|
1359
|
+
secretKey: string;
|
|
1360
|
+
};
|
|
1361
|
+
expect(createApiKeyBody.key.keyId).toBe('alpha-new-key');
|
|
1362
|
+
expect(createApiKeyBody.secretKey).toBe('alpha_secret_key');
|
|
1363
|
+
});
|
|
1364
|
+
});
|