@syncular/server 0.31.0 → 0.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +124 -74
- package/dist/admin.d.ts +394 -50
- package/dist/admin.js +714 -82
- package/dist/d1-storage.d.ts +6 -5
- package/dist/d1-storage.js +136 -21
- package/dist/events-ring.d.ts +12 -2
- package/dist/events-ring.js +27 -2
- package/dist/events.d.ts +21 -3
- package/dist/handler.js +36 -13
- package/dist/operations.js +4 -4
- package/dist/postgres-storage.d.ts +6 -5
- package/dist/postgres-storage.js +102 -18
- package/dist/push.js +28 -7
- package/dist/realtime.d.ts +15 -0
- package/dist/realtime.js +46 -7
- package/dist/seed.js +4 -3
- package/dist/sqlite-storage.d.ts +6 -5
- package/dist/sqlite-storage.js +115 -18
- package/dist/storage.d.ts +58 -7
- package/package.json +2 -2
- package/src/admin.ts +1215 -120
- package/src/d1-storage.ts +199 -25
- package/src/events-ring.ts +31 -2
- package/src/events.ts +27 -4
- package/src/handler.ts +47 -16
- package/src/operations.ts +8 -4
- package/src/postgres-storage.ts +173 -28
- package/src/push.ts +40 -9
- package/src/realtime.ts +60 -7
- package/src/seed.ts +4 -3
- package/src/sqlite-storage.ts +181 -26
- package/src/storage.ts +72 -9
package/src/admin.ts
CHANGED
|
@@ -1,31 +1,61 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `SyncularAdmin` is the operator-facing
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* surface. This host surface is mirrored in the server README.
|
|
2
|
+
* `SyncularAdmin` is the operator-facing surface over the server core: a
|
|
3
|
+
* partition-scoped, JSON-able query layer over `ServerStorage`, the optional
|
|
4
|
+
* segment/blob store stats, and an in-memory event ring, plus three explicit
|
|
5
|
+
* operator actions (retry a dead-lettered reaction, prune the commit log,
|
|
6
|
+
* revoke a client's lease). No separate UI package, no framework, and no
|
|
7
|
+
* wire-protocol surface. This host surface is mirrored in the server README.
|
|
9
8
|
*
|
|
10
|
-
* Nothing here is on the sync hot path.
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* getting a silently-empty console.
|
|
9
|
+
* Nothing here is on the sync hot path. The additive optional storage/store
|
|
10
|
+
* methods it depends on are documented as such and this module fails loud (a
|
|
11
|
+
* thrown `Error`) when a backend lacks one, so a host wiring an unsupported
|
|
12
|
+
* store learns immediately rather than getting a silently-empty console.
|
|
15
13
|
*/
|
|
14
|
+
import {
|
|
15
|
+
bytesToBase64,
|
|
16
|
+
type ColumnType,
|
|
17
|
+
decodeRow,
|
|
18
|
+
type JsonValue,
|
|
19
|
+
type PushOperationResult,
|
|
20
|
+
renderRowValues,
|
|
21
|
+
type ScopeMap,
|
|
22
|
+
} from '@syncular/core';
|
|
16
23
|
import type { BlobStore, BlobStoreStats } from './blob-store';
|
|
17
|
-
import {
|
|
18
|
-
|
|
24
|
+
import {
|
|
25
|
+
clockOf,
|
|
26
|
+
RESOLVER_OUTAGE,
|
|
27
|
+
type ResolveScopes,
|
|
28
|
+
type SyncServerConfig,
|
|
29
|
+
} from './context';
|
|
30
|
+
import { SyncError } from './errors';
|
|
31
|
+
import {
|
|
32
|
+
emitEvent,
|
|
33
|
+
type SyncularServerEvent,
|
|
34
|
+
type SyncularServerEvents,
|
|
35
|
+
} from './events';
|
|
19
36
|
import type { RingBufferEvents, RingEventQuery } from './events-ring';
|
|
20
37
|
import type { LeaseRecord, LeaseStore } from './lease-store';
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
38
|
+
import { RealtimeHub } from './realtime';
|
|
39
|
+
import {
|
|
40
|
+
DEFAULT_RETENTION,
|
|
41
|
+
pruneCommitLog,
|
|
42
|
+
type RetentionPolicy,
|
|
43
|
+
} from './prune';
|
|
44
|
+
import {
|
|
45
|
+
type CompiledSchema,
|
|
46
|
+
compileSchema,
|
|
47
|
+
type ServerSchema,
|
|
48
|
+
} from './schema';
|
|
49
|
+
import { computeEffective, matchesEffective } from './scopes';
|
|
23
50
|
import type { SegmentStore, SegmentStoreStats } from './segment-store';
|
|
24
51
|
import type {
|
|
25
52
|
CheckpointDeclaration,
|
|
26
53
|
ClientRecord,
|
|
27
54
|
CommitMetadata,
|
|
28
55
|
ReactionStatus,
|
|
56
|
+
StoredPushResult,
|
|
57
|
+
StoredRow,
|
|
58
|
+
RowChangeRecord,
|
|
29
59
|
ScopeCommitActivity,
|
|
30
60
|
ServerStorage,
|
|
31
61
|
StoredReaction,
|
|
@@ -35,6 +65,10 @@ import type {
|
|
|
35
65
|
export interface AdminClient {
|
|
36
66
|
readonly clientId: string;
|
|
37
67
|
readonly actorId: string;
|
|
68
|
+
/** SSP2 wire version last accepted from this client. */
|
|
69
|
+
readonly wireVersion: number;
|
|
70
|
+
/** Highest codec schema version among its subscriptions, when recorded. */
|
|
71
|
+
readonly schemaVersion?: number;
|
|
38
72
|
readonly cursor: number;
|
|
39
73
|
/**
|
|
40
74
|
* Commits the client has not pulled yet: `maxCommitSeq − max(cursor, 0)`.
|
|
@@ -49,6 +83,26 @@ export interface AdminClient {
|
|
|
49
83
|
}[];
|
|
50
84
|
/** True when the cursor record was touched within the active window. */
|
|
51
85
|
readonly active: boolean;
|
|
86
|
+
/**
|
|
87
|
+
* True while the client holds an open realtime socket on this process.
|
|
88
|
+
* Absent when the admin has no realtime hub.
|
|
89
|
+
*/
|
|
90
|
+
readonly connected?: boolean;
|
|
91
|
+
/** When the client's oldest open socket opened. */
|
|
92
|
+
readonly connectedSinceMs?: number;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export interface AdminListClientsOptions {
|
|
96
|
+
/** Default 200. */
|
|
97
|
+
readonly limit?: number;
|
|
98
|
+
/** `seen` (default): most recently seen first. `lag`: largest lag first. */
|
|
99
|
+
readonly sort?: 'seen' | 'lag';
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface AdminClientPage {
|
|
103
|
+
readonly clients: readonly AdminClient[];
|
|
104
|
+
/** Known clients in the partition before `limit`. */
|
|
105
|
+
readonly total: number;
|
|
52
106
|
}
|
|
53
107
|
|
|
54
108
|
/** One client's drill-down: record + lease + its slice of the event tail. */
|
|
@@ -63,12 +117,29 @@ export interface AdminClientDetail {
|
|
|
63
117
|
readonly events: readonly SyncularServerEvent[];
|
|
64
118
|
}
|
|
65
119
|
|
|
120
|
+
/**
|
|
121
|
+
* How much of a requested window the ring still holds. `complete` is false
|
|
122
|
+
* when the ring dropped events inside the window; the aggregates then cover
|
|
123
|
+
* only `fromMs` onward.
|
|
124
|
+
*/
|
|
125
|
+
export interface AdminEventCoverage {
|
|
126
|
+
readonly complete: boolean;
|
|
127
|
+
readonly fromMs: number;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export interface AdminPushCounts {
|
|
131
|
+
readonly applied: number;
|
|
132
|
+
readonly rejected: number;
|
|
133
|
+
readonly conflicted: number;
|
|
134
|
+
}
|
|
135
|
+
|
|
66
136
|
/** Ring-derived request/push aggregates over a trailing window. */
|
|
67
137
|
export interface AdminMetrics {
|
|
68
138
|
readonly partition: string;
|
|
69
139
|
readonly windowMs: number;
|
|
70
140
|
/** Wall-clock the aggregation ran at (window end). */
|
|
71
141
|
readonly atMs: number;
|
|
142
|
+
readonly coverage: AdminEventCoverage;
|
|
72
143
|
readonly requests: {
|
|
73
144
|
readonly count: number;
|
|
74
145
|
readonly perMinute: number;
|
|
@@ -78,22 +149,62 @@ export interface AdminMetrics {
|
|
|
78
149
|
readonly p50Ms: number;
|
|
79
150
|
readonly p95Ms: number;
|
|
80
151
|
};
|
|
81
|
-
readonly pushes:
|
|
82
|
-
readonly applied: number;
|
|
83
|
-
readonly rejected: number;
|
|
84
|
-
readonly conflicted: number;
|
|
85
|
-
};
|
|
152
|
+
readonly pushes: AdminPushCounts;
|
|
86
153
|
/**
|
|
87
|
-
*
|
|
88
|
-
*
|
|
154
|
+
* The window split into `counts.length` equal buckets, oldest first: the
|
|
155
|
+
* request count, failed requests, p95 duration (`null` for an empty
|
|
156
|
+
* bucket), and push outcomes per bucket.
|
|
89
157
|
*/
|
|
90
158
|
readonly buckets: {
|
|
91
159
|
readonly widthMs: number;
|
|
92
160
|
readonly counts: readonly number[];
|
|
93
161
|
readonly errors: readonly number[];
|
|
162
|
+
readonly p95Ms: readonly (number | null)[];
|
|
163
|
+
readonly pushes: readonly AdminPushCounts[];
|
|
94
164
|
};
|
|
95
165
|
}
|
|
96
166
|
|
|
167
|
+
/** Failures sharing one source and code within the window. */
|
|
168
|
+
export interface AdminErrorGroup {
|
|
169
|
+
readonly source: 'request' | 'push' | 'reaction' | 'scopes';
|
|
170
|
+
/** The §10.2 code; `scopes.resolve_failed` for a throwing resolver. */
|
|
171
|
+
readonly code: string;
|
|
172
|
+
readonly count: number;
|
|
173
|
+
/** Distinct clients among the events that carry a clientId. */
|
|
174
|
+
readonly clients: number;
|
|
175
|
+
/** Distinct actors among the events that carry an actorId. */
|
|
176
|
+
readonly actors: number;
|
|
177
|
+
readonly lastAtMs: number;
|
|
178
|
+
/** Up to five client ids, most recent first. */
|
|
179
|
+
readonly sampleClientIds: readonly string[];
|
|
180
|
+
/** Up to five rows the failing pushes targeted, most recent first. */
|
|
181
|
+
readonly sampleRows: readonly {
|
|
182
|
+
readonly table: string;
|
|
183
|
+
readonly rowId: string;
|
|
184
|
+
}[];
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** One client's push failures within the window. */
|
|
188
|
+
export interface AdminClientErrors {
|
|
189
|
+
readonly clientId: string;
|
|
190
|
+
readonly actorId: string;
|
|
191
|
+
readonly count: number;
|
|
192
|
+
readonly lastCode: string;
|
|
193
|
+
readonly lastAtMs: number;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** "What is failing": ring failures grouped by code and by client. */
|
|
197
|
+
export interface AdminErrorSummary {
|
|
198
|
+
readonly partition: string;
|
|
199
|
+
readonly windowMs: number;
|
|
200
|
+
readonly atMs: number;
|
|
201
|
+
readonly coverage: AdminEventCoverage;
|
|
202
|
+
/** Largest count first. */
|
|
203
|
+
readonly byCode: readonly AdminErrorGroup[];
|
|
204
|
+
/** Every client with a request or push failure, largest count first. */
|
|
205
|
+
readonly byClient: readonly AdminClientErrors[];
|
|
206
|
+
}
|
|
207
|
+
|
|
97
208
|
/** One partition's row in the fleet view (`listPartitions` + horizon math). */
|
|
98
209
|
export interface AdminPartitionOverview {
|
|
99
210
|
readonly partition: string;
|
|
@@ -106,14 +217,192 @@ export interface AdminPartitionOverview {
|
|
|
106
217
|
readonly recommendation: 'up-to-date' | 'prune-recommended';
|
|
107
218
|
}
|
|
108
219
|
|
|
220
|
+
export interface AdminPartitionPage {
|
|
221
|
+
readonly partitions: readonly AdminPartitionOverview[];
|
|
222
|
+
/** Every registered partition before `offset`/`limit`. */
|
|
223
|
+
readonly total: number;
|
|
224
|
+
}
|
|
225
|
+
|
|
109
226
|
export interface AdminRowInspection {
|
|
110
227
|
readonly table: string;
|
|
111
228
|
readonly rowId: string;
|
|
112
229
|
readonly exists: boolean;
|
|
113
230
|
readonly serverVersion?: number;
|
|
114
231
|
readonly scopes?: Record<string, string>;
|
|
115
|
-
/**
|
|
116
|
-
readonly
|
|
232
|
+
/** Present iff `exists`: the row decoded and rendered per §11. */
|
|
233
|
+
readonly values?: Record<string, JsonValue>;
|
|
234
|
+
/** The retained changes to this row, newest first. */
|
|
235
|
+
readonly history: readonly RowChangeRecord[];
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** One change in a commit drill-down. */
|
|
239
|
+
export interface AdminCommitChange {
|
|
240
|
+
readonly table: string;
|
|
241
|
+
readonly rowId: string;
|
|
242
|
+
readonly op: 'upsert' | 'delete';
|
|
243
|
+
readonly rowVersion?: number;
|
|
244
|
+
readonly scopes: Record<string, string>;
|
|
245
|
+
/** The upserted row rendered per §11; absent for deletes. */
|
|
246
|
+
readonly values?: Record<string, JsonValue>;
|
|
247
|
+
/**
|
|
248
|
+
* True when the stored payload does not decode against the current
|
|
249
|
+
* schema (written under an older column set).
|
|
250
|
+
*/
|
|
251
|
+
readonly undecodable?: boolean;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
export interface AdminCommitDetail extends CommitMetadata {
|
|
255
|
+
readonly changes: readonly AdminCommitChange[];
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* The trace verdict, evaluated in this order:
|
|
260
|
+
* `client-unknown`: no client record for the id.
|
|
261
|
+
* `row-missing`: no row and no retained change to it.
|
|
262
|
+
* `no-subscription`: the client subscribes to nothing on the table.
|
|
263
|
+
* `row-deleted`: the row's newest retained change is a delete the client
|
|
264
|
+
* has pulled.
|
|
265
|
+
* `scope-mismatch`: no subscription requests the row's stored scopes.
|
|
266
|
+
* `scope-denied`: a subscription requests them, but the actor's allowed
|
|
267
|
+
* scopes (the host `resolveScopes`) exclude the row.
|
|
268
|
+
* `pending-pull`: the client's cursor is below the row's newest change.
|
|
269
|
+
* `delivered`: the client has pulled the row's newest change.
|
|
270
|
+
*/
|
|
271
|
+
export type AdminTraceVerdict =
|
|
272
|
+
| 'client-unknown'
|
|
273
|
+
| 'row-missing'
|
|
274
|
+
| 'no-subscription'
|
|
275
|
+
| 'row-deleted'
|
|
276
|
+
| 'scope-mismatch'
|
|
277
|
+
| 'scope-denied'
|
|
278
|
+
| 'pending-pull'
|
|
279
|
+
| 'delivered';
|
|
280
|
+
|
|
281
|
+
/** One of the client's subscriptions on the traced table. */
|
|
282
|
+
export interface AdminTraceSubscription {
|
|
283
|
+
readonly id: string;
|
|
284
|
+
readonly scopes: Record<string, readonly string[]>;
|
|
285
|
+
/** The row's stored scopes satisfy the requested scopes. */
|
|
286
|
+
readonly requestedMatch: boolean;
|
|
287
|
+
/** Each requested variable the row's stored value fails. */
|
|
288
|
+
readonly mismatches: readonly {
|
|
289
|
+
readonly variable: string;
|
|
290
|
+
readonly rowValue: string | null;
|
|
291
|
+
readonly requested: readonly string[];
|
|
292
|
+
}[];
|
|
293
|
+
/**
|
|
294
|
+
* The row also satisfies requested ∩ allowed (§3.2). Absent when the
|
|
295
|
+
* admin has no `resolveScopes` or the resolver reported an outage.
|
|
296
|
+
*/
|
|
297
|
+
readonly effectiveMatch?: boolean;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** "Why does client X (not) see row Y": one read across row and client. */
|
|
301
|
+
export interface AdminRowTrace {
|
|
302
|
+
readonly verdict: AdminTraceVerdict;
|
|
303
|
+
readonly table: string;
|
|
304
|
+
readonly rowId: string;
|
|
305
|
+
readonly clientId: string;
|
|
306
|
+
readonly row: AdminRowInspection;
|
|
307
|
+
/** The row's newest retained change. */
|
|
308
|
+
readonly lastChange?: RowChangeRecord;
|
|
309
|
+
/** The partition horizon: changes at or below it are pruned. */
|
|
310
|
+
readonly horizonSeq: number;
|
|
311
|
+
readonly client?: AdminClient;
|
|
312
|
+
/**
|
|
313
|
+
* The actor's allowed scopes from the host `resolveScopes`, or `outage`
|
|
314
|
+
* when it returned the §7.3.3 outage sentinel. Absent without a resolver.
|
|
315
|
+
*/
|
|
316
|
+
readonly allowed?: ScopeMap | 'outage';
|
|
317
|
+
readonly subscriptions: readonly AdminTraceSubscription[];
|
|
318
|
+
/** The client's retained push rejections and conflicts, newest first. */
|
|
319
|
+
readonly rejections: readonly SyncularServerEvent[];
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** One client's verdict in a row's delivery matrix. */
|
|
323
|
+
export interface AdminRowDeliveryClient {
|
|
324
|
+
readonly client: AdminClient;
|
|
325
|
+
readonly verdict: AdminTraceVerdict;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** Who has one row and who does not: every known client's verdict. */
|
|
329
|
+
export interface AdminRowDelivery {
|
|
330
|
+
readonly table: string;
|
|
331
|
+
readonly rowId: string;
|
|
332
|
+
readonly row: AdminRowInspection;
|
|
333
|
+
readonly lastChange?: RowChangeRecord;
|
|
334
|
+
readonly horizonSeq: number;
|
|
335
|
+
/** Every known client, sorted by verdict then most recently seen. */
|
|
336
|
+
readonly clients: readonly AdminRowDeliveryClient[];
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/** One actor and its devices (clients). */
|
|
340
|
+
export interface AdminActor {
|
|
341
|
+
readonly actorId: string;
|
|
342
|
+
readonly clients: readonly AdminClient[];
|
|
343
|
+
/** Devices with an open socket; absent without a realtime hub. */
|
|
344
|
+
readonly connected?: number;
|
|
345
|
+
readonly lastSeenMs: number;
|
|
346
|
+
readonly maxLag: number;
|
|
347
|
+
/** The actor's failures in the last 5 minutes, as `errors().byClient` counts them. */
|
|
348
|
+
readonly recentFailures: number;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/** A page of decoded rows from one table. */
|
|
352
|
+
export interface AdminRowPage {
|
|
353
|
+
readonly table: string;
|
|
354
|
+
readonly rows: readonly {
|
|
355
|
+
readonly rowId: string;
|
|
356
|
+
readonly serverVersion: number;
|
|
357
|
+
readonly scopes: Record<string, string>;
|
|
358
|
+
readonly values: Record<string, JsonValue>;
|
|
359
|
+
}[];
|
|
360
|
+
/** Pass as `afterRowId` for the next page; absent on the last page. */
|
|
361
|
+
readonly nextAfterRowId?: string;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* One per-operation result of a stored write, JSON-safe: a conflict's
|
|
366
|
+
* `serverRow` and `conflictColumns` bytes are standard base64 (§11).
|
|
367
|
+
*/
|
|
368
|
+
export type AdminWriteResult =
|
|
369
|
+
| Exclude<PushOperationResult, { status: 'conflict' }>
|
|
370
|
+
| (Omit<
|
|
371
|
+
Extract<PushOperationResult, { status: 'conflict' }>,
|
|
372
|
+
'serverRow' | 'conflictColumns'
|
|
373
|
+
> & { readonly serverRow: string; readonly conflictColumns: string });
|
|
374
|
+
|
|
375
|
+
/** One stored outcome of a client write (§2.3 idempotency record). */
|
|
376
|
+
export interface AdminWriteOutcome {
|
|
377
|
+
readonly clientId: string;
|
|
378
|
+
readonly status: StoredPushResult['status'];
|
|
379
|
+
readonly commitSeq?: number;
|
|
380
|
+
readonly recordedAtMs?: number;
|
|
381
|
+
readonly results: readonly AdminWriteResult[];
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** What happened to the write a client logged as `clientCommitId`. */
|
|
385
|
+
export interface AdminWriteLookup {
|
|
386
|
+
readonly clientCommitId: string;
|
|
387
|
+
/** One outcome per client that sent this clientCommitId. */
|
|
388
|
+
readonly outcomes: readonly AdminWriteOutcome[];
|
|
389
|
+
/** Retained ring events carrying this clientCommitId, newest first. */
|
|
390
|
+
readonly events: readonly SyncularServerEvent[];
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** Everything an id pasted into the console search matches. */
|
|
394
|
+
export interface AdminSearchResult {
|
|
395
|
+
readonly query: string;
|
|
396
|
+
/** Clients whose id contains the query, or whose actor equals it. */
|
|
397
|
+
readonly clients: readonly AdminClient[];
|
|
398
|
+
/** Actors whose id contains the query. */
|
|
399
|
+
readonly actors: readonly string[];
|
|
400
|
+
/** Rows whose id equals the query, one per table that holds it. */
|
|
401
|
+
readonly rows: readonly { readonly table: string; readonly rowId: string }[];
|
|
402
|
+
/** The commit whose seq equals the query. */
|
|
403
|
+
readonly commit?: CommitMetadata;
|
|
404
|
+
/** Outcomes of writes whose clientCommitId equals the query. */
|
|
405
|
+
readonly writes: readonly AdminWriteOutcome[];
|
|
117
406
|
}
|
|
118
407
|
|
|
119
408
|
export interface AdminHorizonStatus {
|
|
@@ -129,10 +418,30 @@ export interface AdminHorizonStatus {
|
|
|
129
418
|
readonly recommendation: 'up-to-date' | 'prune-recommended';
|
|
130
419
|
}
|
|
131
420
|
|
|
421
|
+
export interface AdminPruneResult {
|
|
422
|
+
readonly previousHorizonSeq: number;
|
|
423
|
+
readonly horizonSeq: number;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/** One schema table as the console's pickers need it. */
|
|
427
|
+
export interface AdminTableInfo {
|
|
428
|
+
readonly name: string;
|
|
429
|
+
readonly columns: readonly {
|
|
430
|
+
readonly name: string;
|
|
431
|
+
readonly type: ColumnType;
|
|
432
|
+
readonly nullable: boolean;
|
|
433
|
+
readonly primaryKey: boolean;
|
|
434
|
+
}[];
|
|
435
|
+
}
|
|
436
|
+
|
|
132
437
|
export interface AdminListCommitsOptions {
|
|
133
438
|
readonly afterSeq?: number;
|
|
439
|
+
/** Exclusive upper bound: pass the oldest seq of a page for the next. */
|
|
440
|
+
readonly beforeSeq?: number;
|
|
134
441
|
readonly limit?: number;
|
|
135
442
|
readonly table?: string;
|
|
443
|
+
/** Attach each commit's first N changes as `sample`. */
|
|
444
|
+
readonly changeSample?: number;
|
|
136
445
|
}
|
|
137
446
|
|
|
138
447
|
export interface AdminScopeActivityOptions {
|
|
@@ -153,30 +462,40 @@ export interface AdminStats {
|
|
|
153
462
|
export interface SyncularAdminOptions {
|
|
154
463
|
readonly storage: ServerStorage;
|
|
155
464
|
/**
|
|
156
|
-
* The server schema.
|
|
157
|
-
*
|
|
158
|
-
* a storage instance that has not served a sync request yet.
|
|
465
|
+
* The server schema. Row reads (`inspectRow`, `traceRow`, `commitDetail`)
|
|
466
|
+
* require it: it decodes payloads and ensures the relational row tables
|
|
467
|
+
* exist on a storage instance that has not served a sync request yet.
|
|
159
468
|
*/
|
|
160
469
|
readonly schema?: ServerSchema;
|
|
161
470
|
/** RFC 0007: declarations `inspectRow` carries into `ensureSchema`. */
|
|
162
471
|
readonly checkpoints?: readonly CheckpointDeclaration[];
|
|
163
472
|
/** The event ring feeding the event tail. Absent ⇒ `events()` is empty. */
|
|
164
473
|
readonly ring?: RingBufferEvents;
|
|
474
|
+
/** Receives `prune.completed` and `lease.revoked` from operator actions. */
|
|
475
|
+
readonly events?: SyncularServerEvents;
|
|
476
|
+
/** The host resolver: `traceRow` checks the actor's allowed scopes. */
|
|
477
|
+
readonly resolveScopes?: ResolveScopes;
|
|
478
|
+
/** The realtime hub: which clients hold an open socket right now. */
|
|
479
|
+
readonly realtime?: RealtimeHub;
|
|
165
480
|
readonly segments?: SegmentStore;
|
|
166
481
|
readonly blobs?: BlobStore;
|
|
167
|
-
/** The §7.3 lease store —
|
|
482
|
+
/** The §7.3 lease store — the client drill-down and `revokeLease`. */
|
|
168
483
|
readonly leases?: LeaseStore;
|
|
169
|
-
/** Retention policy for horizon recommendation
|
|
484
|
+
/** Retention policy for horizon recommendation and prune (§4.6). */
|
|
170
485
|
readonly retention?: Partial<RetentionPolicy>;
|
|
171
486
|
/** Epoch-ms clock (defaults to `Date.now`) — active-window math. */
|
|
172
487
|
readonly clock?: () => number;
|
|
173
488
|
}
|
|
174
489
|
|
|
490
|
+
const DEFAULT_CLIENT_LIMIT = 200;
|
|
491
|
+
const DEFAULT_PARTITION_LIMIT = 100;
|
|
175
492
|
const DEFAULT_COMMIT_LIMIT = 50;
|
|
176
493
|
const DEFAULT_SCOPE_LIMIT = 50;
|
|
177
494
|
const DEFAULT_REACTION_LIMIT = 100;
|
|
178
495
|
const DEFAULT_CLIENT_EVENT_LIMIT = 100;
|
|
179
|
-
const
|
|
496
|
+
const DEFAULT_ROW_HISTORY_LIMIT = 20;
|
|
497
|
+
const DEFAULT_ROW_PAGE_LIMIT = 50;
|
|
498
|
+
const DEFAULT_WINDOW_MS = 5 * 60 * 1000;
|
|
180
499
|
const DEFAULT_METRICS_BUCKETS = 30;
|
|
181
500
|
|
|
182
501
|
/** Nearest-rank percentile over an unsorted sample; 0 for an empty one. */
|
|
@@ -196,33 +515,89 @@ function required<T>(value: T | undefined, what: string): T {
|
|
|
196
515
|
return value;
|
|
197
516
|
}
|
|
198
517
|
|
|
199
|
-
|
|
518
|
+
/** Verdicts in evaluation order; `rowDelivery` sorts by it. */
|
|
519
|
+
const VERDICT_ORDER: readonly AdminTraceVerdict[] = [
|
|
520
|
+
'client-unknown',
|
|
521
|
+
'row-missing',
|
|
522
|
+
'no-subscription',
|
|
523
|
+
'row-deleted',
|
|
524
|
+
'scope-mismatch',
|
|
525
|
+
'scope-denied',
|
|
526
|
+
'pending-pull',
|
|
527
|
+
'delivered',
|
|
528
|
+
];
|
|
529
|
+
|
|
530
|
+
/** One known client's subscriptions on the row's table and its verdict. */
|
|
531
|
+
function traceVerdict(
|
|
200
532
|
record: ClientRecord,
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
533
|
+
row: AdminRowInspection,
|
|
534
|
+
lastChange: RowChangeRecord | undefined,
|
|
535
|
+
horizonSeq: number,
|
|
536
|
+
allowed: ScopeMap | 'outage' | undefined,
|
|
537
|
+
): {
|
|
538
|
+
readonly subscriptions: AdminTraceSubscription[];
|
|
539
|
+
readonly verdict: AdminTraceVerdict;
|
|
540
|
+
} {
|
|
541
|
+
const rowScopes = row.scopes ?? {};
|
|
542
|
+
const subscriptions = record.subscriptions
|
|
543
|
+
.filter((sub) => sub.table === row.table)
|
|
544
|
+
.map((sub): AdminTraceSubscription => {
|
|
545
|
+
const mismatches = Object.entries(sub.scopes)
|
|
546
|
+
.filter(([variable, values]) => {
|
|
547
|
+
const value = rowScopes[variable];
|
|
548
|
+
return value === undefined || !values.includes(value);
|
|
549
|
+
})
|
|
550
|
+
.map(([variable, values]) => ({
|
|
551
|
+
variable,
|
|
552
|
+
rowValue: rowScopes[variable] ?? null,
|
|
553
|
+
requested: values,
|
|
554
|
+
}));
|
|
555
|
+
const traced = {
|
|
556
|
+
id: sub.id,
|
|
557
|
+
scopes: sub.scopes,
|
|
558
|
+
requestedMatch: mismatches.length === 0,
|
|
559
|
+
mismatches,
|
|
560
|
+
};
|
|
561
|
+
if (allowed === undefined || allowed === 'outage') return traced;
|
|
562
|
+
const effective = computeEffective(sub.scopes, { ok: true, allowed });
|
|
563
|
+
return {
|
|
564
|
+
...traced,
|
|
565
|
+
effectiveMatch:
|
|
566
|
+
effective.status === 'active' &&
|
|
567
|
+
matchesEffective(rowScopes, effective.effective),
|
|
568
|
+
};
|
|
569
|
+
});
|
|
570
|
+
// With no retained change left, the newest one sits at or below the
|
|
571
|
+
// horizon, so a cursor at the horizon has pulled it.
|
|
572
|
+
const pulled = record.cursor >= (lastChange?.commitSeq ?? horizonSeq);
|
|
573
|
+
const verdict: AdminTraceVerdict =
|
|
574
|
+
!row.exists && lastChange === undefined
|
|
575
|
+
? 'row-missing'
|
|
576
|
+
: subscriptions.length === 0
|
|
577
|
+
? 'no-subscription'
|
|
578
|
+
: !row.exists
|
|
579
|
+
? pulled
|
|
580
|
+
? 'row-deleted'
|
|
581
|
+
: 'pending-pull'
|
|
582
|
+
: !subscriptions.some((sub) => sub.requestedMatch)
|
|
583
|
+
? 'scope-mismatch'
|
|
584
|
+
: subscriptions.every((sub) => sub.effectiveMatch === false)
|
|
585
|
+
? 'scope-denied'
|
|
586
|
+
: pulled
|
|
587
|
+
? 'delivered'
|
|
588
|
+
: 'pending-pull';
|
|
589
|
+
return { subscriptions, verdict };
|
|
218
590
|
}
|
|
219
591
|
|
|
220
|
-
/** The
|
|
592
|
+
/** The console query surface plus its operator actions. One per process. */
|
|
221
593
|
export class SyncularAdmin {
|
|
222
594
|
readonly #storage: ServerStorage;
|
|
223
|
-
readonly #schema?:
|
|
595
|
+
readonly #schema?: CompiledSchema;
|
|
224
596
|
readonly #checkpoints?: readonly CheckpointDeclaration[];
|
|
225
597
|
readonly #ring?: RingBufferEvents;
|
|
598
|
+
readonly #events?: SyncularServerEvents;
|
|
599
|
+
readonly #resolveScopes?: ResolveScopes;
|
|
600
|
+
readonly #realtime?: RealtimeHub;
|
|
226
601
|
readonly #segments?: SegmentStore;
|
|
227
602
|
readonly #blobs?: BlobStore;
|
|
228
603
|
readonly #leases?: LeaseStore;
|
|
@@ -231,10 +606,17 @@ export class SyncularAdmin {
|
|
|
231
606
|
|
|
232
607
|
constructor(options: SyncularAdminOptions) {
|
|
233
608
|
this.#storage = options.storage;
|
|
234
|
-
if (options.schema !== undefined)
|
|
609
|
+
if (options.schema !== undefined) {
|
|
610
|
+
this.#schema = compileSchema(options.schema);
|
|
611
|
+
}
|
|
235
612
|
if (options.checkpoints !== undefined)
|
|
236
613
|
this.#checkpoints = options.checkpoints;
|
|
237
614
|
if (options.ring !== undefined) this.#ring = options.ring;
|
|
615
|
+
if (options.events !== undefined) this.#events = options.events;
|
|
616
|
+
if (options.resolveScopes !== undefined) {
|
|
617
|
+
this.#resolveScopes = options.resolveScopes;
|
|
618
|
+
}
|
|
619
|
+
if (options.realtime !== undefined) this.#realtime = options.realtime;
|
|
238
620
|
if (options.segments !== undefined) this.#segments = options.segments;
|
|
239
621
|
if (options.blobs !== undefined) this.#blobs = options.blobs;
|
|
240
622
|
if (options.leases !== undefined) this.#leases = options.leases;
|
|
@@ -243,9 +625,10 @@ export class SyncularAdmin {
|
|
|
243
625
|
}
|
|
244
626
|
|
|
245
627
|
/**
|
|
246
|
-
* Build an admin over a `SyncServerConfig`, reusing its storage
|
|
247
|
-
*
|
|
248
|
-
* composed into the config's `events` by
|
|
628
|
+
* Build an admin over a `SyncServerConfig`, reusing its storage, schema,
|
|
629
|
+
* resolver, events sink, stores, leases and clock. Pass the ring
|
|
630
|
+
* separately (it is an events sink, composed into the config's `events` by
|
|
631
|
+
* the host — see `composeEvents`).
|
|
249
632
|
*/
|
|
250
633
|
static fromConfig(
|
|
251
634
|
config: SyncServerConfig,
|
|
@@ -257,6 +640,11 @@ export class SyncularAdmin {
|
|
|
257
640
|
...(config.checkpoints !== undefined
|
|
258
641
|
? { checkpoints: config.checkpoints }
|
|
259
642
|
: {}),
|
|
643
|
+
resolveScopes: config.resolveScopes,
|
|
644
|
+
...(config.realtime instanceof RealtimeHub
|
|
645
|
+
? { realtime: config.realtime }
|
|
646
|
+
: {}),
|
|
647
|
+
...(config.events !== undefined ? { events: config.events } : {}),
|
|
260
648
|
segments: config.segments,
|
|
261
649
|
...(config.blobs !== undefined ? { blobs: config.blobs } : {}),
|
|
262
650
|
...(config.leases !== undefined ? { leases: config.leases.store } : {}),
|
|
@@ -266,24 +654,48 @@ export class SyncularAdmin {
|
|
|
266
654
|
});
|
|
267
655
|
}
|
|
268
656
|
|
|
269
|
-
/** The
|
|
270
|
-
|
|
657
|
+
/** The server schema version; `undefined` without a schema. */
|
|
658
|
+
get schemaVersion(): number | undefined {
|
|
659
|
+
return this.#schema?.version;
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/** The schema tables and columns, for row and commit pickers. */
|
|
663
|
+
tables(): AdminTableInfo[] {
|
|
664
|
+
return [...(this.#schema?.tables.values() ?? [])].map((table) => ({
|
|
665
|
+
name: table.name,
|
|
666
|
+
columns: table.columns.map((column, i) => ({
|
|
667
|
+
name: column.name,
|
|
668
|
+
type: column.type,
|
|
669
|
+
nullable: column.nullable,
|
|
670
|
+
primaryKey: i === table.primaryKeyIndex,
|
|
671
|
+
})),
|
|
672
|
+
}));
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
/** The known clients for a partition (cursor, lag, versions, subs). */
|
|
676
|
+
async listClients(
|
|
677
|
+
partition: string,
|
|
678
|
+
options: AdminListClientsOptions = {},
|
|
679
|
+
): Promise<AdminClientPage> {
|
|
271
680
|
const list = required(
|
|
272
681
|
this.#storage.listClientRecords?.bind(this.#storage),
|
|
273
682
|
'storage',
|
|
274
683
|
);
|
|
275
|
-
const
|
|
276
|
-
const
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
toAdminClient(record, record.updatedAtMs >= activeFloorMs, maxCommitSeq),
|
|
684
|
+
const view = await this.#clientView(partition);
|
|
685
|
+
const clients = (await list(partition)).map(view);
|
|
686
|
+
clients.sort((a, b) =>
|
|
687
|
+
options.sort === 'lag' ? b.lag - a.lag : b.updatedAtMs - a.updatedAtMs,
|
|
280
688
|
);
|
|
689
|
+
return {
|
|
690
|
+
clients: clients.slice(0, options.limit ?? DEFAULT_CLIENT_LIMIT),
|
|
691
|
+
total: clients.length,
|
|
692
|
+
};
|
|
281
693
|
}
|
|
282
694
|
|
|
283
695
|
/**
|
|
284
696
|
* One client's drill-down: its record (with lag), its §7.3 lease when a
|
|
285
|
-
* lease store is wired, and its recent slice of the
|
|
286
|
-
* "why is this client stale" in a single read.
|
|
697
|
+
* lease store is wired, its realtime state, and its recent slice of the
|
|
698
|
+
* event tail. Answers "why is this client stale" in a single read.
|
|
287
699
|
*/
|
|
288
700
|
async clientDetail(
|
|
289
701
|
partition: string,
|
|
@@ -292,23 +704,18 @@ export class SyncularAdmin {
|
|
|
292
704
|
): Promise<AdminClientDetail> {
|
|
293
705
|
const record = await this.#storage.getClientRecord(partition, clientId);
|
|
294
706
|
const events = this.events({
|
|
707
|
+
partition,
|
|
295
708
|
clientId,
|
|
296
709
|
limit: options.eventLimit ?? DEFAULT_CLIENT_EVENT_LIMIT,
|
|
297
710
|
});
|
|
298
711
|
if (record === undefined) {
|
|
299
712
|
return { clientId, exists: false, events };
|
|
300
713
|
}
|
|
301
|
-
const maxCommitSeq = await this.#storage.getMaxCommitSeq(partition);
|
|
302
|
-
const activeFloorMs = this.#clock() - this.#retention.activeWindowMs;
|
|
303
714
|
const lease = await this.#leases?.get(partition, clientId);
|
|
304
715
|
return {
|
|
305
716
|
clientId,
|
|
306
717
|
exists: true,
|
|
307
|
-
client:
|
|
308
|
-
record,
|
|
309
|
-
record.updatedAtMs >= activeFloorMs,
|
|
310
|
-
maxCommitSeq,
|
|
311
|
-
),
|
|
718
|
+
client: (await this.#clientView(partition))(record),
|
|
312
719
|
...(lease !== undefined ? { lease } : {}),
|
|
313
720
|
events,
|
|
314
721
|
};
|
|
@@ -325,11 +732,68 @@ export class SyncularAdmin {
|
|
|
325
732
|
);
|
|
326
733
|
return read(partition, {
|
|
327
734
|
afterSeq: options.afterSeq ?? 0,
|
|
735
|
+
...(options.beforeSeq !== undefined
|
|
736
|
+
? { beforeSeq: options.beforeSeq }
|
|
737
|
+
: {}),
|
|
328
738
|
limit: options.limit ?? DEFAULT_COMMIT_LIMIT,
|
|
329
739
|
...(options.table !== undefined ? { table: options.table } : {}),
|
|
740
|
+
...(options.changeSample !== undefined
|
|
741
|
+
? { changeSample: options.changeSample }
|
|
742
|
+
: {}),
|
|
330
743
|
});
|
|
331
744
|
}
|
|
332
745
|
|
|
746
|
+
/**
|
|
747
|
+
* One retained commit with every change rendered per §11. Throws
|
|
748
|
+
* `sync.not_found` when the commit does not exist or was pruned.
|
|
749
|
+
*/
|
|
750
|
+
async commitDetail(
|
|
751
|
+
partition: string,
|
|
752
|
+
commitSeq: number,
|
|
753
|
+
): Promise<AdminCommitDetail> {
|
|
754
|
+
const read = required(
|
|
755
|
+
this.#storage.getCommit?.bind(this.#storage),
|
|
756
|
+
'storage',
|
|
757
|
+
);
|
|
758
|
+
const schema = this.#requireSchema();
|
|
759
|
+
const commit = await read(partition, commitSeq);
|
|
760
|
+
if (commit === undefined) {
|
|
761
|
+
throw new SyncError(
|
|
762
|
+
'sync.not_found',
|
|
763
|
+
'commit is not in the retained log',
|
|
764
|
+
JSON.stringify({ commitSeq }),
|
|
765
|
+
);
|
|
766
|
+
}
|
|
767
|
+
return {
|
|
768
|
+
...commit,
|
|
769
|
+
changes: commit.changes.map((change): AdminCommitChange => {
|
|
770
|
+
const base = {
|
|
771
|
+
table: change.table,
|
|
772
|
+
rowId: change.rowId,
|
|
773
|
+
op: change.op,
|
|
774
|
+
...(change.rowVersion !== undefined
|
|
775
|
+
? { rowVersion: change.rowVersion }
|
|
776
|
+
: {}),
|
|
777
|
+
scopes: change.scopes,
|
|
778
|
+
};
|
|
779
|
+
const table = schema.tables.get(change.table);
|
|
780
|
+
if (change.payload === undefined) return base;
|
|
781
|
+
if (table === undefined) return { ...base, undecodable: true };
|
|
782
|
+
try {
|
|
783
|
+
return {
|
|
784
|
+
...base,
|
|
785
|
+
values: renderRowValues(
|
|
786
|
+
table.columns,
|
|
787
|
+
decodeRow(table.columns, change.payload),
|
|
788
|
+
),
|
|
789
|
+
};
|
|
790
|
+
} catch {
|
|
791
|
+
return { ...base, undecodable: true };
|
|
792
|
+
}
|
|
793
|
+
}),
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
|
|
333
797
|
/** Pending, leased, completed, and dead-lettered durable reactions. */
|
|
334
798
|
async listReactions(
|
|
335
799
|
partition: string,
|
|
@@ -347,28 +811,38 @@ export class SyncularAdmin {
|
|
|
347
811
|
}
|
|
348
812
|
|
|
349
813
|
/**
|
|
350
|
-
* Inspect a single row:
|
|
351
|
-
*
|
|
352
|
-
*
|
|
814
|
+
* Inspect a single row: server_version, stored scopes, its values decoded
|
|
815
|
+
* per §11, and its retained change history (newest first). Throws
|
|
816
|
+
* `sync.unknown_table` for a table outside the schema.
|
|
353
817
|
*/
|
|
354
818
|
async inspectRow(
|
|
355
819
|
partition: string,
|
|
356
820
|
table: string,
|
|
357
821
|
rowId: string,
|
|
822
|
+
options: { readonly historyLimit?: number } = {},
|
|
358
823
|
): Promise<AdminRowInspection> {
|
|
359
|
-
const
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
this.#checkpoints,
|
|
824
|
+
const schema = this.#requireSchema();
|
|
825
|
+
const compiled = schema.tables.get(table);
|
|
826
|
+
if (compiled === undefined) {
|
|
827
|
+
throw new SyncError(
|
|
828
|
+
'sync.unknown_table',
|
|
829
|
+
'table is not in the server schema',
|
|
830
|
+
JSON.stringify({ table }),
|
|
367
831
|
);
|
|
368
832
|
}
|
|
369
|
-
|
|
833
|
+
await this.#storage.ensureSchema(schema, this.#checkpoints);
|
|
834
|
+
const listChanges = required(
|
|
835
|
+
this.#storage.listRowChanges?.bind(this.#storage),
|
|
836
|
+
'storage',
|
|
837
|
+
);
|
|
838
|
+
const history = await listChanges(partition, {
|
|
839
|
+
table,
|
|
840
|
+
rowId,
|
|
841
|
+
limit: options.historyLimit ?? DEFAULT_ROW_HISTORY_LIMIT,
|
|
842
|
+
});
|
|
843
|
+
const row = await this.#storage.getRow(partition, table, rowId);
|
|
370
844
|
if (row === undefined) {
|
|
371
|
-
return { table, rowId, exists: false };
|
|
845
|
+
return { table, rowId, exists: false, history };
|
|
372
846
|
}
|
|
373
847
|
return {
|
|
374
848
|
table,
|
|
@@ -376,6 +850,331 @@ export class SyncularAdmin {
|
|
|
376
850
|
exists: true,
|
|
377
851
|
serverVersion: row.serverVersion,
|
|
378
852
|
scopes: row.scopes,
|
|
853
|
+
values: renderRowValues(
|
|
854
|
+
compiled.columns,
|
|
855
|
+
decodeRow(compiled.columns, row.payload),
|
|
856
|
+
),
|
|
857
|
+
history,
|
|
858
|
+
};
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
/**
|
|
862
|
+
* Trace whether one client receives one row, combining the row's stored
|
|
863
|
+
* scopes and newest change with the client's subscriptions, cursor, the
|
|
864
|
+
* actor's allowed scopes, and its retained push failures. See
|
|
865
|
+
* `AdminTraceVerdict` for the verdict order.
|
|
866
|
+
*/
|
|
867
|
+
async traceRow(
|
|
868
|
+
partition: string,
|
|
869
|
+
query: {
|
|
870
|
+
readonly table: string;
|
|
871
|
+
readonly rowId: string;
|
|
872
|
+
readonly clientId: string;
|
|
873
|
+
},
|
|
874
|
+
): Promise<AdminRowTrace> {
|
|
875
|
+
const { table, rowId, clientId } = query;
|
|
876
|
+
const row = await this.inspectRow(partition, table, rowId, {
|
|
877
|
+
historyLimit: 1,
|
|
878
|
+
});
|
|
879
|
+
const lastChange = row.history[0];
|
|
880
|
+
const horizonSeq = await this.#storage.getHorizonSeq(partition);
|
|
881
|
+
const rejections = this.events({ partition, clientId }).filter(
|
|
882
|
+
(event) =>
|
|
883
|
+
event.type === 'push.rejected' || event.type === 'push.conflicted',
|
|
884
|
+
);
|
|
885
|
+
const record = await this.#storage.getClientRecord(partition, clientId);
|
|
886
|
+
const base = {
|
|
887
|
+
table,
|
|
888
|
+
rowId,
|
|
889
|
+
clientId,
|
|
890
|
+
row,
|
|
891
|
+
horizonSeq,
|
|
892
|
+
rejections,
|
|
893
|
+
...(lastChange !== undefined ? { lastChange } : {}),
|
|
894
|
+
};
|
|
895
|
+
if (record === undefined) {
|
|
896
|
+
return { ...base, verdict: 'client-unknown', subscriptions: [] };
|
|
897
|
+
}
|
|
898
|
+
const allowed = await this.#allowedScopes(partition, record);
|
|
899
|
+
return {
|
|
900
|
+
...base,
|
|
901
|
+
client: (await this.#clientView(partition))(record),
|
|
902
|
+
...(allowed !== undefined ? { allowed } : {}),
|
|
903
|
+
...traceVerdict(record, row, lastChange, horizonSeq, allowed),
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
/**
|
|
908
|
+
* Who has one row and who does not: the trace verdict for every known
|
|
909
|
+
* client in the partition, resolving each actor's allowed scopes once.
|
|
910
|
+
*/
|
|
911
|
+
async rowDelivery(
|
|
912
|
+
partition: string,
|
|
913
|
+
table: string,
|
|
914
|
+
rowId: string,
|
|
915
|
+
): Promise<AdminRowDelivery> {
|
|
916
|
+
const list = required(
|
|
917
|
+
this.#storage.listClientRecords?.bind(this.#storage),
|
|
918
|
+
'storage',
|
|
919
|
+
);
|
|
920
|
+
const row = await this.inspectRow(partition, table, rowId, {
|
|
921
|
+
historyLimit: 1,
|
|
922
|
+
});
|
|
923
|
+
const lastChange = row.history[0];
|
|
924
|
+
const horizonSeq = await this.#storage.getHorizonSeq(partition);
|
|
925
|
+
const view = await this.#clientView(partition);
|
|
926
|
+
const allowedByActor = new Map<string, ScopeMap | 'outage' | undefined>();
|
|
927
|
+
const clients: AdminRowDeliveryClient[] = [];
|
|
928
|
+
for (const record of await list(partition)) {
|
|
929
|
+
if (!allowedByActor.has(record.actorId)) {
|
|
930
|
+
allowedByActor.set(
|
|
931
|
+
record.actorId,
|
|
932
|
+
await this.#allowedScopes(partition, record),
|
|
933
|
+
);
|
|
934
|
+
}
|
|
935
|
+
clients.push({
|
|
936
|
+
client: view(record),
|
|
937
|
+
verdict: traceVerdict(
|
|
938
|
+
record,
|
|
939
|
+
row,
|
|
940
|
+
lastChange,
|
|
941
|
+
horizonSeq,
|
|
942
|
+
allowedByActor.get(record.actorId),
|
|
943
|
+
).verdict,
|
|
944
|
+
});
|
|
945
|
+
}
|
|
946
|
+
clients.sort(
|
|
947
|
+
(a, b) =>
|
|
948
|
+
VERDICT_ORDER.indexOf(a.verdict) - VERDICT_ORDER.indexOf(b.verdict) ||
|
|
949
|
+
b.client.updatedAtMs - a.client.updatedAtMs,
|
|
950
|
+
);
|
|
951
|
+
return {
|
|
952
|
+
table,
|
|
953
|
+
rowId,
|
|
954
|
+
row,
|
|
955
|
+
...(lastChange !== undefined ? { lastChange } : {}),
|
|
956
|
+
horizonSeq,
|
|
957
|
+
clients,
|
|
958
|
+
};
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
/**
|
|
962
|
+
* The partition's actors with their devices (clients), open sockets,
|
|
963
|
+
* worst lag, and failures in the last 5 minutes; most recently seen
|
|
964
|
+
* first.
|
|
965
|
+
*/
|
|
966
|
+
async actors(partition: string): Promise<AdminActor[]> {
|
|
967
|
+
const { clients } = await this.listClients(partition, {
|
|
968
|
+
limit: Number.MAX_SAFE_INTEGER,
|
|
969
|
+
});
|
|
970
|
+
// The same per-client tally as `errors()`, so People and the overview's
|
|
971
|
+
// failure panel agree.
|
|
972
|
+
const failures = new Map<string, number>();
|
|
973
|
+
for (const client of this.errors(partition).byClient) {
|
|
974
|
+
failures.set(
|
|
975
|
+
client.actorId,
|
|
976
|
+
(failures.get(client.actorId) ?? 0) + client.count,
|
|
977
|
+
);
|
|
978
|
+
}
|
|
979
|
+
const byActor = new Map<string, AdminClient[]>();
|
|
980
|
+
for (const client of clients) {
|
|
981
|
+
byActor.set(client.actorId, [
|
|
982
|
+
...(byActor.get(client.actorId) ?? []),
|
|
983
|
+
client,
|
|
984
|
+
]);
|
|
985
|
+
}
|
|
986
|
+
return [...byActor].map(([actorId, devices]) => ({
|
|
987
|
+
actorId,
|
|
988
|
+
clients: devices,
|
|
989
|
+
...(this.#realtime !== undefined
|
|
990
|
+
? { connected: devices.filter((device) => device.connected).length }
|
|
991
|
+
: {}),
|
|
992
|
+
lastSeenMs: Math.max(...devices.map((device) => device.updatedAtMs)),
|
|
993
|
+
maxLag: Math.max(...devices.map((device) => device.lag)),
|
|
994
|
+
recentFailures: failures.get(actorId) ?? 0,
|
|
995
|
+
}));
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
/** Row counts per schema table for the partition. */
|
|
999
|
+
async tableCounts(partition: string): Promise<Record<string, number>> {
|
|
1000
|
+
const count = required(
|
|
1001
|
+
this.#storage.countRows?.bind(this.#storage),
|
|
1002
|
+
'storage',
|
|
1003
|
+
);
|
|
1004
|
+
const schema = this.#requireSchema();
|
|
1005
|
+
await this.#storage.ensureSchema(schema, this.#checkpoints);
|
|
1006
|
+
const counts: Record<string, number> = {};
|
|
1007
|
+
for (const table of schema.tables.keys()) {
|
|
1008
|
+
counts[table] = await count(partition, table);
|
|
1009
|
+
}
|
|
1010
|
+
return counts;
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
/**
|
|
1014
|
+
* One page of a table's rows decoded per §11, ordered by rowId. With a
|
|
1015
|
+
* `scope`, only rows whose stored scope matches, read through the scope
|
|
1016
|
+
* index.
|
|
1017
|
+
*/
|
|
1018
|
+
async listRows(
|
|
1019
|
+
partition: string,
|
|
1020
|
+
table: string,
|
|
1021
|
+
options: {
|
|
1022
|
+
readonly afterRowId?: string;
|
|
1023
|
+
readonly limit?: number;
|
|
1024
|
+
readonly scope?: { readonly variable: string; readonly value: string };
|
|
1025
|
+
} = {},
|
|
1026
|
+
): Promise<AdminRowPage> {
|
|
1027
|
+
const schema = this.#requireSchema();
|
|
1028
|
+
const compiled = schema.tables.get(table);
|
|
1029
|
+
if (compiled === undefined) {
|
|
1030
|
+
throw new SyncError(
|
|
1031
|
+
'sync.unknown_table',
|
|
1032
|
+
'table is not in the server schema',
|
|
1033
|
+
JSON.stringify({ table }),
|
|
1034
|
+
);
|
|
1035
|
+
}
|
|
1036
|
+
await this.#storage.ensureSchema(schema, this.#checkpoints);
|
|
1037
|
+
const limit = options.limit ?? DEFAULT_ROW_PAGE_LIMIT;
|
|
1038
|
+
const afterRowId = options.afterRowId ?? null;
|
|
1039
|
+
let rows: StoredRow[];
|
|
1040
|
+
if (options.scope !== undefined) {
|
|
1041
|
+
if (!compiled.declaredVariables.has(options.scope.variable)) {
|
|
1042
|
+
throw new SyncError(
|
|
1043
|
+
'sync.invalid_request',
|
|
1044
|
+
'the table declares no such scope variable',
|
|
1045
|
+
JSON.stringify({ table, variable: options.scope.variable }),
|
|
1046
|
+
);
|
|
1047
|
+
}
|
|
1048
|
+
rows = await this.#storage.scanRows(partition, {
|
|
1049
|
+
table,
|
|
1050
|
+
scopeFilter: { [options.scope.variable]: [options.scope.value] },
|
|
1051
|
+
afterRowId,
|
|
1052
|
+
limit,
|
|
1053
|
+
});
|
|
1054
|
+
} else {
|
|
1055
|
+
const list = required(
|
|
1056
|
+
this.#storage.listRows?.bind(this.#storage),
|
|
1057
|
+
'storage',
|
|
1058
|
+
);
|
|
1059
|
+
rows = await list(partition, { table, afterRowId, limit });
|
|
1060
|
+
}
|
|
1061
|
+
const last = rows.at(-1);
|
|
1062
|
+
return {
|
|
1063
|
+
table,
|
|
1064
|
+
rows: rows.map((row) => ({
|
|
1065
|
+
rowId: row.rowId,
|
|
1066
|
+
serverVersion: row.serverVersion,
|
|
1067
|
+
scopes: row.scopes,
|
|
1068
|
+
values: renderRowValues(
|
|
1069
|
+
compiled.columns,
|
|
1070
|
+
decodeRow(compiled.columns, row.payload),
|
|
1071
|
+
),
|
|
1072
|
+
})),
|
|
1073
|
+
...(rows.length === limit && last !== undefined
|
|
1074
|
+
? { nextAfterRowId: last.rowId }
|
|
1075
|
+
: {}),
|
|
1076
|
+
};
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
/**
|
|
1080
|
+
* What happened to a write: every stored idempotency outcome for the
|
|
1081
|
+
* clientCommitId (applied with its commitSeq, or rejected with the
|
|
1082
|
+
* per-operation results), plus the retained ring events that carry it.
|
|
1083
|
+
*/
|
|
1084
|
+
async findWrite(
|
|
1085
|
+
partition: string,
|
|
1086
|
+
clientCommitId: string,
|
|
1087
|
+
): Promise<AdminWriteLookup> {
|
|
1088
|
+
const list = required(
|
|
1089
|
+
this.#storage.listPushResults?.bind(this.#storage),
|
|
1090
|
+
'storage',
|
|
1091
|
+
);
|
|
1092
|
+
const outcomes = (await list(partition, clientCommitId)).map(
|
|
1093
|
+
({ clientId, result }): AdminWriteOutcome => ({
|
|
1094
|
+
clientId,
|
|
1095
|
+
status: result.status,
|
|
1096
|
+
...(result.commitSeq !== undefined
|
|
1097
|
+
? { commitSeq: result.commitSeq }
|
|
1098
|
+
: {}),
|
|
1099
|
+
...(result.recordedAtMs !== undefined
|
|
1100
|
+
? { recordedAtMs: result.recordedAtMs }
|
|
1101
|
+
: {}),
|
|
1102
|
+
results: result.results.map((entry) =>
|
|
1103
|
+
entry.status === 'conflict'
|
|
1104
|
+
? {
|
|
1105
|
+
...entry,
|
|
1106
|
+
serverRow: bytesToBase64(entry.serverRow),
|
|
1107
|
+
conflictColumns: bytesToBase64(entry.conflictColumns),
|
|
1108
|
+
}
|
|
1109
|
+
: entry,
|
|
1110
|
+
),
|
|
1111
|
+
}),
|
|
1112
|
+
);
|
|
1113
|
+
const events = this.events({ partition }).filter(
|
|
1114
|
+
(event) =>
|
|
1115
|
+
'clientCommitId' in event && event.clientCommitId === clientCommitId,
|
|
1116
|
+
);
|
|
1117
|
+
return { clientCommitId, outcomes, events };
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
/**
|
|
1121
|
+
* Resolve an id pasted from a bug report: clients and actors containing
|
|
1122
|
+
* it, rows with that rowId in any table, the commit with that seq, and
|
|
1123
|
+
* writes with that clientCommitId.
|
|
1124
|
+
*/
|
|
1125
|
+
async search(partition: string, query: string): Promise<AdminSearchResult> {
|
|
1126
|
+
const needle = query.trim();
|
|
1127
|
+
if (needle.length === 0) {
|
|
1128
|
+
throw new SyncError('sync.invalid_request', 'search query is empty');
|
|
1129
|
+
}
|
|
1130
|
+
const { clients } = await this.listClients(partition, {
|
|
1131
|
+
limit: Number.MAX_SAFE_INTEGER,
|
|
1132
|
+
});
|
|
1133
|
+
const schema = this.#requireSchema();
|
|
1134
|
+
await this.#storage.ensureSchema(schema, this.#checkpoints);
|
|
1135
|
+
const rows: { table: string; rowId: string }[] = [];
|
|
1136
|
+
for (const table of schema.tables.keys()) {
|
|
1137
|
+
if (
|
|
1138
|
+
(await this.#storage.getRow(partition, table, needle)) !== undefined
|
|
1139
|
+
) {
|
|
1140
|
+
rows.push({ table, rowId: needle });
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
const read = required(
|
|
1144
|
+
this.#storage.getCommit?.bind(this.#storage),
|
|
1145
|
+
'storage',
|
|
1146
|
+
);
|
|
1147
|
+
const commit = /^\d+$/.test(needle)
|
|
1148
|
+
? await read(partition, Number.parseInt(needle, 10))
|
|
1149
|
+
: undefined;
|
|
1150
|
+
return {
|
|
1151
|
+
query: needle,
|
|
1152
|
+
clients: clients.filter(
|
|
1153
|
+
(client) =>
|
|
1154
|
+
client.clientId.includes(needle) || client.actorId === needle,
|
|
1155
|
+
),
|
|
1156
|
+
actors: [
|
|
1157
|
+
...new Set(
|
|
1158
|
+
clients
|
|
1159
|
+
.map((client) => client.actorId)
|
|
1160
|
+
.filter((actorId) => actorId.includes(needle)),
|
|
1161
|
+
),
|
|
1162
|
+
],
|
|
1163
|
+
rows,
|
|
1164
|
+
...(commit !== undefined
|
|
1165
|
+
? {
|
|
1166
|
+
commit: {
|
|
1167
|
+
commitSeq: commit.commitSeq,
|
|
1168
|
+
clientId: commit.clientId,
|
|
1169
|
+
clientCommitId: commit.clientCommitId,
|
|
1170
|
+
actorId: commit.actorId,
|
|
1171
|
+
createdAtMs: commit.createdAtMs,
|
|
1172
|
+
changeCount: commit.changeCount,
|
|
1173
|
+
tables: commit.tables,
|
|
1174
|
+
},
|
|
1175
|
+
}
|
|
1176
|
+
: {}),
|
|
1177
|
+
writes: (await this.findWrite(partition, needle)).outcomes,
|
|
379
1178
|
};
|
|
380
1179
|
}
|
|
381
1180
|
|
|
@@ -442,14 +1241,9 @@ export class SyncularAdmin {
|
|
|
442
1241
|
};
|
|
443
1242
|
}
|
|
444
1243
|
|
|
445
|
-
/**
|
|
446
|
-
|
|
447
|
-
return this.#
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
/** Blob store counters for a partition (undefined when unsupported/unset). */
|
|
451
|
-
async blobStats(partition: string): Promise<BlobStoreStats | undefined> {
|
|
452
|
-
return this.#blobs?.stats?.(partition);
|
|
1244
|
+
/** True when a realtime hub is wired: clients report `connected`. */
|
|
1245
|
+
get hasRealtime(): boolean {
|
|
1246
|
+
return this.#realtime !== undefined;
|
|
453
1247
|
}
|
|
454
1248
|
|
|
455
1249
|
/** True when this admin has an event ring wired (the event tail is live). */
|
|
@@ -476,57 +1270,200 @@ export class SyncularAdmin {
|
|
|
476
1270
|
/**
|
|
477
1271
|
* Request/push health over a trailing window, derived entirely from the
|
|
478
1272
|
* ring (zero new server state): rates, error share, duration percentiles,
|
|
479
|
-
* and per-bucket
|
|
480
|
-
*
|
|
1273
|
+
* and per-bucket series. Only events carrying this `partition` count.
|
|
1274
|
+
* `coverage` reports when the ring dropped events inside the window.
|
|
481
1275
|
*/
|
|
482
1276
|
metrics(
|
|
483
1277
|
partition: string,
|
|
484
1278
|
options: { readonly windowMs?: number; readonly buckets?: number } = {},
|
|
485
1279
|
): AdminMetrics {
|
|
486
|
-
const windowMs = options.windowMs ??
|
|
1280
|
+
const windowMs = options.windowMs ?? DEFAULT_WINDOW_MS;
|
|
487
1281
|
const bucketCount = options.buckets ?? DEFAULT_METRICS_BUCKETS;
|
|
488
1282
|
const atMs = this.#clock();
|
|
489
1283
|
const sinceMs = atMs - windowMs;
|
|
490
1284
|
const widthMs = windowMs / bucketCount;
|
|
491
1285
|
const counts = new Array<number>(bucketCount).fill(0);
|
|
492
1286
|
const errors = new Array<number>(bucketCount).fill(0);
|
|
1287
|
+
const bucketDurations = Array.from(
|
|
1288
|
+
{ length: bucketCount },
|
|
1289
|
+
(): number[] => [],
|
|
1290
|
+
);
|
|
1291
|
+
const bucketPushes = Array.from({ length: bucketCount }, () => ({
|
|
1292
|
+
applied: 0,
|
|
1293
|
+
rejected: 0,
|
|
1294
|
+
conflicted: 0,
|
|
1295
|
+
}));
|
|
493
1296
|
const durations: number[] = [];
|
|
494
|
-
let requestCount = 0;
|
|
495
1297
|
let errorCount = 0;
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
let conflicted = 0;
|
|
499
|
-
for (const event of this.events({ sinceMs })) {
|
|
500
|
-
if (event.partition !== partition) continue;
|
|
501
|
-
if (event.type === 'push.applied') applied += 1;
|
|
502
|
-
else if (event.type === 'push.rejected') rejected += 1;
|
|
503
|
-
else if (event.type === 'push.conflicted') conflicted += 1;
|
|
504
|
-
if (event.type !== 'request.handled') continue;
|
|
505
|
-
requestCount += 1;
|
|
506
|
-
durations.push(event.durationMs);
|
|
507
|
-
const failed = event.outcome === 'rejected' || event.outcome === 'error';
|
|
508
|
-
if (failed) errorCount += 1;
|
|
1298
|
+
const pushes = { applied: 0, rejected: 0, conflicted: 0 };
|
|
1299
|
+
for (const event of this.events({ partition, sinceMs })) {
|
|
509
1300
|
const bucket = Math.min(
|
|
510
1301
|
bucketCount - 1,
|
|
511
1302
|
Math.max(0, Math.floor((event.atMs - sinceMs) / widthMs)),
|
|
512
1303
|
);
|
|
1304
|
+
const outcome =
|
|
1305
|
+
event.type === 'push.applied'
|
|
1306
|
+
? 'applied'
|
|
1307
|
+
: event.type === 'push.rejected'
|
|
1308
|
+
? 'rejected'
|
|
1309
|
+
: event.type === 'push.conflicted'
|
|
1310
|
+
? 'conflicted'
|
|
1311
|
+
: undefined;
|
|
1312
|
+
if (outcome !== undefined) {
|
|
1313
|
+
pushes[outcome] += 1;
|
|
1314
|
+
const slot = bucketPushes[bucket];
|
|
1315
|
+
if (slot !== undefined) slot[outcome] += 1;
|
|
1316
|
+
}
|
|
1317
|
+
if (event.type !== 'request.handled') continue;
|
|
1318
|
+
durations.push(event.durationMs);
|
|
1319
|
+
bucketDurations[bucket]?.push(event.durationMs);
|
|
513
1320
|
counts[bucket] = (counts[bucket] ?? 0) + 1;
|
|
514
|
-
if (
|
|
1321
|
+
if (event.outcome === 'rejected' || event.outcome === 'error') {
|
|
1322
|
+
errorCount += 1;
|
|
1323
|
+
errors[bucket] = (errors[bucket] ?? 0) + 1;
|
|
1324
|
+
}
|
|
515
1325
|
}
|
|
516
1326
|
return {
|
|
517
1327
|
partition,
|
|
518
1328
|
windowMs,
|
|
519
1329
|
atMs,
|
|
1330
|
+
coverage: this.#coverage(sinceMs),
|
|
520
1331
|
requests: {
|
|
521
|
-
count:
|
|
522
|
-
perMinute:
|
|
1332
|
+
count: durations.length,
|
|
1333
|
+
perMinute: durations.length / (windowMs / 60_000),
|
|
523
1334
|
errorCount,
|
|
524
|
-
errorRate:
|
|
1335
|
+
errorRate: durations.length === 0 ? 0 : errorCount / durations.length,
|
|
525
1336
|
p50Ms: percentile(durations, 0.5),
|
|
526
1337
|
p95Ms: percentile(durations, 0.95),
|
|
527
1338
|
},
|
|
528
|
-
pushes
|
|
529
|
-
buckets: {
|
|
1339
|
+
pushes,
|
|
1340
|
+
buckets: {
|
|
1341
|
+
widthMs,
|
|
1342
|
+
counts,
|
|
1343
|
+
errors,
|
|
1344
|
+
p95Ms: bucketDurations.map((sample) =>
|
|
1345
|
+
sample.length === 0 ? null : percentile(sample, 0.95),
|
|
1346
|
+
),
|
|
1347
|
+
pushes: bucketPushes,
|
|
1348
|
+
},
|
|
1349
|
+
};
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
/**
|
|
1353
|
+
* Failures over a trailing window, grouped by source and code and by
|
|
1354
|
+
* client: failed requests, push rejections and conflicts, dead-lettered
|
|
1355
|
+
* reactions, and throwing scope resolvers. Derived from the ring.
|
|
1356
|
+
*/
|
|
1357
|
+
errors(
|
|
1358
|
+
partition: string,
|
|
1359
|
+
options: { readonly windowMs?: number } = {},
|
|
1360
|
+
): AdminErrorSummary {
|
|
1361
|
+
const windowMs = options.windowMs ?? DEFAULT_WINDOW_MS;
|
|
1362
|
+
const atMs = this.#clock();
|
|
1363
|
+
const sinceMs = atMs - windowMs;
|
|
1364
|
+
const groups = new Map<
|
|
1365
|
+
string,
|
|
1366
|
+
{
|
|
1367
|
+
readonly source: AdminErrorGroup['source'];
|
|
1368
|
+
readonly code: string;
|
|
1369
|
+
count: number;
|
|
1370
|
+
readonly clients: Set<string>;
|
|
1371
|
+
readonly actors: Set<string>;
|
|
1372
|
+
readonly rows: Map<string, { table: string; rowId: string }>;
|
|
1373
|
+
readonly lastAtMs: number;
|
|
1374
|
+
}
|
|
1375
|
+
>();
|
|
1376
|
+
const byClient = new Map<string, AdminClientErrors>();
|
|
1377
|
+
// Newest first, so the first event per group or client is its latest.
|
|
1378
|
+
for (const event of this.events({ partition, sinceMs })) {
|
|
1379
|
+
let source: AdminErrorGroup['source'];
|
|
1380
|
+
let code: string;
|
|
1381
|
+
let clientId: string | undefined;
|
|
1382
|
+
let actorId: string | undefined;
|
|
1383
|
+
if (event.type === 'request.handled') {
|
|
1384
|
+
if (event.outcome !== 'rejected' && event.outcome !== 'error') continue;
|
|
1385
|
+
source = 'request';
|
|
1386
|
+
code = event.errorCode ?? 'sync.internal_error';
|
|
1387
|
+
clientId = event.clientId;
|
|
1388
|
+
actorId = event.actorId;
|
|
1389
|
+
} else if (event.type === 'push.rejected') {
|
|
1390
|
+
source = 'push';
|
|
1391
|
+
code = event.code;
|
|
1392
|
+
clientId = event.clientId;
|
|
1393
|
+
actorId = event.actorId;
|
|
1394
|
+
} else if (event.type === 'push.conflicted') {
|
|
1395
|
+
source = 'push';
|
|
1396
|
+
code = 'sync.version_conflict';
|
|
1397
|
+
clientId = event.clientId;
|
|
1398
|
+
actorId = event.actorId;
|
|
1399
|
+
} else if (event.type === 'reaction.dead_lettered') {
|
|
1400
|
+
source = 'reaction';
|
|
1401
|
+
code = event.errorCode;
|
|
1402
|
+
} else if (event.type === 'scopes.resolve_failed') {
|
|
1403
|
+
source = 'scopes';
|
|
1404
|
+
code = 'scopes.resolve_failed';
|
|
1405
|
+
actorId = event.actorId;
|
|
1406
|
+
} else {
|
|
1407
|
+
continue;
|
|
1408
|
+
}
|
|
1409
|
+
const key = `${source} ${code}`;
|
|
1410
|
+
const group = groups.get(key) ?? {
|
|
1411
|
+
source,
|
|
1412
|
+
code,
|
|
1413
|
+
count: 0,
|
|
1414
|
+
clients: new Set<string>(),
|
|
1415
|
+
actors: new Set<string>(),
|
|
1416
|
+
rows: new Map<string, { table: string; rowId: string }>(),
|
|
1417
|
+
lastAtMs: event.atMs,
|
|
1418
|
+
};
|
|
1419
|
+
group.count += 1;
|
|
1420
|
+
if (
|
|
1421
|
+
(event.type === 'push.rejected' || event.type === 'push.conflicted') &&
|
|
1422
|
+
event.table !== undefined &&
|
|
1423
|
+
event.rowId !== undefined
|
|
1424
|
+
) {
|
|
1425
|
+
group.rows.set(`${event.table} ${event.rowId}`, {
|
|
1426
|
+
table: event.table,
|
|
1427
|
+
rowId: event.rowId,
|
|
1428
|
+
});
|
|
1429
|
+
}
|
|
1430
|
+
if (clientId !== undefined) group.clients.add(clientId);
|
|
1431
|
+
if (actorId !== undefined) group.actors.add(actorId);
|
|
1432
|
+
groups.set(key, group);
|
|
1433
|
+
if (clientId !== undefined && actorId !== undefined) {
|
|
1434
|
+
const seen = byClient.get(clientId);
|
|
1435
|
+
byClient.set(
|
|
1436
|
+
clientId,
|
|
1437
|
+
seen !== undefined
|
|
1438
|
+
? { ...seen, count: seen.count + 1 }
|
|
1439
|
+
: {
|
|
1440
|
+
clientId,
|
|
1441
|
+
actorId,
|
|
1442
|
+
count: 1,
|
|
1443
|
+
lastCode: code,
|
|
1444
|
+
lastAtMs: event.atMs,
|
|
1445
|
+
},
|
|
1446
|
+
);
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1449
|
+
return {
|
|
1450
|
+
partition,
|
|
1451
|
+
windowMs,
|
|
1452
|
+
atMs,
|
|
1453
|
+
coverage: this.#coverage(sinceMs),
|
|
1454
|
+
byCode: [...groups.values()]
|
|
1455
|
+
.sort((a, b) => b.count - a.count)
|
|
1456
|
+
.map((group) => ({
|
|
1457
|
+
source: group.source,
|
|
1458
|
+
code: group.code,
|
|
1459
|
+
count: group.count,
|
|
1460
|
+
clients: group.clients.size,
|
|
1461
|
+
actors: group.actors.size,
|
|
1462
|
+
lastAtMs: group.lastAtMs,
|
|
1463
|
+
sampleClientIds: [...group.clients].slice(0, 5),
|
|
1464
|
+
sampleRows: [...group.rows.values()].slice(0, 5),
|
|
1465
|
+
})),
|
|
1466
|
+
byClient: [...byClient.values()].sort((a, b) => b.count - a.count),
|
|
530
1467
|
};
|
|
531
1468
|
}
|
|
532
1469
|
|
|
@@ -539,14 +1476,23 @@ export class SyncularAdmin {
|
|
|
539
1476
|
|
|
540
1477
|
/**
|
|
541
1478
|
* The fleet view: one row per known partition — retained backlog, client
|
|
542
|
-
* counts, prune recommendation
|
|
543
|
-
*
|
|
1479
|
+
* counts, prune recommendation — paged by `offset`/`limit` (default 100)
|
|
1480
|
+
* because each row costs several reads. A cross-partition read (every
|
|
1481
|
+
* other method is partition-scoped); host authorization should account
|
|
1482
|
+
* for it.
|
|
544
1483
|
*/
|
|
545
|
-
async partitionsOverview(
|
|
1484
|
+
async partitionsOverview(
|
|
1485
|
+
options: { readonly limit?: number; readonly offset?: number } = {},
|
|
1486
|
+
): Promise<AdminPartitionPage> {
|
|
546
1487
|
const partitions = await this.listPartitions();
|
|
1488
|
+
const offset = options.offset ?? 0;
|
|
1489
|
+
const page = partitions.slice(
|
|
1490
|
+
offset,
|
|
1491
|
+
offset + (options.limit ?? DEFAULT_PARTITION_LIMIT),
|
|
1492
|
+
);
|
|
547
1493
|
const activeFloorMs = this.#clock() - this.#retention.activeWindowMs;
|
|
548
1494
|
const out: AdminPartitionOverview[] = [];
|
|
549
|
-
for (const partition of
|
|
1495
|
+
for (const partition of page) {
|
|
550
1496
|
const status = await this.horizonStatus(partition);
|
|
551
1497
|
const cursors = await this.#storage.listClientCursors(partition);
|
|
552
1498
|
out.push({
|
|
@@ -560,6 +1506,155 @@ export class SyncularAdmin {
|
|
|
560
1506
|
recommendation: status.recommendation,
|
|
561
1507
|
});
|
|
562
1508
|
}
|
|
563
|
-
return out;
|
|
1509
|
+
return { partitions: out, total: partitions.length };
|
|
1510
|
+
}
|
|
1511
|
+
|
|
1512
|
+
/**
|
|
1513
|
+
* Operator action: requeue a dead-lettered reaction. Throws
|
|
1514
|
+
* `sync.invalid_request` when the key names no dead-lettered reaction.
|
|
1515
|
+
*/
|
|
1516
|
+
async retryReaction(
|
|
1517
|
+
partition: string,
|
|
1518
|
+
idempotencyKey: string,
|
|
1519
|
+
): Promise<StoredReaction> {
|
|
1520
|
+
const retry = required(
|
|
1521
|
+
this.#storage.retryReaction?.bind(this.#storage),
|
|
1522
|
+
'storage',
|
|
1523
|
+
);
|
|
1524
|
+
const read = required(
|
|
1525
|
+
this.#storage.getReaction?.bind(this.#storage),
|
|
1526
|
+
'storage',
|
|
1527
|
+
);
|
|
1528
|
+
if (!(await retry(partition, idempotencyKey, this.#clock()))) {
|
|
1529
|
+
throw new SyncError(
|
|
1530
|
+
'sync.invalid_request',
|
|
1531
|
+
'no dead-lettered reaction with this key',
|
|
1532
|
+
JSON.stringify({ idempotencyKey }),
|
|
1533
|
+
);
|
|
1534
|
+
}
|
|
1535
|
+
const reaction = await read(partition, idempotencyKey);
|
|
1536
|
+
if (reaction === undefined) {
|
|
1537
|
+
throw new Error('SyncularAdmin: a retried reaction disappeared');
|
|
1538
|
+
}
|
|
1539
|
+
return reaction;
|
|
1540
|
+
}
|
|
1541
|
+
|
|
1542
|
+
/** Operator action: one §4.6 prune pass now, emitting `prune.completed`. */
|
|
1543
|
+
async prune(partition: string): Promise<AdminPruneResult> {
|
|
1544
|
+
const previousHorizonSeq = await this.#storage.getHorizonSeq(partition);
|
|
1545
|
+
const horizonSeq = await pruneCommitLog({
|
|
1546
|
+
storage: this.#storage,
|
|
1547
|
+
partition,
|
|
1548
|
+
nowMs: this.#clock(),
|
|
1549
|
+
retention: this.#retention,
|
|
1550
|
+
...(this.#events !== undefined ? { events: this.#events } : {}),
|
|
1551
|
+
});
|
|
1552
|
+
return { previousHorizonSeq, horizonSeq };
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
/**
|
|
1556
|
+
* Operator action: revoke the client's §7.3 lease (§7.3.4), emitting
|
|
1557
|
+
* `lease.revoked`. Throws `sync.invalid_request` when the client holds no
|
|
1558
|
+
* lease.
|
|
1559
|
+
*/
|
|
1560
|
+
async revokeLease(partition: string, clientId: string): Promise<LeaseRecord> {
|
|
1561
|
+
const leases = required(this.#leases, 'lease store');
|
|
1562
|
+
const lease = await leases.get(partition, clientId);
|
|
1563
|
+
if (lease === undefined) {
|
|
1564
|
+
throw new SyncError(
|
|
1565
|
+
'sync.invalid_request',
|
|
1566
|
+
'the client holds no lease',
|
|
1567
|
+
JSON.stringify({ clientId }),
|
|
1568
|
+
);
|
|
1569
|
+
}
|
|
1570
|
+
await leases.revoke(partition, lease.leaseId);
|
|
1571
|
+
if (this.#events !== undefined) {
|
|
1572
|
+
emitEvent(this.#events, {
|
|
1573
|
+
type: 'lease.revoked',
|
|
1574
|
+
atMs: this.#clock(),
|
|
1575
|
+
partition,
|
|
1576
|
+
leaseId: lease.leaseId,
|
|
1577
|
+
});
|
|
1578
|
+
}
|
|
1579
|
+
return { ...lease, revoked: true };
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
#requireSchema(): CompiledSchema {
|
|
1583
|
+
if (this.#schema === undefined) {
|
|
1584
|
+
throw new Error('SyncularAdmin: row reads require the server schema');
|
|
1585
|
+
}
|
|
1586
|
+
return this.#schema;
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
/**
|
|
1590
|
+
* Map client records to `AdminClient` with one read of the head, the
|
|
1591
|
+
* active floor, and the hub's open sockets for the partition.
|
|
1592
|
+
*/
|
|
1593
|
+
async #clientView(
|
|
1594
|
+
partition: string,
|
|
1595
|
+
): Promise<(record: ClientRecord) => AdminClient> {
|
|
1596
|
+
const maxCommitSeq = await this.#storage.getMaxCommitSeq(partition);
|
|
1597
|
+
const activeFloorMs = this.#clock() - this.#retention.activeWindowMs;
|
|
1598
|
+
const sockets = this.#realtime?.connections(partition);
|
|
1599
|
+
const connectedSince = new Map<string, number>();
|
|
1600
|
+
for (const socket of sockets ?? []) {
|
|
1601
|
+
const since = connectedSince.get(socket.clientId);
|
|
1602
|
+
if (since === undefined || socket.openedAtMs < since) {
|
|
1603
|
+
connectedSince.set(socket.clientId, socket.openedAtMs);
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
return (record) => {
|
|
1607
|
+
let schemaVersion: number | undefined;
|
|
1608
|
+
for (const sub of record.subscriptions) {
|
|
1609
|
+
if (
|
|
1610
|
+
sub.schemaVersion !== undefined &&
|
|
1611
|
+
(schemaVersion === undefined || sub.schemaVersion > schemaVersion)
|
|
1612
|
+
) {
|
|
1613
|
+
schemaVersion = sub.schemaVersion;
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
const connectedSinceMs = connectedSince.get(record.clientId);
|
|
1617
|
+
return {
|
|
1618
|
+
clientId: record.clientId,
|
|
1619
|
+
actorId: record.actorId,
|
|
1620
|
+
wireVersion: record.wireVersion,
|
|
1621
|
+
...(schemaVersion !== undefined ? { schemaVersion } : {}),
|
|
1622
|
+
cursor: record.cursor,
|
|
1623
|
+
// A never-pulled cursor (-1) has seen nothing: it lags the whole log.
|
|
1624
|
+
lag: Math.max(0, maxCommitSeq - Math.max(0, record.cursor)),
|
|
1625
|
+
updatedAtMs: record.updatedAtMs,
|
|
1626
|
+
subscriptions: record.subscriptions.map((sub) => ({
|
|
1627
|
+
id: sub.id,
|
|
1628
|
+
table: sub.table,
|
|
1629
|
+
scopes: sub.scopes,
|
|
1630
|
+
})),
|
|
1631
|
+
active: record.updatedAtMs >= activeFloorMs,
|
|
1632
|
+
...(sockets !== undefined
|
|
1633
|
+
? { connected: connectedSinceMs !== undefined }
|
|
1634
|
+
: {}),
|
|
1635
|
+
...(connectedSinceMs !== undefined ? { connectedSinceMs } : {}),
|
|
1636
|
+
};
|
|
1637
|
+
};
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
/** The actor's allowed scopes from the host resolver, if one is wired. */
|
|
1641
|
+
async #allowedScopes(
|
|
1642
|
+
partition: string,
|
|
1643
|
+
record: ClientRecord,
|
|
1644
|
+
): Promise<ScopeMap | 'outage' | undefined> {
|
|
1645
|
+
if (this.#resolveScopes === undefined) return undefined;
|
|
1646
|
+
const resolved = await this.#resolveScopes({
|
|
1647
|
+
partition,
|
|
1648
|
+
actorId: record.actorId,
|
|
1649
|
+
clientId: record.clientId,
|
|
1650
|
+
});
|
|
1651
|
+
return resolved === RESOLVER_OUTAGE ? 'outage' : resolved;
|
|
1652
|
+
}
|
|
1653
|
+
|
|
1654
|
+
#coverage(sinceMs: number): AdminEventCoverage {
|
|
1655
|
+
const evicted = this.#ring?.evictedThroughMs;
|
|
1656
|
+
return evicted === undefined || evicted < sinceMs
|
|
1657
|
+
? { complete: true, fromMs: sinceMs }
|
|
1658
|
+
: { complete: false, fromMs: evicted };
|
|
564
1659
|
}
|
|
565
1660
|
}
|