@syncular/server 0.31.0 → 0.32.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/admin.ts CHANGED
@@ -1,31 +1,61 @@
1
1
  /**
2
- * `SyncularAdmin` is the operator-facing read surface over the server core.
3
- * It is a read-only, partition-scoped, JSON-able query layer over
4
- * `ServerStorage`, the optional segment/blob store stats, and an in-memory
5
- * event ring. It delivers the 80% operator value (who's connected, what's
6
- * flowing, horizon health, the event tail) as a handful of queries in the
7
- * server package — no separate UI package, no framework, and no wire-protocol
8
- * surface. This host surface is mirrored in the server README.
2
+ * `SyncularAdmin` is the operator-facing surface over the server core: a
3
+ * partition-scoped, JSON-able query layer over `ServerStorage`, the optional
4
+ * segment/blob store stats, and an in-memory event ring, plus three explicit
5
+ * operator actions (retry a dead-lettered reaction, prune the commit log,
6
+ * revoke a client's lease). No separate UI package, no framework, and no
7
+ * wire-protocol surface. This host surface is mirrored in the server README.
9
8
  *
10
- * Nothing here is on the sync hot path. Every method is a plain read; the
11
- * additive optional storage/store methods it depends on are documented as
12
- * such and this module fails loud (a thrown `Error`) when a backend lacks
13
- * one, so a host wiring an unsupported store learns immediately rather than
14
- * getting a silently-empty console.
9
+ * Nothing here is on the sync hot path. The additive optional storage/store
10
+ * methods it depends on are documented as such and this module fails loud (a
11
+ * thrown `Error`) when a backend lacks one, so a host wiring an unsupported
12
+ * store learns immediately rather than getting a silently-empty console.
15
13
  */
14
+ import {
15
+ bytesToBase64,
16
+ type ColumnType,
17
+ decodeRow,
18
+ type JsonValue,
19
+ type PushOperationResult,
20
+ renderRowValues,
21
+ type ScopeMap,
22
+ } from '@syncular/core';
16
23
  import type { BlobStore, BlobStoreStats } from './blob-store';
17
- import { clockOf, type SyncServerConfig } from './context';
18
- import type { SyncularServerEvent } from './events';
24
+ import {
25
+ clockOf,
26
+ RESOLVER_OUTAGE,
27
+ type ResolveScopes,
28
+ type SyncServerConfig,
29
+ } from './context';
30
+ import { SyncError } from './errors';
31
+ import {
32
+ emitEvent,
33
+ type SyncularServerEvent,
34
+ type SyncularServerEvents,
35
+ } from './events';
19
36
  import type { RingBufferEvents, RingEventQuery } from './events-ring';
20
37
  import type { LeaseRecord, LeaseStore } from './lease-store';
21
- import { DEFAULT_RETENTION, type RetentionPolicy } from './prune';
22
- import { compileSchema, type ServerSchema } from './schema';
38
+ import { RealtimeHub } from './realtime';
39
+ import {
40
+ DEFAULT_RETENTION,
41
+ pruneCommitLog,
42
+ type RetentionPolicy,
43
+ } from './prune';
44
+ import {
45
+ type CompiledSchema,
46
+ compileSchema,
47
+ type ServerSchema,
48
+ } from './schema';
49
+ import { computeEffective, matchesEffective } from './scopes';
23
50
  import type { SegmentStore, SegmentStoreStats } from './segment-store';
24
51
  import type {
25
52
  CheckpointDeclaration,
26
53
  ClientRecord,
27
54
  CommitMetadata,
28
55
  ReactionStatus,
56
+ StoredPushResult,
57
+ StoredRow,
58
+ RowChangeRecord,
29
59
  ScopeCommitActivity,
30
60
  ServerStorage,
31
61
  StoredReaction,
@@ -35,6 +65,10 @@ import type {
35
65
  export interface AdminClient {
36
66
  readonly clientId: string;
37
67
  readonly actorId: string;
68
+ /** SSP2 wire version last accepted from this client. */
69
+ readonly wireVersion: number;
70
+ /** Highest codec schema version among its subscriptions, when recorded. */
71
+ readonly schemaVersion?: number;
38
72
  readonly cursor: number;
39
73
  /**
40
74
  * Commits the client has not pulled yet: `maxCommitSeq − max(cursor, 0)`.
@@ -49,6 +83,26 @@ export interface AdminClient {
49
83
  }[];
50
84
  /** True when the cursor record was touched within the active window. */
51
85
  readonly active: boolean;
86
+ /**
87
+ * True while the client holds an open realtime socket on this process.
88
+ * Absent when the admin has no realtime hub.
89
+ */
90
+ readonly connected?: boolean;
91
+ /** When the client's oldest open socket opened. */
92
+ readonly connectedSinceMs?: number;
93
+ }
94
+
95
+ export interface AdminListClientsOptions {
96
+ /** Default 200. */
97
+ readonly limit?: number;
98
+ /** `seen` (default): most recently seen first. `lag`: largest lag first. */
99
+ readonly sort?: 'seen' | 'lag';
100
+ }
101
+
102
+ export interface AdminClientPage {
103
+ readonly clients: readonly AdminClient[];
104
+ /** Known clients in the partition before `limit`. */
105
+ readonly total: number;
52
106
  }
53
107
 
54
108
  /** One client's drill-down: record + lease + its slice of the event tail. */
@@ -63,12 +117,29 @@ export interface AdminClientDetail {
63
117
  readonly events: readonly SyncularServerEvent[];
64
118
  }
65
119
 
120
+ /**
121
+ * How much of a requested window the ring still holds. `complete` is false
122
+ * when the ring dropped events inside the window; the aggregates then cover
123
+ * only `fromMs` onward.
124
+ */
125
+ export interface AdminEventCoverage {
126
+ readonly complete: boolean;
127
+ readonly fromMs: number;
128
+ }
129
+
130
+ export interface AdminPushCounts {
131
+ readonly applied: number;
132
+ readonly rejected: number;
133
+ readonly conflicted: number;
134
+ }
135
+
66
136
  /** Ring-derived request/push aggregates over a trailing window. */
67
137
  export interface AdminMetrics {
68
138
  readonly partition: string;
69
139
  readonly windowMs: number;
70
140
  /** Wall-clock the aggregation ran at (window end). */
71
141
  readonly atMs: number;
142
+ readonly coverage: AdminEventCoverage;
72
143
  readonly requests: {
73
144
  readonly count: number;
74
145
  readonly perMinute: number;
@@ -78,22 +149,62 @@ export interface AdminMetrics {
78
149
  readonly p50Ms: number;
79
150
  readonly p95Ms: number;
80
151
  };
81
- readonly pushes: {
82
- readonly applied: number;
83
- readonly rejected: number;
84
- readonly conflicted: number;
85
- };
152
+ readonly pushes: AdminPushCounts;
86
153
  /**
87
- * Request counts split into `counts.length` equal buckets, oldest first —
88
- * the console's sparkline. `errors` marks the error share per bucket.
154
+ * The window split into `counts.length` equal buckets, oldest first: the
155
+ * request count, failed requests, p95 duration (`null` for an empty
156
+ * bucket), and push outcomes per bucket.
89
157
  */
90
158
  readonly buckets: {
91
159
  readonly widthMs: number;
92
160
  readonly counts: readonly number[];
93
161
  readonly errors: readonly number[];
162
+ readonly p95Ms: readonly (number | null)[];
163
+ readonly pushes: readonly AdminPushCounts[];
94
164
  };
95
165
  }
96
166
 
167
+ /** Failures sharing one source and code within the window. */
168
+ export interface AdminErrorGroup {
169
+ readonly source: 'request' | 'push' | 'reaction' | 'scopes';
170
+ /** The §10.2 code; `scopes.resolve_failed` for a throwing resolver. */
171
+ readonly code: string;
172
+ readonly count: number;
173
+ /** Distinct clients among the events that carry a clientId. */
174
+ readonly clients: number;
175
+ /** Distinct actors among the events that carry an actorId. */
176
+ readonly actors: number;
177
+ readonly lastAtMs: number;
178
+ /** Up to five client ids, most recent first. */
179
+ readonly sampleClientIds: readonly string[];
180
+ /** Up to five rows the failing pushes targeted, most recent first. */
181
+ readonly sampleRows: readonly {
182
+ readonly table: string;
183
+ readonly rowId: string;
184
+ }[];
185
+ }
186
+
187
+ /** One client's push failures within the window. */
188
+ export interface AdminClientErrors {
189
+ readonly clientId: string;
190
+ readonly actorId: string;
191
+ readonly count: number;
192
+ readonly lastCode: string;
193
+ readonly lastAtMs: number;
194
+ }
195
+
196
+ /** "What is failing": ring failures grouped by code and by client. */
197
+ export interface AdminErrorSummary {
198
+ readonly partition: string;
199
+ readonly windowMs: number;
200
+ readonly atMs: number;
201
+ readonly coverage: AdminEventCoverage;
202
+ /** Largest count first. */
203
+ readonly byCode: readonly AdminErrorGroup[];
204
+ /** Every client with a request or push failure, largest count first. */
205
+ readonly byClient: readonly AdminClientErrors[];
206
+ }
207
+
97
208
  /** One partition's row in the fleet view (`listPartitions` + horizon math). */
98
209
  export interface AdminPartitionOverview {
99
210
  readonly partition: string;
@@ -106,14 +217,192 @@ export interface AdminPartitionOverview {
106
217
  readonly recommendation: 'up-to-date' | 'prune-recommended';
107
218
  }
108
219
 
220
+ export interface AdminPartitionPage {
221
+ readonly partitions: readonly AdminPartitionOverview[];
222
+ /** Every registered partition before `offset`/`limit`. */
223
+ readonly total: number;
224
+ }
225
+
109
226
  export interface AdminRowInspection {
110
227
  readonly table: string;
111
228
  readonly rowId: string;
112
229
  readonly exists: boolean;
113
230
  readonly serverVersion?: number;
114
231
  readonly scopes?: Record<string, string>;
115
- /** blobIds this row currently references (§5.9.4), when the store tracks them. */
116
- readonly referencedBlobIds?: readonly string[];
232
+ /** Present iff `exists`: the row decoded and rendered per §11. */
233
+ readonly values?: Record<string, JsonValue>;
234
+ /** The retained changes to this row, newest first. */
235
+ readonly history: readonly RowChangeRecord[];
236
+ }
237
+
238
+ /** One change in a commit drill-down. */
239
+ export interface AdminCommitChange {
240
+ readonly table: string;
241
+ readonly rowId: string;
242
+ readonly op: 'upsert' | 'delete';
243
+ readonly rowVersion?: number;
244
+ readonly scopes: Record<string, string>;
245
+ /** The upserted row rendered per §11; absent for deletes. */
246
+ readonly values?: Record<string, JsonValue>;
247
+ /**
248
+ * True when the stored payload does not decode against the current
249
+ * schema (written under an older column set).
250
+ */
251
+ readonly undecodable?: boolean;
252
+ }
253
+
254
+ export interface AdminCommitDetail extends CommitMetadata {
255
+ readonly changes: readonly AdminCommitChange[];
256
+ }
257
+
258
+ /**
259
+ * The trace verdict, evaluated in this order:
260
+ * `client-unknown`: no client record for the id.
261
+ * `row-missing`: no row and no retained change to it.
262
+ * `no-subscription`: the client subscribes to nothing on the table.
263
+ * `row-deleted`: the row's newest retained change is a delete the client
264
+ * has pulled.
265
+ * `scope-mismatch`: no subscription requests the row's stored scopes.
266
+ * `scope-denied`: a subscription requests them, but the actor's allowed
267
+ * scopes (the host `resolveScopes`) exclude the row.
268
+ * `pending-pull`: the client's cursor is below the row's newest change.
269
+ * `delivered`: the client has pulled the row's newest change.
270
+ */
271
+ export type AdminTraceVerdict =
272
+ | 'client-unknown'
273
+ | 'row-missing'
274
+ | 'no-subscription'
275
+ | 'row-deleted'
276
+ | 'scope-mismatch'
277
+ | 'scope-denied'
278
+ | 'pending-pull'
279
+ | 'delivered';
280
+
281
+ /** One of the client's subscriptions on the traced table. */
282
+ export interface AdminTraceSubscription {
283
+ readonly id: string;
284
+ readonly scopes: Record<string, readonly string[]>;
285
+ /** The row's stored scopes satisfy the requested scopes. */
286
+ readonly requestedMatch: boolean;
287
+ /** Each requested variable the row's stored value fails. */
288
+ readonly mismatches: readonly {
289
+ readonly variable: string;
290
+ readonly rowValue: string | null;
291
+ readonly requested: readonly string[];
292
+ }[];
293
+ /**
294
+ * The row also satisfies requested ∩ allowed (§3.2). Absent when the
295
+ * admin has no `resolveScopes` or the resolver reported an outage.
296
+ */
297
+ readonly effectiveMatch?: boolean;
298
+ }
299
+
300
+ /** "Why does client X (not) see row Y": one read across row and client. */
301
+ export interface AdminRowTrace {
302
+ readonly verdict: AdminTraceVerdict;
303
+ readonly table: string;
304
+ readonly rowId: string;
305
+ readonly clientId: string;
306
+ readonly row: AdminRowInspection;
307
+ /** The row's newest retained change. */
308
+ readonly lastChange?: RowChangeRecord;
309
+ /** The partition horizon: changes at or below it are pruned. */
310
+ readonly horizonSeq: number;
311
+ readonly client?: AdminClient;
312
+ /**
313
+ * The actor's allowed scopes from the host `resolveScopes`, or `outage`
314
+ * when it returned the §7.3.3 outage sentinel. Absent without a resolver.
315
+ */
316
+ readonly allowed?: ScopeMap | 'outage';
317
+ readonly subscriptions: readonly AdminTraceSubscription[];
318
+ /** The client's retained push rejections and conflicts, newest first. */
319
+ readonly rejections: readonly SyncularServerEvent[];
320
+ }
321
+
322
+ /** One client's verdict in a row's delivery matrix. */
323
+ export interface AdminRowDeliveryClient {
324
+ readonly client: AdminClient;
325
+ readonly verdict: AdminTraceVerdict;
326
+ }
327
+
328
+ /** Who has one row and who does not: every known client's verdict. */
329
+ export interface AdminRowDelivery {
330
+ readonly table: string;
331
+ readonly rowId: string;
332
+ readonly row: AdminRowInspection;
333
+ readonly lastChange?: RowChangeRecord;
334
+ readonly horizonSeq: number;
335
+ /** Every known client, sorted by verdict then most recently seen. */
336
+ readonly clients: readonly AdminRowDeliveryClient[];
337
+ }
338
+
339
+ /** One actor and its devices (clients). */
340
+ export interface AdminActor {
341
+ readonly actorId: string;
342
+ readonly clients: readonly AdminClient[];
343
+ /** Devices with an open socket; absent without a realtime hub. */
344
+ readonly connected?: number;
345
+ readonly lastSeenMs: number;
346
+ readonly maxLag: number;
347
+ /** The actor's failures in the last 5 minutes, as `errors().byClient` counts them. */
348
+ readonly recentFailures: number;
349
+ }
350
+
351
+ /** A page of decoded rows from one table. */
352
+ export interface AdminRowPage {
353
+ readonly table: string;
354
+ readonly rows: readonly {
355
+ readonly rowId: string;
356
+ readonly serverVersion: number;
357
+ readonly scopes: Record<string, string>;
358
+ readonly values: Record<string, JsonValue>;
359
+ }[];
360
+ /** Pass as `afterRowId` for the next page; absent on the last page. */
361
+ readonly nextAfterRowId?: string;
362
+ }
363
+
364
+ /**
365
+ * One per-operation result of a stored write, JSON-safe: a conflict's
366
+ * `serverRow` and `conflictColumns` bytes are standard base64 (§11).
367
+ */
368
+ export type AdminWriteResult =
369
+ | Exclude<PushOperationResult, { status: 'conflict' }>
370
+ | (Omit<
371
+ Extract<PushOperationResult, { status: 'conflict' }>,
372
+ 'serverRow' | 'conflictColumns'
373
+ > & { readonly serverRow: string; readonly conflictColumns: string });
374
+
375
+ /** One stored outcome of a client write (§2.3 idempotency record). */
376
+ export interface AdminWriteOutcome {
377
+ readonly clientId: string;
378
+ readonly status: StoredPushResult['status'];
379
+ readonly commitSeq?: number;
380
+ readonly recordedAtMs?: number;
381
+ readonly results: readonly AdminWriteResult[];
382
+ }
383
+
384
+ /** What happened to the write a client logged as `clientCommitId`. */
385
+ export interface AdminWriteLookup {
386
+ readonly clientCommitId: string;
387
+ /** One outcome per client that sent this clientCommitId. */
388
+ readonly outcomes: readonly AdminWriteOutcome[];
389
+ /** Retained ring events carrying this clientCommitId, newest first. */
390
+ readonly events: readonly SyncularServerEvent[];
391
+ }
392
+
393
+ /** Everything an id pasted into the console search matches. */
394
+ export interface AdminSearchResult {
395
+ readonly query: string;
396
+ /** Clients whose id contains the query, or whose actor equals it. */
397
+ readonly clients: readonly AdminClient[];
398
+ /** Actors whose id contains the query. */
399
+ readonly actors: readonly string[];
400
+ /** Rows whose id equals the query, one per table that holds it. */
401
+ readonly rows: readonly { readonly table: string; readonly rowId: string }[];
402
+ /** The commit whose seq equals the query. */
403
+ readonly commit?: CommitMetadata;
404
+ /** Outcomes of writes whose clientCommitId equals the query. */
405
+ readonly writes: readonly AdminWriteOutcome[];
117
406
  }
118
407
 
119
408
  export interface AdminHorizonStatus {
@@ -129,10 +418,30 @@ export interface AdminHorizonStatus {
129
418
  readonly recommendation: 'up-to-date' | 'prune-recommended';
130
419
  }
131
420
 
421
+ export interface AdminPruneResult {
422
+ readonly previousHorizonSeq: number;
423
+ readonly horizonSeq: number;
424
+ }
425
+
426
+ /** One schema table as the console's pickers need it. */
427
+ export interface AdminTableInfo {
428
+ readonly name: string;
429
+ readonly columns: readonly {
430
+ readonly name: string;
431
+ readonly type: ColumnType;
432
+ readonly nullable: boolean;
433
+ readonly primaryKey: boolean;
434
+ }[];
435
+ }
436
+
132
437
  export interface AdminListCommitsOptions {
133
438
  readonly afterSeq?: number;
439
+ /** Exclusive upper bound: pass the oldest seq of a page for the next. */
440
+ readonly beforeSeq?: number;
134
441
  readonly limit?: number;
135
442
  readonly table?: string;
443
+ /** Attach each commit's first N changes as `sample`. */
444
+ readonly changeSample?: number;
136
445
  }
137
446
 
138
447
  export interface AdminScopeActivityOptions {
@@ -153,30 +462,40 @@ export interface AdminStats {
153
462
  export interface SyncularAdminOptions {
154
463
  readonly storage: ServerStorage;
155
464
  /**
156
- * The server schema. When present, row reads (`inspectRow`) ensure the
157
- * relational row tables exist first — needed when the admin runs against
158
- * a storage instance that has not served a sync request yet.
465
+ * The server schema. Row reads (`inspectRow`, `traceRow`, `commitDetail`)
466
+ * require it: it decodes payloads and ensures the relational row tables
467
+ * exist on a storage instance that has not served a sync request yet.
159
468
  */
160
469
  readonly schema?: ServerSchema;
161
470
  /** RFC 0007: declarations `inspectRow` carries into `ensureSchema`. */
162
471
  readonly checkpoints?: readonly CheckpointDeclaration[];
163
472
  /** The event ring feeding the event tail. Absent ⇒ `events()` is empty. */
164
473
  readonly ring?: RingBufferEvents;
474
+ /** Receives `prune.completed` and `lease.revoked` from operator actions. */
475
+ readonly events?: SyncularServerEvents;
476
+ /** The host resolver: `traceRow` checks the actor's allowed scopes. */
477
+ readonly resolveScopes?: ResolveScopes;
478
+ /** The realtime hub: which clients hold an open socket right now. */
479
+ readonly realtime?: RealtimeHub;
165
480
  readonly segments?: SegmentStore;
166
481
  readonly blobs?: BlobStore;
167
- /** The §7.3 lease store — feeds the client drill-down's lease read. */
482
+ /** The §7.3 lease store — the client drill-down and `revokeLease`. */
168
483
  readonly leases?: LeaseStore;
169
- /** Retention policy for horizon recommendation (defaults to §4.6). */
484
+ /** Retention policy for horizon recommendation and prune (§4.6). */
170
485
  readonly retention?: Partial<RetentionPolicy>;
171
486
  /** Epoch-ms clock (defaults to `Date.now`) — active-window math. */
172
487
  readonly clock?: () => number;
173
488
  }
174
489
 
490
+ const DEFAULT_CLIENT_LIMIT = 200;
491
+ const DEFAULT_PARTITION_LIMIT = 100;
175
492
  const DEFAULT_COMMIT_LIMIT = 50;
176
493
  const DEFAULT_SCOPE_LIMIT = 50;
177
494
  const DEFAULT_REACTION_LIMIT = 100;
178
495
  const DEFAULT_CLIENT_EVENT_LIMIT = 100;
179
- const DEFAULT_METRICS_WINDOW_MS = 5 * 60 * 1000;
496
+ const DEFAULT_ROW_HISTORY_LIMIT = 20;
497
+ const DEFAULT_ROW_PAGE_LIMIT = 50;
498
+ const DEFAULT_WINDOW_MS = 5 * 60 * 1000;
180
499
  const DEFAULT_METRICS_BUCKETS = 30;
181
500
 
182
501
  /** Nearest-rank percentile over an unsorted sample; 0 for an empty one. */
@@ -196,33 +515,89 @@ function required<T>(value: T | undefined, what: string): T {
196
515
  return value;
197
516
  }
198
517
 
199
- function toAdminClient(
518
+ /** Verdicts in evaluation order; `rowDelivery` sorts by it. */
519
+ const VERDICT_ORDER: readonly AdminTraceVerdict[] = [
520
+ 'client-unknown',
521
+ 'row-missing',
522
+ 'no-subscription',
523
+ 'row-deleted',
524
+ 'scope-mismatch',
525
+ 'scope-denied',
526
+ 'pending-pull',
527
+ 'delivered',
528
+ ];
529
+
530
+ /** One known client's subscriptions on the row's table and its verdict. */
531
+ function traceVerdict(
200
532
  record: ClientRecord,
201
- active: boolean,
202
- maxCommitSeq: number,
203
- ): AdminClient {
204
- return {
205
- clientId: record.clientId,
206
- actorId: record.actorId,
207
- cursor: record.cursor,
208
- // A never-pulled cursor (-1) has seen nothing: it lags the whole log.
209
- lag: Math.max(0, maxCommitSeq - Math.max(0, record.cursor)),
210
- updatedAtMs: record.updatedAtMs,
211
- subscriptions: record.subscriptions.map((sub) => ({
212
- id: sub.id,
213
- table: sub.table,
214
- scopes: sub.scopes,
215
- })),
216
- active,
217
- };
533
+ row: AdminRowInspection,
534
+ lastChange: RowChangeRecord | undefined,
535
+ horizonSeq: number,
536
+ allowed: ScopeMap | 'outage' | undefined,
537
+ ): {
538
+ readonly subscriptions: AdminTraceSubscription[];
539
+ readonly verdict: AdminTraceVerdict;
540
+ } {
541
+ const rowScopes = row.scopes ?? {};
542
+ const subscriptions = record.subscriptions
543
+ .filter((sub) => sub.table === row.table)
544
+ .map((sub): AdminTraceSubscription => {
545
+ const mismatches = Object.entries(sub.scopes)
546
+ .filter(([variable, values]) => {
547
+ const value = rowScopes[variable];
548
+ return value === undefined || !values.includes(value);
549
+ })
550
+ .map(([variable, values]) => ({
551
+ variable,
552
+ rowValue: rowScopes[variable] ?? null,
553
+ requested: values,
554
+ }));
555
+ const traced = {
556
+ id: sub.id,
557
+ scopes: sub.scopes,
558
+ requestedMatch: mismatches.length === 0,
559
+ mismatches,
560
+ };
561
+ if (allowed === undefined || allowed === 'outage') return traced;
562
+ const effective = computeEffective(sub.scopes, { ok: true, allowed });
563
+ return {
564
+ ...traced,
565
+ effectiveMatch:
566
+ effective.status === 'active' &&
567
+ matchesEffective(rowScopes, effective.effective),
568
+ };
569
+ });
570
+ // With no retained change left, the newest one sits at or below the
571
+ // horizon, so a cursor at the horizon has pulled it.
572
+ const pulled = record.cursor >= (lastChange?.commitSeq ?? horizonSeq);
573
+ const verdict: AdminTraceVerdict =
574
+ !row.exists && lastChange === undefined
575
+ ? 'row-missing'
576
+ : subscriptions.length === 0
577
+ ? 'no-subscription'
578
+ : !row.exists
579
+ ? pulled
580
+ ? 'row-deleted'
581
+ : 'pending-pull'
582
+ : !subscriptions.some((sub) => sub.requestedMatch)
583
+ ? 'scope-mismatch'
584
+ : subscriptions.every((sub) => sub.effectiveMatch === false)
585
+ ? 'scope-denied'
586
+ : pulled
587
+ ? 'delivered'
588
+ : 'pending-pull';
589
+ return { subscriptions, verdict };
218
590
  }
219
591
 
220
- /** The read-only console query surface. Construct one per host process. */
592
+ /** The console query surface plus its operator actions. One per process. */
221
593
  export class SyncularAdmin {
222
594
  readonly #storage: ServerStorage;
223
- readonly #schema?: ServerSchema;
595
+ readonly #schema?: CompiledSchema;
224
596
  readonly #checkpoints?: readonly CheckpointDeclaration[];
225
597
  readonly #ring?: RingBufferEvents;
598
+ readonly #events?: SyncularServerEvents;
599
+ readonly #resolveScopes?: ResolveScopes;
600
+ readonly #realtime?: RealtimeHub;
226
601
  readonly #segments?: SegmentStore;
227
602
  readonly #blobs?: BlobStore;
228
603
  readonly #leases?: LeaseStore;
@@ -231,10 +606,17 @@ export class SyncularAdmin {
231
606
 
232
607
  constructor(options: SyncularAdminOptions) {
233
608
  this.#storage = options.storage;
234
- if (options.schema !== undefined) this.#schema = options.schema;
609
+ if (options.schema !== undefined) {
610
+ this.#schema = compileSchema(options.schema);
611
+ }
235
612
  if (options.checkpoints !== undefined)
236
613
  this.#checkpoints = options.checkpoints;
237
614
  if (options.ring !== undefined) this.#ring = options.ring;
615
+ if (options.events !== undefined) this.#events = options.events;
616
+ if (options.resolveScopes !== undefined) {
617
+ this.#resolveScopes = options.resolveScopes;
618
+ }
619
+ if (options.realtime !== undefined) this.#realtime = options.realtime;
238
620
  if (options.segments !== undefined) this.#segments = options.segments;
239
621
  if (options.blobs !== undefined) this.#blobs = options.blobs;
240
622
  if (options.leases !== undefined) this.#leases = options.leases;
@@ -243,9 +625,10 @@ export class SyncularAdmin {
243
625
  }
244
626
 
245
627
  /**
246
- * Build an admin over a `SyncServerConfig`, reusing its storage / segment
247
- * / blob store / clock. Pass the ring separately (it is an events sink,
248
- * composed into the config's `events` by the host — see `composeEvents`).
628
+ * Build an admin over a `SyncServerConfig`, reusing its storage, schema,
629
+ * resolver, events sink, stores, leases and clock. Pass the ring
630
+ * separately (it is an events sink, composed into the config's `events` by
631
+ * the host — see `composeEvents`).
249
632
  */
250
633
  static fromConfig(
251
634
  config: SyncServerConfig,
@@ -257,6 +640,11 @@ export class SyncularAdmin {
257
640
  ...(config.checkpoints !== undefined
258
641
  ? { checkpoints: config.checkpoints }
259
642
  : {}),
643
+ resolveScopes: config.resolveScopes,
644
+ ...(config.realtime instanceof RealtimeHub
645
+ ? { realtime: config.realtime }
646
+ : {}),
647
+ ...(config.events !== undefined ? { events: config.events } : {}),
260
648
  segments: config.segments,
261
649
  ...(config.blobs !== undefined ? { blobs: config.blobs } : {}),
262
650
  ...(config.leases !== undefined ? { leases: config.leases.store } : {}),
@@ -266,24 +654,48 @@ export class SyncularAdmin {
266
654
  });
267
655
  }
268
656
 
269
- /** The known clients for a partition (cursor, lag, subscriptions). */
270
- async listClients(partition: string): Promise<AdminClient[]> {
657
+ /** The server schema version; `undefined` without a schema. */
658
+ get schemaVersion(): number | undefined {
659
+ return this.#schema?.version;
660
+ }
661
+
662
+ /** The schema tables and columns, for row and commit pickers. */
663
+ tables(): AdminTableInfo[] {
664
+ return [...(this.#schema?.tables.values() ?? [])].map((table) => ({
665
+ name: table.name,
666
+ columns: table.columns.map((column, i) => ({
667
+ name: column.name,
668
+ type: column.type,
669
+ nullable: column.nullable,
670
+ primaryKey: i === table.primaryKeyIndex,
671
+ })),
672
+ }));
673
+ }
674
+
675
+ /** The known clients for a partition (cursor, lag, versions, subs). */
676
+ async listClients(
677
+ partition: string,
678
+ options: AdminListClientsOptions = {},
679
+ ): Promise<AdminClientPage> {
271
680
  const list = required(
272
681
  this.#storage.listClientRecords?.bind(this.#storage),
273
682
  'storage',
274
683
  );
275
- const records = await list(partition);
276
- const maxCommitSeq = await this.#storage.getMaxCommitSeq(partition);
277
- const activeFloorMs = this.#clock() - this.#retention.activeWindowMs;
278
- return records.map((record) =>
279
- toAdminClient(record, record.updatedAtMs >= activeFloorMs, maxCommitSeq),
684
+ const view = await this.#clientView(partition);
685
+ const clients = (await list(partition)).map(view);
686
+ clients.sort((a, b) =>
687
+ options.sort === 'lag' ? b.lag - a.lag : b.updatedAtMs - a.updatedAtMs,
280
688
  );
689
+ return {
690
+ clients: clients.slice(0, options.limit ?? DEFAULT_CLIENT_LIMIT),
691
+ total: clients.length,
692
+ };
281
693
  }
282
694
 
283
695
  /**
284
696
  * One client's drill-down: its record (with lag), its §7.3 lease when a
285
- * lease store is wired, and its recent slice of the event tail. Answers
286
- * "why is this client stale" in a single read.
697
+ * lease store is wired, its realtime state, and its recent slice of the
698
+ * event tail. Answers "why is this client stale" in a single read.
287
699
  */
288
700
  async clientDetail(
289
701
  partition: string,
@@ -292,23 +704,18 @@ export class SyncularAdmin {
292
704
  ): Promise<AdminClientDetail> {
293
705
  const record = await this.#storage.getClientRecord(partition, clientId);
294
706
  const events = this.events({
707
+ partition,
295
708
  clientId,
296
709
  limit: options.eventLimit ?? DEFAULT_CLIENT_EVENT_LIMIT,
297
710
  });
298
711
  if (record === undefined) {
299
712
  return { clientId, exists: false, events };
300
713
  }
301
- const maxCommitSeq = await this.#storage.getMaxCommitSeq(partition);
302
- const activeFloorMs = this.#clock() - this.#retention.activeWindowMs;
303
714
  const lease = await this.#leases?.get(partition, clientId);
304
715
  return {
305
716
  clientId,
306
717
  exists: true,
307
- client: toAdminClient(
308
- record,
309
- record.updatedAtMs >= activeFloorMs,
310
- maxCommitSeq,
311
- ),
718
+ client: (await this.#clientView(partition))(record),
312
719
  ...(lease !== undefined ? { lease } : {}),
313
720
  events,
314
721
  };
@@ -325,11 +732,68 @@ export class SyncularAdmin {
325
732
  );
326
733
  return read(partition, {
327
734
  afterSeq: options.afterSeq ?? 0,
735
+ ...(options.beforeSeq !== undefined
736
+ ? { beforeSeq: options.beforeSeq }
737
+ : {}),
328
738
  limit: options.limit ?? DEFAULT_COMMIT_LIMIT,
329
739
  ...(options.table !== undefined ? { table: options.table } : {}),
740
+ ...(options.changeSample !== undefined
741
+ ? { changeSample: options.changeSample }
742
+ : {}),
330
743
  });
331
744
  }
332
745
 
746
+ /**
747
+ * One retained commit with every change rendered per §11. Throws
748
+ * `sync.not_found` when the commit does not exist or was pruned.
749
+ */
750
+ async commitDetail(
751
+ partition: string,
752
+ commitSeq: number,
753
+ ): Promise<AdminCommitDetail> {
754
+ const read = required(
755
+ this.#storage.getCommit?.bind(this.#storage),
756
+ 'storage',
757
+ );
758
+ const schema = this.#requireSchema();
759
+ const commit = await read(partition, commitSeq);
760
+ if (commit === undefined) {
761
+ throw new SyncError(
762
+ 'sync.not_found',
763
+ 'commit is not in the retained log',
764
+ JSON.stringify({ commitSeq }),
765
+ );
766
+ }
767
+ return {
768
+ ...commit,
769
+ changes: commit.changes.map((change): AdminCommitChange => {
770
+ const base = {
771
+ table: change.table,
772
+ rowId: change.rowId,
773
+ op: change.op,
774
+ ...(change.rowVersion !== undefined
775
+ ? { rowVersion: change.rowVersion }
776
+ : {}),
777
+ scopes: change.scopes,
778
+ };
779
+ const table = schema.tables.get(change.table);
780
+ if (change.payload === undefined) return base;
781
+ if (table === undefined) return { ...base, undecodable: true };
782
+ try {
783
+ return {
784
+ ...base,
785
+ values: renderRowValues(
786
+ table.columns,
787
+ decodeRow(table.columns, change.payload),
788
+ ),
789
+ };
790
+ } catch {
791
+ return { ...base, undecodable: true };
792
+ }
793
+ }),
794
+ };
795
+ }
796
+
333
797
  /** Pending, leased, completed, and dead-lettered durable reactions. */
334
798
  async listReactions(
335
799
  partition: string,
@@ -347,28 +811,38 @@ export class SyncularAdmin {
347
811
  }
348
812
 
349
813
  /**
350
- * Inspect a single row: current server_version, stored scopes, and the
351
- * blobIds it references (when the store tracks references). Payload bytes
352
- * are deliberately NOT decoded — the console shows metadata, not content.
814
+ * Inspect a single row: server_version, stored scopes, its values decoded
815
+ * per §11, and its retained change history (newest first). Throws
816
+ * `sync.unknown_table` for a table outside the schema.
353
817
  */
354
818
  async inspectRow(
355
819
  partition: string,
356
820
  table: string,
357
821
  rowId: string,
822
+ options: { readonly historyLimit?: number } = {},
358
823
  ): Promise<AdminRowInspection> {
359
- const read = required(
360
- this.#storage.getRowScopes?.bind(this.#storage),
361
- 'storage',
362
- );
363
- if (this.#schema !== undefined) {
364
- await this.#storage.ensureSchema(
365
- compileSchema(this.#schema),
366
- this.#checkpoints,
824
+ const schema = this.#requireSchema();
825
+ const compiled = schema.tables.get(table);
826
+ if (compiled === undefined) {
827
+ throw new SyncError(
828
+ 'sync.unknown_table',
829
+ 'table is not in the server schema',
830
+ JSON.stringify({ table }),
367
831
  );
368
832
  }
369
- const row = await read(partition, table, rowId);
833
+ await this.#storage.ensureSchema(schema, this.#checkpoints);
834
+ const listChanges = required(
835
+ this.#storage.listRowChanges?.bind(this.#storage),
836
+ 'storage',
837
+ );
838
+ const history = await listChanges(partition, {
839
+ table,
840
+ rowId,
841
+ limit: options.historyLimit ?? DEFAULT_ROW_HISTORY_LIMIT,
842
+ });
843
+ const row = await this.#storage.getRow(partition, table, rowId);
370
844
  if (row === undefined) {
371
- return { table, rowId, exists: false };
845
+ return { table, rowId, exists: false, history };
372
846
  }
373
847
  return {
374
848
  table,
@@ -376,6 +850,331 @@ export class SyncularAdmin {
376
850
  exists: true,
377
851
  serverVersion: row.serverVersion,
378
852
  scopes: row.scopes,
853
+ values: renderRowValues(
854
+ compiled.columns,
855
+ decodeRow(compiled.columns, row.payload),
856
+ ),
857
+ history,
858
+ };
859
+ }
860
+
861
+ /**
862
+ * Trace whether one client receives one row, combining the row's stored
863
+ * scopes and newest change with the client's subscriptions, cursor, the
864
+ * actor's allowed scopes, and its retained push failures. See
865
+ * `AdminTraceVerdict` for the verdict order.
866
+ */
867
+ async traceRow(
868
+ partition: string,
869
+ query: {
870
+ readonly table: string;
871
+ readonly rowId: string;
872
+ readonly clientId: string;
873
+ },
874
+ ): Promise<AdminRowTrace> {
875
+ const { table, rowId, clientId } = query;
876
+ const row = await this.inspectRow(partition, table, rowId, {
877
+ historyLimit: 1,
878
+ });
879
+ const lastChange = row.history[0];
880
+ const horizonSeq = await this.#storage.getHorizonSeq(partition);
881
+ const rejections = this.events({ partition, clientId }).filter(
882
+ (event) =>
883
+ event.type === 'push.rejected' || event.type === 'push.conflicted',
884
+ );
885
+ const record = await this.#storage.getClientRecord(partition, clientId);
886
+ const base = {
887
+ table,
888
+ rowId,
889
+ clientId,
890
+ row,
891
+ horizonSeq,
892
+ rejections,
893
+ ...(lastChange !== undefined ? { lastChange } : {}),
894
+ };
895
+ if (record === undefined) {
896
+ return { ...base, verdict: 'client-unknown', subscriptions: [] };
897
+ }
898
+ const allowed = await this.#allowedScopes(partition, record);
899
+ return {
900
+ ...base,
901
+ client: (await this.#clientView(partition))(record),
902
+ ...(allowed !== undefined ? { allowed } : {}),
903
+ ...traceVerdict(record, row, lastChange, horizonSeq, allowed),
904
+ };
905
+ }
906
+
907
+ /**
908
+ * Who has one row and who does not: the trace verdict for every known
909
+ * client in the partition, resolving each actor's allowed scopes once.
910
+ */
911
+ async rowDelivery(
912
+ partition: string,
913
+ table: string,
914
+ rowId: string,
915
+ ): Promise<AdminRowDelivery> {
916
+ const list = required(
917
+ this.#storage.listClientRecords?.bind(this.#storage),
918
+ 'storage',
919
+ );
920
+ const row = await this.inspectRow(partition, table, rowId, {
921
+ historyLimit: 1,
922
+ });
923
+ const lastChange = row.history[0];
924
+ const horizonSeq = await this.#storage.getHorizonSeq(partition);
925
+ const view = await this.#clientView(partition);
926
+ const allowedByActor = new Map<string, ScopeMap | 'outage' | undefined>();
927
+ const clients: AdminRowDeliveryClient[] = [];
928
+ for (const record of await list(partition)) {
929
+ if (!allowedByActor.has(record.actorId)) {
930
+ allowedByActor.set(
931
+ record.actorId,
932
+ await this.#allowedScopes(partition, record),
933
+ );
934
+ }
935
+ clients.push({
936
+ client: view(record),
937
+ verdict: traceVerdict(
938
+ record,
939
+ row,
940
+ lastChange,
941
+ horizonSeq,
942
+ allowedByActor.get(record.actorId),
943
+ ).verdict,
944
+ });
945
+ }
946
+ clients.sort(
947
+ (a, b) =>
948
+ VERDICT_ORDER.indexOf(a.verdict) - VERDICT_ORDER.indexOf(b.verdict) ||
949
+ b.client.updatedAtMs - a.client.updatedAtMs,
950
+ );
951
+ return {
952
+ table,
953
+ rowId,
954
+ row,
955
+ ...(lastChange !== undefined ? { lastChange } : {}),
956
+ horizonSeq,
957
+ clients,
958
+ };
959
+ }
960
+
961
+ /**
962
+ * The partition's actors with their devices (clients), open sockets,
963
+ * worst lag, and failures in the last 5 minutes; most recently seen
964
+ * first.
965
+ */
966
+ async actors(partition: string): Promise<AdminActor[]> {
967
+ const { clients } = await this.listClients(partition, {
968
+ limit: Number.MAX_SAFE_INTEGER,
969
+ });
970
+ // The same per-client tally as `errors()`, so People and the overview's
971
+ // failure panel agree.
972
+ const failures = new Map<string, number>();
973
+ for (const client of this.errors(partition).byClient) {
974
+ failures.set(
975
+ client.actorId,
976
+ (failures.get(client.actorId) ?? 0) + client.count,
977
+ );
978
+ }
979
+ const byActor = new Map<string, AdminClient[]>();
980
+ for (const client of clients) {
981
+ byActor.set(client.actorId, [
982
+ ...(byActor.get(client.actorId) ?? []),
983
+ client,
984
+ ]);
985
+ }
986
+ return [...byActor].map(([actorId, devices]) => ({
987
+ actorId,
988
+ clients: devices,
989
+ ...(this.#realtime !== undefined
990
+ ? { connected: devices.filter((device) => device.connected).length }
991
+ : {}),
992
+ lastSeenMs: Math.max(...devices.map((device) => device.updatedAtMs)),
993
+ maxLag: Math.max(...devices.map((device) => device.lag)),
994
+ recentFailures: failures.get(actorId) ?? 0,
995
+ }));
996
+ }
997
+
998
+ /** Row counts per schema table for the partition. */
999
+ async tableCounts(partition: string): Promise<Record<string, number>> {
1000
+ const count = required(
1001
+ this.#storage.countRows?.bind(this.#storage),
1002
+ 'storage',
1003
+ );
1004
+ const schema = this.#requireSchema();
1005
+ await this.#storage.ensureSchema(schema, this.#checkpoints);
1006
+ const counts: Record<string, number> = {};
1007
+ for (const table of schema.tables.keys()) {
1008
+ counts[table] = await count(partition, table);
1009
+ }
1010
+ return counts;
1011
+ }
1012
+
1013
+ /**
1014
+ * One page of a table's rows decoded per §11, ordered by rowId. With a
1015
+ * `scope`, only rows whose stored scope matches, read through the scope
1016
+ * index.
1017
+ */
1018
+ async listRows(
1019
+ partition: string,
1020
+ table: string,
1021
+ options: {
1022
+ readonly afterRowId?: string;
1023
+ readonly limit?: number;
1024
+ readonly scope?: { readonly variable: string; readonly value: string };
1025
+ } = {},
1026
+ ): Promise<AdminRowPage> {
1027
+ const schema = this.#requireSchema();
1028
+ const compiled = schema.tables.get(table);
1029
+ if (compiled === undefined) {
1030
+ throw new SyncError(
1031
+ 'sync.unknown_table',
1032
+ 'table is not in the server schema',
1033
+ JSON.stringify({ table }),
1034
+ );
1035
+ }
1036
+ await this.#storage.ensureSchema(schema, this.#checkpoints);
1037
+ const limit = options.limit ?? DEFAULT_ROW_PAGE_LIMIT;
1038
+ const afterRowId = options.afterRowId ?? null;
1039
+ let rows: StoredRow[];
1040
+ if (options.scope !== undefined) {
1041
+ if (!compiled.declaredVariables.has(options.scope.variable)) {
1042
+ throw new SyncError(
1043
+ 'sync.invalid_request',
1044
+ 'the table declares no such scope variable',
1045
+ JSON.stringify({ table, variable: options.scope.variable }),
1046
+ );
1047
+ }
1048
+ rows = await this.#storage.scanRows(partition, {
1049
+ table,
1050
+ scopeFilter: { [options.scope.variable]: [options.scope.value] },
1051
+ afterRowId,
1052
+ limit,
1053
+ });
1054
+ } else {
1055
+ const list = required(
1056
+ this.#storage.listRows?.bind(this.#storage),
1057
+ 'storage',
1058
+ );
1059
+ rows = await list(partition, { table, afterRowId, limit });
1060
+ }
1061
+ const last = rows.at(-1);
1062
+ return {
1063
+ table,
1064
+ rows: rows.map((row) => ({
1065
+ rowId: row.rowId,
1066
+ serverVersion: row.serverVersion,
1067
+ scopes: row.scopes,
1068
+ values: renderRowValues(
1069
+ compiled.columns,
1070
+ decodeRow(compiled.columns, row.payload),
1071
+ ),
1072
+ })),
1073
+ ...(rows.length === limit && last !== undefined
1074
+ ? { nextAfterRowId: last.rowId }
1075
+ : {}),
1076
+ };
1077
+ }
1078
+
1079
+ /**
1080
+ * What happened to a write: every stored idempotency outcome for the
1081
+ * clientCommitId (applied with its commitSeq, or rejected with the
1082
+ * per-operation results), plus the retained ring events that carry it.
1083
+ */
1084
+ async findWrite(
1085
+ partition: string,
1086
+ clientCommitId: string,
1087
+ ): Promise<AdminWriteLookup> {
1088
+ const list = required(
1089
+ this.#storage.listPushResults?.bind(this.#storage),
1090
+ 'storage',
1091
+ );
1092
+ const outcomes = (await list(partition, clientCommitId)).map(
1093
+ ({ clientId, result }): AdminWriteOutcome => ({
1094
+ clientId,
1095
+ status: result.status,
1096
+ ...(result.commitSeq !== undefined
1097
+ ? { commitSeq: result.commitSeq }
1098
+ : {}),
1099
+ ...(result.recordedAtMs !== undefined
1100
+ ? { recordedAtMs: result.recordedAtMs }
1101
+ : {}),
1102
+ results: result.results.map((entry) =>
1103
+ entry.status === 'conflict'
1104
+ ? {
1105
+ ...entry,
1106
+ serverRow: bytesToBase64(entry.serverRow),
1107
+ conflictColumns: bytesToBase64(entry.conflictColumns),
1108
+ }
1109
+ : entry,
1110
+ ),
1111
+ }),
1112
+ );
1113
+ const events = this.events({ partition }).filter(
1114
+ (event) =>
1115
+ 'clientCommitId' in event && event.clientCommitId === clientCommitId,
1116
+ );
1117
+ return { clientCommitId, outcomes, events };
1118
+ }
1119
+
1120
+ /**
1121
+ * Resolve an id pasted from a bug report: clients and actors containing
1122
+ * it, rows with that rowId in any table, the commit with that seq, and
1123
+ * writes with that clientCommitId.
1124
+ */
1125
+ async search(partition: string, query: string): Promise<AdminSearchResult> {
1126
+ const needle = query.trim();
1127
+ if (needle.length === 0) {
1128
+ throw new SyncError('sync.invalid_request', 'search query is empty');
1129
+ }
1130
+ const { clients } = await this.listClients(partition, {
1131
+ limit: Number.MAX_SAFE_INTEGER,
1132
+ });
1133
+ const schema = this.#requireSchema();
1134
+ await this.#storage.ensureSchema(schema, this.#checkpoints);
1135
+ const rows: { table: string; rowId: string }[] = [];
1136
+ for (const table of schema.tables.keys()) {
1137
+ if (
1138
+ (await this.#storage.getRow(partition, table, needle)) !== undefined
1139
+ ) {
1140
+ rows.push({ table, rowId: needle });
1141
+ }
1142
+ }
1143
+ const read = required(
1144
+ this.#storage.getCommit?.bind(this.#storage),
1145
+ 'storage',
1146
+ );
1147
+ const commit = /^\d+$/.test(needle)
1148
+ ? await read(partition, Number.parseInt(needle, 10))
1149
+ : undefined;
1150
+ return {
1151
+ query: needle,
1152
+ clients: clients.filter(
1153
+ (client) =>
1154
+ client.clientId.includes(needle) || client.actorId === needle,
1155
+ ),
1156
+ actors: [
1157
+ ...new Set(
1158
+ clients
1159
+ .map((client) => client.actorId)
1160
+ .filter((actorId) => actorId.includes(needle)),
1161
+ ),
1162
+ ],
1163
+ rows,
1164
+ ...(commit !== undefined
1165
+ ? {
1166
+ commit: {
1167
+ commitSeq: commit.commitSeq,
1168
+ clientId: commit.clientId,
1169
+ clientCommitId: commit.clientCommitId,
1170
+ actorId: commit.actorId,
1171
+ createdAtMs: commit.createdAtMs,
1172
+ changeCount: commit.changeCount,
1173
+ tables: commit.tables,
1174
+ },
1175
+ }
1176
+ : {}),
1177
+ writes: (await this.findWrite(partition, needle)).outcomes,
379
1178
  };
380
1179
  }
381
1180
 
@@ -442,14 +1241,9 @@ export class SyncularAdmin {
442
1241
  };
443
1242
  }
444
1243
 
445
- /** Segment store counters alone (undefined when unsupported/unset). */
446
- async segmentStats(): Promise<SegmentStoreStats | undefined> {
447
- return this.#segments?.stats?.();
448
- }
449
-
450
- /** Blob store counters for a partition (undefined when unsupported/unset). */
451
- async blobStats(partition: string): Promise<BlobStoreStats | undefined> {
452
- return this.#blobs?.stats?.(partition);
1244
+ /** True when a realtime hub is wired: clients report `connected`. */
1245
+ get hasRealtime(): boolean {
1246
+ return this.#realtime !== undefined;
453
1247
  }
454
1248
 
455
1249
  /** True when this admin has an event ring wired (the event tail is live). */
@@ -476,57 +1270,200 @@ export class SyncularAdmin {
476
1270
  /**
477
1271
  * Request/push health over a trailing window, derived entirely from the
478
1272
  * ring (zero new server state): rates, error share, duration percentiles,
479
- * and per-bucket counts for the console's sparkline. Only events carrying
480
- * this `partition` count. Empty (all zeros) when no ring is wired.
1273
+ * and per-bucket series. Only events carrying this `partition` count.
1274
+ * `coverage` reports when the ring dropped events inside the window.
481
1275
  */
482
1276
  metrics(
483
1277
  partition: string,
484
1278
  options: { readonly windowMs?: number; readonly buckets?: number } = {},
485
1279
  ): AdminMetrics {
486
- const windowMs = options.windowMs ?? DEFAULT_METRICS_WINDOW_MS;
1280
+ const windowMs = options.windowMs ?? DEFAULT_WINDOW_MS;
487
1281
  const bucketCount = options.buckets ?? DEFAULT_METRICS_BUCKETS;
488
1282
  const atMs = this.#clock();
489
1283
  const sinceMs = atMs - windowMs;
490
1284
  const widthMs = windowMs / bucketCount;
491
1285
  const counts = new Array<number>(bucketCount).fill(0);
492
1286
  const errors = new Array<number>(bucketCount).fill(0);
1287
+ const bucketDurations = Array.from(
1288
+ { length: bucketCount },
1289
+ (): number[] => [],
1290
+ );
1291
+ const bucketPushes = Array.from({ length: bucketCount }, () => ({
1292
+ applied: 0,
1293
+ rejected: 0,
1294
+ conflicted: 0,
1295
+ }));
493
1296
  const durations: number[] = [];
494
- let requestCount = 0;
495
1297
  let errorCount = 0;
496
- let applied = 0;
497
- let rejected = 0;
498
- let conflicted = 0;
499
- for (const event of this.events({ sinceMs })) {
500
- if (event.partition !== partition) continue;
501
- if (event.type === 'push.applied') applied += 1;
502
- else if (event.type === 'push.rejected') rejected += 1;
503
- else if (event.type === 'push.conflicted') conflicted += 1;
504
- if (event.type !== 'request.handled') continue;
505
- requestCount += 1;
506
- durations.push(event.durationMs);
507
- const failed = event.outcome === 'rejected' || event.outcome === 'error';
508
- if (failed) errorCount += 1;
1298
+ const pushes = { applied: 0, rejected: 0, conflicted: 0 };
1299
+ for (const event of this.events({ partition, sinceMs })) {
509
1300
  const bucket = Math.min(
510
1301
  bucketCount - 1,
511
1302
  Math.max(0, Math.floor((event.atMs - sinceMs) / widthMs)),
512
1303
  );
1304
+ const outcome =
1305
+ event.type === 'push.applied'
1306
+ ? 'applied'
1307
+ : event.type === 'push.rejected'
1308
+ ? 'rejected'
1309
+ : event.type === 'push.conflicted'
1310
+ ? 'conflicted'
1311
+ : undefined;
1312
+ if (outcome !== undefined) {
1313
+ pushes[outcome] += 1;
1314
+ const slot = bucketPushes[bucket];
1315
+ if (slot !== undefined) slot[outcome] += 1;
1316
+ }
1317
+ if (event.type !== 'request.handled') continue;
1318
+ durations.push(event.durationMs);
1319
+ bucketDurations[bucket]?.push(event.durationMs);
513
1320
  counts[bucket] = (counts[bucket] ?? 0) + 1;
514
- if (failed) errors[bucket] = (errors[bucket] ?? 0) + 1;
1321
+ if (event.outcome === 'rejected' || event.outcome === 'error') {
1322
+ errorCount += 1;
1323
+ errors[bucket] = (errors[bucket] ?? 0) + 1;
1324
+ }
515
1325
  }
516
1326
  return {
517
1327
  partition,
518
1328
  windowMs,
519
1329
  atMs,
1330
+ coverage: this.#coverage(sinceMs),
520
1331
  requests: {
521
- count: requestCount,
522
- perMinute: requestCount / (windowMs / 60_000),
1332
+ count: durations.length,
1333
+ perMinute: durations.length / (windowMs / 60_000),
523
1334
  errorCount,
524
- errorRate: requestCount === 0 ? 0 : errorCount / requestCount,
1335
+ errorRate: durations.length === 0 ? 0 : errorCount / durations.length,
525
1336
  p50Ms: percentile(durations, 0.5),
526
1337
  p95Ms: percentile(durations, 0.95),
527
1338
  },
528
- pushes: { applied, rejected, conflicted },
529
- buckets: { widthMs, counts, errors },
1339
+ pushes,
1340
+ buckets: {
1341
+ widthMs,
1342
+ counts,
1343
+ errors,
1344
+ p95Ms: bucketDurations.map((sample) =>
1345
+ sample.length === 0 ? null : percentile(sample, 0.95),
1346
+ ),
1347
+ pushes: bucketPushes,
1348
+ },
1349
+ };
1350
+ }
1351
+
1352
+ /**
1353
+ * Failures over a trailing window, grouped by source and code and by
1354
+ * client: failed requests, push rejections and conflicts, dead-lettered
1355
+ * reactions, and throwing scope resolvers. Derived from the ring.
1356
+ */
1357
+ errors(
1358
+ partition: string,
1359
+ options: { readonly windowMs?: number } = {},
1360
+ ): AdminErrorSummary {
1361
+ const windowMs = options.windowMs ?? DEFAULT_WINDOW_MS;
1362
+ const atMs = this.#clock();
1363
+ const sinceMs = atMs - windowMs;
1364
+ const groups = new Map<
1365
+ string,
1366
+ {
1367
+ readonly source: AdminErrorGroup['source'];
1368
+ readonly code: string;
1369
+ count: number;
1370
+ readonly clients: Set<string>;
1371
+ readonly actors: Set<string>;
1372
+ readonly rows: Map<string, { table: string; rowId: string }>;
1373
+ readonly lastAtMs: number;
1374
+ }
1375
+ >();
1376
+ const byClient = new Map<string, AdminClientErrors>();
1377
+ // Newest first, so the first event per group or client is its latest.
1378
+ for (const event of this.events({ partition, sinceMs })) {
1379
+ let source: AdminErrorGroup['source'];
1380
+ let code: string;
1381
+ let clientId: string | undefined;
1382
+ let actorId: string | undefined;
1383
+ if (event.type === 'request.handled') {
1384
+ if (event.outcome !== 'rejected' && event.outcome !== 'error') continue;
1385
+ source = 'request';
1386
+ code = event.errorCode ?? 'sync.internal_error';
1387
+ clientId = event.clientId;
1388
+ actorId = event.actorId;
1389
+ } else if (event.type === 'push.rejected') {
1390
+ source = 'push';
1391
+ code = event.code;
1392
+ clientId = event.clientId;
1393
+ actorId = event.actorId;
1394
+ } else if (event.type === 'push.conflicted') {
1395
+ source = 'push';
1396
+ code = 'sync.version_conflict';
1397
+ clientId = event.clientId;
1398
+ actorId = event.actorId;
1399
+ } else if (event.type === 'reaction.dead_lettered') {
1400
+ source = 'reaction';
1401
+ code = event.errorCode;
1402
+ } else if (event.type === 'scopes.resolve_failed') {
1403
+ source = 'scopes';
1404
+ code = 'scopes.resolve_failed';
1405
+ actorId = event.actorId;
1406
+ } else {
1407
+ continue;
1408
+ }
1409
+ const key = `${source} ${code}`;
1410
+ const group = groups.get(key) ?? {
1411
+ source,
1412
+ code,
1413
+ count: 0,
1414
+ clients: new Set<string>(),
1415
+ actors: new Set<string>(),
1416
+ rows: new Map<string, { table: string; rowId: string }>(),
1417
+ lastAtMs: event.atMs,
1418
+ };
1419
+ group.count += 1;
1420
+ if (
1421
+ (event.type === 'push.rejected' || event.type === 'push.conflicted') &&
1422
+ event.table !== undefined &&
1423
+ event.rowId !== undefined
1424
+ ) {
1425
+ group.rows.set(`${event.table} ${event.rowId}`, {
1426
+ table: event.table,
1427
+ rowId: event.rowId,
1428
+ });
1429
+ }
1430
+ if (clientId !== undefined) group.clients.add(clientId);
1431
+ if (actorId !== undefined) group.actors.add(actorId);
1432
+ groups.set(key, group);
1433
+ if (clientId !== undefined && actorId !== undefined) {
1434
+ const seen = byClient.get(clientId);
1435
+ byClient.set(
1436
+ clientId,
1437
+ seen !== undefined
1438
+ ? { ...seen, count: seen.count + 1 }
1439
+ : {
1440
+ clientId,
1441
+ actorId,
1442
+ count: 1,
1443
+ lastCode: code,
1444
+ lastAtMs: event.atMs,
1445
+ },
1446
+ );
1447
+ }
1448
+ }
1449
+ return {
1450
+ partition,
1451
+ windowMs,
1452
+ atMs,
1453
+ coverage: this.#coverage(sinceMs),
1454
+ byCode: [...groups.values()]
1455
+ .sort((a, b) => b.count - a.count)
1456
+ .map((group) => ({
1457
+ source: group.source,
1458
+ code: group.code,
1459
+ count: group.count,
1460
+ clients: group.clients.size,
1461
+ actors: group.actors.size,
1462
+ lastAtMs: group.lastAtMs,
1463
+ sampleClientIds: [...group.clients].slice(0, 5),
1464
+ sampleRows: [...group.rows.values()].slice(0, 5),
1465
+ })),
1466
+ byClient: [...byClient.values()].sort((a, b) => b.count - a.count),
530
1467
  };
531
1468
  }
532
1469
 
@@ -539,14 +1476,23 @@ export class SyncularAdmin {
539
1476
 
540
1477
  /**
541
1478
  * The fleet view: one row per known partition — retained backlog, client
542
- * counts, prune recommendation. A cross-partition read (every other
543
- * method is partition-scoped); host authorization should account for it.
1479
+ * counts, prune recommendation — paged by `offset`/`limit` (default 100)
1480
+ * because each row costs several reads. A cross-partition read (every
1481
+ * other method is partition-scoped); host authorization should account
1482
+ * for it.
544
1483
  */
545
- async partitionsOverview(): Promise<AdminPartitionOverview[]> {
1484
+ async partitionsOverview(
1485
+ options: { readonly limit?: number; readonly offset?: number } = {},
1486
+ ): Promise<AdminPartitionPage> {
546
1487
  const partitions = await this.listPartitions();
1488
+ const offset = options.offset ?? 0;
1489
+ const page = partitions.slice(
1490
+ offset,
1491
+ offset + (options.limit ?? DEFAULT_PARTITION_LIMIT),
1492
+ );
547
1493
  const activeFloorMs = this.#clock() - this.#retention.activeWindowMs;
548
1494
  const out: AdminPartitionOverview[] = [];
549
- for (const partition of partitions) {
1495
+ for (const partition of page) {
550
1496
  const status = await this.horizonStatus(partition);
551
1497
  const cursors = await this.#storage.listClientCursors(partition);
552
1498
  out.push({
@@ -560,6 +1506,155 @@ export class SyncularAdmin {
560
1506
  recommendation: status.recommendation,
561
1507
  });
562
1508
  }
563
- return out;
1509
+ return { partitions: out, total: partitions.length };
1510
+ }
1511
+
1512
+ /**
1513
+ * Operator action: requeue a dead-lettered reaction. Throws
1514
+ * `sync.invalid_request` when the key names no dead-lettered reaction.
1515
+ */
1516
+ async retryReaction(
1517
+ partition: string,
1518
+ idempotencyKey: string,
1519
+ ): Promise<StoredReaction> {
1520
+ const retry = required(
1521
+ this.#storage.retryReaction?.bind(this.#storage),
1522
+ 'storage',
1523
+ );
1524
+ const read = required(
1525
+ this.#storage.getReaction?.bind(this.#storage),
1526
+ 'storage',
1527
+ );
1528
+ if (!(await retry(partition, idempotencyKey, this.#clock()))) {
1529
+ throw new SyncError(
1530
+ 'sync.invalid_request',
1531
+ 'no dead-lettered reaction with this key',
1532
+ JSON.stringify({ idempotencyKey }),
1533
+ );
1534
+ }
1535
+ const reaction = await read(partition, idempotencyKey);
1536
+ if (reaction === undefined) {
1537
+ throw new Error('SyncularAdmin: a retried reaction disappeared');
1538
+ }
1539
+ return reaction;
1540
+ }
1541
+
1542
+ /** Operator action: one §4.6 prune pass now, emitting `prune.completed`. */
1543
+ async prune(partition: string): Promise<AdminPruneResult> {
1544
+ const previousHorizonSeq = await this.#storage.getHorizonSeq(partition);
1545
+ const horizonSeq = await pruneCommitLog({
1546
+ storage: this.#storage,
1547
+ partition,
1548
+ nowMs: this.#clock(),
1549
+ retention: this.#retention,
1550
+ ...(this.#events !== undefined ? { events: this.#events } : {}),
1551
+ });
1552
+ return { previousHorizonSeq, horizonSeq };
1553
+ }
1554
+
1555
+ /**
1556
+ * Operator action: revoke the client's §7.3 lease (§7.3.4), emitting
1557
+ * `lease.revoked`. Throws `sync.invalid_request` when the client holds no
1558
+ * lease.
1559
+ */
1560
+ async revokeLease(partition: string, clientId: string): Promise<LeaseRecord> {
1561
+ const leases = required(this.#leases, 'lease store');
1562
+ const lease = await leases.get(partition, clientId);
1563
+ if (lease === undefined) {
1564
+ throw new SyncError(
1565
+ 'sync.invalid_request',
1566
+ 'the client holds no lease',
1567
+ JSON.stringify({ clientId }),
1568
+ );
1569
+ }
1570
+ await leases.revoke(partition, lease.leaseId);
1571
+ if (this.#events !== undefined) {
1572
+ emitEvent(this.#events, {
1573
+ type: 'lease.revoked',
1574
+ atMs: this.#clock(),
1575
+ partition,
1576
+ leaseId: lease.leaseId,
1577
+ });
1578
+ }
1579
+ return { ...lease, revoked: true };
1580
+ }
1581
+
1582
+ #requireSchema(): CompiledSchema {
1583
+ if (this.#schema === undefined) {
1584
+ throw new Error('SyncularAdmin: row reads require the server schema');
1585
+ }
1586
+ return this.#schema;
1587
+ }
1588
+
1589
+ /**
1590
+ * Map client records to `AdminClient` with one read of the head, the
1591
+ * active floor, and the hub's open sockets for the partition.
1592
+ */
1593
+ async #clientView(
1594
+ partition: string,
1595
+ ): Promise<(record: ClientRecord) => AdminClient> {
1596
+ const maxCommitSeq = await this.#storage.getMaxCommitSeq(partition);
1597
+ const activeFloorMs = this.#clock() - this.#retention.activeWindowMs;
1598
+ const sockets = this.#realtime?.connections(partition);
1599
+ const connectedSince = new Map<string, number>();
1600
+ for (const socket of sockets ?? []) {
1601
+ const since = connectedSince.get(socket.clientId);
1602
+ if (since === undefined || socket.openedAtMs < since) {
1603
+ connectedSince.set(socket.clientId, socket.openedAtMs);
1604
+ }
1605
+ }
1606
+ return (record) => {
1607
+ let schemaVersion: number | undefined;
1608
+ for (const sub of record.subscriptions) {
1609
+ if (
1610
+ sub.schemaVersion !== undefined &&
1611
+ (schemaVersion === undefined || sub.schemaVersion > schemaVersion)
1612
+ ) {
1613
+ schemaVersion = sub.schemaVersion;
1614
+ }
1615
+ }
1616
+ const connectedSinceMs = connectedSince.get(record.clientId);
1617
+ return {
1618
+ clientId: record.clientId,
1619
+ actorId: record.actorId,
1620
+ wireVersion: record.wireVersion,
1621
+ ...(schemaVersion !== undefined ? { schemaVersion } : {}),
1622
+ cursor: record.cursor,
1623
+ // A never-pulled cursor (-1) has seen nothing: it lags the whole log.
1624
+ lag: Math.max(0, maxCommitSeq - Math.max(0, record.cursor)),
1625
+ updatedAtMs: record.updatedAtMs,
1626
+ subscriptions: record.subscriptions.map((sub) => ({
1627
+ id: sub.id,
1628
+ table: sub.table,
1629
+ scopes: sub.scopes,
1630
+ })),
1631
+ active: record.updatedAtMs >= activeFloorMs,
1632
+ ...(sockets !== undefined
1633
+ ? { connected: connectedSinceMs !== undefined }
1634
+ : {}),
1635
+ ...(connectedSinceMs !== undefined ? { connectedSinceMs } : {}),
1636
+ };
1637
+ };
1638
+ }
1639
+
1640
+ /** The actor's allowed scopes from the host resolver, if one is wired. */
1641
+ async #allowedScopes(
1642
+ partition: string,
1643
+ record: ClientRecord,
1644
+ ): Promise<ScopeMap | 'outage' | undefined> {
1645
+ if (this.#resolveScopes === undefined) return undefined;
1646
+ const resolved = await this.#resolveScopes({
1647
+ partition,
1648
+ actorId: record.actorId,
1649
+ clientId: record.clientId,
1650
+ });
1651
+ return resolved === RESOLVER_OUTAGE ? 'outage' : resolved;
1652
+ }
1653
+
1654
+ #coverage(sinceMs: number): AdminEventCoverage {
1655
+ const evicted = this.#ring?.evictedThroughMs;
1656
+ return evicted === undefined || evicted < sinceMs
1657
+ ? { complete: true, fromMs: sinceMs }
1658
+ : { complete: false, fromMs: evicted };
564
1659
  }
565
1660
  }