@spooky-sync/core 0.0.1-canary.237 → 0.0.1-canary.239
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/dist/index.d.ts +134 -5
- package/dist/index.js +498 -83
- package/dist/otel/index.d.ts +1 -1
- package/dist/types.d.ts +40 -27
- package/package.json +3 -3
- package/src/modules/data/data.activity.test.ts +109 -0
- package/src/modules/data/data.authority.test.ts +122 -0
- package/src/modules/data/data.heartbeat-wake.test.ts +115 -0
- package/src/modules/data/data.membership.test.ts +168 -68
- package/src/modules/data/index.ts +248 -83
- package/src/modules/devtools/index.ts +3 -2
- package/src/modules/sync/engine.ts +12 -7
- package/src/modules/sync/sync.health.test.ts +13 -3
- package/src/modules/sync/sync.list-ref-meta.test.ts +193 -0
- package/src/modules/sync/sync.live-removal.test.ts +137 -23
- package/src/modules/sync/sync.ts +326 -60
- package/src/modules/sync/utils.test.ts +1 -1
- package/src/modules/sync/utils.ts +10 -2
- package/src/sp00ky.ts +64 -1
- package/src/types.ts +41 -26
- package/src/utils/error-classification.test.ts +10 -0
- package/src/utils/error-classification.ts +18 -0
|
@@ -28,6 +28,9 @@ import type {
|
|
|
28
28
|
PhaseStat,
|
|
29
29
|
RegistrationTimings,
|
|
30
30
|
RunOptions,
|
|
31
|
+
ServerViewMeta,
|
|
32
|
+
MembershipOutcome,
|
|
33
|
+
QueryAuthorityCallback,
|
|
31
34
|
} from '../../types';
|
|
32
35
|
import { MATERIALIZATION_SAMPLE_WINDOW } from '../../types';
|
|
33
36
|
import {
|
|
@@ -53,13 +56,6 @@ import {
|
|
|
53
56
|
} from './window-query';
|
|
54
57
|
import { mintMutationId } from './mutation-id';
|
|
55
58
|
|
|
56
|
-
/**
|
|
57
|
-
* How many consecutive empty `_00_list_ref` reads it takes to believe a query
|
|
58
|
-
* really is empty, before any non-empty set has been seen this session. One
|
|
59
|
-
* read is the registration race (the SSP flushes a view's initial edges
|
|
60
|
-
* asynchronously); the second comes from the poll a cycle later.
|
|
61
|
-
*/
|
|
62
|
-
const EMPTY_MEMBERSHIP_CONFIRMATIONS = 2;
|
|
63
59
|
|
|
64
60
|
/** Push a timing sample (ms) into a rolling window, capped at the sample window. */
|
|
65
61
|
function pushSample(samples: number[], ms: number): void {
|
|
@@ -98,6 +94,8 @@ export interface DurableMembership {
|
|
|
98
94
|
* is LIVE on those edges, so a swept-but-still-watched view empties the UI.
|
|
99
95
|
*/
|
|
100
96
|
const TTL_HEARTBEAT_FRACTION = 0.5;
|
|
97
|
+
/** A query beaten this recently is skipped by `heartbeatOnWake`. */
|
|
98
|
+
const WAKE_HEARTBEAT_MIN_GAP_MS = 30_000;
|
|
101
99
|
|
|
102
100
|
export class DataModule<S extends SchemaStructure> {
|
|
103
101
|
/** Tab identity baked into mutation ids (shared-tabs rollback routing);
|
|
@@ -107,6 +105,8 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
107
105
|
private pendingQueries: Map<QueryHash, Promise<QueryHash>> = new Map();
|
|
108
106
|
private subscriptions: Map<QueryHash, Set<QueryUpdateCallback>> = new Map();
|
|
109
107
|
private statusSubscriptions: Map<QueryHash, Set<QueryStatusCallback>> = new Map();
|
|
108
|
+
/** Observers of a query's authority (`membershipKnown`) flips, keyed like `statusSubscriptions`. */
|
|
109
|
+
private authoritySubscriptions: Map<QueryHash, Set<QueryAuthorityCallback>> = new Map();
|
|
110
110
|
private mutationCallbacks: Set<MutationCallback> = new Set();
|
|
111
111
|
private debounceTimers: Map<QueryHash, NodeJS.Timeout> = new Map();
|
|
112
112
|
// The update each debounce timer would process on its trailing edge. Kept in a
|
|
@@ -119,6 +119,12 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
119
119
|
// back to `idle` only on 1→0, so an inner cycle finishing can't emit a
|
|
120
120
|
// premature idle mid-registration.
|
|
121
121
|
private fetchDepth: Map<QueryHash, number> = new Map();
|
|
122
|
+
/**
|
|
123
|
+
* Observers of the AGGREGATE fetch activity: how many queries are mid-fetch
|
|
124
|
+
* right now. What a "downloading" indicator in the app chrome hangs off,
|
|
125
|
+
* instead of subscribing to every query's status one by one.
|
|
126
|
+
*/
|
|
127
|
+
private activitySubscriptions: Set<(fetching: number) => void> = new Set();
|
|
122
128
|
private logger: Logger;
|
|
123
129
|
/**
|
|
124
130
|
* Optional observer notified whenever a query's fetch status changes.
|
|
@@ -136,6 +142,15 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
136
142
|
* DataModule is constructed — mirrors `onQueryStatusChange`.
|
|
137
143
|
*/
|
|
138
144
|
public onHeartbeat?: (hash: QueryHash) => void;
|
|
145
|
+
/**
|
|
146
|
+
* Fired once per episode when the server's `_00_query` row for a query we
|
|
147
|
+
* hold membership for is found missing (see `updateQueryRemoteArray`). Wired
|
|
148
|
+
* by Sp00kyClient to enqueue a `register` down-event so the SSP rebuilds the
|
|
149
|
+
* view; the client keeps rendering what it has meanwhile.
|
|
150
|
+
*/
|
|
151
|
+
public onViewLost?: (hash: QueryHash) => void;
|
|
152
|
+
/** DevTools observer for authority flips; mirrors `onQueryStatusChange`. */
|
|
153
|
+
public onQueryAuthorityChange?: (hash: QueryHash, known: boolean) => void;
|
|
139
154
|
/**
|
|
140
155
|
* Optional hook fired by {@link deregisterQuery} when an opt-in query (e.g. a
|
|
141
156
|
* viewport-windowed list cancelling an off-screen window) loses its last
|
|
@@ -349,6 +364,61 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
349
364
|
};
|
|
350
365
|
}
|
|
351
366
|
|
|
367
|
+
/**
|
|
368
|
+
* Subscribe to a query's authority flips: whether server membership is known
|
|
369
|
+
* for it. With `{ immediate: true }` the callback fires synchronously with
|
|
370
|
+
* the current value (`false` for an unknown query).
|
|
371
|
+
*/
|
|
372
|
+
subscribeAuthority(
|
|
373
|
+
queryHash: string,
|
|
374
|
+
callback: QueryAuthorityCallback,
|
|
375
|
+
options: { immediate?: boolean } = {}
|
|
376
|
+
): () => void {
|
|
377
|
+
if (!this.authoritySubscriptions.has(queryHash)) {
|
|
378
|
+
this.authoritySubscriptions.set(queryHash, new Set());
|
|
379
|
+
}
|
|
380
|
+
this.authoritySubscriptions.get(queryHash)?.add(callback);
|
|
381
|
+
|
|
382
|
+
if (options.immediate) {
|
|
383
|
+
callback(this.activeQueries.get(queryHash)?.config.membershipKnown === true);
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
return () => {
|
|
387
|
+
const subs = this.authoritySubscriptions.get(queryHash);
|
|
388
|
+
if (subs) {
|
|
389
|
+
subs.delete(callback);
|
|
390
|
+
if (subs.size === 0) {
|
|
391
|
+
this.authoritySubscriptions.delete(queryHash);
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/** Whether server membership is known for `queryHash` (see `QueryConfig.membershipKnown`). */
|
|
398
|
+
isAuthoritative(queryHash: string): boolean {
|
|
399
|
+
return this.activeQueries.get(queryHash)?.config.membershipKnown === true;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/**
|
|
403
|
+
* The single writer of `config.membershipKnown` for a live query. Fans out
|
|
404
|
+
* to `subscribeAuthority` listeners and DevTools on a change; no-op when
|
|
405
|
+
* unchanged or the query is unknown.
|
|
406
|
+
*/
|
|
407
|
+
private setMembershipKnown(queryHash: string, known: boolean): void {
|
|
408
|
+
const queryState = this.activeQueries.get(queryHash);
|
|
409
|
+
if (!queryState) return;
|
|
410
|
+
const was = queryState.config.membershipKnown === true;
|
|
411
|
+
queryState.config.membershipKnown = known;
|
|
412
|
+
if (was === known) return;
|
|
413
|
+
this.onQueryAuthorityChange?.(queryHash, known);
|
|
414
|
+
const subs = this.authoritySubscriptions.get(queryHash);
|
|
415
|
+
if (subs) {
|
|
416
|
+
for (const callback of subs) {
|
|
417
|
+
callback(known);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
352
422
|
/**
|
|
353
423
|
* Set a query's fetch status and notify status observers (DevTools +
|
|
354
424
|
* `subscribeStatus` listeners). No-op when the status is unchanged or the
|
|
@@ -382,6 +452,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
382
452
|
this.fetchDepth.set(queryHash, depth + 1);
|
|
383
453
|
if (depth === 0) {
|
|
384
454
|
this.setQueryStatus(queryHash, 'fetching');
|
|
455
|
+
this.notifyActivity();
|
|
385
456
|
}
|
|
386
457
|
}
|
|
387
458
|
|
|
@@ -391,11 +462,49 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
391
462
|
if (depth <= 1) {
|
|
392
463
|
this.fetchDepth.delete(queryHash);
|
|
393
464
|
this.setQueryStatus(queryHash, 'idle');
|
|
465
|
+
this.notifyActivity();
|
|
394
466
|
return;
|
|
395
467
|
}
|
|
396
468
|
this.fetchDepth.set(queryHash, depth - 1);
|
|
397
469
|
}
|
|
398
470
|
|
|
471
|
+
/** How many queries are mid-fetch right now (see {@link subscribeActivity}). */
|
|
472
|
+
get fetchingQueryCount(): number {
|
|
473
|
+
return this.fetchDepth.size;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/**
|
|
477
|
+
* Observe the number of queries currently fetching. Fires on every change of
|
|
478
|
+
* that number (a query entering or leaving its outermost fetch cycle), and
|
|
479
|
+
* synchronously with the current value when `immediate` is set. One
|
|
480
|
+
* subscription per indicator, however many queries the page holds.
|
|
481
|
+
*/
|
|
482
|
+
subscribeActivity(
|
|
483
|
+
callback: (fetching: number) => void,
|
|
484
|
+
options: { immediate?: boolean } = {}
|
|
485
|
+
): () => void {
|
|
486
|
+
this.activitySubscriptions.add(callback);
|
|
487
|
+
if (options.immediate) callback(this.fetchDepth.size);
|
|
488
|
+
return () => {
|
|
489
|
+
this.activitySubscriptions.delete(callback);
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
private notifyActivity(): void {
|
|
494
|
+
if (this.activitySubscriptions.size === 0) return;
|
|
495
|
+
const n = this.fetchDepth.size;
|
|
496
|
+
for (const cb of this.activitySubscriptions) {
|
|
497
|
+
try {
|
|
498
|
+
cb(n);
|
|
499
|
+
} catch (err) {
|
|
500
|
+
this.logger.warn(
|
|
501
|
+
{ err, Category: 'sp00ky-client::DataModule::notifyActivity' },
|
|
502
|
+
'Activity subscriber threw'
|
|
503
|
+
);
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
|
|
399
508
|
/**
|
|
400
509
|
* Subscribe to mutations (for sync)
|
|
401
510
|
*/
|
|
@@ -1147,7 +1256,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1147
1256
|
// known and rendering can stop falling back to a predicate scan. The durable
|
|
1148
1257
|
// `_00_window` mirror is deliberately left to `updateQueryRemoteArray` (the
|
|
1149
1258
|
// single write point) — the registration that follows lands there anyway.
|
|
1150
|
-
|
|
1259
|
+
this.setMembershipKnown(hash, true);
|
|
1151
1260
|
|
|
1152
1261
|
queryState.records = await this.materializeRecords(queryState);
|
|
1153
1262
|
const subscribers = this.subscriptions.get(hash);
|
|
@@ -1429,6 +1538,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1429
1538
|
this.cache.unregisterQuery(hash);
|
|
1430
1539
|
this.activeQueries.delete(hash);
|
|
1431
1540
|
this.subscriptions.delete(hash);
|
|
1541
|
+
this.authoritySubscriptions.delete(hash);
|
|
1432
1542
|
}
|
|
1433
1543
|
|
|
1434
1544
|
/**
|
|
@@ -1479,91 +1589,97 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1479
1589
|
hash: string,
|
|
1480
1590
|
remoteArray: RecordVersionArray,
|
|
1481
1591
|
opts?: {
|
|
1482
|
-
/** `_00_query
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1592
|
+
/** The `_00_query` row as read in the same round trip as the edges. */
|
|
1593
|
+
meta?: ServerViewMeta;
|
|
1594
|
+
/**
|
|
1595
|
+
* The caller verified upstream that the ids missing from `remoteArray`
|
|
1596
|
+
* no longer exist (see `Sp00kySync` LIVE removal handling); an empty set
|
|
1597
|
+
* is then a real answer even without a readable row.
|
|
1598
|
+
*/
|
|
1599
|
+
verifiedRemoval?: boolean;
|
|
1600
|
+
}
|
|
1601
|
+
): Promise<MembershipOutcome> {
|
|
1486
1602
|
const queryState = this.getQueryByHash(hash);
|
|
1487
1603
|
if (!queryState) {
|
|
1488
1604
|
this.logger.warn(
|
|
1489
1605
|
{ hash, Category: 'sp00ky-client::DataModule::updateQueryRemoteArray' },
|
|
1490
1606
|
'Query to update remote array not found'
|
|
1491
1607
|
);
|
|
1492
|
-
return;
|
|
1608
|
+
return 'ignored';
|
|
1493
1609
|
}
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
//
|
|
1499
|
-
//
|
|
1500
|
-
//
|
|
1501
|
-
// durable `_00_window` row, which kept the list blank across reloads.
|
|
1610
|
+
const meta = opts?.meta;
|
|
1611
|
+
if (meta) queryState.serverState = meta.present ? meta.state : null;
|
|
1612
|
+
const held = queryState.config.remoteArray ?? [];
|
|
1613
|
+
|
|
1614
|
+
// A NON-EMPTY set is always the server's answer. An EMPTY set is only
|
|
1615
|
+
// believed when the server stands behind it, because "no edges" has four
|
|
1616
|
+
// causes and only one of them is "no rows":
|
|
1502
1617
|
//
|
|
1503
|
-
//
|
|
1504
|
-
//
|
|
1505
|
-
//
|
|
1506
|
-
//
|
|
1507
|
-
//
|
|
1508
|
-
// `
|
|
1509
|
-
//
|
|
1510
|
-
//
|
|
1511
|
-
//
|
|
1512
|
-
//
|
|
1618
|
+
// - the `_00_query` row is gone (`meta.present === false`): a TTL sweep,
|
|
1619
|
+
// an SSP reset or a scheduler wipe took the view. The client keeps what
|
|
1620
|
+
// it renders and asks for a re-registration ("view lost"). Believing
|
|
1621
|
+
// the empty here is what blanked lists after every sweep and, via the
|
|
1622
|
+
// durable `_00_window` mirror, kept them blank across reloads.
|
|
1623
|
+
// - `state === 'materializing'`: the SSP has computed the set and its
|
|
1624
|
+
// edges are in flight (the flusher window). Not an answer yet.
|
|
1625
|
+
// - `state === 'ready'` with `rowCount > 0` (or unreadable): the row and
|
|
1626
|
+
// the edges disagree (metrics lag, a stranded view). Not trusted.
|
|
1627
|
+
// - `state === 'ready'` with `rowCount === 0`: a real empty result. The
|
|
1628
|
+
// SSP flips `ready` inside the transaction that writes the edges, so
|
|
1629
|
+
// this cannot be observed mid-publish.
|
|
1513
1630
|
//
|
|
1514
|
-
//
|
|
1515
|
-
//
|
|
1516
|
-
//
|
|
1517
|
-
//
|
|
1518
|
-
//
|
|
1519
|
-
//
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
queryState.config.emptyReads = emptyReads;
|
|
1533
|
-
const exhausted =
|
|
1534
|
-
serverRowCount === null || serverRowCount === undefined
|
|
1535
|
-
? emptyReads >= EMPTY_MEMBERSHIP_CONFIRMATIONS
|
|
1536
|
-
: false; // a positive row count is never "confirmed empty"
|
|
1537
|
-
if (!exhausted) {
|
|
1538
|
-
this.logger.debug(
|
|
1539
|
-
{
|
|
1540
|
-
hash,
|
|
1541
|
-
emptyReads,
|
|
1542
|
-
serverRowCount,
|
|
1543
|
-
Category: 'sp00ky-client::DataModule::updateQueryRemoteArray',
|
|
1544
|
-
},
|
|
1545
|
-
'Ignoring empty membership: the server still reports rows for this query'
|
|
1546
|
-
);
|
|
1547
|
-
return;
|
|
1631
|
+
// A server that predates `state` (`state === null`) can only be judged on
|
|
1632
|
+
// `rowCount`: `0` is empty, anything else is the flush-window ambiguity and
|
|
1633
|
+
// is left alone. There is deliberately NO retry budget that eventually
|
|
1634
|
+
// accepts an unexplained empty: the register lifecycle, the poll and the
|
|
1635
|
+
// heartbeat all retry, and a server that cannot explain an empty must not
|
|
1636
|
+
// be able to erase a device's data.
|
|
1637
|
+
if (remoteArray.length === 0 && !opts?.verifiedRemoval) {
|
|
1638
|
+
if (!meta || !meta.present) {
|
|
1639
|
+
if (held.length > 0 || queryState.config.membershipKnown) {
|
|
1640
|
+
if (!queryState.viewLost) {
|
|
1641
|
+
queryState.viewLost = true;
|
|
1642
|
+
this.logger.warn(
|
|
1643
|
+
{ hash, held: held.length, Category: 'sp00ky-client::DataModule::updateQueryRemoteArray' },
|
|
1644
|
+
'Server has no _00_query row for a query we hold membership for; keeping local state and re-registering'
|
|
1645
|
+
);
|
|
1646
|
+
this.onViewLost?.(hash);
|
|
1647
|
+
}
|
|
1648
|
+
return 'view-lost';
|
|
1548
1649
|
}
|
|
1650
|
+
this.logger.debug(
|
|
1651
|
+
{ hash, Category: 'sp00ky-client::DataModule::updateQueryRemoteArray' },
|
|
1652
|
+
'Ignoring empty membership: the _00_query row is not readable yet'
|
|
1653
|
+
);
|
|
1654
|
+
return 'ignored';
|
|
1655
|
+
}
|
|
1656
|
+
const knownEmpty = meta.rowCount === 0 && meta.state !== 'materializing';
|
|
1657
|
+
if (!knownEmpty) {
|
|
1658
|
+
this.logger.debug(
|
|
1659
|
+
{
|
|
1660
|
+
hash,
|
|
1661
|
+
state: meta.state,
|
|
1662
|
+
rowCount: meta.rowCount,
|
|
1663
|
+
Category: 'sp00ky-client::DataModule::updateQueryRemoteArray',
|
|
1664
|
+
},
|
|
1665
|
+
'Ignoring empty membership: the server has not confirmed this query empty'
|
|
1666
|
+
);
|
|
1667
|
+
return 'ignored';
|
|
1549
1668
|
}
|
|
1550
1669
|
}
|
|
1670
|
+
if (meta?.present) queryState.viewLost = false;
|
|
1551
1671
|
|
|
1552
1672
|
const epoch = this.local.epoch;
|
|
1553
1673
|
queryState.config.remoteArray = remoteArray;
|
|
1554
|
-
// The single point where authoritative membership arrives (registration
|
|
1555
|
-
// the `_00_list_ref` poll
|
|
1556
|
-
// membership" is latched and where the durable
|
|
1557
|
-
|
|
1674
|
+
// The single point where authoritative membership arrives (registration,
|
|
1675
|
+
// the `_00_list_ref` poll and verified LIVE removals all land here), so it
|
|
1676
|
+
// is where "we now know the membership" is latched and where the durable
|
|
1677
|
+
// mirror is written. Every set that reaches this line is server-backed, so
|
|
1678
|
+
// the mirror is always `confirmed`: a reload may trust it even when empty.
|
|
1679
|
+
this.setMembershipKnown(hash, true);
|
|
1558
1680
|
queryState.serverMembership = true;
|
|
1559
|
-
if (remoteArray.length > 0) {
|
|
1560
|
-
// Earns the right to believe a later empty set — that transition is a
|
|
1561
|
-
// genuine removal and must be honoured, or removed rows resurrect.
|
|
1562
|
-
queryState.config.remoteSeen = true;
|
|
1563
|
-
queryState.config.emptyReads = 0;
|
|
1564
|
-
}
|
|
1565
1681
|
if (queryState.config.membershipKey) {
|
|
1566
|
-
await this.writeWindowMembership(queryState.config.membershipKey, remoteArray,
|
|
1682
|
+
await this.writeWindowMembership(queryState.config.membershipKey, remoteArray, true);
|
|
1567
1683
|
}
|
|
1568
1684
|
try {
|
|
1569
1685
|
await this.local.query(
|
|
@@ -1575,9 +1691,10 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1575
1691
|
{ epoch }
|
|
1576
1692
|
);
|
|
1577
1693
|
} catch (err) {
|
|
1578
|
-
if (err instanceof StaleEpochError) return;
|
|
1694
|
+
if (err instanceof StaleEpochError) return 'applied';
|
|
1579
1695
|
throw err;
|
|
1580
1696
|
}
|
|
1697
|
+
return 'applied';
|
|
1581
1698
|
}
|
|
1582
1699
|
|
|
1583
1700
|
/**
|
|
@@ -1592,7 +1709,9 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1592
1709
|
}
|
|
1593
1710
|
this.debounceTimers.clear();
|
|
1594
1711
|
this.pendingStreamUpdates.clear();
|
|
1712
|
+
const wasFetching = this.fetchDepth.size > 0;
|
|
1595
1713
|
this.fetchDepth.clear();
|
|
1714
|
+
if (wasFetching) this.notifyActivity();
|
|
1596
1715
|
for (const queryState of this.activeQueries.values()) {
|
|
1597
1716
|
if (queryState.ttlTimer) {
|
|
1598
1717
|
clearTimeout(queryState.ttlTimer);
|
|
@@ -1626,18 +1745,18 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1626
1745
|
// bucket's durable row if it has one (a returning user), otherwise mark it
|
|
1627
1746
|
// unknown so the first paint falls back to a scan instead of rendering an
|
|
1628
1747
|
// empty list until the server answers.
|
|
1629
|
-
|
|
1630
|
-
//
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1748
|
+
// Via the single writer so `subscribeAuthority` observers see the reset
|
|
1749
|
+
// BEFORE the `[]` records notify below, and the re-latch after it.
|
|
1750
|
+
this.setMembershipKnown(hash, false);
|
|
1751
|
+
queryState.viewLost = false;
|
|
1752
|
+
queryState.serverState = null;
|
|
1634
1753
|
if (config.membershipKey) {
|
|
1635
1754
|
const durable = await this.getWindowMembership(config.membershipKey);
|
|
1636
1755
|
// Same rule as the cold-start read: a non-empty row, or an empty one
|
|
1637
1756
|
// the server confirmed, is membership; an unmarked empty row is not.
|
|
1638
1757
|
if (durable && (durable.ids.length > 0 || durable.confirmed)) {
|
|
1639
1758
|
config.remoteArray = durable.ids;
|
|
1640
|
-
|
|
1759
|
+
this.setMembershipKnown(hash, true);
|
|
1641
1760
|
}
|
|
1642
1761
|
}
|
|
1643
1762
|
queryState.hydrated = false;
|
|
@@ -2522,6 +2641,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
2522
2641
|
);
|
|
2523
2642
|
return;
|
|
2524
2643
|
}
|
|
2644
|
+
queryState.lastHeartbeatAt = Date.now();
|
|
2525
2645
|
this.onHeartbeat?.(hash);
|
|
2526
2646
|
this.logger.debug(
|
|
2527
2647
|
{
|
|
@@ -2535,6 +2655,51 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
2535
2655
|
}, heartbeatTime);
|
|
2536
2656
|
}
|
|
2537
2657
|
|
|
2658
|
+
/**
|
|
2659
|
+
* Heartbeat every watched query NOW and re-anchor its timer.
|
|
2660
|
+
*
|
|
2661
|
+
* The TTL timer is the only thing keeping a view alive on the server, and
|
|
2662
|
+
* browsers throttle timers in hidden tabs (Chrome aligns them to one-minute
|
|
2663
|
+
* ticks after five minutes hidden, and suspends them outright on some
|
|
2664
|
+
* mobile platforms). A tab that comes back after a long stretch may have
|
|
2665
|
+
* missed its beat by more than the server's slack, and the sweep will have
|
|
2666
|
+
* taken the row and its edges. The client does not blank on that any more
|
|
2667
|
+
* (`updateQueryRemoteArray` treats a missing row as "view lost"), but the
|
|
2668
|
+
* sooner it finds out, the sooner the view is rebuilt: `heartbeatQuery`
|
|
2669
|
+
* re-registers a query whose row is gone. Called on `visibilitychange` (to
|
|
2670
|
+
* visible) and `online`.
|
|
2671
|
+
*
|
|
2672
|
+
* Queries beaten within the last `minGapMs` are skipped, so a flurry of
|
|
2673
|
+
* wake events costs one beat per query. There is deliberately no local
|
|
2674
|
+
* expiry anywhere in this module: the TTL is the server's, and the client's
|
|
2675
|
+
* only duty is to keep asserting liveness while something is watching.
|
|
2676
|
+
*/
|
|
2677
|
+
heartbeatOnWake(minGapMs: number = WAKE_HEARTBEAT_MIN_GAP_MS): number {
|
|
2678
|
+
const now = Date.now();
|
|
2679
|
+
let beaten = 0;
|
|
2680
|
+
for (const [hash, queryState] of this.activeQueries.entries()) {
|
|
2681
|
+
if ((this.subscriptions.get(hash)?.size ?? 0) === 0) continue;
|
|
2682
|
+
if (queryState.lastHeartbeatAt !== undefined && now - queryState.lastHeartbeatAt < minGapMs) {
|
|
2683
|
+
continue;
|
|
2684
|
+
}
|
|
2685
|
+
if (queryState.ttlTimer) {
|
|
2686
|
+
clearTimeout(queryState.ttlTimer);
|
|
2687
|
+
queryState.ttlTimer = null;
|
|
2688
|
+
}
|
|
2689
|
+
queryState.lastHeartbeatAt = now;
|
|
2690
|
+
this.onHeartbeat?.(hash);
|
|
2691
|
+
this.startTTLHeartbeat(queryState, hash);
|
|
2692
|
+
beaten++;
|
|
2693
|
+
}
|
|
2694
|
+
if (beaten > 0) {
|
|
2695
|
+
this.logger.debug(
|
|
2696
|
+
{ beaten, Category: 'sp00ky-client::DataModule::heartbeatOnWake' },
|
|
2697
|
+
'Wake heartbeat sent for watched queries'
|
|
2698
|
+
);
|
|
2699
|
+
}
|
|
2700
|
+
return beaten;
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2538
2703
|
private async replaceRecordInQueries(record: Record<string, any>): Promise<void> {
|
|
2539
2704
|
for (const [queryHash, queryState] of this.activeQueries.entries()) {
|
|
2540
2705
|
const index = queryState.records.findIndex((r) => r.id === record.id);
|
|
@@ -229,8 +229,9 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
229
229
|
// the panel: is the server's set known, has a non-empty one been seen
|
|
230
230
|
// this session, how many empty reads were ignored.
|
|
231
231
|
membershipKnown: q.config.membershipKnown === true,
|
|
232
|
-
|
|
233
|
-
|
|
232
|
+
authoritative: q.config.membershipKnown === true,
|
|
233
|
+
viewLost: q.viewLost === true,
|
|
234
|
+
serverState: q.serverState ?? null,
|
|
234
235
|
// Detailed per-phase processing-time breakdown (SSP sub-phases, local/
|
|
235
236
|
// remote record fetch, frontend reconcile, registration). Flows to both
|
|
236
237
|
// the DevTools panel and the MCP (which returns activeQueries verbatim).
|
|
@@ -37,7 +37,7 @@ export class SyncEngine {
|
|
|
37
37
|
*/
|
|
38
38
|
async syncRecords(
|
|
39
39
|
diff: RecordVersionDiff
|
|
40
|
-
): Promise<{ remoteFetchMs: number; stillRemoteIds: string[] }> {
|
|
40
|
+
): Promise<{ remoteFetchMs: number; stillRemoteIds: string[]; confirmedRemovedIds: string[] }> {
|
|
41
41
|
const { added, updated, removed } = diff;
|
|
42
42
|
|
|
43
43
|
this.logger.debug(
|
|
@@ -54,8 +54,9 @@ export class SyncEngine {
|
|
|
54
54
|
// locally. Returns ids that LEFT the view's list_ref but still exist upstream
|
|
55
55
|
// (so they weren't deleted) — the caller converges localArray to drop them.
|
|
56
56
|
let stillRemoteIds: string[] = [];
|
|
57
|
+
let confirmedRemovedIds: string[] = [];
|
|
57
58
|
if (removed.length > 0) {
|
|
58
|
-
stillRemoteIds = await this.handleRemovedRecords(removed);
|
|
59
|
+
({ stillRemoteIds, confirmedRemovedIds } = await this.handleRemovedRecords(removed));
|
|
59
60
|
}
|
|
60
61
|
|
|
61
62
|
// Fetch added/updated records from remote. Skip ids whose body the local
|
|
@@ -68,7 +69,7 @@ export class SyncEngine {
|
|
|
68
69
|
return !(localVersion && item.version <= localVersion);
|
|
69
70
|
});
|
|
70
71
|
if (toFetch.length === 0) {
|
|
71
|
-
return { remoteFetchMs: 0, stillRemoteIds };
|
|
72
|
+
return { remoteFetchMs: 0, stillRemoteIds, confirmedRemovedIds };
|
|
72
73
|
}
|
|
73
74
|
|
|
74
75
|
// Build a version map from the diff (versions come from _00_list_ref)
|
|
@@ -151,7 +152,7 @@ export class SyncEngine {
|
|
|
151
152
|
records: remoteResults,
|
|
152
153
|
});
|
|
153
154
|
|
|
154
|
-
return { remoteFetchMs, stillRemoteIds };
|
|
155
|
+
return { remoteFetchMs, stillRemoteIds, confirmedRemovedIds };
|
|
155
156
|
}
|
|
156
157
|
|
|
157
158
|
/**
|
|
@@ -168,7 +169,9 @@ export class SyncEngine {
|
|
|
168
169
|
* row to a later sync round is recoverable; deleting a fresh row that
|
|
169
170
|
* upstream still has is not.
|
|
170
171
|
*/
|
|
171
|
-
private async handleRemovedRecords(
|
|
172
|
+
private async handleRemovedRecords(
|
|
173
|
+
removed: RecordId[]
|
|
174
|
+
): Promise<{ stillRemoteIds: string[]; confirmedRemovedIds: string[] }> {
|
|
172
175
|
this.logger.debug(
|
|
173
176
|
{
|
|
174
177
|
removed: removed.map((r) => r.toString()),
|
|
@@ -216,7 +219,7 @@ export class SyncEngine {
|
|
|
216
219
|
},
|
|
217
220
|
'Remote existence check failed, skipping deletion to avoid clobbering fresh data'
|
|
218
221
|
);
|
|
219
|
-
return [];
|
|
222
|
+
return { stillRemoteIds: [], confirmedRemovedIds: [] };
|
|
220
223
|
}
|
|
221
224
|
|
|
222
225
|
// Ids that left the view's list_ref but STILL exist upstream — not deletions,
|
|
@@ -224,6 +227,7 @@ export class SyncEngine {
|
|
|
224
227
|
// longer matches the query). The caller drops these from `localArray` so the
|
|
225
228
|
// poll's diff stops re-flagging them every tick (the `job:` churn).
|
|
226
229
|
const stillRemoteIds: string[] = [];
|
|
230
|
+
const confirmedRemovedIds: string[] = [];
|
|
227
231
|
for (const recordId of removed) {
|
|
228
232
|
const recordIdStr = encodeRecordId(recordId);
|
|
229
233
|
if (!existingRemoteIds.has(recordIdStr)) {
|
|
@@ -236,10 +240,11 @@ export class SyncEngine {
|
|
|
236
240
|
);
|
|
237
241
|
// Use CacheModule to handle both local DB and DBSP deletion
|
|
238
242
|
await this.cache.delete(recordId.table.name, recordIdStr);
|
|
243
|
+
confirmedRemovedIds.push(recordIdStr);
|
|
239
244
|
} else {
|
|
240
245
|
stillRemoteIds.push(recordIdStr);
|
|
241
246
|
}
|
|
242
247
|
}
|
|
243
|
-
return stillRemoteIds;
|
|
248
|
+
return { stillRemoteIds, confirmedRemovedIds };
|
|
244
249
|
}
|
|
245
250
|
}
|
|
@@ -38,11 +38,21 @@ function makeSync(hashes: string[]) {
|
|
|
38
38
|
// (`applyListRefSnapshot`). Stub both so the test controls per-cycle
|
|
39
39
|
// reachability. `poll` invokes the private method.
|
|
40
40
|
const fetch = vi.fn();
|
|
41
|
-
const apply = vi.fn().mockResolvedValue(false);
|
|
41
|
+
const apply = vi.fn().mockResolvedValue({ changed: false, outcome: 'unchanged' });
|
|
42
42
|
(sync as any).fetchListRefSnapshots = fetch;
|
|
43
43
|
(sync as any).applyListRefSnapshot = apply;
|
|
44
44
|
const snapshotsFor = (hs: string[]) =>
|
|
45
|
-
new Map(
|
|
45
|
+
new Map(
|
|
46
|
+
hs.map((h) => [
|
|
47
|
+
h,
|
|
48
|
+
{
|
|
49
|
+
primary: [],
|
|
50
|
+
subquery: [],
|
|
51
|
+
rowCount: null,
|
|
52
|
+
meta: { present: true, rowCount: null, state: null },
|
|
53
|
+
},
|
|
54
|
+
])
|
|
55
|
+
);
|
|
46
56
|
fetch.mockImplementation(async (hs: string[]) => snapshotsFor(hs));
|
|
47
57
|
const poll = () => (sync as any).pollListRefForActiveQueries() as Promise<boolean>;
|
|
48
58
|
|
|
@@ -102,7 +112,7 @@ describe('sync health via idle poll', () => {
|
|
|
102
112
|
// snapshots then fails on its own network error → still a reached cycle.
|
|
103
113
|
fetch.mockImplementation(async (hs: string[]) => snapshotsFor(hs));
|
|
104
114
|
apply
|
|
105
|
-
.mockResolvedValueOnce(true)
|
|
115
|
+
.mockResolvedValueOnce({ changed: true, outcome: 'applied' })
|
|
106
116
|
.mockRejectedValueOnce(new Error(CONNECTION_UNAVAILABLE));
|
|
107
117
|
await expect(poll()).resolves.toBe(true);
|
|
108
118
|
expect(sync.syncHealth.status).toBe('healthy');
|