@spooky-sync/core 0.0.1-canary.256 → 0.0.1-canary.258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +10 -0
- package/dist/index.js +109 -18
- package/package.json +3 -3
- package/src/client/router.ts +3 -1
- package/src/kernel/constants.ts +7 -0
- package/src/kernel/events.ts +1 -0
- package/src/query/membership.saga.test.ts +80 -9
- package/src/query/membership.saga.ts +61 -7
- package/src/query/membership.ts +22 -9
- package/src/query/register.saga.test.ts +30 -22
- package/src/query/register.saga.ts +19 -9
- package/src/state/client-state.ts +7 -0
- package/src/state/reducers.ts +12 -0
- package/src/testing/build.ts +2 -0
package/dist/index.d.ts
CHANGED
|
@@ -1127,6 +1127,9 @@ type RuntimeEvent = {
|
|
|
1127
1127
|
type: 'AckPrune';
|
|
1128
1128
|
} | {
|
|
1129
1129
|
type: 'ReadDirtyMembership';
|
|
1130
|
+
} | {
|
|
1131
|
+
type: 'RecoverLostView';
|
|
1132
|
+
hash: QueryHash;
|
|
1130
1133
|
} | {
|
|
1131
1134
|
type: 'ReadMembership';
|
|
1132
1135
|
hashes: QueryHash[];
|
|
@@ -1501,6 +1504,13 @@ interface QueryEntry {
|
|
|
1501
1504
|
readonly lastHeartbeatAt: number | null;
|
|
1502
1505
|
readonly lastPolledAt: number | null;
|
|
1503
1506
|
readonly registerAttempts: number;
|
|
1507
|
+
/**
|
|
1508
|
+
* Re-registrations issued because the `_00_query` row read as gone, since
|
|
1509
|
+
* the row was last seen. Paces the next one (see `applyMembership`).
|
|
1510
|
+
*/
|
|
1511
|
+
readonly viewLostCount: number;
|
|
1512
|
+
/** When the paced re-registration is due; null when none is pending. */
|
|
1513
|
+
readonly viewLostRetryAt: number | null;
|
|
1504
1514
|
readonly telemetry: QueryTelemetry;
|
|
1505
1515
|
}
|
|
1506
1516
|
type OutboxStatus = 'pending' | 'acked';
|
package/dist/index.js
CHANGED
|
@@ -1073,8 +1073,8 @@ function selfAllowlistedVariant(flag, userId) {
|
|
|
1073
1073
|
|
|
1074
1074
|
//#endregion
|
|
1075
1075
|
//#region src/modules/devtools/index.ts
|
|
1076
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
1077
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
1076
|
+
const CORE_VERSION = "0.0.1-canary.258";
|
|
1077
|
+
const WASM_VERSION = "0.0.1-canary.258";
|
|
1078
1078
|
const SURREAL_VERSION = "3.0.3";
|
|
1079
1079
|
var DevToolsService = class DevToolsService {
|
|
1080
1080
|
eventsHistory = [];
|
|
@@ -1686,6 +1686,13 @@ const RETRY_BASE_MS = 500;
|
|
|
1686
1686
|
const RETRY_MAX_MS = 15e3;
|
|
1687
1687
|
/** Registration attempts before a query is reported as failed to settle. */
|
|
1688
1688
|
const REGISTER_MAX_RETRIES = 3;
|
|
1689
|
+
/**
|
|
1690
|
+
* Pacing of repeated view-lost recovery. The first re-registration after a
|
|
1691
|
+
* `_00_query` row reads as gone is immediate; each further one for the same
|
|
1692
|
+
* query, before the row has been seen again, waits on this backoff.
|
|
1693
|
+
*/
|
|
1694
|
+
const VIEW_LOST_RETRY_BASE_MS = 1e3;
|
|
1695
|
+
const VIEW_LOST_RETRY_MAX_MS = 3e4;
|
|
1689
1696
|
/** `_00_list_ref` poll cadence (fallback when LIVE is quiet). */
|
|
1690
1697
|
const LIST_REF_POLL_BASE_MS = 500;
|
|
1691
1698
|
/** Self-heal cadence while sync health is degraded. */
|
|
@@ -1877,6 +1884,12 @@ const bumpRegisterAttempts = (hash) => (s) => withEntry(s, hash, (e) => ({
|
|
|
1877
1884
|
...e,
|
|
1878
1885
|
registerAttempts: e.registerAttempts + 1
|
|
1879
1886
|
}));
|
|
1887
|
+
/** View-lost recovery pacing; see `QueryEntry.viewLostCount`. */
|
|
1888
|
+
const setViewLost = (hash, viewLostCount, viewLostRetryAt) => (s) => withEntry(s, hash, (e) => e.viewLostCount === viewLostCount && e.viewLostRetryAt === viewLostRetryAt ? e : {
|
|
1889
|
+
...e,
|
|
1890
|
+
viewLostCount,
|
|
1891
|
+
viewLostRetryAt
|
|
1892
|
+
});
|
|
1880
1893
|
const resetRegisterAttempts = (hash) => (s) => withEntry(s, hash, (e) => e.registerAttempts === 0 ? e : {
|
|
1881
1894
|
...e,
|
|
1882
1895
|
registerAttempts: 0
|
|
@@ -2087,7 +2100,9 @@ const rebindQuery = (hash, next) => (s) => {
|
|
|
2087
2100
|
serverState: null,
|
|
2088
2101
|
lastHeartbeatAt: null,
|
|
2089
2102
|
lastPolledAt: null,
|
|
2090
|
-
registerAttempts: 0
|
|
2103
|
+
registerAttempts: 0,
|
|
2104
|
+
viewLostCount: 0,
|
|
2105
|
+
viewLostRetryAt: null
|
|
2091
2106
|
}));
|
|
2092
2107
|
return rebound === s ? s : {
|
|
2093
2108
|
...rebound,
|
|
@@ -2658,10 +2673,21 @@ function decideMembershipOutcome(input) {
|
|
|
2658
2673
|
if (!meta || !meta.present) return held > 0 || phase !== "cold" ? "view-lost" : "ignored";
|
|
2659
2674
|
return meta.rowCount === 0 && meta.state !== "materializing" ? "applied" : "ignored";
|
|
2660
2675
|
}
|
|
2676
|
+
/**
|
|
2677
|
+
* A statement that did not answer (an ERR, or no result at all). Kept apart
|
|
2678
|
+
* from a `null` result, which is the server saying "no such row": reading a
|
|
2679
|
+
* failed statement as a missing row turned every timeout under load into a
|
|
2680
|
+
* lost view and re-registered it, over and over (whitepawn, 2026-09-16).
|
|
2681
|
+
*/
|
|
2682
|
+
const FAILED = Symbol("statement failed");
|
|
2661
2683
|
const toPairs = (rows) => dedupeRecordVersions(Array.isArray(rows) ? rows.map((r) => [encodeRecordId(r.out), r.version]) : []);
|
|
2662
|
-
/**
|
|
2684
|
+
/**
|
|
2685
|
+
* Fold the single-query statement batch (edges, meta, children) into a
|
|
2686
|
+
* snapshot. `null` when any of the three did not answer: half an answer is
|
|
2687
|
+
* not one, and a missing meta would read as a lost view.
|
|
2688
|
+
*/
|
|
2663
2689
|
function snapshotFromSingle(items, metaRow, children) {
|
|
2664
|
-
if (!Array.isArray(items)) return null;
|
|
2690
|
+
if (!Array.isArray(items) || metaRow === FAILED || children === FAILED) return null;
|
|
2665
2691
|
return {
|
|
2666
2692
|
primary: toPairs(items),
|
|
2667
2693
|
subquery: toPairs(children),
|
|
@@ -2671,11 +2697,11 @@ function snapshotFromSingle(items, metaRow, children) {
|
|
|
2671
2697
|
/**
|
|
2672
2698
|
* Fold the many-query statement batch into one snapshot per hash. Every hash
|
|
2673
2699
|
* in `hashById` gets a snapshot; a query whose row did not come back reads
|
|
2674
|
-
* `present: false`.
|
|
2700
|
+
* `present: false`. Empty when either statement did not answer.
|
|
2675
2701
|
*/
|
|
2676
2702
|
function snapshotsFromBatch(edges, counts, hashById) {
|
|
2677
2703
|
const out = /* @__PURE__ */ new Map();
|
|
2678
|
-
if (!Array.isArray(edges)) return out;
|
|
2704
|
+
if (!Array.isArray(edges) || counts === FAILED) return out;
|
|
2679
2705
|
for (const hash of hashById.values()) out.set(hash, {
|
|
2680
2706
|
primary: [],
|
|
2681
2707
|
subquery: [],
|
|
@@ -2938,6 +2964,7 @@ function* applyMembership(hash, remoteArray, meta, verifiedRemoval) {
|
|
|
2938
2964
|
const entry = yield fx.state.read((s) => s.queries.get(hash));
|
|
2939
2965
|
if (!entry) return "ignored";
|
|
2940
2966
|
if (meta) yield fx.state.update(setServerState(hash, meta.present ? meta.state : null));
|
|
2967
|
+
if (meta?.present && (entry.viewLostCount > 0 || entry.viewLostRetryAt !== null)) yield fx.state.update(setViewLost(hash, 0, null));
|
|
2941
2968
|
const outcome = decideMembershipOutcome({
|
|
2942
2969
|
phase: entry.lifecycle.phase,
|
|
2943
2970
|
held: entry.remoteArray.length,
|
|
@@ -2954,8 +2981,30 @@ function* applyMembership(hash, remoteArray, meta, verifiedRemoval) {
|
|
|
2954
2981
|
hash
|
|
2955
2982
|
});
|
|
2956
2983
|
}
|
|
2957
|
-
if (entry.
|
|
2958
|
-
|
|
2984
|
+
if (entry.viewLostRetryAt !== null) return outcome;
|
|
2985
|
+
if (entry.viewLostCount === 0) {
|
|
2986
|
+
yield fx.state.update(setViewLost(hash, 1, null));
|
|
2987
|
+
if (entry.lifecycle.remote !== "unregistered") yield fx.state.update(applyLifecycle(hash, { type: "remote-dropped" }));
|
|
2988
|
+
yield fx.dispatch({ type: "EnsureRegistered" });
|
|
2989
|
+
return outcome;
|
|
2990
|
+
}
|
|
2991
|
+
const delay = backoffMs(entry.viewLostCount - 1, VIEW_LOST_RETRY_BASE_MS, VIEW_LOST_RETRY_MAX_MS);
|
|
2992
|
+
const now = yield fx.now();
|
|
2993
|
+
yield fx.state.update(setViewLost(hash, entry.viewLostCount + 1, now + delay));
|
|
2994
|
+
yield fx.emit({
|
|
2995
|
+
type: "log",
|
|
2996
|
+
level: "debug",
|
|
2997
|
+
message: "view still lost; re-registering after a delay",
|
|
2998
|
+
data: {
|
|
2999
|
+
hash,
|
|
3000
|
+
attempt: entry.viewLostCount + 1,
|
|
3001
|
+
delayMs: delay
|
|
3002
|
+
}
|
|
3003
|
+
});
|
|
3004
|
+
yield fx.timer.set(`view-lost:${hash}`, delay, {
|
|
3005
|
+
type: "RecoverLostView",
|
|
3006
|
+
hash
|
|
3007
|
+
});
|
|
2959
3008
|
return outcome;
|
|
2960
3009
|
}
|
|
2961
3010
|
const wasAuthoritative = isAuthoritative(entry.lifecycle);
|
|
@@ -2982,6 +3031,21 @@ function* applyMembership(hash, remoteArray, meta, verifiedRemoval) {
|
|
|
2982
3031
|
yield fx.dispatch({ type: "FetchRows" });
|
|
2983
3032
|
return outcome;
|
|
2984
3033
|
}
|
|
3034
|
+
/**
|
|
3035
|
+
* The paced view-lost re-registration. A no-op when the row has been seen
|
|
3036
|
+
* since (the pacing was reset) or the query recovered or went away.
|
|
3037
|
+
*/
|
|
3038
|
+
function* recoverLostView(hash) {
|
|
3039
|
+
const entry = yield fx.state.read((s) => s.queries.get(hash));
|
|
3040
|
+
if (!entry || entry.viewLostRetryAt === null) return;
|
|
3041
|
+
yield fx.state.update(setViewLost(hash, entry.viewLostCount, null));
|
|
3042
|
+
if (entry.lifecycle.phase !== "view-lost") return;
|
|
3043
|
+
if (entry.lifecycle.remote !== "unregistered") yield fx.state.update(applyLifecycle(hash, { type: "remote-dropped" }));
|
|
3044
|
+
yield fx.dispatch({
|
|
3045
|
+
type: "RegisterRemote",
|
|
3046
|
+
hash
|
|
3047
|
+
});
|
|
3048
|
+
}
|
|
2985
3049
|
/** Replace the subquery child set; bodies follow through the fetch plan. */
|
|
2986
3050
|
function* applySubqueryChildren(hash, children) {
|
|
2987
3051
|
const entry = yield fx.state.read((s) => s.queries.get(hash));
|
|
@@ -2990,10 +3054,11 @@ function* applySubqueryChildren(hash, children) {
|
|
|
2990
3054
|
yield fx.dispatch({ type: "FetchRows" });
|
|
2991
3055
|
}
|
|
2992
3056
|
const ok = (r) => r.ok;
|
|
3057
|
+
/** One statement's result, or `FAILED` when it did not answer (see `FAILED`). */
|
|
2993
3058
|
const stmt$1 = (results, i) => {
|
|
2994
|
-
if (!Array.isArray(results)) return
|
|
3059
|
+
if (!Array.isArray(results)) return FAILED;
|
|
2995
3060
|
const r = results[i];
|
|
2996
|
-
return r && r.status === "OK" ? r.result :
|
|
3061
|
+
return r && r.status === "OK" ? r.result : FAILED;
|
|
2997
3062
|
};
|
|
2998
3063
|
/**
|
|
2999
3064
|
* Read the server membership of many queries in as few round trips as the
|
|
@@ -3031,11 +3096,17 @@ function* readMembership(env, hashes, opts = {}) {
|
|
|
3031
3096
|
if (chunk.length === 1) {
|
|
3032
3097
|
const snap = snapshotFromSingle(stmt$1(r.value, 0), stmt$1(r.value, 1), stmt$1(r.value, 2));
|
|
3033
3098
|
if (snap) snapshots.set(chunk[0], snap);
|
|
3099
|
+
else failed = true;
|
|
3034
3100
|
return;
|
|
3035
3101
|
}
|
|
3036
3102
|
const hashById = new Map(chunk.map((h) => [encodeRecordId(byHash.get(h).def.id), h]));
|
|
3037
3103
|
const edges = stmt$1(r.value, 0);
|
|
3038
|
-
const
|
|
3104
|
+
const counts = stmt$1(r.value, 1);
|
|
3105
|
+
if (!Array.isArray(edges) || counts === FAILED) {
|
|
3106
|
+
failed = true;
|
|
3107
|
+
return;
|
|
3108
|
+
}
|
|
3109
|
+
const batch = snapshotsFromBatch(edges, counts, hashById);
|
|
3039
3110
|
const held = new Map(chunk.map((h) => [h, byHash.get(h).remoteArray.length]));
|
|
3040
3111
|
const suspect = new Set(suspectHashes(batch, held, Array.isArray(edges) ? edges.length : 0));
|
|
3041
3112
|
for (const [h, snap] of batch) if (suspect.has(h)) rereads.push(byHash.get(h));
|
|
@@ -3048,6 +3119,7 @@ function* readMembership(env, hashes, opts = {}) {
|
|
|
3048
3119
|
}
|
|
3049
3120
|
const snap = snapshotFromSingle(stmt$1(r.value, 0), stmt$1(r.value, 1), stmt$1(r.value, 2));
|
|
3050
3121
|
if (snap) snapshots.set(rereads[i].def.hash, snap);
|
|
3122
|
+
else failed = true;
|
|
3051
3123
|
});
|
|
3052
3124
|
let changed = false;
|
|
3053
3125
|
const stillMaterializing = [];
|
|
@@ -3164,6 +3236,8 @@ function* registerLocal(env, input) {
|
|
|
3164
3236
|
lastHeartbeatAt: null,
|
|
3165
3237
|
lastPolledAt: null,
|
|
3166
3238
|
registerAttempts: 0,
|
|
3239
|
+
viewLostCount: 0,
|
|
3240
|
+
viewLostRetryAt: null,
|
|
3167
3241
|
telemetry: {
|
|
3168
3242
|
...emptyTelemetry(),
|
|
3169
3243
|
registrationTimings: reg.timings
|
|
@@ -3249,11 +3323,24 @@ function* registerRemote(env, hash, retry = false) {
|
|
|
3249
3323
|
if (!(yield fx.state.read((s) => s.queries.has(hash)))) return;
|
|
3250
3324
|
const register = stmt(results, 0);
|
|
3251
3325
|
if (!register || register.status === "ERR") throw new Error(register?.status === "ERR" ? register.error : "register returned nothing");
|
|
3252
|
-
const
|
|
3253
|
-
const
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3326
|
+
const answer = (r) => r?.status === "OK" ? r.result : FAILED;
|
|
3327
|
+
const snap = snapshotFromSingle(answer(stmt(results, 1)), answer(stmt(results, 2)), answer(stmt(results, 3)));
|
|
3328
|
+
if (!snap) {
|
|
3329
|
+
yield fx.state.update(compose(applyLifecycle(hash, { type: "remote-registered" }), resetRegisterAttempts(hash)));
|
|
3330
|
+
yield fx.emit({
|
|
3331
|
+
type: "log",
|
|
3332
|
+
level: "debug",
|
|
3333
|
+
message: "registration read-back did not answer; re-reading membership",
|
|
3334
|
+
data: { hash }
|
|
3335
|
+
});
|
|
3336
|
+
yield* markMembershipDirty([hash]);
|
|
3337
|
+
yield fx.dispatch({
|
|
3338
|
+
type: "SyncOutcome",
|
|
3339
|
+
ok: false,
|
|
3340
|
+
error: "registration read-back failed"
|
|
3341
|
+
});
|
|
3342
|
+
return;
|
|
3343
|
+
}
|
|
3257
3344
|
if ((yield* applyMembership(hash, snap.primary, snap.meta)) === "ignored" && snap.meta.present && snap.meta.state === "materializing") {
|
|
3258
3345
|
yield fx.state.update(compose(markMembershipDirty$1([hash]), setMembershipReread(hash, 1)));
|
|
3259
3346
|
yield fx.timer.set("membership", 150, { type: "ReadDirtyMembership" });
|
|
@@ -5574,6 +5661,10 @@ function route(env, event) {
|
|
|
5574
5661
|
saga: registerRemote(env, event.hash, event.retry),
|
|
5575
5662
|
lane: dedupe(`register:${event.hash}`)
|
|
5576
5663
|
};
|
|
5664
|
+
case "RecoverLostView": return {
|
|
5665
|
+
saga: recoverLostView(event.hash),
|
|
5666
|
+
lane: dedupe(`view-lost:${event.hash}`)
|
|
5667
|
+
};
|
|
5577
5668
|
case "ReadDirtyMembership": return {
|
|
5578
5669
|
saga: readDirtyMembership(env),
|
|
5579
5670
|
lane: serial("membership")
|
|
@@ -12544,7 +12635,7 @@ function createTabsCoordinator(config, s, host) {
|
|
|
12544
12635
|
return new TabsCoordinator({
|
|
12545
12636
|
tabId: s.tabId,
|
|
12546
12637
|
fingerprint: computeTabsFingerprint({
|
|
12547
|
-
coreVersion: "0.0.1-canary.
|
|
12638
|
+
coreVersion: "0.0.1-canary.258",
|
|
12548
12639
|
schemaHash: hash53(config.schemaSurql),
|
|
12549
12640
|
endpoint: config.database.endpoint ?? "",
|
|
12550
12641
|
namespace: config.database.namespace,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.258",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -63,8 +63,8 @@
|
|
|
63
63
|
}
|
|
64
64
|
},
|
|
65
65
|
"dependencies": {
|
|
66
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
67
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
66
|
+
"@spooky-sync/query-builder": "0.0.1-canary.258",
|
|
67
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.258",
|
|
68
68
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
69
69
|
"@surrealdb/wasm": "^3.0.3",
|
|
70
70
|
"blurhash": "^2.0.5",
|
package/src/client/router.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { RuntimeEvent } from '../kernel/events';
|
|
|
2
2
|
import type { Lane, Saga } from '../kernel/saga';
|
|
3
3
|
import type { SagaEnv } from '../query/env';
|
|
4
4
|
import { ensureRegistered, registerRemote } from '../query/register.saga';
|
|
5
|
-
import { readDirtyMembership, readMembership } from '../query/membership.saga';
|
|
5
|
+
import { readDirtyMembership, readMembership, recoverLostView } from '../query/membership.saga';
|
|
6
6
|
import { fetchRows } from '../query/fetch.saga';
|
|
7
7
|
import { materialize, streamUpdate } from '../query/materialize.saga';
|
|
8
8
|
import { ackPrune, gcTick, lifecycleTick } from '../query/lifecycle.saga';
|
|
@@ -37,6 +37,8 @@ export function route(env: SagaEnv, event: RuntimeEvent): RouteTarget {
|
|
|
37
37
|
return { saga: ensureRegistered(env, { requireAuth: event.requireAuth, attempt: event.attempt }), lane: serial('ensure') };
|
|
38
38
|
case 'RegisterRemote':
|
|
39
39
|
return { saga: registerRemote(env, event.hash, event.retry), lane: dedupe(`register:${event.hash}`) };
|
|
40
|
+
case 'RecoverLostView':
|
|
41
|
+
return { saga: recoverLostView(event.hash), lane: dedupe(`view-lost:${event.hash}`) };
|
|
40
42
|
case 'ReadDirtyMembership':
|
|
41
43
|
return { saga: readDirtyMembership(env), lane: serial('membership') };
|
|
42
44
|
case 'ReadMembership':
|
package/src/kernel/constants.ts
CHANGED
|
@@ -23,6 +23,13 @@ export const RETRY_BASE_MS = 500;
|
|
|
23
23
|
export const RETRY_MAX_MS = 15_000;
|
|
24
24
|
/** Registration attempts before a query is reported as failed to settle. */
|
|
25
25
|
export const REGISTER_MAX_RETRIES = 3;
|
|
26
|
+
/**
|
|
27
|
+
* Pacing of repeated view-lost recovery. The first re-registration after a
|
|
28
|
+
* `_00_query` row reads as gone is immediate; each further one for the same
|
|
29
|
+
* query, before the row has been seen again, waits on this backoff.
|
|
30
|
+
*/
|
|
31
|
+
export const VIEW_LOST_RETRY_BASE_MS = 1_000;
|
|
32
|
+
export const VIEW_LOST_RETRY_MAX_MS = 30_000;
|
|
26
33
|
/** `_00_list_ref` poll cadence (fallback when LIVE is quiet). */
|
|
27
34
|
export const LIST_REF_POLL_BASE_MS = 500;
|
|
28
35
|
export const LIST_REF_POLL_MAX_MS = 5_000;
|
package/src/kernel/events.ts
CHANGED
|
@@ -12,6 +12,7 @@ export type RuntimeEvent =
|
|
|
12
12
|
| { type: 'SyncOutcome'; ok: boolean; error?: unknown }
|
|
13
13
|
| { type: 'AckPrune' }
|
|
14
14
|
| { type: 'ReadDirtyMembership' }
|
|
15
|
+
| { type: 'RecoverLostView'; hash: QueryHash }
|
|
15
16
|
| { type: 'ReadMembership'; hashes: QueryHash[] }
|
|
16
17
|
| { type: 'FetchRows' }
|
|
17
18
|
| { type: 'Materialize'; hash: QueryHash }
|
|
@@ -4,7 +4,7 @@ import { runPure } from '../testing/run-pure';
|
|
|
4
4
|
import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
|
|
5
5
|
import * as R from '../state/reducers';
|
|
6
6
|
import { defaultEnv } from './env';
|
|
7
|
-
import { applyMembership, applySubqueryChildren, markMembershipDirty, readDirtyMembership, readMembership } from './membership.saga';
|
|
7
|
+
import { applyMembership, applySubqueryChildren, markMembershipDirty, readDirtyMembership, readMembership, recoverLostView } from './membership.saga';
|
|
8
8
|
import type { StatementResult } from '../kernel/effects';
|
|
9
9
|
|
|
10
10
|
const env = defaultEnv({ tables: [] } as any);
|
|
@@ -26,9 +26,51 @@ describe('applyMembership', () => {
|
|
|
26
26
|
expect(out.state.queries.get('a')!.lifecycle).toMatchObject({ phase: 'view-lost', remote: 'unregistered' });
|
|
27
27
|
expect(out.emitted).toEqual([{ type: 'query:view-lost', hash: 'a' }]);
|
|
28
28
|
expect(out.dispatched).toEqual([{ type: 'EnsureRegistered' }]);
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
expect(out.state.queries.get('a')!).toMatchObject({ viewLostCount: 1, viewLostRetryAt: null });
|
|
30
|
+
});
|
|
31
|
+
it('view-lost again before the row is seen: paced, and reads while it waits do nothing', async () => {
|
|
32
|
+
const lost = buildState([
|
|
33
|
+
buildEntry({ def: { hash: 'a' }, lifecycle: { phase: 'view-lost', remote: 'registered' }, remoteArray: [['t:1', 1]], viewLostCount: 1 }),
|
|
34
|
+
]);
|
|
35
|
+
const again = await runPure(applyMembership('a', []), { state: lost, now: 1_000 });
|
|
36
|
+
expect(again.result).toBe('view-lost');
|
|
37
|
+
expect(again.emitted.filter((e) => e.type === 'query:view-lost')).toEqual([]);
|
|
38
|
+
expect(again.dispatched).toEqual([]);
|
|
39
|
+
expect(again.state.queries.get('a')!).toMatchObject({ viewLostCount: 2, viewLostRetryAt: 2_000 });
|
|
40
|
+
expect(again.state.queries.get('a')!.lifecycle.remote).toBe('registered');
|
|
41
|
+
expect(again.timers.get('view-lost:a')).toEqual({ ms: 1_000, event: { type: 'RecoverLostView', hash: 'a' } });
|
|
42
|
+
|
|
43
|
+
const waiting = await runPure(applyMembership('a', []), { state: again.state, now: 1_500 });
|
|
44
|
+
expect(waiting.dispatched).toEqual([]);
|
|
45
|
+
expect(waiting.timers.size).toBe(0);
|
|
46
|
+
expect(waiting.state.queries.get('a')!).toMatchObject({ viewLostCount: 2, viewLostRetryAt: 2_000 });
|
|
47
|
+
|
|
48
|
+
// The next retry waits longer.
|
|
49
|
+
const third = buildState([
|
|
50
|
+
buildEntry({ def: { hash: 'a' }, lifecycle: { phase: 'view-lost', remote: 'registered' }, remoteArray: [['t:1', 1]], viewLostCount: 3 }),
|
|
51
|
+
]);
|
|
52
|
+
expect((await runPure(applyMembership('a', []), { state: third })).timers.get('view-lost:a')!.ms).toBe(4_000);
|
|
53
|
+
});
|
|
54
|
+
it('recoverLostView re-registers only while still lost and still scheduled', async () => {
|
|
55
|
+
const scheduled = (phase: 'view-lost' | 'live', retryAt: number | null) =>
|
|
56
|
+
buildState([
|
|
57
|
+
buildEntry({ def: { hash: 'a' }, lifecycle: { phase, remote: 'registered' }, remoteArray: [['t:1', 1]], viewLostCount: 2, viewLostRetryAt: retryAt }),
|
|
58
|
+
]);
|
|
59
|
+
const due = await runPure(recoverLostView('a'), { state: scheduled('view-lost', 2_000) });
|
|
60
|
+
expect(due.dispatched).toEqual([{ type: 'RegisterRemote', hash: 'a' }]);
|
|
61
|
+
expect(due.state.queries.get('a')!).toMatchObject({ viewLostRetryAt: null, viewLostCount: 2 });
|
|
62
|
+
expect(due.state.queries.get('a')!.lifecycle.remote).toBe('unregistered');
|
|
63
|
+
expect((await runPure(recoverLostView('a'), { state: scheduled('view-lost', null) })).dispatched).toEqual([]);
|
|
64
|
+
expect((await runPure(recoverLostView('a'), { state: scheduled('live', 2_000) })).dispatched).toEqual([]);
|
|
65
|
+
expect((await runPure(recoverLostView('zz'), { state: scheduled('view-lost', 2_000) })).dispatched).toEqual([]);
|
|
66
|
+
});
|
|
67
|
+
it('seeing the row again resets the pacing', async () => {
|
|
68
|
+
const s = buildState([
|
|
69
|
+
buildEntry({ def: { hash: 'a' }, lifecycle: { phase: 'view-lost', remote: 'registered' }, remoteArray: [['t:1', 1]], viewLostCount: 3, viewLostRetryAt: 9_000 }),
|
|
70
|
+
]);
|
|
71
|
+
const out = await runPure(applyMembership('a', [], { present: true, rowCount: 1, state: 'materializing' }), { state: s });
|
|
72
|
+
expect(out.result).toBe('ignored');
|
|
73
|
+
expect(out.state.queries.get('a')!).toMatchObject({ viewLostCount: 0, viewLostRetryAt: null });
|
|
32
74
|
});
|
|
33
75
|
it('applied: commits, writes the view row, flips authority once, releases acked items, asks for bodies', async () => {
|
|
34
76
|
const s = buildState(
|
|
@@ -100,6 +142,32 @@ describe('readMembership', () => {
|
|
|
100
142
|
expect(out.state.membershipDirty.size).toBe(0);
|
|
101
143
|
expect(out.dispatched.at(-1)).toEqual({ type: 'SyncOutcome', ok: true, error: undefined });
|
|
102
144
|
});
|
|
145
|
+
it('a meta statement that errored is a failed read, not a lost row', async () => {
|
|
146
|
+
const s = buildState([buildEntry({ def: { hash: 'a', id: qid('a') }, lifecycle: { phase: 'live', remote: 'registered' }, remoteArray: [['thing:1', 1]] })]);
|
|
147
|
+
const out = await runPure(readMembership(env, ['a']), {
|
|
148
|
+
state: s,
|
|
149
|
+
handlers: {
|
|
150
|
+
'remote.query': () => [ok([]), { status: 'ERR', error: 'The query was not executed due to a failed transaction' }, ok([])],
|
|
151
|
+
},
|
|
152
|
+
});
|
|
153
|
+
expect(out.result).toEqual({ changed: false, failed: true });
|
|
154
|
+
expect(out.state.queries.get('a')!.lifecycle).toMatchObject({ phase: 'live', remote: 'registered' });
|
|
155
|
+
expect(out.dispatched.filter((e) => e.type === 'EnsureRegistered')).toEqual([]);
|
|
156
|
+
expect(out.dispatched.at(-1)).toEqual({ type: 'SyncOutcome', ok: false, error: 'membership read failed' });
|
|
157
|
+
});
|
|
158
|
+
it('a batch whose meta statement errored is a failed read for every query in it', async () => {
|
|
159
|
+
const s = buildState([
|
|
160
|
+
buildEntry({ def: { hash: 'a', id: qid('a') }, lifecycle: { phase: 'cached', remote: 'registered' } }),
|
|
161
|
+
buildEntry({ def: { hash: 'b', id: qid('b') }, lifecycle: { phase: 'cached', remote: 'registered' } }),
|
|
162
|
+
]);
|
|
163
|
+
const out = await runPure(readMembership(env, ['a', 'b']), {
|
|
164
|
+
state: s,
|
|
165
|
+
handlers: { 'remote.query': () => [ok([]), { status: 'ERR', error: 'timeout' }] },
|
|
166
|
+
});
|
|
167
|
+
expect(out.result.failed).toBe(true);
|
|
168
|
+
for (const h of ['a', 'b']) expect(out.state.queries.get(h)!.lifecycle.phase).toBe('cached');
|
|
169
|
+
expect(out.dispatched.filter((e) => e.type === 'EnsureRegistered')).toEqual([]);
|
|
170
|
+
});
|
|
103
171
|
it('equal set on a live query applies nothing; a cached query with the same set still flips to live', async () => {
|
|
104
172
|
const live = buildState([buildEntry({ def: { hash: 'a', id: qid('a') }, lifecycle: { phase: 'live' }, remoteArray: [['thing:1', 1]], serverState: 'ready' })]);
|
|
105
173
|
const handlers = { 'remote.query': () => [ok([out1('1')]), ok({ rowCount: 1, state: 'ready' }), ok([])], 'local.upsert': () => undefined };
|
|
@@ -137,7 +205,7 @@ describe('readMembership', () => {
|
|
|
137
205
|
expect(out.state.queries.get('b')!.remoteArray).toEqual([['thing:2', 1], ['thing:3', 1]]);
|
|
138
206
|
expect(out.result.changed).toBe(true);
|
|
139
207
|
});
|
|
140
|
-
it('failures: a failed chunk or re-read reports failed; a non-array primary is
|
|
208
|
+
it('failures: a failed chunk or re-read reports failed; a non-array primary is a failed read', async () => {
|
|
141
209
|
const s = buildState([
|
|
142
210
|
buildEntry({ def: { hash: 'a', id: qid('a') }, lifecycle: { phase: 'live' }, remoteArray: [['thing:1', 1]] }),
|
|
143
211
|
buildEntry({ def: { hash: 'b', id: qid('b') }, lifecycle: { phase: 'live' }, remoteArray: [['thing:2', 1]] }),
|
|
@@ -167,13 +235,16 @@ describe('readMembership', () => {
|
|
|
167
235
|
expect(n).toBe(3);
|
|
168
236
|
const single = buildState([buildEntry({ def: { hash: 'a', id: qid('a') }, lifecycle: { phase: 'live' }, remoteArray: [['thing:1', 1]] })]);
|
|
169
237
|
const notArray = await runPure(readMembership(env, ['a']), { state: single, handlers: { 'remote.query': () => [ok('x'), ok(null), ok(null)] } });
|
|
170
|
-
expect(notArray.result).toEqual({ changed: false, failed:
|
|
238
|
+
expect(notArray.result).toEqual({ changed: false, failed: true });
|
|
239
|
+
expect(notArray.state.queries.get('a')!.lifecycle.phase).toBe('live');
|
|
171
240
|
const batchNotArray = await runPure(readMembership(env, ['a', 'b']), { state: s, handlers: { 'remote.query': () => [ok(null), ok(null)] } });
|
|
172
|
-
expect(batchNotArray.result).toEqual({ changed: false, failed:
|
|
241
|
+
expect(batchNotArray.result).toEqual({ changed: false, failed: true });
|
|
173
242
|
const undefinedResult = await runPure(readMembership(env, ['a']), { state: single, handlers: { 'remote.query': () => undefined } });
|
|
174
|
-
expect(undefinedResult.result).toEqual({ changed: false, failed:
|
|
243
|
+
expect(undefinedResult.result).toEqual({ changed: false, failed: true });
|
|
175
244
|
const errStmt = await runPure(readMembership(env, ['a']), { state: single, handlers: { 'remote.query': () => [{ status: 'ERR', error: 'x' }, ok(null), ok(null)] } });
|
|
176
|
-
expect(errStmt.result).toEqual({ changed: false, failed:
|
|
245
|
+
expect(errStmt.result).toEqual({ changed: false, failed: true });
|
|
246
|
+
// None of those is evidence the view is gone.
|
|
247
|
+
for (const r of [notArray, undefinedResult, errStmt]) expect(r.dispatched.filter((d) => d.type === 'EnsureRegistered')).toEqual([]);
|
|
177
248
|
});
|
|
178
249
|
it('materializing views walk the re-read ladder then stop', async () => {
|
|
179
250
|
const s = buildState([buildEntry({ def: { hash: 'a', id: qid('a') }, lifecycle: { phase: 'cold' } })]);
|
|
@@ -2,7 +2,13 @@ import type { QueryHash, RecordVersionArray, ServerViewMeta } from '../types';
|
|
|
2
2
|
import type { Saga } from '../kernel/saga';
|
|
3
3
|
import type { Settled, StatementResult } from '../kernel/effects';
|
|
4
4
|
import { fx } from '../kernel/effects';
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
MATERIALIZING_REREAD_LADDER_MS,
|
|
7
|
+
MEMBERSHIP_COALESCE_MS,
|
|
8
|
+
VIEW_LOST_RETRY_BASE_MS,
|
|
9
|
+
VIEW_LOST_RETRY_MAX_MS,
|
|
10
|
+
backoffMs,
|
|
11
|
+
} from '../kernel/constants';
|
|
6
12
|
import type { ClientState, QueryEntry } from '../state/client-state';
|
|
7
13
|
import { isAuthoritative } from '../state/lifecycle';
|
|
8
14
|
import * as R from '../state/reducers';
|
|
@@ -12,6 +18,7 @@ import type { SagaEnv } from './env';
|
|
|
12
18
|
import { listRefTable } from './env';
|
|
13
19
|
import * as sql from './sql';
|
|
14
20
|
import {
|
|
21
|
+
FAILED,
|
|
15
22
|
decideMembershipOutcome,
|
|
16
23
|
snapshotFromSingle,
|
|
17
24
|
snapshotsFromBatch,
|
|
@@ -34,6 +41,10 @@ export function* applyMembership(
|
|
|
34
41
|
const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
|
|
35
42
|
if (!entry) return 'ignored';
|
|
36
43
|
if (meta) yield fx.state.update(R.setServerState(hash, meta.present ? meta.state : null));
|
|
44
|
+
// The row is there: whatever made it read as gone has passed.
|
|
45
|
+
if (meta?.present && (entry.viewLostCount > 0 || entry.viewLostRetryAt !== null)) {
|
|
46
|
+
yield fx.state.update(R.setViewLost(hash, 0, null));
|
|
47
|
+
}
|
|
37
48
|
const outcome = decideMembershipOutcome({
|
|
38
49
|
phase: entry.lifecycle.phase,
|
|
39
50
|
held: entry.remoteArray.length,
|
|
@@ -47,10 +58,30 @@ export function* applyMembership(
|
|
|
47
58
|
yield fx.state.update(R.applyLifecycle(hash, { type: 'row-missing' }));
|
|
48
59
|
yield fx.emit({ type: 'query:view-lost', hash });
|
|
49
60
|
}
|
|
50
|
-
|
|
51
|
-
|
|
61
|
+
// A recovery is already scheduled: every read in the meantime would
|
|
62
|
+
// otherwise re-register again.
|
|
63
|
+
if (entry.viewLostRetryAt !== null) return outcome;
|
|
64
|
+
if (entry.viewLostCount === 0) {
|
|
65
|
+
yield fx.state.update(R.setViewLost(hash, 1, null));
|
|
66
|
+
if (entry.lifecycle.remote !== 'unregistered') {
|
|
67
|
+
yield fx.state.update(R.applyLifecycle(hash, { type: 'remote-dropped' }));
|
|
68
|
+
}
|
|
69
|
+
yield fx.dispatch({ type: 'EnsureRegistered' });
|
|
70
|
+
return outcome;
|
|
52
71
|
}
|
|
53
|
-
|
|
72
|
+
// Re-registered already and the row still reads as gone. Under load that
|
|
73
|
+
// repeated every read, each one costing the server a registration while it
|
|
74
|
+
// was still publishing the last, so from here on it is paced.
|
|
75
|
+
const delay = backoffMs(entry.viewLostCount - 1, VIEW_LOST_RETRY_BASE_MS, VIEW_LOST_RETRY_MAX_MS);
|
|
76
|
+
const now = (yield fx.now()) as number;
|
|
77
|
+
yield fx.state.update(R.setViewLost(hash, entry.viewLostCount + 1, now + delay));
|
|
78
|
+
yield fx.emit({
|
|
79
|
+
type: 'log',
|
|
80
|
+
level: 'debug',
|
|
81
|
+
message: 'view still lost; re-registering after a delay',
|
|
82
|
+
data: { hash, attempt: entry.viewLostCount + 1, delayMs: delay },
|
|
83
|
+
});
|
|
84
|
+
yield fx.timer.set(`view-lost:${hash}`, delay, { type: 'RecoverLostView', hash });
|
|
54
85
|
return outcome;
|
|
55
86
|
}
|
|
56
87
|
const wasAuthoritative = isAuthoritative(entry.lifecycle);
|
|
@@ -66,6 +97,21 @@ export function* applyMembership(
|
|
|
66
97
|
return outcome;
|
|
67
98
|
}
|
|
68
99
|
|
|
100
|
+
/**
|
|
101
|
+
* The paced view-lost re-registration. A no-op when the row has been seen
|
|
102
|
+
* since (the pacing was reset) or the query recovered or went away.
|
|
103
|
+
*/
|
|
104
|
+
export function* recoverLostView(hash: QueryHash): Saga<void> {
|
|
105
|
+
const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
|
|
106
|
+
if (!entry || entry.viewLostRetryAt === null) return;
|
|
107
|
+
yield fx.state.update(R.setViewLost(hash, entry.viewLostCount, null));
|
|
108
|
+
if (entry.lifecycle.phase !== 'view-lost') return;
|
|
109
|
+
if (entry.lifecycle.remote !== 'unregistered') {
|
|
110
|
+
yield fx.state.update(R.applyLifecycle(hash, { type: 'remote-dropped' }));
|
|
111
|
+
}
|
|
112
|
+
yield fx.dispatch({ type: 'RegisterRemote', hash });
|
|
113
|
+
}
|
|
114
|
+
|
|
69
115
|
/** Replace the subquery child set; bodies follow through the fetch plan. */
|
|
70
116
|
export function* applySubqueryChildren(hash: QueryHash, children: RecordVersionArray): Saga<void> {
|
|
71
117
|
const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
|
|
@@ -75,10 +121,11 @@ export function* applySubqueryChildren(hash: QueryHash, children: RecordVersionA
|
|
|
75
121
|
}
|
|
76
122
|
|
|
77
123
|
const ok = (r: Settled): r is { ok: true; value: unknown } => r.ok;
|
|
124
|
+
/** One statement's result, or `FAILED` when it did not answer (see `FAILED`). */
|
|
78
125
|
const stmt = (results: unknown, i: number): unknown => {
|
|
79
|
-
if (!Array.isArray(results)) return
|
|
126
|
+
if (!Array.isArray(results)) return FAILED;
|
|
80
127
|
const r = (results as StatementResult[])[i];
|
|
81
|
-
return r && r.status === 'OK' ? r.result :
|
|
128
|
+
return r && r.status === 'OK' ? r.result : FAILED;
|
|
82
129
|
};
|
|
83
130
|
|
|
84
131
|
/**
|
|
@@ -123,11 +170,17 @@ export function* readMembership(
|
|
|
123
170
|
if (chunk.length === 1) {
|
|
124
171
|
const snap = snapshotFromSingle(stmt(r.value, 0) as never, stmt(r.value, 1) as never, stmt(r.value, 2) as never);
|
|
125
172
|
if (snap) snapshots.set(chunk[0], snap);
|
|
173
|
+
else failed = true;
|
|
126
174
|
return;
|
|
127
175
|
}
|
|
128
176
|
const hashById = new Map(chunk.map((h) => [encodeRecordId(byHash.get(h)!.def.id), h]));
|
|
129
177
|
const edges = stmt(r.value, 0) as never[] | null;
|
|
130
|
-
const
|
|
178
|
+
const counts = stmt(r.value, 1);
|
|
179
|
+
if (!Array.isArray(edges) || counts === FAILED) {
|
|
180
|
+
failed = true;
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
const batch = snapshotsFromBatch(edges, counts as never, hashById);
|
|
131
184
|
const held = new Map(chunk.map((h) => [h, byHash.get(h)!.remoteArray.length]));
|
|
132
185
|
const suspect = new Set(suspectHashes(batch, held, Array.isArray(edges) ? edges.length : 0));
|
|
133
186
|
for (const [h, snap] of batch) {
|
|
@@ -144,6 +197,7 @@ export function* readMembership(
|
|
|
144
197
|
}
|
|
145
198
|
const snap = snapshotFromSingle(stmt(r.value, 0) as never, stmt(r.value, 1) as never, stmt(r.value, 2) as never);
|
|
146
199
|
if (snap) snapshots.set(rereads[i].def.hash, snap);
|
|
200
|
+
else failed = true;
|
|
147
201
|
});
|
|
148
202
|
}
|
|
149
203
|
|
package/src/query/membership.ts
CHANGED
|
@@ -76,6 +76,15 @@ export function decideMembershipOutcome(input: MembershipDecisionInput): Members
|
|
|
76
76
|
|
|
77
77
|
// ---- list_ref snapshots ------------------------------------------------------
|
|
78
78
|
|
|
79
|
+
/**
|
|
80
|
+
* A statement that did not answer (an ERR, or no result at all). Kept apart
|
|
81
|
+
* from a `null` result, which is the server saying "no such row": reading a
|
|
82
|
+
* failed statement as a missing row turned every timeout under load into a
|
|
83
|
+
* lost view and re-registered it, over and over (whitepawn, 2026-09-16).
|
|
84
|
+
*/
|
|
85
|
+
export const FAILED: unique symbol = Symbol('statement failed');
|
|
86
|
+
export type Answer<T> = T | null | typeof FAILED;
|
|
87
|
+
|
|
79
88
|
export interface ListRefEdgeRow {
|
|
80
89
|
in: RecordId<string>;
|
|
81
90
|
out: RecordId<string>;
|
|
@@ -92,28 +101,32 @@ export interface ListRefSnapshot {
|
|
|
92
101
|
const toPairs = (rows: Array<{ out: RecordId<string>; version: number }> | null | undefined): RecordVersionArray =>
|
|
93
102
|
dedupeRecordVersions(Array.isArray(rows) ? rows.map((r) => [encodeRecordId(r.out), r.version]) : []);
|
|
94
103
|
|
|
95
|
-
/**
|
|
104
|
+
/**
|
|
105
|
+
* Fold the single-query statement batch (edges, meta, children) into a
|
|
106
|
+
* snapshot. `null` when any of the three did not answer: half an answer is
|
|
107
|
+
* not one, and a missing meta would read as a lost view.
|
|
108
|
+
*/
|
|
96
109
|
export function snapshotFromSingle(
|
|
97
|
-
items: Array<{ out: RecordId<string>; version: number }
|
|
98
|
-
metaRow: QueryMetaRow
|
|
99
|
-
children: Array<{ out: RecordId<string>; version: number }
|
|
110
|
+
items: Answer<Array<{ out: RecordId<string>; version: number }>>,
|
|
111
|
+
metaRow: Answer<QueryMetaRow>,
|
|
112
|
+
children: Answer<Array<{ out: RecordId<string>; version: number }>>
|
|
100
113
|
): ListRefSnapshot | null {
|
|
101
|
-
if (!Array.isArray(items)) return null;
|
|
114
|
+
if (!Array.isArray(items) || metaRow === FAILED || children === FAILED) return null;
|
|
102
115
|
return { primary: toPairs(items), subquery: toPairs(children), meta: metaFromRow(metaRow) };
|
|
103
116
|
}
|
|
104
117
|
|
|
105
118
|
/**
|
|
106
119
|
* Fold the many-query statement batch into one snapshot per hash. Every hash
|
|
107
120
|
* in `hashById` gets a snapshot; a query whose row did not come back reads
|
|
108
|
-
* `present: false`.
|
|
121
|
+
* `present: false`. Empty when either statement did not answer.
|
|
109
122
|
*/
|
|
110
123
|
export function snapshotsFromBatch(
|
|
111
|
-
edges: ListRefEdgeRow[]
|
|
112
|
-
counts: Array<QueryMetaRow | null
|
|
124
|
+
edges: Answer<ListRefEdgeRow[]>,
|
|
125
|
+
counts: Answer<Array<QueryMetaRow | null>>,
|
|
113
126
|
hashById: ReadonlyMap<string, string>
|
|
114
127
|
): Map<string, ListRefSnapshot> {
|
|
115
128
|
const out = new Map<string, ListRefSnapshot>();
|
|
116
|
-
if (!Array.isArray(edges)) return out;
|
|
129
|
+
if (!Array.isArray(edges) || counts === FAILED) return out;
|
|
117
130
|
for (const hash of hashById.values()) {
|
|
118
131
|
out.set(hash, { primary: [], subquery: [], meta: { present: false, rowCount: null, state: null } });
|
|
119
132
|
}
|
|
@@ -193,16 +193,20 @@ describe('registerRemote', () => {
|
|
|
193
193
|
handlers: { 'remote.query': () => [{ status: 'ERR', error: 'denied' }] },
|
|
194
194
|
});
|
|
195
195
|
expect(errStmt.state.queries.get('a')!.registerAttempts).toBe(1);
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
state: buildState([entry]),
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
196
|
+
// The register statement answered but the read-back did not: the query
|
|
197
|
+
// IS registered, and its membership is read again rather than guessed.
|
|
198
|
+
for (const readBack of [
|
|
199
|
+
[okStmt(null), { status: 'ERR', error: 'no table' }, okStmt(null), okStmt(null)],
|
|
200
|
+
[okStmt(null), okStmt('nope'), okStmt(null), okStmt(null)],
|
|
201
|
+
] as StatementResult[][]) {
|
|
202
|
+
const out = await runPure(registerRemote(env, 'a'), { state: buildState([entry]), handlers: { 'remote.query': () => readBack } });
|
|
203
|
+
const e = out.state.queries.get('a')!;
|
|
204
|
+
expect(e.registerAttempts).toBe(0);
|
|
205
|
+
expect(e.lifecycle).toMatchObject({ remote: 'registered', fetchDepth: 0 });
|
|
206
|
+
expect(out.state.membershipDirty.has('a')).toBe(true);
|
|
207
|
+
expect(out.timers.get('membership')).toEqual({ ms: 50, event: { type: 'ReadDirtyMembership' } });
|
|
208
|
+
expect(out.dispatched).toContainEqual({ type: 'SyncOutcome', ok: false, error: 'registration read-back failed' });
|
|
209
|
+
}
|
|
206
210
|
const empty = await runPure(registerRemote(env, 'a'), { state: buildState([entry]), handlers: { 'remote.query': () => [] } });
|
|
207
211
|
expect(empty.state.queries.get('a')!.registerAttempts).toBe(1);
|
|
208
212
|
const last = await runPure(registerRemote(env, 'a'), {
|
|
@@ -246,18 +250,22 @@ describe('registerRemote', () => {
|
|
|
246
250
|
expect(isNotAllowlisted('HTTP 4030')).toBe(false);
|
|
247
251
|
});
|
|
248
252
|
|
|
249
|
-
it('
|
|
250
|
-
const
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
253
|
+
it('an ERR meta or children statement applies nothing and re-reads, instead of reading as a lost row', async () => {
|
|
254
|
+
const lostBefore = buildEntry({ def: { hash: 'a' }, lifecycle: { phase: 'view-lost' }, remoteArray: [['thing:1', 1]], viewLostCount: 1 });
|
|
255
|
+
for (const [meta, kids] of [
|
|
256
|
+
[{ status: 'ERR', error: 'meta' }, okStmt([])],
|
|
257
|
+
[okStmt({ rowCount: 1, state: 'ready' }), { status: 'ERR', error: 'kids' }],
|
|
258
|
+
] as StatementResult[][]) {
|
|
259
|
+
const out = await runPure(registerRemote(env, 'a'), {
|
|
260
|
+
state: buildState([lostBefore]),
|
|
261
|
+
handlers: { 'remote.query': () => [okStmt(null), okStmt([{ out: new RecordId('thing', '1'), version: 1 }]), meta, kids] },
|
|
262
|
+
});
|
|
263
|
+
const e = out.state.queries.get('a')!;
|
|
264
|
+
expect(e.lifecycle).toMatchObject({ phase: 'view-lost', remote: 'registered' });
|
|
265
|
+
expect(e.viewLostCount).toBe(1);
|
|
266
|
+
expect(out.dispatched.filter((d) => d.type === 'EnsureRegistered')).toEqual([]);
|
|
267
|
+
expect(out.state.membershipDirty.has('a')).toBe(true);
|
|
268
|
+
}
|
|
261
269
|
});
|
|
262
270
|
it('stops when the entry disappears mid-flight', async () => {
|
|
263
271
|
const out = await runPure(registerRemote(env, 'a'), {
|
|
@@ -14,8 +14,8 @@ import { parseDuration } from '../utils/index';
|
|
|
14
14
|
import type { SagaEnv } from './env';
|
|
15
15
|
import { listRefTable } from './env';
|
|
16
16
|
import { queryHashInput, viewKeyInput } from './hash';
|
|
17
|
-
import { isResolvedBefore, parseViewRow, snapshotFromSingle } from './membership';
|
|
18
|
-
import { applyMembership, applySubqueryChildren } from './membership.saga';
|
|
17
|
+
import { FAILED, isResolvedBefore, parseViewRow, snapshotFromSingle } from './membership';
|
|
18
|
+
import { applyMembership, applySubqueryChildren, markMembershipDirty } from './membership.saga';
|
|
19
19
|
import * as sql from './sql';
|
|
20
20
|
|
|
21
21
|
export interface RegisterInput {
|
|
@@ -86,6 +86,8 @@ export function* registerLocal(env: SagaEnv, input: RegisterInput): Saga<QueryHa
|
|
|
86
86
|
lastHeartbeatAt: null,
|
|
87
87
|
lastPolledAt: null,
|
|
88
88
|
registerAttempts: 0,
|
|
89
|
+
viewLostCount: 0,
|
|
90
|
+
viewLostRetryAt: null,
|
|
89
91
|
telemetry: { ...emptyTelemetry(), registrationTimings: reg.timings },
|
|
90
92
|
};
|
|
91
93
|
yield fx.state.update(R.putQuery(entry));
|
|
@@ -163,15 +165,23 @@ export function* registerRemote(env: SagaEnv, hash: QueryHash, retry = false): S
|
|
|
163
165
|
if (!stillThere) return;
|
|
164
166
|
const register = stmt(results, 0);
|
|
165
167
|
if (!register || register.status === 'ERR') throw new Error(register?.status === 'ERR' ? register.error : 'register returned nothing');
|
|
166
|
-
const
|
|
167
|
-
const meta = stmt(results, 2);
|
|
168
|
-
const children = stmt(results, 3);
|
|
168
|
+
const answer = (r: StatementResult | undefined): unknown => (r?.status === 'OK' ? r.result : FAILED);
|
|
169
169
|
const snap = snapshotFromSingle(
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
170
|
+
answer(stmt(results, 1)) as never,
|
|
171
|
+
answer(stmt(results, 2)) as never,
|
|
172
|
+
answer(stmt(results, 3)) as never
|
|
173
173
|
);
|
|
174
|
-
if (!snap)
|
|
174
|
+
if (!snap) {
|
|
175
|
+
// Registered, but the read-back did not answer. That says nothing about
|
|
176
|
+
// the row, so read membership again rather than guess: a guess of
|
|
177
|
+
// "gone" re-registered the query, and under load the next read-back
|
|
178
|
+
// failed the same way.
|
|
179
|
+
yield fx.state.update(R.compose(R.applyLifecycle(hash, { type: 'remote-registered' }), R.resetRegisterAttempts(hash)));
|
|
180
|
+
yield fx.emit({ type: 'log', level: 'debug', message: 'registration read-back did not answer; re-reading membership', data: { hash } });
|
|
181
|
+
yield* markMembershipDirty([hash]);
|
|
182
|
+
yield fx.dispatch({ type: 'SyncOutcome', ok: false, error: 'registration read-back failed' });
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
175
185
|
const outcome = yield* applyMembership(hash, snap.primary, snap.meta);
|
|
176
186
|
if (outcome === 'ignored' && snap.meta.present && snap.meta.state === 'materializing') {
|
|
177
187
|
yield fx.state.update(R.compose(R.markMembershipDirty([hash]), R.setMembershipReread(hash, 1)));
|
|
@@ -58,6 +58,13 @@ export interface QueryEntry {
|
|
|
58
58
|
readonly lastHeartbeatAt: number | null;
|
|
59
59
|
readonly lastPolledAt: number | null;
|
|
60
60
|
readonly registerAttempts: number;
|
|
61
|
+
/**
|
|
62
|
+
* Re-registrations issued because the `_00_query` row read as gone, since
|
|
63
|
+
* the row was last seen. Paces the next one (see `applyMembership`).
|
|
64
|
+
*/
|
|
65
|
+
readonly viewLostCount: number;
|
|
66
|
+
/** When the paced re-registration is due; null when none is pending. */
|
|
67
|
+
readonly viewLostRetryAt: number | null;
|
|
61
68
|
readonly telemetry: QueryTelemetry;
|
|
62
69
|
}
|
|
63
70
|
|
package/src/state/reducers.ts
CHANGED
|
@@ -188,6 +188,16 @@ export const bumpRegisterAttempts =
|
|
|
188
188
|
(s) =>
|
|
189
189
|
withEntry(s, hash, (e) => ({ ...e, registerAttempts: e.registerAttempts + 1 }));
|
|
190
190
|
|
|
191
|
+
/** View-lost recovery pacing; see `QueryEntry.viewLostCount`. */
|
|
192
|
+
export const setViewLost =
|
|
193
|
+
(hash: QueryHash, viewLostCount: number, viewLostRetryAt: number | null): Reducer =>
|
|
194
|
+
(s) =>
|
|
195
|
+
withEntry(s, hash, (e) =>
|
|
196
|
+
e.viewLostCount === viewLostCount && e.viewLostRetryAt === viewLostRetryAt
|
|
197
|
+
? e
|
|
198
|
+
: { ...e, viewLostCount, viewLostRetryAt }
|
|
199
|
+
);
|
|
200
|
+
|
|
191
201
|
export const resetRegisterAttempts =
|
|
192
202
|
(hash: QueryHash): Reducer =>
|
|
193
203
|
(s) =>
|
|
@@ -421,6 +431,8 @@ export const rebindQuery =
|
|
|
421
431
|
lastHeartbeatAt: null,
|
|
422
432
|
lastPolledAt: null,
|
|
423
433
|
registerAttempts: 0,
|
|
434
|
+
viewLostCount: 0,
|
|
435
|
+
viewLostRetryAt: null,
|
|
424
436
|
}));
|
|
425
437
|
return rebound === s ? s : { ...rebound, dirty: addAll(rebound.dirty, [hash]) };
|
|
426
438
|
};
|
package/src/testing/build.ts
CHANGED
|
@@ -38,6 +38,8 @@ export function buildEntry(over: DeepPartial<QueryEntry> & { def?: Partial<Query
|
|
|
38
38
|
lastHeartbeatAt: over.lastHeartbeatAt ?? null,
|
|
39
39
|
lastPolledAt: over.lastPolledAt ?? null,
|
|
40
40
|
registerAttempts: over.registerAttempts ?? 0,
|
|
41
|
+
viewLostCount: over.viewLostCount ?? 0,
|
|
42
|
+
viewLostRetryAt: over.viewLostRetryAt ?? null,
|
|
41
43
|
telemetry: { ...emptyTelemetry(), ...(over.telemetry as object) },
|
|
42
44
|
};
|
|
43
45
|
}
|