@spooky-sync/core 0.0.1-canary.34 → 0.0.1-canary.340

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (269) hide show
  1. package/AGENTS.md +90 -0
  2. package/README.md +59 -16
  3. package/dist/index.d.ts +2115 -142
  4. package/dist/index.js +16427 -2575
  5. package/dist/live.d.ts +149 -0
  6. package/dist/live.js +739 -0
  7. package/dist/otel/index.d.ts +1 -1
  8. package/dist/otel/index.js +3 -3
  9. package/dist/pure.d.ts +389 -0
  10. package/dist/pure.js +24 -0
  11. package/dist/sqlite-open.js +392 -0
  12. package/dist/sqlite-worker.d.ts +1 -0
  13. package/dist/sqlite-worker.js +439 -0
  14. package/dist/sw.d.ts +207 -0
  15. package/dist/sw.js +393 -0
  16. package/dist/tabs-broker-worker.d.ts +8 -0
  17. package/dist/tabs-broker-worker.js +477 -0
  18. package/dist/types.d.ts +1917 -38
  19. package/dist/worker-edges.js +480 -0
  20. package/dist/worker-impersonation.js +60 -0
  21. package/dist/worker-ref-tables.d.ts +48 -0
  22. package/dist/worker-types.d.ts +112 -0
  23. package/dist/worker-types.js +73 -0
  24. package/package.json +28 -7
  25. package/scripts/check-broker-bundle.mjs +33 -0
  26. package/scripts/check-worker-entries.mjs +173 -0
  27. package/skills/sp00ky-core/SKILL.md +37 -7
  28. package/skills/sp00ky-core/references/config.md +7 -1
  29. package/src/boot/auth-flip.saga.test.ts +135 -0
  30. package/src/boot/auth-flip.saga.ts +77 -0
  31. package/src/boot/boot.saga.test.ts +203 -0
  32. package/src/boot/boot.saga.ts +146 -0
  33. package/src/boot/bucket-switch.saga.test.ts +189 -0
  34. package/src/boot/bucket-switch.saga.ts +136 -0
  35. package/src/boot/preload.saga.test.ts +80 -0
  36. package/src/boot/preload.saga.ts +40 -0
  37. package/src/bucket-blurhash.test.ts +148 -0
  38. package/src/build-globals.d.ts +12 -0
  39. package/src/client/bucket-handle.test.ts +195 -0
  40. package/src/client/bucket-handle.ts +352 -0
  41. package/src/client/router.test.ts +58 -0
  42. package/src/client/router.ts +100 -0
  43. package/src/client/runtime.test.ts +235 -0
  44. package/src/client/runtime.ts +296 -0
  45. package/src/client/services.test.ts +74 -0
  46. package/src/client/services.ts +435 -0
  47. package/src/client/sp00ky-client.test.ts +341 -0
  48. package/src/client/sp00ky-client.ts +625 -0
  49. package/src/events/events.test.ts +2 -1
  50. package/src/events/index.ts +3 -0
  51. package/src/index.ts +69 -1
  52. package/src/kernel/constants.test.ts +19 -0
  53. package/src/kernel/constants.ts +80 -0
  54. package/src/kernel/effects.test.ts +40 -0
  55. package/src/kernel/effects.ts +167 -0
  56. package/src/kernel/events.ts +57 -0
  57. package/src/kernel/interpreter.test.ts +116 -0
  58. package/src/kernel/interpreter.ts +148 -0
  59. package/src/kernel/saga.test.ts +89 -0
  60. package/src/kernel/saga.ts +73 -0
  61. package/src/live/index.test.ts +512 -0
  62. package/src/live/index.ts +1059 -0
  63. package/src/modules/app-release/index.test.ts +125 -0
  64. package/src/modules/app-release/index.ts +209 -0
  65. package/src/modules/auth/auth.local-first.test.ts +228 -0
  66. package/src/modules/auth/auth.session-recheck.test.ts +140 -0
  67. package/src/modules/auth/events/index.ts +9 -2
  68. package/src/modules/auth/impersonation-banner.ts +443 -0
  69. package/src/modules/auth/impersonation.integration.test.ts +205 -0
  70. package/src/modules/auth/impersonation.test.ts +377 -0
  71. package/src/modules/auth/impersonation.ts +179 -0
  72. package/src/modules/auth/index.ts +543 -22
  73. package/src/modules/crdt/acceptance.ts +75 -0
  74. package/src/modules/crdt/bytes.ts +50 -0
  75. package/src/modules/crdt/crdt-field.ts +125 -0
  76. package/src/modules/crdt/crdt.test.ts +515 -0
  77. package/src/modules/crdt/hash.ts +103 -0
  78. package/src/modules/crdt/index.ts +793 -0
  79. package/src/modules/crdt/loro-loader.ts +25 -0
  80. package/src/modules/crdt/mirror.ts +37 -0
  81. package/src/modules/devtools/flags.ts +349 -0
  82. package/src/modules/devtools/index.ts +748 -95
  83. package/src/modules/devtools/mutations.test.ts +92 -0
  84. package/src/modules/devtools/mutations.ts +160 -0
  85. package/src/modules/devtools/notify-throttle.test.ts +166 -0
  86. package/src/modules/devtools/state-shape.test.ts +263 -0
  87. package/src/modules/devtools/storage-info.test.ts +79 -0
  88. package/src/modules/devtools/storage-info.ts +168 -0
  89. package/src/modules/devtools/versions.test.ts +74 -0
  90. package/src/modules/devtools/versions.ts +110 -0
  91. package/src/modules/feature-flag/index.test.ts +298 -0
  92. package/src/modules/feature-flag/index.ts +316 -0
  93. package/src/modules/ref-tables.test.ts +93 -0
  94. package/src/modules/ref-tables.ts +91 -0
  95. package/src/modules/web-push/index.test.ts +824 -0
  96. package/src/modules/web-push/index.ts +1027 -0
  97. package/src/mutation/ingest-outage.integration.test.ts +400 -0
  98. package/src/mutation/jobs.test.ts +25 -0
  99. package/src/mutation/jobs.ts +41 -0
  100. package/src/mutation/mutation-id.test.ts +37 -0
  101. package/src/mutation/mutation-id.ts +35 -0
  102. package/src/mutation/pending-writes.integration.test.ts +175 -0
  103. package/src/mutation/push.saga.test.ts +361 -0
  104. package/src/mutation/push.saga.ts +245 -0
  105. package/src/mutation/rows.test.ts +278 -0
  106. package/src/mutation/rows.ts +499 -0
  107. package/src/mutation/tray.saga.test.ts +63 -0
  108. package/src/mutation/tray.saga.ts +48 -0
  109. package/src/mutation/write.saga.test.ts +316 -0
  110. package/src/mutation/write.saga.ts +268 -0
  111. package/src/otel/index.ts +10 -7
  112. package/src/pure.ts +66 -0
  113. package/src/push/types.ts +174 -0
  114. package/src/query/anon-query-isolation.integration.test.ts +531 -0
  115. package/src/query/env.test.ts +23 -0
  116. package/src/query/env.ts +65 -0
  117. package/src/query/fetch.saga.test.ts +199 -0
  118. package/src/query/fetch.saga.ts +120 -0
  119. package/src/query/hash.test.ts +14 -0
  120. package/src/query/hash.ts +18 -0
  121. package/src/query/id-set-plan.test.ts +132 -0
  122. package/src/query/lifecycle.saga.test.ts +274 -0
  123. package/src/query/lifecycle.saga.ts +186 -0
  124. package/src/query/list-ref-live.integration.test.ts +203 -0
  125. package/src/query/materialize.saga.test.ts +188 -0
  126. package/src/query/materialize.saga.ts +96 -0
  127. package/src/query/materialize.test.ts +58 -0
  128. package/src/query/materialize.ts +57 -0
  129. package/src/query/membership.saga.test.ts +482 -0
  130. package/src/query/membership.saga.ts +315 -0
  131. package/src/query/membership.test.ts +151 -0
  132. package/src/query/membership.ts +234 -0
  133. package/src/query/register.saga.test.ts +312 -0
  134. package/src/query/register.saga.ts +235 -0
  135. package/src/query/render-set.test.ts +47 -0
  136. package/src/query/render-set.ts +71 -0
  137. package/src/query/session-guard.integration.test.ts +304 -0
  138. package/src/query/sql.test.ts +118 -0
  139. package/src/query/sql.ts +157 -0
  140. package/src/query/window-query.test.ts +70 -0
  141. package/src/query/window-query.ts +208 -0
  142. package/src/services/blobs/blob-cache.test.ts +359 -0
  143. package/src/services/blobs/blob-cache.ts +603 -0
  144. package/src/services/blobs/blob-manifest.ts +227 -0
  145. package/src/services/blobs/blob-store.test.ts +77 -0
  146. package/src/services/blobs/blob-store.ts +359 -0
  147. package/src/services/blobs/blob.fixture.ts +90 -0
  148. package/src/services/blobs/index.ts +70 -0
  149. package/src/services/database/cache-engine.ts +209 -0
  150. package/src/services/database/connection-supervisor.test.ts +318 -0
  151. package/src/services/database/connection-supervisor.ts +422 -0
  152. package/src/services/database/database.query-timeout.test.ts +140 -0
  153. package/src/services/database/database.responses.test.ts +43 -0
  154. package/src/services/database/database.ts +156 -61
  155. package/src/services/database/engine-factory.ts +33 -0
  156. package/src/services/database/errors.ts +34 -0
  157. package/src/services/database/events/index.ts +2 -1
  158. package/src/services/database/index.ts +7 -0
  159. package/src/services/database/landed-body-clear.integration.test.ts +236 -0
  160. package/src/services/database/local-migrator.integration.test.ts +110 -0
  161. package/src/services/database/local-migrator.ts +61 -16
  162. package/src/services/database/local.test.ts +150 -0
  163. package/src/services/database/local.ts +508 -63
  164. package/src/services/database/plan-render.test.ts +159 -0
  165. package/src/services/database/plan-render.ts +108 -0
  166. package/src/services/database/relation-resolver.test.ts +484 -0
  167. package/src/services/database/relation-resolver.ts +0 -0
  168. package/src/services/database/remote.ts +285 -6
  169. package/src/services/database/sqlite-cache-engine.test.ts +628 -0
  170. package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
  171. package/src/services/database/sqlite-cache-engine.ts +1401 -0
  172. package/src/services/database/sqlite-column-reset.integration.test.ts +100 -0
  173. package/src/services/database/sqlite-column-reset.ts +91 -0
  174. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  175. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  176. package/src/services/database/sqlite-lock-verify.test.ts +33 -0
  177. package/src/services/database/sqlite-lock-verify.ts +45 -0
  178. package/src/services/database/sqlite-open.test.ts +150 -0
  179. package/src/services/database/sqlite-open.ts +164 -0
  180. package/src/services/database/sqlite-plan-sql.test.ts +164 -0
  181. package/src/services/database/sqlite-plan-sql.ts +196 -0
  182. package/src/services/database/sqlite-projection.test.ts +121 -0
  183. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  184. package/src/services/database/sqlite-select.test.ts +261 -0
  185. package/src/services/database/sqlite-select.ts +131 -0
  186. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  187. package/src/services/database/sqlite-transport.ts +224 -0
  188. package/src/services/database/sqlite-worker.ts +437 -0
  189. package/src/services/database/surql-translate.ts +417 -0
  190. package/src/services/database/surreal-cache-engine.ts +163 -0
  191. package/src/services/logger/index.test.ts +132 -0
  192. package/src/services/logger/index.ts +189 -7
  193. package/src/services/persistence/localstorage.ts +2 -2
  194. package/src/services/persistence/resilient.ts +10 -3
  195. package/src/services/persistence/surrealdb.ts +4 -4
  196. package/src/services/stream-processor/index.ts +806 -67
  197. package/src/services/stream-processor/permissions.test.ts +47 -0
  198. package/src/services/stream-processor/permissions.ts +53 -0
  199. package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
  200. package/src/services/stream-processor/stream-processor.prime.test.ts +234 -0
  201. package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
  202. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  203. package/src/services/stream-processor/wasm-types.ts +58 -2
  204. package/src/services/tabs/broker-client.ts +284 -0
  205. package/src/services/tabs/broker.test.ts +381 -0
  206. package/src/services/tabs/coordinator.test.ts +386 -0
  207. package/src/services/tabs/coordinator.ts +646 -0
  208. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  209. package/src/services/tabs/leader-locks.ts +75 -0
  210. package/src/services/tabs/protocol.ts +262 -0
  211. package/src/services/tabs/support.ts +36 -0
  212. package/src/services/tabs/tabs-broker-worker.ts +652 -0
  213. package/src/sp00ky.ts +5 -395
  214. package/src/state/client-state.ts +216 -0
  215. package/src/state/lifecycle.test.ts +184 -0
  216. package/src/state/lifecycle.ts +127 -0
  217. package/src/state/reducers.test.ts +319 -0
  218. package/src/state/reducers.ts +550 -0
  219. package/src/state/selectors.test.ts +188 -0
  220. package/src/state/selectors.ts +259 -0
  221. package/src/sw/index.test.ts +582 -0
  222. package/src/sw/index.ts +607 -0
  223. package/src/sw/store.ts +96 -0
  224. package/src/sync/connection.saga.test.ts +97 -0
  225. package/src/sync/connection.saga.ts +89 -0
  226. package/src/sync/live.saga.test.ts +229 -0
  227. package/src/sync/live.saga.ts +116 -0
  228. package/src/sync/policy.health.test.ts +56 -0
  229. package/src/sync/policy.test.ts +663 -0
  230. package/src/sync/policy.ts +515 -0
  231. package/src/sync/poll.saga.test.ts +51 -0
  232. package/src/sync/poll.saga.ts +36 -0
  233. package/src/sync/tabs.saga.test.ts +78 -0
  234. package/src/sync/tabs.saga.ts +96 -0
  235. package/src/testing/adapters.ts +137 -0
  236. package/src/testing/build.ts +63 -0
  237. package/src/testing/fake-services.ts +63 -0
  238. package/src/testing/run-pure.test.ts +92 -0
  239. package/src/testing/run-pure.ts +128 -0
  240. package/src/testing/services.ts +17 -0
  241. package/src/types.ts +661 -10
  242. package/src/utils/blurhash.ts +90 -0
  243. package/src/utils/edges.ts +57 -0
  244. package/src/utils/error-classification.test.ts +105 -0
  245. package/src/utils/error-classification.ts +64 -0
  246. package/src/utils/index.ts +89 -12
  247. package/src/utils/parser.test.ts +67 -102
  248. package/src/utils/parser.ts +64 -2
  249. package/src/utils/semver.test.ts +32 -0
  250. package/src/utils/semver.ts +30 -0
  251. package/src/utils/sha256.ts +15 -0
  252. package/src/utils/surql.ts +29 -18
  253. package/src/utils/utils.test.ts +3 -2
  254. package/tsdown.config.ts +117 -3
  255. package/vitest.config.ts +48 -0
  256. package/src/modules/cache/index.ts +0 -241
  257. package/src/modules/cache/types.ts +0 -19
  258. package/src/modules/data/data.test.ts +0 -58
  259. package/src/modules/data/index.ts +0 -879
  260. package/src/modules/sync/engine.ts +0 -173
  261. package/src/modules/sync/events/index.ts +0 -77
  262. package/src/modules/sync/index.ts +0 -3
  263. package/src/modules/sync/queue/index.ts +0 -2
  264. package/src/modules/sync/queue/queue-down.ts +0 -89
  265. package/src/modules/sync/queue/queue-up.ts +0 -223
  266. package/src/modules/sync/scheduler.ts +0 -84
  267. package/src/modules/sync/sync.ts +0 -421
  268. package/src/modules/sync/utils.test.ts +0 -311
  269. package/src/modules/sync/utils.ts +0 -167
@@ -0,0 +1,315 @@
1
+ import type { EdgeDelta, QueryHash, RecordVersionArray, ServerViewMeta } from '../types';
2
+ import type { Saga } from '../kernel/saga';
3
+ import type { Settled, StatementResult } from '../kernel/effects';
4
+ import { fx } from '../kernel/effects';
5
+ import {
6
+ MATERIALIZING_REREAD_LADDER_MS,
7
+ MEMBERSHIP_COALESCE_MS,
8
+ VIEW_LOST_RETRY_BASE_MS,
9
+ VIEW_LOST_RETRY_MAX_MS,
10
+ backoffMs,
11
+ } from '../kernel/constants';
12
+ import type { ClientState, QueryEntry } from '../state/client-state';
13
+ import * as R from '../state/reducers';
14
+ import { encodeRecordId } from '../utils/index';
15
+ import { departedIds } from '../state/selectors';
16
+ import { forgetDeparted } from './lifecycle.saga';
17
+ import { planListRefPollChunks, recordVersionArraysEqual } from '../sync/policy';
18
+ import type { SagaEnv } from './env';
19
+ import { listRefTable } from './env';
20
+ import * as sql from './sql';
21
+ import {
22
+ FAILED,
23
+ decideMembershipOutcome,
24
+ rebaseRead,
25
+ withDeltas,
26
+ snapshotFromSingle,
27
+ snapshotsFromBatch,
28
+ suspectHashes,
29
+ type ListRefSnapshot,
30
+ } from './membership';
31
+
32
+ type MembershipOutcome = 'applied' | 'ignored' | 'view-lost';
33
+
34
+ /**
35
+ * Accept (or refuse) a server id-set for one query. The only writer of
36
+ * `remoteArray`, the `_00_view` row and the cold -> live transition.
37
+ */
38
+ export function* applyMembership(
39
+ hash: QueryHash,
40
+ remoteArray: RecordVersionArray,
41
+ meta?: ServerViewMeta,
42
+ verifiedRemoval?: boolean
43
+ ): Saga<MembershipOutcome> {
44
+ const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
45
+ if (!entry) return 'ignored';
46
+ if (meta) yield fx.state.update(R.setServerState(hash, meta.present ? meta.state : null));
47
+ // The row is there: whatever made it read as gone has passed.
48
+ if (meta?.present && (entry.viewLostCount > 0 || entry.viewLostRetryAt !== null)) {
49
+ yield fx.state.update(R.setViewLost(hash, 0, null));
50
+ }
51
+ const outcome = decideMembershipOutcome({
52
+ phase: entry.lifecycle.phase,
53
+ held: entry.remoteArray.length,
54
+ remoteArray,
55
+ meta,
56
+ verifiedRemoval,
57
+ });
58
+ if (outcome === 'ignored') return outcome;
59
+ if (outcome === 'view-lost') {
60
+ if (entry.lifecycle.phase !== 'view-lost') {
61
+ yield fx.state.update(R.applyLifecycle(hash, { type: 'row-missing' }));
62
+ yield fx.emit({ type: 'query:view-lost', hash });
63
+ }
64
+ // A recovery is already scheduled: every read in the meantime would
65
+ // otherwise re-register again.
66
+ if (entry.viewLostRetryAt !== null) return outcome;
67
+ if (entry.viewLostCount === 0) {
68
+ yield fx.state.update(R.setViewLost(hash, 1, null));
69
+ if (entry.lifecycle.remote !== 'unregistered') {
70
+ yield fx.state.update(R.applyLifecycle(hash, { type: 'remote-dropped' }));
71
+ }
72
+ yield fx.dispatch({ type: 'EnsureRegistered' });
73
+ return outcome;
74
+ }
75
+ // Re-registered already and the row still reads as gone. Under load that
76
+ // repeated every read, each one costing the server a registration while it
77
+ // was still publishing the last, so from here on it is paced.
78
+ const delay = backoffMs(entry.viewLostCount - 1, VIEW_LOST_RETRY_BASE_MS, VIEW_LOST_RETRY_MAX_MS);
79
+ const now = (yield fx.now()) as number;
80
+ yield fx.state.update(R.setViewLost(hash, entry.viewLostCount + 1, now + delay));
81
+ yield fx.emit({
82
+ type: 'log',
83
+ level: 'debug',
84
+ message: 'view still lost; re-registering after a delay',
85
+ data: { hash, attempt: entry.viewLostCount + 1, delayMs: delay },
86
+ });
87
+ yield fx.timer.set(`view-lost:${hash}`, delay, { type: 'RecoverLostView', hash });
88
+ return outcome;
89
+ }
90
+ yield fx.state.update(R.commitMembership(hash, remoteArray, meta?.present ?? true));
91
+ yield* forgetDeparted(hash, departedIds(entry.remoteArray, remoteArray));
92
+ // No authority flip here: the set is known but its rows are not painted
93
+ // yet (bodies may still be on their way). `materialize` flips it once they are.
94
+ const now = (yield fx.now()) as number;
95
+ try {
96
+ yield fx.local.upsert(
97
+ sql.VIEW_TABLE,
98
+ sql.viewRecordId(entry.def.viewKey),
99
+ sql.viewRow(remoteArray, entry.subqueryRemoteArray, true, now),
100
+ 'replace'
101
+ );
102
+ } catch (err) {
103
+ yield fx.emit({ type: 'log', level: 'debug', message: 'view row write failed', data: { hash, err } });
104
+ }
105
+ yield fx.dispatch({ type: 'FetchRows' });
106
+ return outcome;
107
+ }
108
+
109
+ /**
110
+ * The paced view-lost re-registration. A no-op when the row has been seen
111
+ * since (the pacing was reset) or the query recovered or went away.
112
+ */
113
+ export function* recoverLostView(hash: QueryHash): Saga<void> {
114
+ const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
115
+ if (!entry || entry.viewLostRetryAt === null) return;
116
+ yield fx.state.update(R.setViewLost(hash, entry.viewLostCount, null));
117
+ if (entry.lifecycle.phase !== 'view-lost') return;
118
+ if (entry.lifecycle.remote !== 'unregistered') {
119
+ yield fx.state.update(R.applyLifecycle(hash, { type: 'remote-dropped' }));
120
+ }
121
+ yield fx.dispatch({ type: 'RegisterRemote', hash });
122
+ }
123
+
124
+ /**
125
+ * Replace the subquery child set; bodies follow through the fetch plan. A
126
+ * query with a `_00_view` row (past `cold`) records the children there too,
127
+ * so the GC keeps their bodies and a cold scan may paint them.
128
+ */
129
+ export function* applySubqueryChildren(hash: QueryHash, children: RecordVersionArray): Saga<void> {
130
+ const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
131
+ if (!entry || recordVersionArraysEqual(entry.subqueryRemoteArray, children)) return;
132
+ yield fx.state.update(R.setSubqueryRemoteArray(hash, children));
133
+ yield* forgetDeparted(hash, departedIds(entry.subqueryRemoteArray, children));
134
+ if (entry.lifecycle.phase !== 'cold') {
135
+ try {
136
+ yield fx.local.upsert(sql.VIEW_TABLE, sql.viewRecordId(entry.def.viewKey), { children }, 'merge');
137
+ } catch (err) {
138
+ yield fx.emit({ type: 'log', level: 'debug', message: 'view row write failed', data: { hash, err } });
139
+ }
140
+ }
141
+ yield fx.dispatch({ type: 'FetchRows' });
142
+ }
143
+
144
+ const ok = (r: Settled): r is { ok: true; value: unknown } => r.ok;
145
+ /** One statement's result, or `FAILED` when it did not answer (see `FAILED`). */
146
+ const stmt = (results: unknown, i: number): unknown => {
147
+ if (!Array.isArray(results)) return FAILED;
148
+ const r = (results as StatementResult[])[i];
149
+ return r && r.status === 'OK' ? r.result : FAILED;
150
+ };
151
+
152
+ /**
153
+ * Commit LIVE edge deltas (rows joining a view, or moving to a newer
154
+ * version) straight to membership, without re-reading the edges: the
155
+ * notification already says what the read would. Returns the hashes it
156
+ * settled; any other change still goes through {@link readMembership}.
157
+ *
158
+ * Only for a query whose membership is known and current (`live`, not
159
+ * materializing, no view-lost recovery pending): before that the read decides
160
+ * what the set is. Removals are never applied here, so a view whose edges
161
+ * are being rewritten never loses (and garbage-collects) bodies mid-rewrite.
162
+ * Nor for a query with a `LIMIT`: a row joining it pushes another out, and
163
+ * rendering the joiner before the read removes the leaver would show one row
164
+ * too many (the id-set render drops the `LIMIT`).
165
+ */
166
+ export function* applyEdgeDeltas(deltas: EdgeDelta[]): Saga<Set<QueryHash>> {
167
+ const settled = new Set<QueryHash>();
168
+ const byHash = new Map<QueryHash, EdgeDelta[]>();
169
+ for (const d of deltas) byHash.set(d.hash, [...(byHash.get(d.hash) ?? []), d]);
170
+ for (const [hash, list] of byHash) {
171
+ const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
172
+ if (!entry || entry.lifecycle.phase !== 'live' || entry.serverState === 'materializing' || entry.viewLostRetryAt !== null) continue;
173
+ if (entry.def.plan ? entry.def.plan.limit != null : /\bLIMIT\b/i.test(entry.def.surql)) continue;
174
+ const next = withDeltas(entry.remoteArray, list);
175
+ // Never empty, so always applied.
176
+ if (next !== entry.remoteArray) yield* applyMembership(hash, next);
177
+ settled.add(hash);
178
+ }
179
+ return settled;
180
+ }
181
+
182
+ /**
183
+ * Read the server membership of many queries in as few round trips as the
184
+ * row budget allows, then apply each changed set. Every server id-set enters
185
+ * state here, except the rows LIVE reports joining (see
186
+ * {@link applyEdgeDeltas}): registration, the poll, LIVE dirt and view-lost
187
+ * recovery all come through here.
188
+ */
189
+ export function* readMembership(
190
+ env: SagaEnv,
191
+ hashes: QueryHash[],
192
+ opts: { force?: boolean } = {}
193
+ ): Saga<{ changed: boolean; failed: boolean }> {
194
+ const state = (yield fx.state.read((s) => s)) as ClientState;
195
+ const entries = hashes.map((h) => state.queries.get(h)).filter((e): e is QueryEntry => !!e);
196
+ if (entries.length === 0) return { changed: false, failed: false };
197
+ const table = listRefTable(env, state);
198
+ const now = (yield fx.now()) as number;
199
+ const chunks = planListRefPollChunks(
200
+ entries.map((e) => ({ hash: e.def.hash, rows: e.remoteArray.length, lastPolledAt: opts.force ? 0 : (e.lastPolledAt ?? 0) })),
201
+ { now }
202
+ );
203
+ const byHash = new Map(entries.map((e) => [e.def.hash, e]));
204
+ const single = (e: QueryEntry) =>
205
+ fx.remote.query(sql.singleSnapshotSelect(table), { in: e.def.id }, env.remoteTimeoutMs);
206
+ const requests = chunks.map((chunk) =>
207
+ chunk.length === 1
208
+ ? single(byHash.get(chunk[0])!)
209
+ : fx.remote.query(sql.batchSnapshotSelect(table), { ins: chunk.map((h) => byHash.get(h)!.def.id) }, env.remoteTimeoutMs)
210
+ );
211
+ const results = (yield fx.all(requests)) as Settled[];
212
+
213
+ const snapshots = new Map<QueryHash, ListRefSnapshot>();
214
+ let failed = false;
215
+ const rereads: QueryEntry[] = [];
216
+ chunks.forEach((chunk, i) => {
217
+ const r = results[i];
218
+ if (!ok(r)) {
219
+ failed = true;
220
+ return;
221
+ }
222
+ if (chunk.length === 1) {
223
+ const snap = snapshotFromSingle(stmt(r.value, 0) as never, stmt(r.value, 1) as never, stmt(r.value, 2) as never);
224
+ if (snap) snapshots.set(chunk[0], snap);
225
+ else failed = true;
226
+ return;
227
+ }
228
+ const hashById = new Map(chunk.map((h) => [encodeRecordId(byHash.get(h)!.def.id), h]));
229
+ const edges = stmt(r.value, 0) as never[] | null;
230
+ const counts = stmt(r.value, 1);
231
+ if (!Array.isArray(edges) || counts === FAILED) {
232
+ failed = true;
233
+ return;
234
+ }
235
+ const batch = snapshotsFromBatch(edges, counts as never, hashById);
236
+ const held = new Map(chunk.map((h) => [h, byHash.get(h)!.remoteArray.length]));
237
+ const suspect = new Set(suspectHashes(batch, held, edges.length));
238
+ for (const [h, snap] of batch) {
239
+ if (suspect.has(h)) rereads.push(byHash.get(h)!);
240
+ else snapshots.set(h, snap);
241
+ }
242
+ });
243
+ if (rereads.length > 0) {
244
+ const again = (yield fx.all(rereads.map(single))) as Settled[];
245
+ again.forEach((r, i) => {
246
+ if (!ok(r)) {
247
+ failed = true;
248
+ return;
249
+ }
250
+ const snap = snapshotFromSingle(stmt(r.value, 0) as never, stmt(r.value, 1) as never, stmt(r.value, 2) as never);
251
+ if (snap) snapshots.set(rereads[i].def.hash, snap);
252
+ else failed = true;
253
+ });
254
+ }
255
+
256
+ let changed = false;
257
+ const stillMaterializing: QueryHash[] = [];
258
+ for (const [hash, snap] of snapshots) {
259
+ const current = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
260
+ if (!current) continue;
261
+ const primary = rebaseRead(snap.primary, byHash.get(hash)!.remoteArray, current.remoteArray);
262
+ const metaChanged = current.serverState !== (snap.meta.present ? snap.meta.state : null);
263
+ if (!recordVersionArraysEqual(primary, current.remoteArray) || metaChanged || current.lifecycle.phase === 'cold' || current.lifecycle.phase === 'cached') {
264
+ const outcome = yield* applyMembership(hash, primary, snap.meta);
265
+ if (outcome === 'applied') changed = true;
266
+ if (outcome === 'ignored' && snap.meta.present && snap.meta.state === 'materializing') stillMaterializing.push(hash);
267
+ }
268
+ yield* applySubqueryChildren(hash, snap.subquery);
269
+ }
270
+ yield fx.state.update(R.compose(R.clearMembershipDirty([...snapshots.keys()]), R.stampPolled(snapshots.keys(), now)));
271
+
272
+ // A view whose edges are still in flight: re-read on a short ladder rather
273
+ // than waiting for a backed-off poll tick.
274
+ for (const hash of stillMaterializing) {
275
+ const attempt = (yield fx.state.read((s) => s.membershipReread.get(hash) ?? 0)) as number;
276
+ if (attempt >= MATERIALIZING_REREAD_LADDER_MS.length) {
277
+ yield fx.state.update(R.setMembershipReread(hash, null));
278
+ continue;
279
+ }
280
+ yield fx.state.update(R.compose(R.setMembershipReread(hash, attempt + 1), R.markMembershipDirty([hash])));
281
+ yield fx.timer.set('membership', MATERIALIZING_REREAD_LADDER_MS[attempt], { type: 'ReadDirtyMembership' });
282
+ }
283
+ for (const hash of snapshots.keys()) {
284
+ if (!stillMaterializing.includes(hash)) yield fx.state.update(R.setMembershipReread(hash, null));
285
+ }
286
+ yield fx.dispatch({ type: 'SyncOutcome', ok: !failed, error: failed ? 'membership read failed' : undefined });
287
+ return { changed, failed };
288
+ }
289
+
290
+ /**
291
+ * LIVE handler: mark dirty and coalesce into one batched re-read.
292
+ *
293
+ * The window is armed ONCE per burst, when the dirty set goes from empty to
294
+ * non-empty. `timer.set` replaces a pending timer, so re-arming on every event
295
+ * would push the read out for as long as events keep arriving: on a busy
296
+ * `_00_list_ref` table (registrations, an import, another tab syncing) the
297
+ * events never stop for a whole 50 ms and the read simply never ran, leaving
298
+ * membership frozen until a poll tick happened to cover it.
299
+ */
300
+ export function* markMembershipDirty(hashes: QueryHash[]): Saga<void> {
301
+ const armed = (yield fx.state.read((s) => s.membershipDirty.size > 0)) as boolean;
302
+ yield fx.state.update(R.markMembershipDirty(hashes));
303
+ if (!armed) yield fx.timer.set('membership', MEMBERSHIP_COALESCE_MS, { type: 'ReadDirtyMembership' });
304
+ }
305
+
306
+ export function* readDirtyMembership(env: SagaEnv): Saga<void> {
307
+ const dirty = (yield fx.state.read((s) => [...s.membershipDirty])) as QueryHash[];
308
+ if (dirty.length === 0) return;
309
+ yield* readMembership(env, dirty, { force: true });
310
+ // Dirt that arrived while the read was in flight, or a hash the read could
311
+ // not answer, holds the set non-empty - and with the window armed only on the
312
+ // empty -> non-empty edge, nothing else would arm it again.
313
+ const left = (yield fx.state.read((s) => s.membershipDirty.size)) as number;
314
+ if (left > 0) yield fx.timer.set('membership', MEMBERSHIP_COALESCE_MS, { type: 'ReadDirtyMembership' });
315
+ }
@@ -0,0 +1,151 @@
1
+ import { describe, expect, it } from 'vitest';
2
+ import { RecordId } from 'surrealdb';
3
+ import {
4
+ decideMembershipOutcome,
5
+ dedupeRecordVersions,
6
+ isResolvedBefore,
7
+ metaFromRow,
8
+ parseViewRow,
9
+ rebaseRead,
10
+ snapshotFromSingle,
11
+ snapshotsFromBatch,
12
+ suspectHashes,
13
+ withDeltas,
14
+ } from './membership';
15
+
16
+ describe('durable view row', () => {
17
+ it('parses and gates resolved-before', () => {
18
+ expect(parseViewRow(null)).toBeNull();
19
+ expect(parseViewRow({ ids: 'nope' })).toBeNull();
20
+ expect(parseViewRow({ ids: [['t:1', 1]], confirmed: true })).toEqual({ ids: [['t:1', 1]], confirmed: true });
21
+ expect(parseViewRow({ ids: [] })).toEqual({ ids: [], confirmed: false });
22
+ expect(isResolvedBefore(null)).toBe(false);
23
+ expect(isResolvedBefore({ ids: [], confirmed: false })).toBe(false);
24
+ expect(isResolvedBefore({ ids: [], confirmed: true })).toBe(true);
25
+ expect(isResolvedBefore({ ids: [['t:1', 1]], confirmed: false })).toBe(true);
26
+ });
27
+ });
28
+
29
+ describe('metaFromRow / dedupe', () => {
30
+ it('folds the row-count select', () => {
31
+ expect(metaFromRow(null)).toEqual({ present: false, rowCount: null, state: null });
32
+ expect(metaFromRow({ rowCount: 3, state: 'ready' })).toEqual({ present: true, rowCount: 3, state: 'ready' });
33
+ expect(metaFromRow({ rowCount: null, state: 'weird' })).toEqual({ present: true, rowCount: null, state: null });
34
+ expect(metaFromRow({ state: 'materializing' })).toEqual({ present: true, rowCount: null, state: 'materializing' });
35
+ });
36
+ it('keeps the highest version per id', () => {
37
+ const single: [string, number][] = [['t:1', 1]];
38
+ expect(dedupeRecordVersions(single)).toBe(single);
39
+ const clean: [string, number][] = [['t:1', 1], ['t:2', 1]];
40
+ expect(dedupeRecordVersions(clean)).toBe(clean);
41
+ expect(dedupeRecordVersions([['t:1', 1], ['t:1', 3], ['t:1', 2]])).toEqual([['t:1', 3]]);
42
+ });
43
+ });
44
+
45
+ describe('decideMembershipOutcome', () => {
46
+ const ready0 = { present: true, rowCount: 0, state: 'ready' as const };
47
+ it('non-empty or verified removal is always applied', () => {
48
+ expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [['t:1', 1]] })).toBe('applied');
49
+ expect(decideMembershipOutcome({ phase: 'live', held: 2, remoteArray: [], verifiedRemoval: true })).toBe('applied');
50
+ });
51
+ it('empty with no readable row: view-lost when holding, ignored when cold and empty', () => {
52
+ expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [] })).toBe('ignored');
53
+ expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [], meta: { present: false, rowCount: null, state: null } })).toBe('ignored');
54
+ expect(decideMembershipOutcome({ phase: 'cold', held: 2, remoteArray: [] })).toBe('view-lost');
55
+ expect(decideMembershipOutcome({ phase: 'cached', held: 0, remoteArray: [] })).toBe('view-lost');
56
+ expect(decideMembershipOutcome({ phase: 'live', held: 1, remoteArray: [], meta: { present: false, rowCount: null, state: null } })).toBe('view-lost');
57
+ });
58
+ it('empty with a present row: applied only when ready and zero rows', () => {
59
+ expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [], meta: ready0 })).toBe('applied');
60
+ expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [], meta: { present: true, rowCount: 0, state: null } })).toBe('applied');
61
+ expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [], meta: { present: true, rowCount: 0, state: 'materializing' } })).toBe('ignored');
62
+ expect(decideMembershipOutcome({ phase: 'live', held: 3, remoteArray: [], meta: { present: true, rowCount: 3, state: 'ready' } })).toBe('ignored');
63
+ expect(decideMembershipOutcome({ phase: 'live', held: 3, remoteArray: [], meta: { present: true, rowCount: null, state: 'ready' } })).toBe('ignored');
64
+ });
65
+ });
66
+
67
+ describe('snapshots', () => {
68
+ const q1 = new RecordId('_00_query', 'a');
69
+ const q2 = new RecordId('_00_query', 'b');
70
+ const out = (id: string) => new RecordId('thing', id);
71
+ it('single: edges, meta, children; null when the primary select is not an array', () => {
72
+ expect(snapshotFromSingle(null, null, null)).toBeNull();
73
+ const snap = snapshotFromSingle(
74
+ [{ out: out('1'), version: 1 }, { out: out('1'), version: 2 }],
75
+ { rowCount: 1, state: 'ready' },
76
+ [{ out: out('c'), version: 1 }]
77
+ );
78
+ expect(snap).toEqual({
79
+ primary: [['thing:1', 2]],
80
+ subquery: [['thing:c', 1]],
81
+ meta: { present: true, rowCount: 1, state: 'ready' },
82
+ });
83
+ expect(snapshotFromSingle([], null, null)!.subquery).toEqual([]);
84
+ });
85
+ it('batch: splits by hash and parent, fills meta, marks missing rows absent', () => {
86
+ const hashById = new Map([['_00_query:a', 'ha'], ['_00_query:b', 'hb']]);
87
+ expect(snapshotsFromBatch(null, null, hashById).size).toBe(0);
88
+ const snaps = snapshotsFromBatch(
89
+ [
90
+ { in: q1, out: out('1'), version: 1 },
91
+ { in: q1, out: out('1'), version: 1 },
92
+ { in: q1, out: out('c'), version: 1, parent: q1 },
93
+ { in: new RecordId('_00_query', 'zz'), out: out('9'), version: 1 },
94
+ ],
95
+ [{ id: q1, rowCount: 1, state: 'ready' }, null, { rowCount: 5 }, { id: new RecordId('_00_query', 'zz') }],
96
+ hashById
97
+ );
98
+ expect(snaps.get('ha')).toEqual({
99
+ primary: [['thing:1', 1]],
100
+ subquery: [['thing:c', 1]],
101
+ meta: { present: true, rowCount: 1, state: 'ready' },
102
+ });
103
+ expect(snaps.get('hb')).toEqual({ primary: [], subquery: [], meta: { present: false, rowCount: null, state: null } });
104
+ expect(snapshotsFromBatch([], null, hashById).get('ha')!.meta.present).toBe(false);
105
+ });
106
+ it('suspectHashes flags held queries with a missing row or a silently empty batch', () => {
107
+ const snaps = new Map([
108
+ ['missing', { primary: [], subquery: [], meta: { present: false, rowCount: null, state: null } }],
109
+ ['empty', { primary: [], subquery: [], meta: { present: true, rowCount: 4, state: 'ready' as const } }],
110
+ ['fine', { primary: [], subquery: [], meta: { present: true, rowCount: 0, state: 'ready' as const } }],
111
+ ['unheld', { primary: [], subquery: [], meta: { present: false, rowCount: null, state: null } }],
112
+ ]);
113
+ const held = new Map([['missing', 2], ['empty', 4], ['fine', 1], ['unheld', 0]]);
114
+ expect(suspectHashes(snaps, held, 0)).toEqual(['missing', 'empty']);
115
+ expect(suspectHashes(snaps, held, 1)).toEqual(['missing']);
116
+ const nullCount = new Map([['q', { primary: [], subquery: [], meta: { present: true, rowCount: null, state: 'ready' as const } }]]);
117
+ expect(suspectHashes(nullCount, new Map([['q', 3]]), 0)).toEqual([]);
118
+ expect(suspectHashes(nullCount, new Map(), 0)).toEqual([]);
119
+ });
120
+ });
121
+
122
+ describe('withDeltas', () => {
123
+ it('adds a joining row and moves a held one to a newer version', () => {
124
+ const held: Array<[string, number]> = [['t:1', 1], ['t:2', 1]];
125
+ expect(withDeltas(held, [{ id: 't:3', version: 1 }, { id: 't:2', version: 4 }])).toEqual([['t:1', 1], ['t:2', 4], ['t:3', 1]]);
126
+ });
127
+ it('hands back the held set itself when nothing moves', () => {
128
+ const held: Array<[string, number]> = [['t:1', 5]];
129
+ expect(withDeltas(held, [{ id: 't:1', version: 5 }, { id: 't:1', version: 2 }])).toBe(held);
130
+ expect(withDeltas(held, [])).toBe(held);
131
+ });
132
+ });
133
+
134
+ describe('rebaseRead', () => {
135
+ it('is the read itself when state did not move while it was in flight', () => {
136
+ const before: Array<[string, number]> = [['t:1', 1]];
137
+ const read: Array<[string, number]> = [['t:2', 1]];
138
+ expect(rebaseRead(read, before, before)).toBe(read);
139
+ });
140
+ it('keeps a row that joined (or moved) after the read was sent', () => {
141
+ const before: Array<[string, number]> = [['t:1', 1]];
142
+ const now: Array<[string, number]> = [['t:1', 3], ['t:2', 1]];
143
+ // The server answered before either change.
144
+ expect(rebaseRead([['t:1', 1]], before, now)).toEqual([['t:1', 3], ['t:2', 1]]);
145
+ // ...or after them: nothing is doubled or moved back.
146
+ expect(rebaseRead([['t:1', 4], ['t:2', 2]], before, now)).toEqual([['t:1', 4], ['t:2', 2]]);
147
+ });
148
+ it('drops a row that left in the meantime', () => {
149
+ expect(rebaseRead([['t:1', 1], ['t:2', 1]], [['t:1', 1], ['t:2', 1]], [['t:2', 1]])).toEqual([['t:2', 1]]);
150
+ });
151
+ });