@spooky-sync/core 0.0.1-canary.33 → 0.0.1-canary.331

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (266) hide show
  1. package/AGENTS.md +84 -0
  2. package/README.md +56 -16
  3. package/dist/index.d.ts +2080 -142
  4. package/dist/index.js +16009 -2480
  5. package/dist/live.d.ts +149 -0
  6. package/dist/live.js +739 -0
  7. package/dist/otel/index.d.ts +1 -1
  8. package/dist/otel/index.js +3 -3
  9. package/dist/pure.d.ts +373 -0
  10. package/dist/pure.js +24 -0
  11. package/dist/sqlite-open.js +392 -0
  12. package/dist/sqlite-worker.d.ts +1 -0
  13. package/dist/sqlite-worker.js +439 -0
  14. package/dist/sw.d.ts +207 -0
  15. package/dist/sw.js +393 -0
  16. package/dist/tabs-broker-worker.d.ts +8 -0
  17. package/dist/tabs-broker-worker.js +477 -0
  18. package/dist/types.d.ts +1857 -38
  19. package/dist/worker-edges.js +480 -0
  20. package/dist/worker-impersonation.js +60 -0
  21. package/dist/worker-ref-tables.d.ts +48 -0
  22. package/dist/worker-types.d.ts +112 -0
  23. package/dist/worker-types.js +73 -0
  24. package/package.json +28 -7
  25. package/scripts/check-broker-bundle.mjs +33 -0
  26. package/scripts/check-worker-entries.mjs +173 -0
  27. package/skills/sp00ky-core/SKILL.md +32 -7
  28. package/skills/sp00ky-core/references/config.md +7 -1
  29. package/src/boot/auth-flip.saga.test.ts +135 -0
  30. package/src/boot/auth-flip.saga.ts +77 -0
  31. package/src/boot/boot.saga.test.ts +203 -0
  32. package/src/boot/boot.saga.ts +146 -0
  33. package/src/boot/bucket-switch.saga.test.ts +187 -0
  34. package/src/boot/bucket-switch.saga.ts +133 -0
  35. package/src/boot/preload.saga.test.ts +80 -0
  36. package/src/boot/preload.saga.ts +40 -0
  37. package/src/bucket-blurhash.test.ts +148 -0
  38. package/src/build-globals.d.ts +12 -0
  39. package/src/client/bucket-handle.test.ts +195 -0
  40. package/src/client/bucket-handle.ts +352 -0
  41. package/src/client/router.test.ts +58 -0
  42. package/src/client/router.ts +100 -0
  43. package/src/client/runtime.test.ts +195 -0
  44. package/src/client/runtime.ts +296 -0
  45. package/src/client/services.test.ts +74 -0
  46. package/src/client/services.ts +434 -0
  47. package/src/client/sp00ky-client.test.ts +341 -0
  48. package/src/client/sp00ky-client.ts +625 -0
  49. package/src/events/events.test.ts +2 -1
  50. package/src/events/index.ts +3 -0
  51. package/src/index.ts +69 -1
  52. package/src/kernel/constants.test.ts +19 -0
  53. package/src/kernel/constants.ts +80 -0
  54. package/src/kernel/effects.test.ts +40 -0
  55. package/src/kernel/effects.ts +167 -0
  56. package/src/kernel/events.ts +57 -0
  57. package/src/kernel/interpreter.test.ts +116 -0
  58. package/src/kernel/interpreter.ts +148 -0
  59. package/src/kernel/saga.test.ts +89 -0
  60. package/src/kernel/saga.ts +73 -0
  61. package/src/live/index.test.ts +512 -0
  62. package/src/live/index.ts +1059 -0
  63. package/src/modules/app-release/index.test.ts +125 -0
  64. package/src/modules/app-release/index.ts +209 -0
  65. package/src/modules/auth/auth.local-first.test.ts +228 -0
  66. package/src/modules/auth/events/index.ts +9 -2
  67. package/src/modules/auth/impersonation-banner.ts +443 -0
  68. package/src/modules/auth/impersonation.integration.test.ts +205 -0
  69. package/src/modules/auth/impersonation.test.ts +377 -0
  70. package/src/modules/auth/impersonation.ts +179 -0
  71. package/src/modules/auth/index.ts +476 -22
  72. package/src/modules/crdt/acceptance.ts +75 -0
  73. package/src/modules/crdt/bytes.ts +50 -0
  74. package/src/modules/crdt/crdt-field.ts +125 -0
  75. package/src/modules/crdt/crdt.test.ts +469 -0
  76. package/src/modules/crdt/hash.ts +103 -0
  77. package/src/modules/crdt/index.ts +757 -0
  78. package/src/modules/crdt/loro-loader.ts +25 -0
  79. package/src/modules/crdt/mirror.ts +37 -0
  80. package/src/modules/devtools/flags.ts +349 -0
  81. package/src/modules/devtools/index.ts +746 -95
  82. package/src/modules/devtools/mutations.test.ts +92 -0
  83. package/src/modules/devtools/mutations.ts +160 -0
  84. package/src/modules/devtools/notify-throttle.test.ts +166 -0
  85. package/src/modules/devtools/state-shape.test.ts +263 -0
  86. package/src/modules/devtools/storage-info.test.ts +79 -0
  87. package/src/modules/devtools/storage-info.ts +168 -0
  88. package/src/modules/devtools/versions.test.ts +74 -0
  89. package/src/modules/devtools/versions.ts +110 -0
  90. package/src/modules/feature-flag/index.test.ts +298 -0
  91. package/src/modules/feature-flag/index.ts +316 -0
  92. package/src/modules/ref-tables.test.ts +93 -0
  93. package/src/modules/ref-tables.ts +91 -0
  94. package/src/modules/web-push/index.test.ts +824 -0
  95. package/src/modules/web-push/index.ts +1027 -0
  96. package/src/mutation/jobs.test.ts +25 -0
  97. package/src/mutation/jobs.ts +41 -0
  98. package/src/mutation/mutation-id.test.ts +37 -0
  99. package/src/mutation/mutation-id.ts +35 -0
  100. package/src/mutation/pending-writes.integration.test.ts +175 -0
  101. package/src/mutation/push.saga.test.ts +301 -0
  102. package/src/mutation/push.saga.ts +225 -0
  103. package/src/mutation/rows.test.ts +231 -0
  104. package/src/mutation/rows.ts +427 -0
  105. package/src/mutation/tray.saga.test.ts +63 -0
  106. package/src/mutation/tray.saga.ts +48 -0
  107. package/src/mutation/write.saga.test.ts +316 -0
  108. package/src/mutation/write.saga.ts +268 -0
  109. package/src/otel/index.ts +10 -7
  110. package/src/pure.ts +66 -0
  111. package/src/push/types.ts +174 -0
  112. package/src/query/anon-query-isolation.integration.test.ts +527 -0
  113. package/src/query/env.test.ts +23 -0
  114. package/src/query/env.ts +65 -0
  115. package/src/query/fetch.saga.test.ts +199 -0
  116. package/src/query/fetch.saga.ts +120 -0
  117. package/src/query/hash.test.ts +14 -0
  118. package/src/query/hash.ts +18 -0
  119. package/src/query/id-set-plan.test.ts +132 -0
  120. package/src/query/lifecycle.saga.test.ts +274 -0
  121. package/src/query/lifecycle.saga.ts +186 -0
  122. package/src/query/list-ref-live.integration.test.ts +203 -0
  123. package/src/query/materialize.saga.test.ts +115 -0
  124. package/src/query/materialize.saga.ts +84 -0
  125. package/src/query/materialize.test.ts +58 -0
  126. package/src/query/materialize.ts +57 -0
  127. package/src/query/membership.saga.test.ts +407 -0
  128. package/src/query/membership.saga.ts +282 -0
  129. package/src/query/membership.test.ts +118 -0
  130. package/src/query/membership.ts +201 -0
  131. package/src/query/register.saga.test.ts +311 -0
  132. package/src/query/register.saga.ts +235 -0
  133. package/src/query/render-set.test.ts +47 -0
  134. package/src/query/render-set.ts +71 -0
  135. package/src/query/sql.test.ts +118 -0
  136. package/src/query/sql.ts +157 -0
  137. package/src/query/window-query.test.ts +70 -0
  138. package/src/query/window-query.ts +208 -0
  139. package/src/services/blobs/blob-cache.test.ts +359 -0
  140. package/src/services/blobs/blob-cache.ts +603 -0
  141. package/src/services/blobs/blob-manifest.ts +227 -0
  142. package/src/services/blobs/blob-store.test.ts +77 -0
  143. package/src/services/blobs/blob-store.ts +359 -0
  144. package/src/services/blobs/blob.fixture.ts +90 -0
  145. package/src/services/blobs/index.ts +70 -0
  146. package/src/services/database/cache-engine.ts +209 -0
  147. package/src/services/database/connection-supervisor.test.ts +318 -0
  148. package/src/services/database/connection-supervisor.ts +422 -0
  149. package/src/services/database/database.query-timeout.test.ts +140 -0
  150. package/src/services/database/database.responses.test.ts +43 -0
  151. package/src/services/database/database.ts +156 -61
  152. package/src/services/database/engine-factory.ts +33 -0
  153. package/src/services/database/errors.ts +34 -0
  154. package/src/services/database/events/index.ts +2 -1
  155. package/src/services/database/index.ts +7 -0
  156. package/src/services/database/landed-body-clear.integration.test.ts +236 -0
  157. package/src/services/database/local-migrator.integration.test.ts +110 -0
  158. package/src/services/database/local-migrator.ts +61 -16
  159. package/src/services/database/local.test.ts +150 -0
  160. package/src/services/database/local.ts +508 -63
  161. package/src/services/database/plan-render.test.ts +159 -0
  162. package/src/services/database/plan-render.ts +108 -0
  163. package/src/services/database/relation-resolver.test.ts +484 -0
  164. package/src/services/database/relation-resolver.ts +0 -0
  165. package/src/services/database/remote.ts +285 -6
  166. package/src/services/database/sqlite-cache-engine.test.ts +628 -0
  167. package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
  168. package/src/services/database/sqlite-cache-engine.ts +1401 -0
  169. package/src/services/database/sqlite-column-reset.integration.test.ts +100 -0
  170. package/src/services/database/sqlite-column-reset.ts +91 -0
  171. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  172. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  173. package/src/services/database/sqlite-lock-verify.test.ts +33 -0
  174. package/src/services/database/sqlite-lock-verify.ts +45 -0
  175. package/src/services/database/sqlite-open.test.ts +150 -0
  176. package/src/services/database/sqlite-open.ts +164 -0
  177. package/src/services/database/sqlite-plan-sql.test.ts +164 -0
  178. package/src/services/database/sqlite-plan-sql.ts +196 -0
  179. package/src/services/database/sqlite-projection.test.ts +121 -0
  180. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  181. package/src/services/database/sqlite-select.test.ts +261 -0
  182. package/src/services/database/sqlite-select.ts +131 -0
  183. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  184. package/src/services/database/sqlite-transport.ts +224 -0
  185. package/src/services/database/sqlite-worker.ts +437 -0
  186. package/src/services/database/surql-translate.ts +417 -0
  187. package/src/services/database/surreal-cache-engine.ts +163 -0
  188. package/src/services/logger/index.test.ts +132 -0
  189. package/src/services/logger/index.ts +189 -7
  190. package/src/services/persistence/localstorage.ts +2 -2
  191. package/src/services/persistence/resilient.ts +10 -3
  192. package/src/services/persistence/surrealdb.ts +4 -4
  193. package/src/services/stream-processor/index.ts +806 -67
  194. package/src/services/stream-processor/permissions.test.ts +47 -0
  195. package/src/services/stream-processor/permissions.ts +53 -0
  196. package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
  197. package/src/services/stream-processor/stream-processor.prime.test.ts +234 -0
  198. package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
  199. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  200. package/src/services/stream-processor/wasm-types.ts +58 -2
  201. package/src/services/tabs/broker-client.ts +284 -0
  202. package/src/services/tabs/broker.test.ts +381 -0
  203. package/src/services/tabs/coordinator.test.ts +386 -0
  204. package/src/services/tabs/coordinator.ts +646 -0
  205. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  206. package/src/services/tabs/leader-locks.ts +75 -0
  207. package/src/services/tabs/protocol.ts +262 -0
  208. package/src/services/tabs/support.ts +36 -0
  209. package/src/services/tabs/tabs-broker-worker.ts +652 -0
  210. package/src/sp00ky.ts +5 -395
  211. package/src/state/client-state.ts +208 -0
  212. package/src/state/lifecycle.test.ts +141 -0
  213. package/src/state/lifecycle.ts +99 -0
  214. package/src/state/reducers.test.ts +306 -0
  215. package/src/state/reducers.ts +531 -0
  216. package/src/state/selectors.test.ts +188 -0
  217. package/src/state/selectors.ts +258 -0
  218. package/src/sw/index.test.ts +582 -0
  219. package/src/sw/index.ts +607 -0
  220. package/src/sw/store.ts +96 -0
  221. package/src/sync/connection.saga.test.ts +97 -0
  222. package/src/sync/connection.saga.ts +89 -0
  223. package/src/sync/live.saga.test.ts +178 -0
  224. package/src/sync/live.saga.ts +107 -0
  225. package/src/sync/policy.health.test.ts +56 -0
  226. package/src/sync/policy.test.ts +663 -0
  227. package/src/sync/policy.ts +515 -0
  228. package/src/sync/poll.saga.test.ts +51 -0
  229. package/src/sync/poll.saga.ts +36 -0
  230. package/src/sync/tabs.saga.test.ts +78 -0
  231. package/src/sync/tabs.saga.ts +96 -0
  232. package/src/testing/adapters.ts +137 -0
  233. package/src/testing/build.ts +63 -0
  234. package/src/testing/fake-services.ts +63 -0
  235. package/src/testing/run-pure.test.ts +92 -0
  236. package/src/testing/run-pure.ts +128 -0
  237. package/src/testing/services.ts +17 -0
  238. package/src/types.ts +633 -10
  239. package/src/utils/blurhash.ts +90 -0
  240. package/src/utils/edges.ts +39 -0
  241. package/src/utils/error-classification.test.ts +54 -0
  242. package/src/utils/error-classification.ts +25 -0
  243. package/src/utils/index.ts +89 -12
  244. package/src/utils/parser.test.ts +67 -102
  245. package/src/utils/parser.ts +64 -2
  246. package/src/utils/semver.test.ts +32 -0
  247. package/src/utils/semver.ts +30 -0
  248. package/src/utils/sha256.ts +15 -0
  249. package/src/utils/surql.ts +29 -18
  250. package/src/utils/utils.test.ts +3 -2
  251. package/tsdown.config.ts +117 -3
  252. package/vitest.config.ts +48 -0
  253. package/src/modules/cache/index.ts +0 -241
  254. package/src/modules/cache/types.ts +0 -19
  255. package/src/modules/data/data.test.ts +0 -58
  256. package/src/modules/data/index.ts +0 -879
  257. package/src/modules/sync/engine.ts +0 -173
  258. package/src/modules/sync/events/index.ts +0 -77
  259. package/src/modules/sync/index.ts +0 -3
  260. package/src/modules/sync/queue/index.ts +0 -2
  261. package/src/modules/sync/queue/queue-down.ts +0 -89
  262. package/src/modules/sync/queue/queue-up.ts +0 -223
  263. package/src/modules/sync/scheduler.ts +0 -84
  264. package/src/modules/sync/sync.ts +0 -421
  265. package/src/modules/sync/utils.test.ts +0 -311
  266. package/src/modules/sync/utils.ts +0 -167
@@ -0,0 +1,199 @@
1
+ import { describe, expect, it } from 'vitest';
2
+ import { RecordId } from 'surrealdb';
3
+ import { runPure } from '../testing/run-pure';
4
+ import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
5
+ import * as R from '../state/reducers';
6
+ import { defaultEnv } from './env';
7
+ import { fetchRows, landChunk } from './fetch.saga';
8
+ import type { StatementResult } from '../kernel/effects';
9
+
10
+ const env = defaultEnv({ tables: [{ name: 'thing', columns: { a: {} } }] } as any);
11
+ const ok = (result: unknown): StatementResult => ({ status: 'OK', result });
12
+ const live = { phase: 'live' } as const;
13
+ const primed = (s: ReturnType<typeof buildState>) => ({ ...s, primed: true });
14
+
15
+ describe('fetchRows', () => {
16
+ it('waits for the prime and returns on an empty plan (resetting the attempt counter)', async () => {
17
+ const s = R.patchSync({ fetchAttempts: 3 })(buildState());
18
+ const out = await runPure(fetchRows(env), { state: s, handlers: { 'state.wait': (_e, ctx) => void (ctx.state = { ...ctx.state, primed: true }) } });
19
+ expect(out.state.sync.fetchAttempts).toBe(0);
20
+ expect(out.log.filter((e) => e.kind === 'remote.query')).toHaveLength(0);
21
+ });
22
+
23
+ it('fetches the deduped set once, writes + ingests bodies, records versions, balances fetch depth', async () => {
24
+ const s = primed(
25
+ buildState(
26
+ [
27
+ buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 2], ['thing:2', 1]] }),
28
+ buildEntry({ def: { hash: 'b' }, lifecycle: live, remoteArray: [['thing:2', 1], ['thing:3', 1]] }),
29
+ ],
30
+ R.setVersions([['thing:3', 1], ['thing:1', 1]]),
31
+ R.outboxReplace([buildOutboxItem({ type: 'delete', recordId: 'thing:2' })])
32
+ )
33
+ );
34
+ const depths: number[] = [];
35
+ const out = await runPure(fetchRows(env), {
36
+ state: s,
37
+ epoch: 7,
38
+ handlers: {
39
+ 'remote.query': (e: any, ctx) => {
40
+ depths.push(ctx.state.queries.get('a')!.lifecycle.fetchDepth);
41
+ expect(e.sql).toBe('SELECT * FROM $ids');
42
+ expect(e.vars.ids).toEqual([new RecordId('thing', '1')]);
43
+ return [ok([{ id: new RecordId('thing', '1'), a: 'x', junk: true }])];
44
+ },
45
+ 'local.execute': () => undefined,
46
+ 'ssp.ingest': () => undefined,
47
+ },
48
+ });
49
+ expect(depths).toEqual([1]);
50
+ expect(out.state.queries.get('a')!.lifecycle.fetchDepth).toBe(0);
51
+ expect(out.state.versions.get('thing:1')).toBe(2);
52
+ const exec = out.log.find((e) => e.kind === 'local.execute') as any;
53
+ expect(exec.vars.content0).toEqual({ a: 'x', _00_rv: 2 });
54
+ expect(exec.epoch).toBe(7);
55
+ expect(out.log.findIndex((e) => e.kind === 'local.epoch')).toBeLessThan(out.log.findIndex((e) => e.kind === 'all'));
56
+ const ingest = out.log.find((e) => e.kind === 'ssp.ingest') as any;
57
+ expect(ingest.records).toEqual([{ table: 'thing', op: 'UPDATE', id: 'thing:1', record: { id: new RecordId('thing', '1'), a: 'x', _00_rv: 2 } }]);
58
+ expect(out.dispatched).toEqual([{ type: 'SyncOutcome', ok: true, error: undefined }]);
59
+ expect(out.state.dirty.has('a')).toBe(true);
60
+ });
61
+
62
+ it('a body the server does not return is remembered at the requested version so the plan converges', async () => {
63
+ const s = primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1], ['other:2', 1]] })]));
64
+ let calls = 0;
65
+ const out = await runPure(fetchRows(env), {
66
+ state: s,
67
+ handlers: {
68
+ 'remote.query': () => {
69
+ calls++;
70
+ return [ok([{ id: new RecordId('other', '2'), z: 1 }, { nope: true }])];
71
+ },
72
+ 'local.execute': () => undefined,
73
+ 'ssp.ingest': () => undefined,
74
+ },
75
+ });
76
+ expect(calls).toBe(1);
77
+ expect(out.state.versions.get('thing:1')).toBe(1);
78
+ expect(out.state.versions.get('other:2')).toBe(1);
79
+ const extra = await runPure(fetchRows(env), {
80
+ state: s,
81
+ handlers: {
82
+ 'remote.query': () => [ok([{ id: new RecordId('thing', '1') }, { id: new RecordId('thing', 'unrequested') }])],
83
+ 'local.execute': () => undefined,
84
+ 'ssp.ingest': (e: any) => {
85
+ expect(e.records[1]).toMatchObject({ id: 'thing:unrequested', record: { _00_rv: 0 } });
86
+ },
87
+ },
88
+ });
89
+ expect(extra.state.versions.has('thing:unrequested')).toBe(false);
90
+ const ingest = out.log.find((e) => e.kind === 'ssp.ingest') as any;
91
+ expect(ingest.records[0]).toMatchObject({ table: 'other', op: 'CREATE', record: { z: 1, _00_rv: 1 } });
92
+ });
93
+
94
+ it('failures back off: transport error, ERR statement, non-array result, local write error; ingest errors are logged only', async () => {
95
+ const s = primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1]] })]));
96
+ const expectBackoff = (out: Awaited<ReturnType<typeof runPure>>, attempts: number) => {
97
+ expect(out.state.sync.fetchAttempts).toBe(attempts);
98
+ expect(out.timers.get('fetch')).toEqual({ ms: 500 * 2 ** (attempts - 1), event: { type: 'FetchRows' } });
99
+ expect(out.state.queries.get('a')!.lifecycle.fetchDepth).toBe(0);
100
+ expect(out.dispatched.at(-1)).toMatchObject({ type: 'SyncOutcome', ok: false });
101
+ };
102
+ expectBackoff(await runPure(fetchRows(env), { state: s, handlers: { 'remote.query': () => { throw new Error('offline'); } } }), 1);
103
+ expectBackoff(await runPure(fetchRows(env), { state: s, handlers: { 'remote.query': () => [{ status: 'ERR', error: 'x' }] } }), 1);
104
+ expectBackoff(await runPure(fetchRows(env), { state: s, handlers: { 'remote.query': () => [ok('nope')] } }), 1);
105
+ expectBackoff(
106
+ await runPure(fetchRows(env), {
107
+ state: R.patchSync({ fetchAttempts: 1 })(s),
108
+ handlers: {
109
+ 'remote.query': () => [ok([{ id: new RecordId('thing', '1') }])],
110
+ 'local.execute': () => {
111
+ throw new Error('stale epoch');
112
+ },
113
+ },
114
+ }),
115
+ 2
116
+ );
117
+ const ingestFail = await runPure(fetchRows(env), {
118
+ state: s,
119
+ handlers: {
120
+ 'remote.query': () => [ok([{ id: new RecordId('thing', '1') }])],
121
+ 'local.execute': () => undefined,
122
+ 'ssp.ingest': () => {
123
+ throw new Error('wasm');
124
+ },
125
+ },
126
+ });
127
+ expect(ingestFail.state.sync.fetchAttempts).toBe(0);
128
+ expect(ingestFail.emitted).toContainEqual(expect.objectContaining({ message: 'circuit ingest failed' }));
129
+ });
130
+
131
+ it('a leader relays landed bodies to followers', async () => {
132
+ const s = { ...primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1]] })])), tabRole: 'leader' as const };
133
+ const out = await runPure(fetchRows(env), {
134
+ state: s,
135
+ handlers: { 'remote.query': () => [ok([{ id: new RecordId('thing', '1') }])], 'local.execute': () => undefined, 'ssp.ingest': () => undefined },
136
+ });
137
+ expect(out.emitted).toContainEqual({ type: 'tabs:broadcast', message: { type: 'ingest', records: [expect.objectContaining({ id: 'thing:1' })] } });
138
+ });
139
+
140
+ it('loops until the plan is empty', async () => {
141
+ const s = primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1]] })]));
142
+ let calls = 0;
143
+ const out = await runPure(fetchRows(env), {
144
+ state: s,
145
+ handlers: {
146
+ 'remote.query': (_e, ctx) => {
147
+ calls++;
148
+ if (calls === 1) ctx.state = R.commitMembership('a', [['thing:1', 1], ['thing:2', 1]], true)(ctx.state);
149
+ return [ok([])];
150
+ },
151
+ },
152
+ });
153
+ expect(calls).toBe(2);
154
+ expect(out.state.versions.get('thing:2')).toBe(1);
155
+ });
156
+ });
157
+
158
+ describe('landChunk', () => {
159
+ const typed = defaultEnv({
160
+ tables: [
161
+ {
162
+ name: 'broadcast',
163
+ columns: {
164
+ title: { type: 'string', optional: false },
165
+ owner: { type: 'string', recordId: true, optional: false },
166
+ club: { type: 'string', recordId: true, optional: true },
167
+ note: { type: 'string', optional: true },
168
+ },
169
+ },
170
+ ],
171
+ } as any);
172
+
173
+ // `UPDATE broadcast:x UNSET club` leaves no `club` key in the body the server
174
+ // returns, and a MERGE that does not name it kept the stale local club.
175
+ it('names each optional column the body lacks as undefined so the MERGE clears it', async () => {
176
+ const rid = new RecordId('broadcast', 'x');
177
+ const out = await runPure(landChunk(typed, ['broadcast:x'], [{ id: rid, title: 't', note: 'n', _00_extra: 1 }], new Map([['broadcast:x', 4]]), buildState(), 3), {
178
+ handlers: { 'local.execute': () => undefined, 'ssp.ingest': () => undefined },
179
+ });
180
+ expect(out.result).toBe(true);
181
+ const exec = out.log.find((e) => e.kind === 'local.execute') as any;
182
+ expect(exec.query.sql).toContain('UPSERT ONLY $id0 MERGE $content0');
183
+ expect(exec.vars.content0).toStrictEqual({ club: undefined, title: 't', note: 'n', _00_extra: 1, _00_rv: 4 });
184
+ expect(Object.prototype.hasOwnProperty.call(exec.vars.content0, 'club')).toBe(true);
185
+ // A required column (`owner`) is never cleared: its absence means hidden, not unset.
186
+ expect(Object.prototype.hasOwnProperty.call(exec.vars.content0, 'owner')).toBe(false);
187
+ // The circuit record carries no clear: CREATE/UPDATE replace the circuit row.
188
+ const ingest = out.log.find((e) => e.kind === 'ssp.ingest') as any;
189
+ expect(ingest.records[0].record).toStrictEqual({ id: rid, title: 't', note: 'n', _00_extra: 1, _00_rv: 4 });
190
+ });
191
+
192
+ it('names nothing for a table without a schema', async () => {
193
+ const out = await runPure(landChunk(typed, ['other:1'], [{ id: new RecordId('other', '1'), z: 1 }], new Map([['other:1', 1]]), buildState(), 0), {
194
+ handlers: { 'local.execute': () => undefined, 'ssp.ingest': () => undefined },
195
+ });
196
+ const exec = out.log.find((e) => e.kind === 'local.execute') as any;
197
+ expect(exec.vars.content0).toStrictEqual({ z: 1, _00_rv: 1 });
198
+ });
199
+ });
@@ -0,0 +1,120 @@
1
+ import type { RecordId } from 'surrealdb';
2
+ import type { Saga } from '../kernel/saga';
3
+ import type { Settled, StatementResult } from '../kernel/effects';
4
+ import { fx } from '../kernel/effects';
5
+ import { backoffMs } from '../kernel/constants';
6
+ import type { IngestRecord } from '../services/stream-processor/index';
7
+ import type { ClientState } from '../state/client-state';
8
+ import * as R from '../state/reducers';
9
+ import { planFetch } from '../state/selectors';
10
+ import { encodeRecordId, parseRecordIdString } from '../utils/index';
11
+ import { absentOptionalColumns, cleanRecord } from '../utils/parser';
12
+ import type { SagaEnv } from './env';
13
+ import * as sql from './sql';
14
+
15
+ /**
16
+ * The unified row fetcher (the poke analogue). Runs on the serial `fetch`
17
+ * lane: computes the cross-query set of bodies missing or stale locally,
18
+ * pulls them in parallel chunks, writes + ingests them, and loops until the
19
+ * plan is empty. A row shared by N queries is fetched once. Bodies are never
20
+ * deleted here; ids that left membership simply stop rendering.
21
+ */
22
+ export function* fetchRows(env: SagaEnv): Saga<void> {
23
+ yield fx.state.wait((s) => s.primed);
24
+ for (;;) {
25
+ const state = (yield fx.state.read((s) => s)) as ClientState;
26
+ const plan = planFetch(state);
27
+ if (plan.chunks.length === 0) {
28
+ if (state.sync.fetchAttempts !== 0) yield fx.state.update(R.patchSync({ fetchAttempts: 0 }));
29
+ return;
30
+ }
31
+ // The engine's epoch, not a mirrored counter: it also moves on transport
32
+ // failover, and a stale value makes the store drop every body write.
33
+ const epoch = (yield fx.local.epoch()) as number;
34
+ yield fx.state.update(R.compose(...plan.hashes.map((h) => R.applyLifecycle(h, { type: 'fetch-begin' }))));
35
+ let failed = 0;
36
+ try {
37
+ const results = (yield fx.all(
38
+ plan.chunks.map((ids) => fx.remote.query(sql.bodySelect(), { ids: ids.map(parseRecordIdString) }, env.remoteTimeoutMs))
39
+ )) as Settled[];
40
+ for (let i = 0; i < results.length; i++) {
41
+ const r = results[i];
42
+ const first = r.ok ? (r.value as StatementResult[])[0] : undefined;
43
+ if (!r.ok || !first || first.status === 'ERR' || !Array.isArray(first.result)) {
44
+ failed++;
45
+ continue;
46
+ }
47
+ const landed = yield* landChunk(env, plan.chunks[i], first.result as Array<Record<string, unknown>>, plan.versions, state, epoch);
48
+ if (!landed) failed++;
49
+ }
50
+ } finally {
51
+ yield fx.state.update(R.compose(...plan.hashes.map((h) => R.applyLifecycle(h, { type: 'fetch-end' }))));
52
+ }
53
+ yield fx.dispatch({ type: 'SyncOutcome', ok: failed === 0, error: failed > 0 ? 'body fetch failed' : undefined });
54
+ if (failed > 0) {
55
+ const attempt = (yield fx.state.read((s) => s.sync.fetchAttempts)) as number;
56
+ yield fx.state.update(R.patchSync({ fetchAttempts: attempt + 1 }));
57
+ yield fx.timer.set('fetch', backoffMs(attempt), { type: 'FetchRows' });
58
+ return;
59
+ }
60
+ }
61
+ }
62
+
63
+ /**
64
+ * Write one chunk's bodies to the store and circuit; record their versions.
65
+ *
66
+ * Shared with the LIVE path: a body that arrives on a notification is landed
67
+ * through here so it is indistinguishable from a fetched one, which is what
68
+ * makes `planFetch` stop asking for it.
69
+ */
70
+ export function* landChunk(
71
+ env: SagaEnv,
72
+ requested: string[],
73
+ rows: Array<Record<string, unknown>>,
74
+ versions: ReadonlyMap<string, number>,
75
+ state: ClientState,
76
+ epoch: number
77
+ ): Saga<boolean> {
78
+ const bodies: Array<{ id: unknown; content: Record<string, unknown> }> = [];
79
+ const ingest: IngestRecord[] = [];
80
+ const seen = new Set<string>();
81
+ for (const row of rows) {
82
+ const rid = row?.id as RecordId<string> | undefined;
83
+ if (!rid || typeof rid !== 'object') continue;
84
+ const id = encodeRecordId(rid);
85
+ const table = rid.table.name;
86
+ const version = versions.get(id) ?? 0;
87
+ const columns = env.schema.tables.find((t) => t.name === table)?.columns;
88
+ const cleaned = columns ? cleanRecord(columns, row) : row;
89
+ const { id: _id, ...content } = cleaned;
90
+ // The row is the server's whole body, so an optional column it lacks was
91
+ // cleared upstream: name it (as `undefined`) so the MERGE removes the local
92
+ // copy instead of keeping it. `_00_*` local bookkeeping is never named, so
93
+ // MERGE (not REPLACE) still preserves it. The circuit record below needs no
94
+ // such key: CREATE/UPDATE replace the circuit's row outright.
95
+ bodies.push({ id: rid, content: { ...absentOptionalColumns(columns, row), ...content, _00_rv: version } });
96
+ ingest.push({ table, op: state.versions.has(id) ? 'UPDATE' : 'CREATE', id, record: { ...cleaned, _00_rv: version } });
97
+ seen.add(id);
98
+ }
99
+ if (bodies.length > 0) {
100
+ try {
101
+ const tx = sql.upsertBodiesTx(bodies);
102
+ yield fx.local.execute(tx.query, tx.vars, epoch);
103
+ } catch (error) {
104
+ yield fx.emit({ type: 'log', level: 'warn', message: 'body write failed', data: { error } });
105
+ return false;
106
+ }
107
+ try {
108
+ yield fx.ssp.ingest(ingest);
109
+ } catch (error) {
110
+ yield fx.emit({ type: 'log', level: 'warn', message: 'circuit ingest failed', data: { error } });
111
+ }
112
+ if (state.tabRole === 'leader') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'ingest', records: ingest } });
113
+ }
114
+ // Ids the server did not return (deleted or unreadable upstream) count as
115
+ // known at the requested version, so the plan stops asking for them; their
116
+ // absence from the store is what the render shows.
117
+ const entries: Array<readonly [string, number]> = requested.map((id) => [id, versions.get(id) as number]);
118
+ yield fx.state.update(R.setVersions(entries));
119
+ return true;
120
+ }
@@ -0,0 +1,14 @@
1
+ import { describe, expect, it } from 'vitest';
2
+ import { queryHashInput, viewKeyInput } from './hash';
3
+
4
+ describe('hash inputs (golden against the previous DataModule formulas)', () => {
5
+ const input = { surql: 'SELECT * FROM t WHERE a = $a', params: { a: 1, b: 'x' } };
6
+ it('salted key is JSON.stringify({surql, params, sessionId}) in that key order', () => {
7
+ expect(queryHashInput(input, 'sess')).toBe(JSON.stringify({ ...input, sessionId: 'sess' }));
8
+ expect(queryHashInput(input, null)).toBe(JSON.stringify({ ...input, sessionId: null }));
9
+ });
10
+ it('view key is the same input minus the salt', () => {
11
+ expect(viewKeyInput(input)).toBe(JSON.stringify(input));
12
+ expect(viewKeyInput(input)).not.toBe(queryHashInput(input, 'sess'));
13
+ });
14
+ });
@@ -0,0 +1,18 @@
1
+ /**
2
+ * Hash inputs for the two query keys. Kept byte-identical to the previous
3
+ * implementation (`DataModule.calculateHash` / `calculateMembershipKey`):
4
+ * the salted key names the remote `_00_query` row, the unsalted one names the
5
+ * durable `_00_view` row, and both are `sha256(JSON.stringify(input))`.
6
+ */
7
+ export interface QueryKeyInput {
8
+ surql: string;
9
+ params: Record<string, unknown>;
10
+ }
11
+
12
+ /** Session-salted: two tabs of one user must not share a remote view. */
13
+ export const queryHashInput = (input: QueryKeyInput, sessionId: string | null): string =>
14
+ JSON.stringify({ surql: input.surql, params: input.params, sessionId });
15
+
16
+ /** Session-independent: survives reload and bucket switch. */
17
+ export const viewKeyInput = (input: QueryKeyInput): string =>
18
+ JSON.stringify({ surql: input.surql, params: input.params });
@@ -0,0 +1,132 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import type { QueryPlan } from '@spooky-sync/query-builder';
3
+ import { buildIdSetPlan, buildIdSetSurql, buildWindowMaterializationPlan } from './window-query';
4
+ import { executeSelect, type SelectDb } from '../services/database/sqlite-select';
5
+ import type { Row } from '../services/database/cache-engine';
6
+
7
+ describe('buildWindowMaterializationPlan', () => {
8
+ it('returns null without an offset and an id-set plan with one', () => {
9
+ const plan: QueryPlan = { table: 't', where: [{ field: 'a', op: '=', value: 1 } as any], limit: 5 };
10
+ expect(buildWindowMaterializationPlan(plan, ['t:1'])).toBeNull();
11
+ expect(buildWindowMaterializationPlan({ ...plan, offset: 0 }, ['t:1'])).toBeNull();
12
+ const win = buildWindowMaterializationPlan({ ...plan, offset: 5 }, ['t:1']);
13
+ expect(win).toMatchObject({ ids: ['t:1'], where: undefined, limit: undefined, offset: undefined });
14
+ });
15
+ });
16
+
17
+ // `buildIdSetPlan` is what makes membership-authoritative rendering possible on
18
+ // any local engine: it turns a query's plan into "these exact ids, ordered,
19
+ // relations intact". Dropping `where` is the point — re-applying the predicate
20
+ // against stale local bodies is what let removed rows keep rendering.
21
+
22
+ const BASE: QueryPlan = {
23
+ table: 'thread',
24
+ where: [{ field: 'done', op: '=', value: false }],
25
+ limit: 50,
26
+ offset: 0,
27
+ orderBy: [['title', 'asc']],
28
+ relations: [
29
+ { alias: 'comments', table: 'comment', cardinality: 'many', foreignKeyField: 'thread' },
30
+ ],
31
+ } as any;
32
+
33
+ describe('buildIdSetPlan', () => {
34
+ it('binds the id-set and drops where/limit/offset, keeping orderBy + relations', () => {
35
+ const out = buildIdSetPlan(BASE, ['thread:A']);
36
+ expect(out.ids).toEqual(['thread:A']);
37
+ expect(out.where).toBeUndefined();
38
+ expect(out.limit).toBeUndefined();
39
+ expect(out.offset).toBeUndefined();
40
+ expect(out.orderBy).toEqual([['title', 'asc']]);
41
+ expect(out.relations).toEqual(BASE.relations);
42
+ expect(out.table).toBe('thread');
43
+ });
44
+
45
+ it('applies to non-offset plans, unlike the windowed-only helper', () => {
46
+ // The windowed helper deliberately bails so the old caller kept its scan;
47
+ // membership rendering needs the same rewrite for every query.
48
+ expect(buildWindowMaterializationPlan(BASE, ['thread:A'])).toBeNull();
49
+ expect(buildIdSetPlan(BASE, ['thread:A']).ids).toEqual(['thread:A']);
50
+ });
51
+
52
+ it('does not mutate the input plan', () => {
53
+ buildIdSetPlan(BASE, ['thread:A']);
54
+ expect(BASE.where).toEqual([{ field: 'done', op: '=', value: false }]);
55
+ expect(BASE.limit).toBe(50);
56
+ });
57
+ });
58
+
59
+ describe('buildIdSetSurql', () => {
60
+ it('rewrites a non-offset query to the id-set source, keeping ORDER BY', () => {
61
+ const out = buildIdSetSurql('SELECT * FROM thread WHERE done = false ORDER BY title ASC LIMIT 50;');
62
+ expect(out?.query).toBe('SELECT * FROM (SELECT * FROM $__win) ORDER BY title ASC');
63
+ });
64
+
65
+ it('preserves the projection', () => {
66
+ const out = buildIdSetSurql('SELECT id, title FROM thread WHERE done = false;');
67
+ expect(out?.query).toBe('SELECT id, title FROM (SELECT * FROM $__win)');
68
+ });
69
+
70
+ it('returns null when there is no top-level FROM to replace', () => {
71
+ expect(buildIdSetSurql('RETURN true;')).toBeNull();
72
+ });
73
+ });
74
+
75
+ // ---- Engine parity -------------------------------------------------------
76
+ //
77
+ // Both local engines short-circuit to `selectByIds` when `plan.ids` is set. This
78
+ // pins that the id-set plan really does filter AND still resolves `.related()`
79
+ // on the SQLite engine (the `where`-dropping path the SurrealDB engine shares).
80
+
81
+ const FIXTURE: Record<string, Row[]> = {
82
+ thread: [
83
+ { id: 'thread:A', title: 'a thread', done: false },
84
+ { id: 'thread:B', title: 'b thread', done: false },
85
+ ],
86
+ comment: [
87
+ { id: 'comment:1', thread: 'thread:A', body: 'on A' },
88
+ { id: 'comment:2', thread: 'thread:B', body: 'on B' },
89
+ ],
90
+ };
91
+
92
+ function makeSelectDb(): SelectDb {
93
+ const respond = (sql: string, bind: unknown[] = []): { data: string }[] => {
94
+ const table = /FROM "([^"]+)"/.exec(sql)?.[1];
95
+ let rows = table ? (FIXTURE[table] ?? []) : [];
96
+ const rel = /WHERE json_extract\(data, '\$\.([A-Za-z0-9_.]+)'\) IN/.exec(sql);
97
+ if (rel) rows = rows.filter((r) => bind.includes(r[rel[1]]));
98
+ else if (/WHERE id IN/.test(sql)) rows = rows.filter((r) => bind.includes(r.id));
99
+ return rows.map((r) => ({ data: JSON.stringify(r) }));
100
+ };
101
+ return {
102
+ exec: (sql, bind) => respond(sql, bind ?? []),
103
+ run: () => {},
104
+ knownTables: new Set<string>(),
105
+ };
106
+ }
107
+
108
+ describe('id-set plan on the SQLite engine', () => {
109
+ it('returns only the id-set even though both bodies match the predicate', async () => {
110
+ const { rows } = await executeSelect(
111
+ buildIdSetPlan(BASE, ['thread:A']),
112
+ {},
113
+ makeSelectDb()
114
+ );
115
+ expect(rows.map((r) => r.id)).toEqual(['thread:A']);
116
+ });
117
+
118
+ it('still resolves .related() children for the id-set rows', async () => {
119
+ const { rows, relationFetches } = await executeSelect(
120
+ buildIdSetPlan(BASE, ['thread:A']),
121
+ {},
122
+ makeSelectDb()
123
+ );
124
+ expect(relationFetches).toBe(1);
125
+ expect((rows[0] as any).comments.map((c: Row) => c.id)).toEqual(['comment:1']);
126
+ });
127
+
128
+ it('renders nothing for an empty id-set (known-and-empty membership)', async () => {
129
+ const { rows } = await executeSelect(buildIdSetPlan(BASE, []), {}, makeSelectDb());
130
+ expect(rows).toEqual([]);
131
+ });
132
+ });