@spooky-sync/core 0.0.1-canary.31 → 0.0.1-canary.310

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/AGENTS.md +81 -0
  2. package/README.md +56 -16
  3. package/dist/index.d.ts +2026 -137
  4. package/dist/index.js +14956 -2434
  5. package/dist/live.d.ts +141 -0
  6. package/dist/live.js +709 -0
  7. package/dist/otel/index.d.ts +1 -1
  8. package/dist/otel/index.js +3 -3
  9. package/dist/pure.d.ts +329 -0
  10. package/dist/pure.js +24 -0
  11. package/dist/sqlite-open.js +358 -0
  12. package/dist/sqlite-worker.d.ts +1 -0
  13. package/dist/sqlite-worker.js +439 -0
  14. package/dist/sw.d.ts +207 -0
  15. package/dist/sw.js +393 -0
  16. package/dist/tabs-broker-worker.d.ts +8 -0
  17. package/dist/tabs-broker-worker.js +472 -0
  18. package/dist/types.d.ts +1765 -38
  19. package/dist/worker-edges.js +423 -0
  20. package/dist/worker-impersonation.js +60 -0
  21. package/dist/worker-ref-tables.d.ts +46 -0
  22. package/dist/worker-types.d.ts +112 -0
  23. package/dist/worker-types.js +73 -0
  24. package/package.json +27 -7
  25. package/scripts/check-broker-bundle.mjs +33 -0
  26. package/scripts/check-worker-entries.mjs +172 -0
  27. package/skills/sp00ky-core/SKILL.md +31 -6
  28. package/skills/sp00ky-core/references/config.md +6 -0
  29. package/src/boot/auth-flip.saga.test.ts +135 -0
  30. package/src/boot/auth-flip.saga.ts +77 -0
  31. package/src/boot/boot.saga.test.ts +158 -0
  32. package/src/boot/boot.saga.ts +131 -0
  33. package/src/boot/bucket-switch.saga.test.ts +187 -0
  34. package/src/boot/bucket-switch.saga.ts +133 -0
  35. package/src/boot/preload.saga.test.ts +51 -0
  36. package/src/boot/preload.saga.ts +31 -0
  37. package/src/bucket-blurhash.test.ts +148 -0
  38. package/src/build-globals.d.ts +12 -0
  39. package/src/client/bucket-handle.test.ts +117 -0
  40. package/src/client/bucket-handle.ts +227 -0
  41. package/src/client/router.test.ts +58 -0
  42. package/src/client/router.ts +100 -0
  43. package/src/client/runtime.test.ts +195 -0
  44. package/src/client/runtime.ts +296 -0
  45. package/src/client/services.ts +370 -0
  46. package/src/client/sp00ky-client.test.ts +303 -0
  47. package/src/client/sp00ky-client.ts +586 -0
  48. package/src/events/events.test.ts +2 -1
  49. package/src/events/index.ts +3 -0
  50. package/src/index.ts +69 -1
  51. package/src/kernel/constants.test.ts +19 -0
  52. package/src/kernel/constants.ts +80 -0
  53. package/src/kernel/effects.test.ts +40 -0
  54. package/src/kernel/effects.ts +161 -0
  55. package/src/kernel/events.ts +57 -0
  56. package/src/kernel/interpreter.test.ts +116 -0
  57. package/src/kernel/interpreter.ts +148 -0
  58. package/src/kernel/saga.test.ts +89 -0
  59. package/src/kernel/saga.ts +73 -0
  60. package/src/live/index.test.ts +478 -0
  61. package/src/live/index.ts +1013 -0
  62. package/src/modules/app-release/index.test.ts +125 -0
  63. package/src/modules/app-release/index.ts +209 -0
  64. package/src/modules/auth/auth.local-first.test.ts +141 -0
  65. package/src/modules/auth/events/index.ts +9 -2
  66. package/src/modules/auth/impersonation-banner.ts +443 -0
  67. package/src/modules/auth/impersonation.test.ts +355 -0
  68. package/src/modules/auth/impersonation.ts +179 -0
  69. package/src/modules/auth/index.ts +386 -17
  70. package/src/modules/crdt/crdt-field.ts +294 -0
  71. package/src/modules/crdt/crdt-hydration.test.ts +210 -0
  72. package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
  73. package/src/modules/crdt/index.ts +463 -0
  74. package/src/modules/crdt/loro-loader.ts +25 -0
  75. package/src/modules/devtools/flags.ts +349 -0
  76. package/src/modules/devtools/index.ts +746 -95
  77. package/src/modules/devtools/mutations.test.ts +92 -0
  78. package/src/modules/devtools/mutations.ts +160 -0
  79. package/src/modules/devtools/notify-throttle.test.ts +166 -0
  80. package/src/modules/devtools/state-shape.test.ts +263 -0
  81. package/src/modules/devtools/storage-info.test.ts +79 -0
  82. package/src/modules/devtools/storage-info.ts +168 -0
  83. package/src/modules/devtools/versions.test.ts +74 -0
  84. package/src/modules/devtools/versions.ts +110 -0
  85. package/src/modules/feature-flag/index.test.ts +251 -0
  86. package/src/modules/feature-flag/index.ts +316 -0
  87. package/src/modules/ref-tables.test.ts +91 -0
  88. package/src/modules/ref-tables.ts +88 -0
  89. package/src/modules/web-push/index.test.ts +824 -0
  90. package/src/modules/web-push/index.ts +1027 -0
  91. package/src/mutation/jobs.test.ts +25 -0
  92. package/src/mutation/jobs.ts +41 -0
  93. package/src/mutation/mutation-id.test.ts +37 -0
  94. package/src/mutation/mutation-id.ts +35 -0
  95. package/src/mutation/pending-writes.integration.test.ts +175 -0
  96. package/src/mutation/push.saga.test.ts +285 -0
  97. package/src/mutation/push.saga.ts +217 -0
  98. package/src/mutation/rows.test.ts +231 -0
  99. package/src/mutation/rows.ts +427 -0
  100. package/src/mutation/tray.saga.test.ts +63 -0
  101. package/src/mutation/tray.saga.ts +48 -0
  102. package/src/mutation/write.saga.test.ts +316 -0
  103. package/src/mutation/write.saga.ts +268 -0
  104. package/src/otel/index.ts +10 -7
  105. package/src/pure.ts +65 -0
  106. package/src/push/types.ts +174 -0
  107. package/src/query/env.test.ts +23 -0
  108. package/src/query/env.ts +56 -0
  109. package/src/query/fetch.saga.test.ts +199 -0
  110. package/src/query/fetch.saga.ts +120 -0
  111. package/src/query/hash.test.ts +14 -0
  112. package/src/query/hash.ts +18 -0
  113. package/src/query/id-set-plan.test.ts +132 -0
  114. package/src/query/lifecycle.saga.test.ts +274 -0
  115. package/src/query/lifecycle.saga.ts +159 -0
  116. package/src/query/materialize.saga.test.ts +115 -0
  117. package/src/query/materialize.saga.ts +84 -0
  118. package/src/query/materialize.test.ts +58 -0
  119. package/src/query/materialize.ts +57 -0
  120. package/src/query/membership.saga.test.ts +350 -0
  121. package/src/query/membership.saga.ts +278 -0
  122. package/src/query/membership.test.ts +118 -0
  123. package/src/query/membership.ts +201 -0
  124. package/src/query/register.saga.test.ts +293 -0
  125. package/src/query/register.saga.ts +226 -0
  126. package/src/query/render-set.test.ts +47 -0
  127. package/src/query/render-set.ts +71 -0
  128. package/src/query/sql.test.ts +63 -0
  129. package/src/query/sql.ts +93 -0
  130. package/src/query/window-query.test.ts +70 -0
  131. package/src/query/window-query.ts +195 -0
  132. package/src/services/blobs/blob-cache.test.ts +359 -0
  133. package/src/services/blobs/blob-cache.ts +603 -0
  134. package/src/services/blobs/blob-manifest.ts +227 -0
  135. package/src/services/blobs/blob-store.test.ts +77 -0
  136. package/src/services/blobs/blob-store.ts +359 -0
  137. package/src/services/blobs/blob.fixture.ts +90 -0
  138. package/src/services/blobs/index.ts +70 -0
  139. package/src/services/database/cache-engine.ts +199 -0
  140. package/src/services/database/connection-supervisor.test.ts +318 -0
  141. package/src/services/database/connection-supervisor.ts +422 -0
  142. package/src/services/database/database.query-timeout.test.ts +140 -0
  143. package/src/services/database/database.responses.test.ts +43 -0
  144. package/src/services/database/database.ts +146 -61
  145. package/src/services/database/engine-factory.ts +33 -0
  146. package/src/services/database/errors.ts +34 -0
  147. package/src/services/database/events/index.ts +2 -1
  148. package/src/services/database/index.ts +7 -0
  149. package/src/services/database/landed-body-clear.integration.test.ts +236 -0
  150. package/src/services/database/local-migrator.integration.test.ts +110 -0
  151. package/src/services/database/local-migrator.ts +59 -16
  152. package/src/services/database/local.test.ts +64 -0
  153. package/src/services/database/local.ts +487 -63
  154. package/src/services/database/plan-render.test.ts +159 -0
  155. package/src/services/database/plan-render.ts +108 -0
  156. package/src/services/database/relation-resolver.test.ts +467 -0
  157. package/src/services/database/relation-resolver.ts +0 -0
  158. package/src/services/database/remote.ts +213 -6
  159. package/src/services/database/sqlite-cache-engine.test.ts +617 -0
  160. package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
  161. package/src/services/database/sqlite-cache-engine.ts +1372 -0
  162. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  163. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  164. package/src/services/database/sqlite-lock-verify.test.ts +33 -0
  165. package/src/services/database/sqlite-lock-verify.ts +45 -0
  166. package/src/services/database/sqlite-open.test.ts +150 -0
  167. package/src/services/database/sqlite-open.ts +164 -0
  168. package/src/services/database/sqlite-plan-sql.test.ts +139 -0
  169. package/src/services/database/sqlite-plan-sql.ts +179 -0
  170. package/src/services/database/sqlite-projection.test.ts +121 -0
  171. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  172. package/src/services/database/sqlite-select.test.ts +246 -0
  173. package/src/services/database/sqlite-select.ts +131 -0
  174. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  175. package/src/services/database/sqlite-transport.ts +224 -0
  176. package/src/services/database/sqlite-worker.ts +437 -0
  177. package/src/services/database/surql-translate.ts +416 -0
  178. package/src/services/database/surreal-cache-engine.ts +161 -0
  179. package/src/services/logger/index.test.ts +132 -0
  180. package/src/services/logger/index.ts +189 -7
  181. package/src/services/persistence/localstorage.ts +2 -2
  182. package/src/services/persistence/resilient.ts +10 -3
  183. package/src/services/persistence/surrealdb.ts +4 -4
  184. package/src/services/stream-processor/index.ts +779 -67
  185. package/src/services/stream-processor/permissions.test.ts +47 -0
  186. package/src/services/stream-processor/permissions.ts +53 -0
  187. package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
  188. package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
  189. package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
  190. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  191. package/src/services/stream-processor/wasm-types.ts +58 -2
  192. package/src/services/tabs/broker-client.ts +283 -0
  193. package/src/services/tabs/broker.test.ts +327 -0
  194. package/src/services/tabs/coordinator.test.ts +386 -0
  195. package/src/services/tabs/coordinator.ts +638 -0
  196. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  197. package/src/services/tabs/leader-locks.ts +75 -0
  198. package/src/services/tabs/protocol.ts +262 -0
  199. package/src/services/tabs/support.ts +36 -0
  200. package/src/services/tabs/tabs-broker-worker.ts +640 -0
  201. package/src/sp00ky.ts +5 -395
  202. package/src/state/client-state.ts +200 -0
  203. package/src/state/lifecycle.test.ts +118 -0
  204. package/src/state/lifecycle.ts +86 -0
  205. package/src/state/reducers.test.ts +304 -0
  206. package/src/state/reducers.ts +526 -0
  207. package/src/state/selectors.test.ts +167 -0
  208. package/src/state/selectors.ts +216 -0
  209. package/src/sw/index.test.ts +582 -0
  210. package/src/sw/index.ts +607 -0
  211. package/src/sw/store.ts +96 -0
  212. package/src/sync/connection.saga.test.ts +86 -0
  213. package/src/sync/connection.saga.ts +85 -0
  214. package/src/sync/live.saga.test.ts +178 -0
  215. package/src/sync/live.saga.ts +107 -0
  216. package/src/sync/policy.health.test.ts +56 -0
  217. package/src/sync/policy.test.ts +659 -0
  218. package/src/sync/policy.ts +502 -0
  219. package/src/sync/poll.saga.test.ts +51 -0
  220. package/src/sync/poll.saga.ts +36 -0
  221. package/src/sync/tabs.saga.test.ts +78 -0
  222. package/src/sync/tabs.saga.ts +96 -0
  223. package/src/testing/adapters.ts +133 -0
  224. package/src/testing/build.ts +63 -0
  225. package/src/testing/fake-services.ts +48 -0
  226. package/src/testing/run-pure.test.ts +92 -0
  227. package/src/testing/run-pure.ts +128 -0
  228. package/src/testing/services.ts +17 -0
  229. package/src/types.ts +624 -10
  230. package/src/utils/blurhash.ts +90 -0
  231. package/src/utils/edges.ts +39 -0
  232. package/src/utils/error-classification.test.ts +54 -0
  233. package/src/utils/error-classification.ts +25 -0
  234. package/src/utils/index.ts +78 -12
  235. package/src/utils/parser.test.ts +67 -102
  236. package/src/utils/parser.ts +64 -2
  237. package/src/utils/semver.test.ts +32 -0
  238. package/src/utils/semver.ts +30 -0
  239. package/src/utils/sha256.ts +15 -0
  240. package/src/utils/surql.ts +30 -18
  241. package/src/utils/utils.test.ts +3 -2
  242. package/tsdown.config.ts +117 -3
  243. package/vitest.config.ts +48 -0
  244. package/src/modules/cache/index.ts +0 -241
  245. package/src/modules/cache/types.ts +0 -19
  246. package/src/modules/data/data.test.ts +0 -58
  247. package/src/modules/data/index.ts +0 -879
  248. package/src/modules/sync/engine.ts +0 -173
  249. package/src/modules/sync/events/index.ts +0 -77
  250. package/src/modules/sync/index.ts +0 -3
  251. package/src/modules/sync/queue/index.ts +0 -2
  252. package/src/modules/sync/queue/queue-down.ts +0 -89
  253. package/src/modules/sync/queue/queue-up.ts +0 -223
  254. package/src/modules/sync/scheduler.ts +0 -84
  255. package/src/modules/sync/sync.ts +0 -421
  256. package/src/modules/sync/utils.test.ts +0 -311
  257. package/src/modules/sync/utils.ts +0 -167
@@ -0,0 +1,92 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import type { OutboxItem, PendingWrite } from '../../state/client-state';
3
+ import { initialHealth } from '../../state/client-state';
4
+ import { buildMutationsState, trimHistory, type MutationHistoryEntry } from './mutations';
5
+
6
+ const id = (ts: number, n = 1) => `_00_pending_mutations:${String(ts).padStart(13, '0')}_${String(n).padStart(4, '0')}_tab`;
7
+
8
+ const item = (over: Partial<OutboxItem> & { id: string }): OutboxItem => ({
9
+ type: 'update',
10
+ recordId: 'thread:a',
11
+ table: 'thread',
12
+ status: 'pending',
13
+ ackedAt: null,
14
+ attempts: 0,
15
+ ...over,
16
+ });
17
+
18
+ const build = (over: Partial<Parameters<typeof buildMutationsState>[0]> = {}) =>
19
+ buildMutationsState({
20
+ outbox: [],
21
+ pendingWrites: [],
22
+ failedCount: 0,
23
+ tabRole: 'solo',
24
+ health: initialHealth('connected'),
25
+ history: [],
26
+ cap: 200,
27
+ ...over,
28
+ });
29
+
30
+ describe('buildMutationsState', () => {
31
+ it('reads status off the outbox, queued time off the id', () => {
32
+ const s = build({
33
+ outbox: [
34
+ item({ id: id(1000) }),
35
+ item({ id: id(2000), attempts: 3 }),
36
+ item({ id: id(3000), status: 'acked', ackedAt: 3500 }),
37
+ ],
38
+ });
39
+ expect(s.entries.map((e) => [e.queuedAt, e.status])).toEqual([
40
+ [3000, 'synced'],
41
+ [2000, 'retrying'],
42
+ [1000, 'pending'],
43
+ ]);
44
+ expect(s.entries[0].settledAt).toBe(3500);
45
+ expect(s.counts).toMatchObject({ pending: 1, retrying: 1, synced: 1, rolledBack: 0 });
46
+ });
47
+
48
+ it('keeps what left the outbox, and the live outbox wins over history', () => {
49
+ const history: MutationHistoryEntry[] = [
50
+ { id: id(1000), op: 'create', recordId: 'thread:a', table: 'thread', fields: ['title'], queuedAt: 1000, outcome: { status: 'synced', at: 1100 } },
51
+ { id: id(2000), op: 'update', recordId: 'thread:b', table: 'thread', fields: ['body'], queuedAt: 2000, outcome: { status: 'rolled-back', at: 2100, error: 'denied' } },
52
+ { id: id(3000), op: 'update', recordId: 'thread:c', table: 'thread', fields: ['x'], queuedAt: 3000 },
53
+ { id: id(4000), op: 'delete', recordId: 'thread:d', table: 'thread', queuedAt: 4000 },
54
+ ];
55
+ const s = build({ history, outbox: [item({ id: id(3000), recordId: 'thread:c', attempts: 1 })] });
56
+ const byId = new Map(s.entries.map((e) => [e.id, e]));
57
+ expect(byId.get(id(1000))).toMatchObject({ status: 'synced', settledAt: 1100, fields: ['title'] });
58
+ expect(byId.get(id(2000))).toMatchObject({ status: 'rolled-back', error: 'denied' });
59
+ expect(byId.get(id(3000))).toMatchObject({ status: 'retrying', attempts: 1, fields: ['x'] });
60
+ expect(byId.get(id(4000))).toMatchObject({ status: 'dropped' });
61
+ });
62
+
63
+ it('caps entries but counts and totals everything', () => {
64
+ const outbox = Array.from({ length: 5 }, (_, i) => item({ id: id(1000 + i) }));
65
+ const s = build({ outbox, cap: 2 });
66
+ expect(s.entries.map((e) => e.queuedAt)).toEqual([1004, 1003]);
67
+ expect(s.total).toBe(5);
68
+ expect(s.counts.pending).toBe(5);
69
+ });
70
+
71
+ it('lists debounced writes and carries role, health and tray count', () => {
72
+ const w: PendingWrite = { key: 'k', table: 'doc', recordId: 'doc:1', data: { title: 't', body: 'b' }, before: null, firstAt: 42, mirrorId: null };
73
+ const s = build({
74
+ pendingWrites: [w],
75
+ failedCount: 3,
76
+ tabRole: 'follower',
77
+ health: { ...initialHealth('reconnecting'), status: 'degraded', consecutiveFailures: 4, error: 'socket closed' },
78
+ });
79
+ expect(s.debounced).toEqual([{ key: 'k', recordId: 'doc:1', table: 'doc', fields: ['title', 'body'], since: 42, durable: false }]);
80
+ expect(s).toMatchObject({ role: 'follower', connection: 'reconnecting', health: 'degraded', consecutiveFailures: 4, lastError: 'socket closed' });
81
+ expect(s.counts).toMatchObject({ failed: 3, debounced: 1 });
82
+ });
83
+ });
84
+
85
+ describe('trimHistory', () => {
86
+ it('drops the oldest entries first', () => {
87
+ const h = new Map<string, MutationHistoryEntry>();
88
+ for (const n of [1, 2, 3]) h.set(`m${n}`, { id: `m${n}`, op: 'create', recordId: 'a:1', table: 'a', queuedAt: n });
89
+ trimHistory(h, 2);
90
+ expect([...h.keys()]).toEqual(['m2', 'm3']);
91
+ });
92
+ });
@@ -0,0 +1,160 @@
1
+ import type { ConnectionState, MutationEventType, SyncHealth } from '../../types';
2
+ import type { OutboxItem, PendingWrite, TabRole } from '../../state/client-state';
3
+ import { createdAtFromId } from '../../mutation/rows';
4
+
5
+ /**
6
+ * Where a mutation is, as the DevTools Mutations tab shows it:
7
+ * - `pending` in the outbox, not sent yet (or sent and not answered)
8
+ * - `retrying` in the outbox after at least one failed push
9
+ * - `synced` the server accepted it
10
+ * - `rolled-back` the server rejected it; the local change was undone and the
11
+ * row moved to the failed-writes tray
12
+ * - `dropped` left the outbox without an answer this tab saw (a bucket
13
+ * switch, a sign-out, or its row was already gone)
14
+ */
15
+ export type DevToolsMutationStatus = 'pending' | 'retrying' | 'synced' | 'rolled-back' | 'dropped';
16
+
17
+ export interface DevToolsMutation {
18
+ /** `_00_pending_mutations:<ts>_<seq>_<tab>`, escaping removed. */
19
+ id: string;
20
+ op: MutationEventType;
21
+ recordId: string;
22
+ table: string;
23
+ /** Field names of the payload (the values stay in the page; see `mutationOp('get')`). */
24
+ fields?: string[];
25
+ status: DevToolsMutationStatus;
26
+ attempts: number;
27
+ queuedAt: number;
28
+ settledAt?: number;
29
+ error?: string;
30
+ }
31
+
32
+ /** What DevTools remembers about one mutation after it left the outbox. */
33
+ export interface MutationHistoryEntry {
34
+ id: string;
35
+ op: MutationEventType;
36
+ recordId: string;
37
+ table: string;
38
+ fields?: string[];
39
+ queuedAt: number;
40
+ outcome?: { status: 'synced' | 'rolled-back'; at: number; error?: string };
41
+ }
42
+
43
+ export interface DevToolsDebouncedWrite {
44
+ key: string;
45
+ recordId: string;
46
+ table: string;
47
+ fields: string[];
48
+ since: number;
49
+ /** Mirrored to `_00_pending_writes`, so a reload inside the delay keeps it. */
50
+ durable: boolean;
51
+ }
52
+
53
+ export interface DevToolsMutationsState {
54
+ /** `follower` tabs never drain: the leader pushes their writes. */
55
+ role: TabRole;
56
+ connection: ConnectionState;
57
+ health: SyncHealth['status'];
58
+ consecutiveFailures: number;
59
+ lastError?: string;
60
+ counts: {
61
+ pending: number;
62
+ retrying: number;
63
+ debounced: number;
64
+ /** Rows in the persistent failed-writes tray, any session. */
65
+ failed: number;
66
+ /** This session, since DevTools attached. */
67
+ synced: number;
68
+ rolledBack: number;
69
+ };
70
+ /** Newest first, capped at `cap`. */
71
+ entries: DevToolsMutation[];
72
+ total: number;
73
+ debounced: DevToolsDebouncedWrite[];
74
+ }
75
+
76
+ export interface MutationsInput {
77
+ outbox: ReadonlyArray<OutboxItem>;
78
+ pendingWrites: Iterable<PendingWrite>;
79
+ failedCount: number;
80
+ tabRole: TabRole;
81
+ health: SyncHealth;
82
+ history: Iterable<MutationHistoryEntry>;
83
+ cap: number;
84
+ }
85
+
86
+ /** When a mutation was queued, from the timestamp its id starts with (0 for a legacy id). */
87
+ export const queuedAtOf = (id: string): number => createdAtFromId(id) ?? 0;
88
+
89
+ /** The tab's mutation picture: the live outbox over what DevTools saw leave it. */
90
+ export function buildMutationsState(input: MutationsInput): DevToolsMutationsState {
91
+ const byId = new Map<string, DevToolsMutation>();
92
+ for (const h of input.history) {
93
+ byId.set(h.id, {
94
+ id: h.id,
95
+ op: h.op,
96
+ recordId: h.recordId,
97
+ table: h.table,
98
+ fields: h.fields,
99
+ status: h.outcome ? h.outcome.status : 'dropped',
100
+ attempts: 0,
101
+ queuedAt: h.queuedAt,
102
+ settledAt: h.outcome?.at,
103
+ error: h.outcome?.error,
104
+ });
105
+ }
106
+ for (const item of input.outbox) {
107
+ const seen = byId.get(item.id);
108
+ const status: DevToolsMutationStatus =
109
+ item.status === 'acked' ? 'synced' : item.attempts > 0 ? 'retrying' : 'pending';
110
+ byId.set(item.id, {
111
+ id: item.id,
112
+ op: item.type,
113
+ recordId: item.recordId,
114
+ table: item.table,
115
+ fields: seen?.fields,
116
+ status,
117
+ attempts: item.attempts,
118
+ queuedAt: seen?.queuedAt || queuedAtOf(item.id),
119
+ settledAt: item.ackedAt ?? seen?.settledAt,
120
+ });
121
+ }
122
+
123
+ const all = [...byId.values()].toSorted((a, b) => b.queuedAt - a.queuedAt || (a.id < b.id ? 1 : -1));
124
+ const count = (s: DevToolsMutationStatus) => all.filter((m) => m.status === s).length;
125
+ const debounced = [...input.pendingWrites].map((w) => ({
126
+ key: w.key,
127
+ recordId: w.recordId,
128
+ table: w.table,
129
+ fields: Object.keys(w.data),
130
+ since: w.firstAt,
131
+ durable: w.mirrorId !== null,
132
+ }));
133
+
134
+ return {
135
+ role: input.tabRole,
136
+ connection: input.health.connection,
137
+ health: input.health.status,
138
+ consecutiveFailures: input.health.consecutiveFailures,
139
+ lastError: input.health.error,
140
+ counts: {
141
+ pending: count('pending'),
142
+ retrying: count('retrying'),
143
+ debounced: debounced.length,
144
+ failed: input.failedCount,
145
+ synced: count('synced'),
146
+ rolledBack: count('rolled-back'),
147
+ },
148
+ entries: all.slice(0, input.cap),
149
+ total: all.length,
150
+ debounced,
151
+ };
152
+ }
153
+
154
+ /** Keep the newest `cap` entries of an insertion-ordered history map. */
155
+ export function trimHistory(history: Map<string, MutationHistoryEntry>, cap: number): void {
156
+ for (const id of history.keys()) {
157
+ if (history.size <= cap) return;
158
+ history.delete(id);
159
+ }
160
+ }
@@ -0,0 +1,166 @@
1
+ import { describe, it, expect, vi, afterEach } from 'vitest';
2
+ import { RecordId } from 'surrealdb';
3
+ import { DevToolsService } from './index';
4
+
5
+ /**
6
+ * A state push serializes EVERY active query's full record set and postMessage
7
+ * clones it again, so its cost scales with the whole client dataset. It is
8
+ * requested per event — including one per local DB query — so without
9
+ * coalescing a page load's few hundred local queries turn a few MB of rows into
10
+ * GBs of short-lived large-object garbage and OOM the renderer. These tests pin
11
+ * the coalescing, not the payload. `onStateChanged` stands in for any trigger.
12
+ */
13
+
14
+ function harness(recordCount = 500) {
15
+ const posted: any[] = [];
16
+ const listeners: ((e: any) => void)[] = [];
17
+ const fakeWindow: any = {
18
+ postMessage: (msg: any) => posted.push(msg),
19
+ addEventListener: (_type: string, cb: (e: any) => void) => listeners.push(cb),
20
+ dispatchEvent: () => true,
21
+ };
22
+ fakeWindow.self = fakeWindow;
23
+ vi.stubGlobal('window', fakeWindow);
24
+ vi.stubGlobal('CustomEvent', class {
25
+ type: string;
26
+ constructor(type: string) {
27
+ this.type = type;
28
+ }
29
+ });
30
+
31
+ const noop = () => {};
32
+ const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
33
+ logger.child = () => logger;
34
+
35
+ const local: any = {
36
+ query: async () => [],
37
+ getConfig: () => ({ store: 'memory' }),
38
+ currentBucketId: 'anon',
39
+ storageHealth: { status: 'memory', fallback: false },
40
+ };
41
+ const remote: any = { query: async () => [] };
42
+ const auth: any = {
43
+ isAuthenticated: false,
44
+ currentUser: undefined,
45
+ eventSystem: { subscribe: noop },
46
+ };
47
+ // One query holding a lot of rows: the thing whose repeated serialization is
48
+ // what actually blows the heap.
49
+ const records = Array.from({ length: recordCount }, (_, i) => ({ id: `game:${i}`, pgn: 'x' }));
50
+ const dataManager: any = {
51
+ getActiveQueries: () => [
52
+ {
53
+ config: { id: new RecordId('_00_query', 'q1'), params: {} },
54
+ status: 'idle',
55
+ records,
56
+ updateCount: 1,
57
+ },
58
+ ],
59
+ phaseTimings: () => ({}),
60
+ };
61
+
62
+ const service = new DevToolsService(local, remote, logger, { tables: [] } as any, auth, dataManager);
63
+ // Whatever the next push should carry; tests move it between requests.
64
+ const mutations: any = {
65
+ outbox: [],
66
+ pendingWrites: new Map(),
67
+ failedCount: 0,
68
+ tabRole: 'solo',
69
+ sync: { health: { status: 'healthy', consecutiveFailures: 0, everConnected: true, connection: 'connected' } },
70
+ };
71
+ service.setMutationSource({
72
+ state: () => mutations,
73
+ listFailed: async () => [],
74
+ retryFailed: async () => false,
75
+ discardFailed: async () => false,
76
+ });
77
+ // Announce a consumer, exactly like the extension's page-script does.
78
+ for (const cb of listeners) {
79
+ cb({ source: fakeWindow, data: { type: 'SP00KY_DEVTOOLS_CONNECT' } });
80
+ }
81
+ const statePushes = () => posted.filter((m) => m.type === 'SP00KY_STATE_CHANGED');
82
+ return { service, statePushes, posted, mutations };
83
+ }
84
+
85
+ afterEach(() => {
86
+ vi.useRealTimers();
87
+ vi.unstubAllGlobals();
88
+ });
89
+
90
+ describe('DevToolsService state-push coalescing', () => {
91
+ it('pushes once on connect, on the next macrotask', () => {
92
+ vi.useFakeTimers();
93
+ const { statePushes } = harness();
94
+ // Never inline: a push serializes the state, and the request can come
95
+ // from inside an ingest or a mutation the app is awaiting.
96
+ expect(statePushes().length).toBe(0);
97
+ vi.advanceTimersByTime(0);
98
+ expect(statePushes().length).toBe(1);
99
+ });
100
+
101
+ it('collapses a burst of change requests into ONE trailing push', () => {
102
+ vi.useFakeTimers();
103
+ const { service, statePushes } = harness();
104
+ const before = statePushes().length;
105
+
106
+ // What a page load looks like: hundreds of requests, each of which used to
107
+ // serialize the entire query state.
108
+ for (let i = 0; i < 400; i++) service.onStateChanged();
109
+ // Nothing extra yet — the burst is queued, not serialized 400 times.
110
+ expect(statePushes().length).toBe(before);
111
+
112
+ vi.advanceTimersByTime(300);
113
+ expect(statePushes().length).toBe(before + 1);
114
+ });
115
+
116
+ it('still pushes again once the window has passed', () => {
117
+ vi.useFakeTimers();
118
+ const { service, statePushes } = harness();
119
+ const before = statePushes().length;
120
+
121
+ service.onStateChanged();
122
+ vi.advanceTimersByTime(300);
123
+ expect(statePushes().length).toBe(before + 1);
124
+
125
+ // A request arriving right after that flush is still inside the window, so it
126
+ // queues rather than pushing again...
127
+ service.onStateChanged();
128
+ expect(statePushes().length).toBe(before + 1);
129
+ vi.advanceTimersByTime(300);
130
+ expect(statePushes().length).toBe(before + 2);
131
+
132
+ // ...but once the tab has been idle past the window, the next request pushes
133
+ // straight away, so the panel never waits on a quiet app.
134
+ vi.advanceTimersByTime(1000);
135
+ service.onStateChanged();
136
+ vi.advanceTimersByTime(0);
137
+ expect(statePushes().length).toBe(before + 3);
138
+ });
139
+
140
+ it('drops a queued push when the consumer disconnects mid-window', () => {
141
+ vi.useFakeTimers();
142
+ const { service, statePushes, posted } = harness();
143
+ const before = statePushes().length;
144
+
145
+ service.onStateChanged();
146
+ (service as any).enabled = false; // panel closed while the push was queued
147
+ vi.advanceTimersByTime(300);
148
+ expect(statePushes().length).toBe(before);
149
+ expect(posted.some((m) => m.type === 'SP00KY_STATE_CHANGED' && m.state === undefined)).toBe(false);
150
+ });
151
+
152
+ it('serializes the LATEST state at flush time, not at request time', () => {
153
+ vi.useFakeTimers();
154
+ const { service, statePushes, mutations } = harness();
155
+ vi.advanceTimersByTime(300);
156
+
157
+ service.onStateChanged();
158
+ mutations.failedCount = 1;
159
+ service.onStateChanged();
160
+ mutations.failedCount = 2;
161
+ vi.advanceTimersByTime(300);
162
+
163
+ // One push for the window, carrying what was true when it flushed.
164
+ expect(statePushes().at(-1).state.mutations.counts.failed).toBe(2);
165
+ });
166
+ });
@@ -0,0 +1,263 @@
1
+ import { describe, it, expect, vi, afterEach } from 'vitest';
2
+ import { RecordId } from 'surrealdb';
3
+ import { DevToolsService } from './index';
4
+ import { initialHealth } from '../../state/client-state';
5
+ import { LogTap, createLogger } from '../../services/logger/index';
6
+
7
+ // Vitest runs in node, where `pino` resolves to its node build and ignores the
8
+ // `browser` options the tap hangs off. Apps bundle the browser build.
9
+ // The specifier is widened to `string`: pino ships no types for that entry.
10
+ vi.mock('pino', async () => ({ default: (await import('pino/browser.js' as string)).default }));
11
+
12
+ /**
13
+ * The pushed state used to carry every view's full record set (and both
14
+ * membership arrays), deep-cloned once by the serializer and again by
15
+ * postMessage, on every sync event, from inside the ingest call stack. On a
16
+ * client holding a few thousand rows that is what the main thread was doing
17
+ * instead of finishing the write the app was awaiting. Pushes now carry counts
18
+ * and capped ids; rows are pulled per view on demand.
19
+ */
20
+ function harness(recordCount = 500) {
21
+ const posted: any[] = [];
22
+ const listeners: ((e: any) => void)[] = [];
23
+ const fakeWindow: any = {
24
+ postMessage: (msg: any) => posted.push(msg),
25
+ addEventListener: (_type: string, cb: (e: any) => void) => listeners.push(cb),
26
+ dispatchEvent: () => true,
27
+ };
28
+ fakeWindow.self = fakeWindow;
29
+ vi.stubGlobal('window', fakeWindow);
30
+ vi.stubGlobal('CustomEvent', class {
31
+ type: string;
32
+ constructor(type: string) {
33
+ this.type = type;
34
+ }
35
+ });
36
+ const noop = () => {};
37
+ const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
38
+ logger.child = () => logger;
39
+ const infoQueries: string[] = [];
40
+ const local: any = {
41
+ query: async (sql: string) => {
42
+ infoQueries.push(sql);
43
+ return [];
44
+ },
45
+ getConfig: () => ({ store: 'memory' }),
46
+ currentBucketId: 'anon',
47
+ storageHealth: { status: 'memory', fallback: false },
48
+ };
49
+ const remote: any = { query: async () => [] };
50
+ const auth: any = { isAuthenticated: false, currentUser: undefined, eventSystem: { subscribe: noop } };
51
+ const records = Array.from({ length: recordCount }, (_, i) => ({ id: `game:${i}`, pgn: 'x' }));
52
+ const localArray = records.map((r) => [r.id, 1] as [string, number]);
53
+ const id = new RecordId('_00_query', 'q1');
54
+ const query = {
55
+ config: { id, params: {}, localArray, remoteArray: localArray, surql: 'SELECT * FROM game' },
56
+ status: 'idle',
57
+ records,
58
+ updateCount: 1,
59
+ };
60
+ const dataManager: any = {
61
+ getActiveQueries: () => [query],
62
+ getQueryById: (rid: RecordId<string>) => (String(rid) === String(id) ? query : undefined),
63
+ phaseTimings: () => ({}),
64
+ };
65
+ const service = new DevToolsService(local, remote, logger, { tables: [] } as any, auth, dataManager);
66
+ for (const cb of listeners) cb({ source: fakeWindow, data: { type: 'SP00KY_DEVTOOLS_CONNECT' } });
67
+ const statePushes = () => posted.filter((m) => m.type === 'SP00KY_STATE_CHANGED');
68
+ return { service, statePushes, posted, fakeWindow, infoQueries, logger };
69
+ }
70
+
71
+ describe('DevTools pushed state shape', () => {
72
+ afterEach(() => {
73
+ vi.unstubAllGlobals();
74
+ vi.useRealTimers();
75
+ });
76
+
77
+ it('carries counts and capped ids, never the rows', async () => {
78
+ vi.useFakeTimers();
79
+ const { service, statePushes } = harness(500);
80
+ service.onStateChanged();
81
+ await vi.advanceTimersByTimeAsync(300);
82
+ const push = statePushes().at(-1);
83
+ expect(push).toBeDefined();
84
+ const q: any = Object.values(push.state.activeQueries)[0];
85
+ expect(q.data).toBeUndefined();
86
+ expect(q.localArray).toBeUndefined();
87
+ expect(q.remoteArray).toBeUndefined();
88
+ expect(q.dataSize).toBe(500);
89
+ expect(q.localCount).toBe(500);
90
+ expect(q.remoteCount).toBe(500);
91
+ expect(q.localIds).toHaveLength(200);
92
+ expect(q.idsTruncated).toBe(true);
93
+ });
94
+
95
+ it('never pushes synchronously inside the call that requested it', async () => {
96
+ vi.useFakeTimers();
97
+ const { service, statePushes } = harness(5);
98
+ // Connect already queued one push; drain it so the next request is "idle".
99
+ await vi.advanceTimersByTimeAsync(300);
100
+ const before = statePushes().length;
101
+ service.onStreamUpdate({ queryHash: 'q1', localArray: [], op: 'CREATE' });
102
+ expect(statePushes().length).toBe(before);
103
+ await vi.advanceTimersByTimeAsync(0);
104
+ expect(statePushes().length).toBe(before + 1);
105
+ });
106
+
107
+ it('serves the rows of one view on demand', () => {
108
+ const { service, fakeWindow } = harness(3);
109
+ void service;
110
+ const state = fakeWindow.__00__.getState();
111
+ const hash = Number(Object.keys(state.activeQueries)[0]);
112
+ const rows = fakeWindow.__00__.getQueryRows(hash);
113
+ expect(rows.data).toHaveLength(3);
114
+ expect(rows.localArray).toHaveLength(3);
115
+ expect(fakeWindow.__00__.getQueryRows(12345)).toBeNull();
116
+ });
117
+
118
+ it('logs a stream update as counts and timings, never the membership array', () => {
119
+ const { service, logger } = harness(2);
120
+ const debug = vi.spyOn(logger, 'debug');
121
+ service.onStreamUpdate({ queryHash: 'q1', localArray: [['a', 1], ['b', 1]], op: 'UPDATE', storeApplyMs: 1.5 });
122
+ const [fields, msg] = debug.mock.calls.at(-1) as [any, string];
123
+ expect(msg).toBe('StreamUpdate');
124
+ expect(fields).toMatchObject({ queryHash: 'q1', op: 'UPDATE', localCount: 2, storeApplyMs: 1.5 });
125
+ expect(fields.localArray).toBeUndefined();
126
+ });
127
+
128
+ it('no longer keeps an event log, but still sends the field older panels key on', () => {
129
+ const { fakeWindow } = harness(1);
130
+ expect(fakeWindow.__00__.getState().eventsHistory).toEqual([]);
131
+ expect(fakeWindow.__00__.clearHistory).toBeUndefined();
132
+ });
133
+
134
+ it('ignores a synthetic re-materialize', async () => {
135
+ vi.useFakeTimers();
136
+ const { service, statePushes } = harness(1);
137
+ await vi.advanceTimersByTimeAsync(300);
138
+ const before = statePushes().length;
139
+ service.onStreamUpdate({ queryHash: 'q1', localArray: [], op: 'UPDATE', synthetic: true });
140
+ await vi.advanceTimersByTimeAsync(300);
141
+ expect(statePushes().length).toBe(before);
142
+ });
143
+
144
+ it('refreshes the table list on an explicit pull, not on a push', async () => {
145
+ vi.useFakeTimers();
146
+ const { service, fakeWindow, infoQueries } = harness(1);
147
+ const connectRefreshes = infoQueries.filter((q) => q.includes('INFO FOR DB')).length;
148
+ for (let i = 0; i < 10; i++) {
149
+ service.onStreamUpdate({ queryHash: 'q1', localArray: [], op: 'UPDATE' });
150
+ await vi.advanceTimersByTimeAsync(300);
151
+ }
152
+ expect(infoQueries.filter((q) => q.includes('INFO FOR DB')).length).toBe(connectRefreshes);
153
+ vi.setSystemTime(Date.now() + 60_000);
154
+ fakeWindow.__00__.getState();
155
+ expect(infoQueries.filter((q) => q.includes('INFO FOR DB')).length).toBe(connectRefreshes + 1);
156
+ });
157
+ });
158
+
159
+ describe('DevTools mutations and logs', () => {
160
+ afterEach(() => {
161
+ vi.restoreAllMocks();
162
+ vi.unstubAllGlobals();
163
+ vi.useRealTimers();
164
+ });
165
+
166
+ const mutationState = (): any => ({
167
+ outbox: [
168
+ {
169
+ id: '_00_pending_mutations:0000000001000_0001_t',
170
+ type: 'create',
171
+ recordId: 'thread:a',
172
+ table: 'thread',
173
+ status: 'pending',
174
+ ackedAt: null,
175
+ attempts: 0,
176
+ },
177
+ ],
178
+ pendingWrites: new Map(),
179
+ failedCount: 1,
180
+ tabRole: 'solo',
181
+ sync: { health: initialHealth('connected') },
182
+ });
183
+
184
+ it('pushes the outbox and what already left it', async () => {
185
+ vi.useFakeTimers();
186
+ const { service, statePushes } = harness(0);
187
+ service.setMutationSource({
188
+ state: mutationState,
189
+ listFailed: async () => [],
190
+ retryFailed: async () => true,
191
+ discardFailed: async () => false,
192
+ });
193
+ service.onMutation([
194
+ {
195
+ type: 'update',
196
+ mutation_id: new RecordId('_00_pending_mutations', '0000000002000_0002_t'),
197
+ record_id: new RecordId('thread', 'b'),
198
+ data: { title: 'a very long payload the push must not carry' },
199
+ tableName: 'thread',
200
+ },
201
+ ]);
202
+ service.onMutationOutcome({
203
+ mutationId: '_00_pending_mutations:0000000002000_0002_t',
204
+ recordId: 'thread:b',
205
+ eventType: 'update',
206
+ status: 'rolled-back',
207
+ error: 'denied',
208
+ });
209
+ await vi.advanceTimersByTimeAsync(300);
210
+ const m = statePushes().at(-1).state.mutations;
211
+ expect(m.counts).toMatchObject({ pending: 1, rolledBack: 1, failed: 1 });
212
+ expect(m.entries[0]).toMatchObject({
213
+ recordId: 'thread:b',
214
+ status: 'rolled-back',
215
+ error: 'denied',
216
+ fields: ['title'],
217
+ queuedAt: 2000,
218
+ });
219
+ expect(JSON.stringify(m)).not.toContain('very long payload');
220
+ });
221
+
222
+ it('answers the tray ops, with a reason when the row is gone', async () => {
223
+ const { service, fakeWindow } = harness(0);
224
+ service.setMutationSource({
225
+ state: mutationState,
226
+ listFailed: async () => [],
227
+ retryFailed: async () => true,
228
+ discardFailed: async () => false,
229
+ });
230
+ expect(await fakeWindow.__00__.mutationOp('retry', { id: 'm1' })).toEqual({ success: true });
231
+ expect(await fakeWindow.__00__.mutationOp('discard', { id: 'm1' })).toMatchObject({
232
+ success: false,
233
+ error: expect.stringContaining('not in the failed tray'),
234
+ });
235
+ expect(await fakeWindow.__00__.mutationOp('listFailed')).toEqual({ success: true, failed: [] });
236
+ });
237
+
238
+ it('pushes new log lines as deltas and keeps them out of the state', async () => {
239
+ vi.useFakeTimers();
240
+ vi.spyOn(console, 'log').mockImplementation(() => {});
241
+ const { service, posted, fakeWindow } = harness(0);
242
+ const tap = new LogTap('info');
243
+ const logger = createLogger('info', undefined, tap);
244
+ logger.info('before attach');
245
+ service.setLogTap(tap);
246
+ logger.warn('one');
247
+ logger.info('two');
248
+ await vi.advanceTimersByTimeAsync(300);
249
+ const pushes = posted.filter((m) => m.type === 'SP00KY_LOGS');
250
+ expect(pushes).toHaveLength(1);
251
+ expect(pushes[0].entries.map((e: any) => e.msg)).toEqual(['one', 'two']);
252
+ expect(fakeWindow.__00__.getState().logs).toEqual({ head: 3, consoleLevel: 'info', captureLevel: 'info' });
253
+ const backlog = fakeWindow.__00__.logOp('read', {});
254
+ expect(backlog.entries.map((e: any) => e.msg)).toEqual(['before attach', 'one', 'two']);
255
+ expect(fakeWindow.__00__.logOp('setCaptureLevel', { level: 'debug' })).toMatchObject({
256
+ success: true,
257
+ captureLevel: 'debug',
258
+ });
259
+ logger.debug('now captured');
260
+ expect(fakeWindow.__00__.logOp('read', { after: 3 }).entries.map((e: any) => e.msg)).toEqual(['now captured']);
261
+ expect(fakeWindow.__00__.logOp('setCaptureLevel', { level: 'loud' }).success).toBe(false);
262
+ });
263
+ });