@spooky-sync/core 0.0.1-canary.24 → 0.0.1-canary.241

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/AGENTS.md +77 -0
  2. package/README.md +56 -16
  3. package/dist/index.d.ts +2176 -80
  4. package/dist/index.js +12557 -2588
  5. package/dist/otel/index.d.ts +2 -2
  6. package/dist/otel/index.js +6 -6
  7. package/dist/sqlite-open.js +322 -0
  8. package/dist/sqlite-worker.d.ts +1 -0
  9. package/dist/sqlite-worker.js +439 -0
  10. package/dist/tabs-broker-worker.d.ts +8 -0
  11. package/dist/tabs-broker-worker.js +472 -0
  12. package/dist/types.d.ts +747 -46
  13. package/package.json +16 -8
  14. package/scripts/check-broker-bundle.mjs +33 -0
  15. package/skills/{spooky-core → sp00ky-core}/SKILL.md +30 -12
  16. package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
  17. package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
  18. package/src/boot/auth-flip.saga.test.ts +82 -0
  19. package/src/boot/auth-flip.saga.ts +62 -0
  20. package/src/boot/boot.saga.test.ts +151 -0
  21. package/src/boot/boot.saga.ts +128 -0
  22. package/src/boot/bucket-switch.saga.test.ts +158 -0
  23. package/src/boot/bucket-switch.saga.ts +115 -0
  24. package/src/boot/preload.saga.test.ts +51 -0
  25. package/src/boot/preload.saga.ts +31 -0
  26. package/src/bucket-blurhash.test.ts +148 -0
  27. package/src/build-globals.d.ts +12 -0
  28. package/src/client/bucket-handle.test.ts +117 -0
  29. package/src/client/bucket-handle.ts +227 -0
  30. package/src/client/router.test.ts +56 -0
  31. package/src/client/router.ts +94 -0
  32. package/src/client/runtime.test.ts +192 -0
  33. package/src/client/runtime.ts +296 -0
  34. package/src/client/services.ts +349 -0
  35. package/src/client/sp00ky-client.test.ts +241 -0
  36. package/src/client/sp00ky-client.ts +482 -0
  37. package/src/events/events.test.ts +2 -1
  38. package/src/events/index.ts +3 -0
  39. package/src/index.ts +37 -2
  40. package/src/kernel/constants.test.ts +19 -0
  41. package/src/kernel/constants.ts +54 -0
  42. package/src/kernel/effects.test.ts +40 -0
  43. package/src/kernel/effects.ts +159 -0
  44. package/src/kernel/events.ts +55 -0
  45. package/src/kernel/interpreter.test.ts +116 -0
  46. package/src/kernel/interpreter.ts +143 -0
  47. package/src/kernel/saga.test.ts +89 -0
  48. package/src/kernel/saga.ts +73 -0
  49. package/src/modules/app-release/index.test.ts +125 -0
  50. package/src/modules/app-release/index.ts +209 -0
  51. package/src/modules/auth/auth.local-first.test.ts +141 -0
  52. package/src/modules/auth/events/index.ts +2 -1
  53. package/src/modules/auth/index.ts +138 -25
  54. package/src/modules/crdt/crdt-field.ts +294 -0
  55. package/src/modules/crdt/crdt-hydration.test.ts +210 -0
  56. package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
  57. package/src/modules/crdt/index.ts +463 -0
  58. package/src/modules/crdt/loro-loader.ts +25 -0
  59. package/src/modules/devtools/flags.ts +349 -0
  60. package/src/modules/devtools/index.ts +459 -47
  61. package/src/modules/devtools/notify-throttle.test.ts +154 -0
  62. package/src/modules/devtools/state-shape.test.ts +149 -0
  63. package/src/modules/devtools/storage-info.test.ts +79 -0
  64. package/src/modules/devtools/storage-info.ts +168 -0
  65. package/src/modules/devtools/versions.test.ts +74 -0
  66. package/src/modules/devtools/versions.ts +110 -0
  67. package/src/modules/feature-flag/index.test.ts +251 -0
  68. package/src/modules/feature-flag/index.ts +316 -0
  69. package/src/modules/ref-tables.test.ts +91 -0
  70. package/src/modules/ref-tables.ts +88 -0
  71. package/src/mutation/jobs.test.ts +25 -0
  72. package/src/mutation/jobs.ts +41 -0
  73. package/src/mutation/mutation-id.test.ts +37 -0
  74. package/src/mutation/mutation-id.ts +35 -0
  75. package/src/mutation/push.saga.test.ts +285 -0
  76. package/src/mutation/push.saga.ts +215 -0
  77. package/src/mutation/rows.test.ts +201 -0
  78. package/src/mutation/rows.ts +377 -0
  79. package/src/mutation/tray.saga.test.ts +63 -0
  80. package/src/mutation/tray.saga.ts +48 -0
  81. package/src/mutation/write.saga.test.ts +163 -0
  82. package/src/mutation/write.saga.ts +166 -0
  83. package/src/otel/index.ts +13 -10
  84. package/src/query/env.test.ts +23 -0
  85. package/src/query/env.ts +56 -0
  86. package/src/query/fetch.saga.test.ts +156 -0
  87. package/src/query/fetch.saga.ts +109 -0
  88. package/src/query/hash.test.ts +14 -0
  89. package/src/query/hash.ts +18 -0
  90. package/src/query/id-set-plan.test.ts +132 -0
  91. package/src/query/lifecycle.saga.test.ts +122 -0
  92. package/src/query/lifecycle.saga.ts +101 -0
  93. package/src/query/materialize.saga.test.ts +82 -0
  94. package/src/query/materialize.saga.ts +75 -0
  95. package/src/query/materialize.test.ts +50 -0
  96. package/src/query/materialize.ts +44 -0
  97. package/src/query/membership.saga.test.ts +246 -0
  98. package/src/query/membership.saga.ts +208 -0
  99. package/src/query/membership.test.ts +118 -0
  100. package/src/query/membership.ts +158 -0
  101. package/src/query/register.saga.test.ts +251 -0
  102. package/src/query/register.saga.ts +190 -0
  103. package/src/query/render-set.test.ts +36 -0
  104. package/src/query/render-set.ts +60 -0
  105. package/src/query/sql.test.ts +62 -0
  106. package/src/query/sql.ts +79 -0
  107. package/src/query/window-query.test.ts +70 -0
  108. package/src/query/window-query.ts +195 -0
  109. package/src/services/blobs/blob-cache.test.ts +359 -0
  110. package/src/services/blobs/blob-cache.ts +603 -0
  111. package/src/services/blobs/blob-manifest.ts +227 -0
  112. package/src/services/blobs/blob-store.test.ts +77 -0
  113. package/src/services/blobs/blob-store.ts +359 -0
  114. package/src/services/blobs/blob.fixture.ts +90 -0
  115. package/src/services/blobs/index.ts +70 -0
  116. package/src/services/database/cache-engine.ts +193 -0
  117. package/src/services/database/connection-supervisor.test.ts +318 -0
  118. package/src/services/database/connection-supervisor.ts +422 -0
  119. package/src/services/database/database.query-timeout.test.ts +140 -0
  120. package/src/services/database/database.responses.test.ts +43 -0
  121. package/src/services/database/database.ts +145 -62
  122. package/src/services/database/engine-factory.ts +33 -0
  123. package/src/services/database/errors.ts +34 -0
  124. package/src/services/database/events/index.ts +2 -1
  125. package/src/services/database/index.ts +7 -0
  126. package/src/services/database/local-migrator.ts +30 -27
  127. package/src/services/database/local.test.ts +64 -0
  128. package/src/services/database/local.ts +484 -67
  129. package/src/services/database/plan-render.test.ts +159 -0
  130. package/src/services/database/plan-render.ts +108 -0
  131. package/src/services/database/relation-resolver.test.ts +467 -0
  132. package/src/services/database/relation-resolver.ts +0 -0
  133. package/src/services/database/remote.ts +222 -15
  134. package/src/services/database/sqlite-cache-engine.test.ts +617 -0
  135. package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
  136. package/src/services/database/sqlite-cache-engine.ts +1349 -0
  137. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  138. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  139. package/src/services/database/sqlite-lock-verify.test.ts +33 -0
  140. package/src/services/database/sqlite-lock-verify.ts +45 -0
  141. package/src/services/database/sqlite-open.test.ts +150 -0
  142. package/src/services/database/sqlite-open.ts +164 -0
  143. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  144. package/src/services/database/sqlite-plan-sql.ts +138 -0
  145. package/src/services/database/sqlite-projection.test.ts +99 -0
  146. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  147. package/src/services/database/sqlite-select.test.ts +246 -0
  148. package/src/services/database/sqlite-select.ts +131 -0
  149. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  150. package/src/services/database/sqlite-transport.ts +224 -0
  151. package/src/services/database/sqlite-worker.ts +437 -0
  152. package/src/services/database/surql-translate.ts +416 -0
  153. package/src/services/database/surreal-cache-engine.ts +161 -0
  154. package/src/services/logger/index.ts +3 -2
  155. package/src/services/persistence/localstorage.ts +2 -2
  156. package/src/services/persistence/resilient.ts +11 -4
  157. package/src/services/persistence/surrealdb.ts +10 -10
  158. package/src/services/stream-processor/index.ts +796 -84
  159. package/src/services/stream-processor/permissions.test.ts +47 -0
  160. package/src/services/stream-processor/permissions.ts +53 -0
  161. package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
  162. package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
  163. package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
  164. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  165. package/src/services/stream-processor/wasm-types.ts +59 -3
  166. package/src/services/tabs/broker-client.ts +283 -0
  167. package/src/services/tabs/broker.test.ts +327 -0
  168. package/src/services/tabs/coordinator.test.ts +365 -0
  169. package/src/services/tabs/coordinator.ts +633 -0
  170. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  171. package/src/services/tabs/leader-locks.ts +75 -0
  172. package/src/services/tabs/protocol.ts +262 -0
  173. package/src/services/tabs/support.ts +36 -0
  174. package/src/services/tabs/tabs-broker-worker.ts +640 -0
  175. package/src/sp00ky.ts +5 -0
  176. package/src/state/client-state.ts +184 -0
  177. package/src/state/lifecycle.test.ts +118 -0
  178. package/src/state/lifecycle.ts +86 -0
  179. package/src/state/reducers.test.ts +280 -0
  180. package/src/state/reducers.ts +476 -0
  181. package/src/state/selectors.test.ts +149 -0
  182. package/src/state/selectors.ts +184 -0
  183. package/src/sync/connection.saga.test.ts +86 -0
  184. package/src/sync/connection.saga.ts +85 -0
  185. package/src/sync/live.saga.test.ts +69 -0
  186. package/src/sync/live.saga.ts +49 -0
  187. package/src/sync/policy.health.test.ts +56 -0
  188. package/src/sync/policy.test.ts +659 -0
  189. package/src/sync/policy.ts +502 -0
  190. package/src/sync/poll.saga.test.ts +51 -0
  191. package/src/sync/poll.saga.ts +36 -0
  192. package/src/sync/tabs.saga.test.ts +74 -0
  193. package/src/sync/tabs.saga.ts +92 -0
  194. package/src/testing/adapters.ts +133 -0
  195. package/src/testing/build.ts +61 -0
  196. package/src/testing/fake-services.ts +34 -0
  197. package/src/testing/run-pure.test.ts +92 -0
  198. package/src/testing/run-pure.ts +128 -0
  199. package/src/testing/services.ts +17 -0
  200. package/src/types.ts +550 -14
  201. package/src/utils/blurhash.ts +90 -0
  202. package/src/utils/error-classification.test.ts +54 -0
  203. package/src/utils/error-classification.ts +25 -0
  204. package/src/utils/index.ts +79 -13
  205. package/src/utils/parser.test.ts +49 -120
  206. package/src/utils/parser.ts +32 -2
  207. package/src/utils/semver.test.ts +32 -0
  208. package/src/utils/semver.ts +30 -0
  209. package/src/utils/surql.ts +30 -18
  210. package/src/utils/utils.test.ts +2 -1
  211. package/src/utils/withRetry.test.ts +1 -1
  212. package/tsdown.config.ts +86 -1
  213. package/vitest.config.ts +48 -0
  214. package/src/modules/cache/index.ts +0 -241
  215. package/src/modules/cache/types.ts +0 -19
  216. package/src/modules/data/data.test.ts +0 -58
  217. package/src/modules/data/index.ts +0 -864
  218. package/src/modules/sync/engine.ts +0 -166
  219. package/src/modules/sync/events/index.ts +0 -77
  220. package/src/modules/sync/index.ts +0 -3
  221. package/src/modules/sync/queue/index.ts +0 -2
  222. package/src/modules/sync/queue/queue-down.ts +0 -89
  223. package/src/modules/sync/queue/queue-up.ts +0 -223
  224. package/src/modules/sync/scheduler.ts +0 -84
  225. package/src/modules/sync/sync.ts +0 -409
  226. package/src/modules/sync/utils.test.ts +0 -311
  227. package/src/modules/sync/utils.ts +0 -170
  228. package/src/spooky.ts +0 -395
@@ -1,8 +1,8 @@
1
- import { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
2
- import { Logger } from '../../services/logger/index';
3
- import { SchemaStructure } from '@spooky-sync/query-builder';
1
+ import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
2
+ import type { Logger } from '../../services/logger/index';
3
+ import type { SchemaStructure } from '@spooky-sync/query-builder';
4
4
  import { RecordId } from 'surrealdb';
5
- import { StreamUpdate, StreamUpdateReceiver } from '../../services/stream-processor/index';
5
+ import type { StreamUpdate, StreamUpdateReceiver } from '../../services/stream-processor/index';
6
6
  import { encodeRecordId } from '../../utils/index';
7
7
 
8
8
  // DevTools interfaces (matching extension expectations)
@@ -13,31 +13,180 @@ export interface DevToolsEvent {
13
13
  payload: any;
14
14
  }
15
15
 
16
- import { DataModule } from '../data/index';
17
- import { AuthService } from '../auth/index';
16
+ import type { QueryState, QueryTimings } from '../../types';
17
+ import type { AuthService } from '../auth/index';
18
18
  import { AuthEventTypes } from '../auth/events/index';
19
+ import {
20
+ type BackendInfo,
21
+ emptyBackendInfo,
22
+ parseBackendInfo,
23
+ UNAVAILABLE,
24
+ } from './versions';
25
+ import { walkOpfs, type BlobCacheInfo, type SharedTabsInfo, type StorageInfo } from './storage-info';
26
+ import { FlagsAdminService, type LocalOverrideStore } from './flags';
27
+
28
+ // Real bundled frontend versions, injected at build time by tsdown's
29
+ // version-define plugin (see tsdown.config.ts). The `typeof` guard keeps these
30
+ // from throwing a ReferenceError when a downstream app bundles core from source
31
+ // (where the plugin never runs); in that case they fall back to 'unknown' and
32
+ // DevTools simply reports an unknown frontend version instead of crashing.
33
+ const CORE_VERSION =
34
+ typeof __SP00KY_CORE_VERSION__ !== 'undefined' ? __SP00KY_CORE_VERSION__ : 'unknown';
35
+ const WASM_VERSION =
36
+ typeof __SP00KY_WASM_VERSION__ !== 'undefined' ? __SP00KY_WASM_VERSION__ : 'unknown';
37
+ const SURREAL_VERSION =
38
+ typeof __SP00KY_SURREAL_VERSION__ !== 'undefined' ? __SP00KY_SURREAL_VERSION__ : 'unknown';
39
+
40
+ /** What DevTools reads about live queries; the runtime derives it from state. */
41
+ export interface DevToolsQuerySource {
42
+ getActiveQueries(): QueryState[];
43
+ getQueryById(id: RecordId<string>): QueryState | undefined;
44
+ phaseTimings(q: QueryState): QueryTimings;
45
+ }
19
46
 
20
47
  export class DevToolsService implements StreamUpdateReceiver {
21
48
  private eventsHistory: DevToolsEvent[] = [];
22
49
  private eventIdCounter = 0;
23
- private version = '1.0.0';
50
+ // Real bundled frontend version (injected at build time via tsdown `define`).
51
+ private version = CORE_VERSION;
52
+ // Backend stack info (versions + per-entity status), read via the
53
+ // `fn::spooky::info()` SurrealQL function; empty/'unavailable' until resolved.
54
+ private backendInfo: BackendInfo = emptyBackendInfo();
55
+ // Dormant until a devtools consumer (extension panel or MCP) handshakes via
56
+ // `SP00KY_DEVTOOLS_CONNECT`. While false, `notifyDevTools()`/`addEvent()` do no
57
+ // work, so prod pays zero serialization/postMessage cost for an unwatched panel.
58
+ // `window.__00__.getState()` stays live regardless, so the panel's first paint
59
+ // (the on-demand GET_STATE pull) still works before the push channel turns on.
60
+ private enabled = false;
61
+
62
+ // A state push serializes EVERY active query's full record set (see
63
+ // `getActiveQueries`) and postMessage clones it again, so its cost scales with
64
+ // the whole client dataset — and it is triggered per event, including one per
65
+ // local DB query (`DATABASE_LOCAL_QUERY` → `logEvent`). Unthrottled, a single
66
+ // page load's few hundred local queries turn a handful of MB of rows into GBs
67
+ // of short-lived large-object garbage and OOM the renderer (V8
68
+ // "young object promotion failed"). Coalesce instead: push immediately when
69
+ // idle, then at most once per window, always serializing the LATEST state.
70
+ private static readonly NOTIFY_MIN_INTERVAL_MS = 250;
71
+ private notifyTimer: ReturnType<typeof setTimeout> | null = null;
72
+ private lastNotifyAt = 0;
73
+ /** How many ids a pushed state carries per view; the rest is on demand. */
74
+ private static readonly STATE_IDS_CAP = 200;
75
+ /** devtools numeric hash -> the query's `_00_query` id, for on-demand rows. */
76
+ private hashToQuery = new Map<number, unknown>();
77
+
78
+ /** Shared-tabs snapshot for the panel, wired by Sp00kyClient whenever the
79
+ * feature was REQUESTED (so an inactive/degraded tab still reports why). */
80
+ private tabsInfoProvider: (() => SharedTabsInfo | null) | null = null;
81
+
82
+ setTabsInfoProvider(provider: () => SharedTabsInfo | null): void {
83
+ this.tabsInfoProvider = provider;
84
+ }
85
+
86
+ /** Blob cache counters for the panel, wired by Sp00kyClient. */
87
+ private blobInfoProvider: (() => BlobCacheInfo) | null = null;
88
+
89
+ setBlobInfoProvider(provider: () => BlobCacheInfo): void {
90
+ this.blobInfoProvider = provider;
91
+ }
92
+
93
+ // Full local table list (incl. internal `_00_*`), enumerated from the local DB
94
+ // via our own reliable `service.query` — the DevTools panel's page-eval query
95
+ // bridge is unreliable under load, so the panel relies on this instead.
96
+ private localTables: string[] = [];
97
+ private localTablesFetching = false;
98
+ private localTablesAt = 0;
99
+
100
+ // Feature flag admin, backing the panel's Access tab. The local-override
101
+ // store is injected later (`setFeatureFlagOverrides`) because the
102
+ // FeatureFlagModule is built after this service.
103
+ private featureOverrides: LocalOverrideStore | null = null;
104
+ private readonly flagsAdmin: FlagsAdminService;
24
105
 
25
106
  constructor(
26
- private databaseService: LocalDatabaseService,
107
+ private databaseService: LocalStore,
27
108
  private remoteDatabaseService: RemoteDatabaseService,
28
109
  private logger: Logger,
29
110
  private schema: SchemaStructure,
30
111
  private authService: AuthService<SchemaStructure>,
31
- private dataManager?: DataModule<SchemaStructure>
112
+ private dataManager?: DevToolsQuerySource
32
113
  ) {
114
+ this.flagsAdmin = new FlagsAdminService({
115
+ remote: this.remoteDatabaseService,
116
+ local: this.databaseService,
117
+ logger: this.logger,
118
+ currentUserId: () => {
119
+ const id = this.authService.currentUser?.id;
120
+ if (!id) return null;
121
+ return id instanceof RecordId ? encodeRecordId(id) : String(id);
122
+ },
123
+ overrides: () => this.featureOverrides,
124
+ });
125
+
33
126
  this.exposeToWindow();
34
127
 
35
- // Subscribe to auth events
128
+ // Stay dormant until a devtools consumer announces itself. The extension's
129
+ // page-script posts this once it detects `window.__00__`; the panel can also
130
+ // disconnect to return us to dormant. Until then we skip all serialization.
131
+ if (typeof window !== 'undefined') {
132
+ window.addEventListener('message', (e) => {
133
+ if (e.source !== window) return;
134
+ const type = (e.data as { type?: string } | undefined)?.type;
135
+ if (type === 'SP00KY_DEVTOOLS_CONNECT') {
136
+ this.enabled = true;
137
+ this.refreshLocalTables();
138
+ this.notifyDevTools();
139
+ } else if (type === 'SP00KY_DEVTOOLS_DISCONNECT') {
140
+ this.enabled = false;
141
+ }
142
+ });
143
+ }
144
+
145
+ // Subscribe to auth events. The initial fire-and-forget version fetch (below)
146
+ // races the remote connection; on the free plan the remote DB (SurrealDB
147
+ // Cloud) has no guest access, so `fn::spooky::info()` is only callable once
148
+ // signed in. Re-fetch when auth resolves — until the versions actually land —
149
+ // instead of leaving them 'unavailable' forever.
36
150
  this.authService.eventSystem.subscribe(AuthEventTypes.AuthStateChanged, () => {
37
- this.notifyDevTools();
151
+ if (this.authService.isAuthenticated && this.backendInfo.versions.ssp === UNAVAILABLE) {
152
+ void this.refreshBackendVersions();
153
+ } else {
154
+ this.notifyDevTools();
155
+ }
38
156
  });
39
157
 
40
- this.logger.debug({ Category: 'spooky-client::DevToolsService::init' }, 'Service initialized');
158
+ // Push state when the local store reports its durability (the open happens
159
+ // during connect, typically before a panel attaches, so this mostly matters
160
+ // for a later bucket switch that loses OPFS).
161
+ this.databaseService.subscribeToStorageHealth?.(() => this.notifyDevTools());
162
+
163
+ // Fire-and-forget backend version discovery; re-push state when it lands.
164
+ void this.refreshBackendVersions();
165
+
166
+ this.logger.debug({ Category: 'sp00ky-client::DevToolsService::init' }, 'Service initialized');
167
+ }
168
+
169
+ /**
170
+ * Re-read backend stack info via the `fn::spooky::info()` SurrealQL function
171
+ * over the open remote connection (no HTTP/CORS), then notify the panel.
172
+ * Never throws: on failure the info stays empty/'unavailable'.
173
+ */
174
+ private async refreshBackendVersions(): Promise<void> {
175
+ try {
176
+ // `RETURN fn::spooky::info()` → one statement result: the /info entity array.
177
+ const result = await this.remoteDatabaseService.query<unknown[]>(
178
+ 'RETURN fn::spooky::info()'
179
+ );
180
+ const first = Array.isArray(result) ? result[0] : result;
181
+ this.backendInfo = parseBackendInfo(first);
182
+ } catch (err) {
183
+ this.logger.debug(
184
+ { err, Category: 'sp00ky-client::DevToolsService::versions' },
185
+ 'fn::spooky::info() unavailable; backend versions stay unavailable'
186
+ );
187
+ this.backendInfo = emptyBackendInfo();
188
+ }
189
+ this.notifyDevTools();
41
190
  }
42
191
 
43
192
  // Get active queries directly from DataManager (single source of truth)
@@ -48,21 +197,52 @@ export class DevToolsService implements StreamUpdateReceiver {
48
197
  const queries = this.dataManager.getActiveQueries();
49
198
  queries.forEach((q) => {
50
199
  const queryHash = this.hashString(encodeRecordId(q.config.id));
200
+ const createdAt =
201
+ q.config.lastActiveAt instanceof Date
202
+ ? q.config.lastActiveAt.getTime()
203
+ : new Date(q.config.lastActiveAt || Date.now()).getTime();
204
+ this.hashToQuery.set(queryHash, q.config.id);
205
+ const localArray = q.config.localArray ?? [];
206
+ const remoteArray = q.config.remoteArray ?? [];
51
207
  result.set(queryHash, {
52
208
  queryHash,
53
209
  status: 'active',
54
- createdAt:
55
- q.config.lastActiveAt instanceof Date
56
- ? q.config.lastActiveAt.getTime()
57
- : new Date(q.config.lastActiveAt || Date.now()).getTime(),
58
- lastUpdate: Date.now(),
210
+ // Runtime fetch status, distinct from the `status: 'active'`
211
+ // registration flag above. `fetchStatus` is 'idle' | 'fetching'.
212
+ fetchStatus: q.status,
213
+ isFetching: q.status === 'fetching',
214
+ createdAt,
215
+ // Real last-update time; before the first update it equals createdAt.
216
+ // (Previously Date.now(), which reset the column on every state push.)
217
+ lastUpdate: q.lastUpdatedAt ?? createdAt,
59
218
  updateCount: q.updateCount,
219
+ ttl: q.config.ttl,
60
220
  query: q.config.surql,
61
221
  variables: q.config.params || {},
62
222
  dataSize: q.records?.length || 0,
63
- data: q.records,
64
- localArray: q.config.localArray,
65
- remoteArray: q.config.remoteArray,
223
+ // Counts and (capped) ids only. The rows themselves used to ride along
224
+ // here: every push then deep-cloned every view's records twice
225
+ // (serialize + postMessage), on a 4k-row client dataset up to four
226
+ // times a second, from inside the ingest call stack. The panel pulls
227
+ // rows on demand through `getQueryRows` instead.
228
+ localCount: localArray.length,
229
+ remoteCount: remoteArray.length,
230
+ localIds: localArray.slice(0, DevToolsService.STATE_IDS_CAP).map(([id]) => id),
231
+ remoteIds: remoteArray.slice(0, DevToolsService.STATE_IDS_CAP).map(([id]) => id),
232
+ idsTruncated:
233
+ localArray.length > DevToolsService.STATE_IDS_CAP ||
234
+ remoteArray.length > DevToolsService.STATE_IDS_CAP,
235
+ // Membership state, so "why is this list empty" is answerable from
236
+ // the panel: is the server's set known, has a non-empty one been seen
237
+ // this session, how many empty reads were ignored.
238
+ membershipKnown: q.config.membershipKnown === true,
239
+ authoritative: q.config.membershipKnown === true,
240
+ viewLost: q.viewLost === true,
241
+ serverState: q.serverState ?? null,
242
+ // Detailed per-phase processing-time breakdown (SSP sub-phases, local/
243
+ // remote record fetch, frontend reconcile, registration). Flows to both
244
+ // the DevTools panel and the MCP (which returns activeQueries verbatim).
245
+ timings: this.dataManager!.phaseTimings(q),
66
246
  });
67
247
  });
68
248
  return result;
@@ -70,7 +250,7 @@ export class DevToolsService implements StreamUpdateReceiver {
70
250
 
71
251
  public onQueryInitialized(payload: any) {
72
252
  this.logger.debug(
73
- { payload, Category: 'spooky-client::DevToolsService::onQueryInitialized' },
253
+ { payload, Category: 'sp00ky-client::DevToolsService::onQueryInitialized' },
74
254
  'QueryInitialized'
75
255
  );
76
256
  const queryHash = this.hashString(payload.queryId.toString());
@@ -87,7 +267,7 @@ export class DevToolsService implements StreamUpdateReceiver {
87
267
  this.logger.debug(
88
268
  {
89
269
  id: payload.queryId?.toString(),
90
- Category: 'spooky-client::DevToolsService::onQueryUpdated',
270
+ Category: 'sp00ky-client::DevToolsService::onQueryUpdated',
91
271
  },
92
272
  'QueryUpdated'
93
273
  );
@@ -95,18 +275,28 @@ export class DevToolsService implements StreamUpdateReceiver {
95
275
 
96
276
  this.addEvent('QUERY_UPDATED', {
97
277
  queryHash,
98
- data: payload.records,
278
+ recordCount: Array.isArray(payload.records) ? payload.records.length : 0,
99
279
  });
100
280
  this.notifyDevTools();
101
281
  }
102
282
 
103
283
  public onStreamUpdate(update: StreamUpdate) {
284
+ // A synthetic re-materialize is not an ingest (DataModule.scheduleRematerialize).
285
+ if (update.synthetic) return;
104
286
  this.logger.debug(
105
- { update, Category: 'spooky-client::DevToolsService::onStreamUpdate' },
287
+ { queryHash: update.queryHash, Category: 'sp00ky-client::DevToolsService::onStreamUpdate' },
106
288
  'StreamUpdate'
107
289
  );
290
+ // Counts and timings, not the `localArray` itself: that is one [id, version]
291
+ // pair per row of the view, serialized on every single update.
108
292
  this.addEvent('STREAM_UPDATE', {
109
- updates: [update],
293
+ queryHash: update.queryHash,
294
+ op: update.op,
295
+ localCount: update.localArray?.length ?? 0,
296
+ materializationTimeMs: update.materializationTimeMs,
297
+ storeApplyMs: update.storeApplyMs,
298
+ circuitStepMs: update.circuitStepMs,
299
+ transformMs: update.transformMs,
110
300
  });
111
301
  this.notifyDevTools();
112
302
  }
@@ -116,8 +306,10 @@ export class DevToolsService implements StreamUpdateReceiver {
116
306
  payloads.forEach((p) => {
117
307
  this.addEvent('MUTATION_REQUEST_EXECUTION', {
118
308
  mutation: {
119
- type: 'create', // simplifying
120
- data: 'data' in p ? p.data : undefined,
309
+ type: p.type ?? 'create',
310
+ // Field names only; a payload body (a PGN, a document) is not
311
+ // something to clone on every write.
312
+ fields: 'data' in p && p.data && typeof p.data === 'object' ? Object.keys(p.data) : [],
121
313
  selector: encodeRecordId(p.record_id),
122
314
  },
123
315
  });
@@ -142,6 +334,8 @@ export class DevToolsService implements StreamUpdateReceiver {
142
334
  }
143
335
 
144
336
  private addEvent(eventType: string, payload: any) {
337
+ // No consumer attached → skip recording (and the recursive serialize it does).
338
+ if (!this.enabled) return;
145
339
  this.eventsHistory.push({
146
340
  id: this.eventIdCounter++,
147
341
  timestamp: Date.now(),
@@ -151,7 +345,55 @@ export class DevToolsService implements StreamUpdateReceiver {
151
345
  if (this.eventsHistory.length > 100) this.eventsHistory.shift();
152
346
  }
153
347
 
154
- private getState() {
348
+ /** Unwrap a SurrealDB `INFO FOR DB` result to its `{ tables, ... }` object. */
349
+ private unwrapInfo(res: any): any {
350
+ if (!Array.isArray(res) || !res[0]) return null;
351
+ const first = res[0];
352
+ if (first && typeof first === 'object' && 'result' in first) return first.result;
353
+ if (Array.isArray(first)) return first[0];
354
+ return first;
355
+ }
356
+
357
+ /**
358
+ * Refresh the cached full local-table list from `INFO FOR DB`. Fire-and-forget
359
+ * and throttled — called from getState() so the panel gets every table
360
+ * (including internal `_00_*`) without running its own (flaky) queries.
361
+ */
362
+ private refreshLocalTables(): void {
363
+ if (this.localTablesFetching) return;
364
+ const now = Date.now();
365
+ if (now - this.localTablesAt < 30_000) return;
366
+ this.localTablesFetching = true;
367
+ void this.databaseService
368
+ .query<any>('INFO FOR DB')
369
+ .then((res) => {
370
+ const info = this.unwrapInfo(res);
371
+ this.localTablesAt = Date.now();
372
+ if (info && info.tables) {
373
+ // The circuit snapshot table holds a BLOB, not JSON rows; the
374
+ // explorer cannot render it and has nothing to show for it.
375
+ const names = Object.keys(info.tables).filter((n) => n !== '_00_circuit_snapshot');
376
+ const changed =
377
+ names.length !== this.localTables.length ||
378
+ names.some((n, i) => n !== this.localTables[i]);
379
+ this.localTables = names;
380
+ if (changed) this.notifyDevTools();
381
+ }
382
+ })
383
+ .catch(() => {
384
+ // Ignore — fall back to the declared app schema below.
385
+ })
386
+ .finally(() => {
387
+ this.localTablesFetching = false;
388
+ });
389
+ }
390
+
391
+ private getState(opts: { refreshTables?: boolean } = {}) {
392
+ // The local-table list (`INFO FOR DB`, a local round trip) is refreshed on
393
+ // an explicit pull only, never by the push path: pushes follow every sync
394
+ // event, and a local query per push competed with the app's own writes for
395
+ // the single local op queue.
396
+ if (opts.refreshTables) this.refreshLocalTables();
155
397
  return this.serializeForDevTools({
156
398
  eventsHistory: [...this.eventsHistory],
157
399
  activeQueries: Object.fromEntries(this.getActiveQueries()),
@@ -160,24 +402,148 @@ export class DevToolsService implements StreamUpdateReceiver {
160
402
  userId: this.authService.currentUser?.id,
161
403
  },
162
404
  version: this.version,
405
+ versions: {
406
+ frontend: {
407
+ core: CORE_VERSION,
408
+ wasm: WASM_VERSION,
409
+ surrealdb: SURREAL_VERSION,
410
+ },
411
+ backend: this.backendInfo.versions,
412
+ entities: this.backendInfo.entities,
413
+ },
163
414
  database: {
164
- tables: this.schema.tables.map((t) => t.name),
415
+ // Prefer the live local-table list (includes internal `_00_*`); fall
416
+ // back to the declared app schema until the first enumeration lands.
417
+ tables: this.localTables.length
418
+ ? this.localTables
419
+ : this.schema.tables.map((t) => t.name),
165
420
  tableData: {},
421
+ // Which backend answers "Local". The Database explorer labels its source
422
+ // picker with it and explains translation failures against `sqlite`,
423
+ // whose SurrealQL vocabulary is a bounded subset.
424
+ engine: this.databaseService.engineKind ?? 'custom',
425
+ // Durability of the local store. `fallback: true` means persistence was
426
+ // requested but the dataset is actually sitting in RAM.
427
+ storage: this.databaseService.storageHealth ?? { status: 'unknown', fallback: false },
428
+ // Shared-tabs role state (null when the feature is off / fell back).
429
+ tabs: this.tabsInfoProvider?.() ?? null,
166
430
  },
167
431
  });
168
432
  }
169
433
 
170
- private notifyDevTools() {
171
- if (typeof window !== 'undefined') {
172
- window.postMessage(
173
- {
174
- type: 'SPOOKY_STATE_CHANGED',
175
- source: 'spooky-devtools-page',
176
- state: this.getState(),
177
- },
178
- '*'
434
+ /**
435
+ * Full storage diagnostics for the DevTools Storage tab. Every section is
436
+ * gathered independently and failures land in that section's `error` field,
437
+ * so one broken source (a mid-switch worker, a browser without OPFS) never
438
+ * blanks the whole panel.
439
+ */
440
+ public async getStorageInfo(opts?: { tableCounts?: boolean }): Promise<StorageInfo> {
441
+ const nav = typeof navigator !== 'undefined' ? navigator : undefined;
442
+
443
+ const info: StorageInfo = {
444
+ at: Date.now(),
445
+ engine: {
446
+ kind: this.databaseService.engineKind ?? 'custom',
447
+ store: this.databaseService.getConfig()?.store ?? 'memory',
448
+ bucketId: this.databaseService.currentBucketId,
449
+ },
450
+ health: this.databaseService.storageHealth ?? { status: 'unknown', fallback: false },
451
+ tabs: this.tabsInfoProvider?.() ?? null,
452
+ browser: {},
453
+ opfs: { supported: false, entries: [], totalBytes: 0, truncated: false },
454
+ };
455
+
456
+ try {
457
+ if (nav?.storage?.estimate) {
458
+ const est = await nav.storage.estimate();
459
+ info.browser.usage = est.usage;
460
+ info.browser.quota = est.quota;
461
+ // Chrome-only per-storage-system breakdown; absent elsewhere.
462
+ const details = (est as any).usageDetails;
463
+ if (details && typeof details === 'object') info.browser.usageDetails = details;
464
+ }
465
+ if (nav?.storage?.persisted) {
466
+ info.browser.persisted = await nav.storage.persisted();
467
+ }
468
+ } catch (e) {
469
+ info.browser.error = e instanceof Error ? e.message : String(e);
470
+ }
471
+
472
+ info.opfs = await walkOpfs();
473
+
474
+ try {
475
+ info.blobs = this.blobInfoProvider?.();
476
+ } catch (e) {
477
+ this.logger.warn(
478
+ { err: e, Category: 'sp00ky-client::DevToolsService::getStorageInfo' },
479
+ 'Blob cache diagnostics failed'
480
+ );
481
+ }
482
+
483
+ const stats = (globalThis as any).__sqliteStats;
484
+ if (stats && typeof stats === 'object') {
485
+ info.sqliteStats = { ...stats, byType: { ...(stats.byType ?? {}) } };
486
+ }
487
+
488
+ try {
489
+ info.engineDiagnostics = await this.databaseService.getStorageDiagnostics?.(opts);
490
+ } catch (e) {
491
+ this.logger.warn(
492
+ { err: e, Category: 'sp00ky-client::DevToolsService::getStorageInfo' },
493
+ 'Engine storage diagnostics failed'
179
494
  );
180
495
  }
496
+
497
+ return this.serializeForDevTools(info);
498
+ }
499
+
500
+ /** Ask the browser to exempt this origin's storage from eviction. */
501
+ public async requestPersistentStorage(): Promise<{ granted: boolean }> {
502
+ try {
503
+ const granted = (await navigator.storage?.persist?.()) ?? false;
504
+ return { granted };
505
+ } catch {
506
+ return { granted: false };
507
+ }
508
+ }
509
+
510
+ /**
511
+ * Request a state push. Coalesced (see {@link NOTIFY_MIN_INTERVAL_MS}): the
512
+ * first call after an idle period pushes straight away so the panel stays
513
+ * responsive, and any calls during the window collapse into ONE trailing push
514
+ * that serializes the state as of the flush, not as of the request. Callers
515
+ * stay fire-and-forget.
516
+ */
517
+ private notifyDevTools() {
518
+ // No consumer attached → no getState() serialization, no postMessage broadcast.
519
+ if (!this.enabled) return;
520
+ if (typeof window === 'undefined') return;
521
+ // A trailing push is already queued; it will carry this change too.
522
+ if (this.notifyTimer !== null) return;
523
+
524
+ // Always a macrotask, never inline: this is called from inside the ingest
525
+ // and mutation call stacks (onStreamUpdate / onMutation), i.e. inside the
526
+ // `await db.create(...)` the app is waiting on. A push that serializes the
527
+ // state right there charged every write for the panel's refresh.
528
+ const waited = Date.now() - this.lastNotifyAt;
529
+ const delay = Math.max(0, DevToolsService.NOTIFY_MIN_INTERVAL_MS - waited);
530
+ this.notifyTimer = setTimeout(() => {
531
+ this.notifyTimer = null;
532
+ // Still gated on `enabled`: the panel may have disconnected while queued.
533
+ if (this.enabled) this.flushNotify();
534
+ }, delay);
535
+ }
536
+
537
+ private flushNotify() {
538
+ this.lastNotifyAt = Date.now();
539
+ window.postMessage(
540
+ {
541
+ type: 'SP00KY_STATE_CHANGED',
542
+ source: 'sp00ky-devtools-page',
543
+ state: this.getState(),
544
+ },
545
+ '*'
546
+ );
181
547
  }
182
548
 
183
549
  private serializeForDevTools(data: any, seen = new WeakSet<object>()): any {
@@ -218,6 +584,10 @@ export class DevToolsService implements StreamUpdateReceiver {
218
584
  const result: Record<string, any> = {};
219
585
  for (const key in data) {
220
586
  if (Object.prototype.hasOwnProperty.call(data, key)) {
587
+ // Skip absent optional fields: recursing them would emit the STRING
588
+ // 'undefined' (the top-level mapping below), which panels then have
589
+ // to filter back out (see 3d84fe8a).
590
+ if (data[key] === undefined) continue;
221
591
  result[key] = this.serializeForDevTools(data[key], seen);
222
592
  }
223
593
  }
@@ -227,15 +597,54 @@ export class DevToolsService implements StreamUpdateReceiver {
227
597
  return data;
228
598
  }
229
599
 
600
+ /**
601
+ * Hand the FeatureFlagModule to the Access tab so it can read and write local
602
+ * overrides. Called from `Sp00kyClient` once both are constructed; until then
603
+ * the override methods are no-ops that report an empty map.
604
+ */
605
+ public setFeatureFlagOverrides(store: LocalOverrideStore): void {
606
+ this.featureOverrides = store;
607
+ }
608
+
230
609
  private exposeToWindow() {
231
610
  if (typeof window !== 'undefined') {
232
- (window as any).__SPOOKY__ = {
611
+ (window as any).__00__ = {
233
612
  version: this.version,
234
- getState: () => this.getState(),
613
+ getState: () => this.getState({ refreshTables: true }),
614
+ // The rows of ONE view, on demand. The pushed state carries counts and
615
+ // capped ids only (see getActiveQueries); the panel's Data tab and the
616
+ // MCP fetch the rows here when somebody actually looks at them.
617
+ getQueryRows: (queryHash: number) => {
618
+ const id = this.hashToQuery.get(Number(queryHash));
619
+ const q = id !== undefined ? this.dataManager?.getQueryById(id as any) : undefined;
620
+ if (!q) return null;
621
+ return this.serializeForDevTools({
622
+ queryHash: Number(queryHash),
623
+ data: q.records,
624
+ localArray: q.config.localArray,
625
+ remoteArray: q.config.remoteArray,
626
+ });
627
+ },
628
+ // ---- Feature flags (Access tab) --------------------------------
629
+ // Remote reads/writes are admin-gated by SurrealDB, not here: a
630
+ // non-admin gets an empty flag list, and the `fn::feature::*` calls
631
+ // are denied outright. The override methods are purely local and
632
+ // work signed out.
633
+ getFlags: () => this.flagsAdmin.getFlags(),
634
+ setFlagEnabled: (key: string, enabled: boolean) =>
635
+ this.flagsAdmin.setFlagEnabled(key, enabled),
636
+ setFlagUserVariant: (key: string, variant: string, remove: boolean, userId?: string) =>
637
+ this.flagsAdmin.setFlagUserVariant(key, variant, remove, userId),
638
+ setLocalFlagOverride: (key: string, variant: string | null, payload?: unknown) =>
639
+ this.flagsAdmin.setLocalFlagOverride(key, variant, payload),
640
+ clearLocalFlagOverrides: () => this.flagsAdmin.clearLocalFlagOverrides(),
235
641
  clearHistory: () => {
236
642
  this.eventsHistory = [];
237
643
  this.notifyDevTools();
238
644
  },
645
+ refreshVersions: () => this.refreshBackendVersions(),
646
+ getStorageInfo: (opts?: { tableCounts?: boolean }) => this.getStorageInfo(opts),
647
+ requestPersistentStorage: () => this.requestPersistentStorage(),
239
648
  getTableData: async (tableName: string) => {
240
649
  try {
241
650
  // Returns the first statement result as T.
@@ -270,7 +679,7 @@ export class DevToolsService implements StreamUpdateReceiver {
270
679
  return this.serializeForDevTools(records) || [];
271
680
  } catch (e) {
272
681
  this.logger.error(
273
- { err: e, Category: 'spooky-client::DevToolsService::exposeToWindow' },
682
+ { err: e, Category: 'sp00ky-client::DevToolsService::exposeToWindow' },
274
683
  'Failed to get table data'
275
684
  );
276
685
  return [];
@@ -299,7 +708,7 @@ export class DevToolsService implements StreamUpdateReceiver {
299
708
  runQuery: async (query: string, target: 'local' | 'remote' = 'local') => {
300
709
  try {
301
710
  this.logger.debug(
302
- { query, target, Category: 'spooky-client::DevToolsService::runQuery' },
711
+ { query, target, Category: 'sp00ky-client::DevToolsService::runQuery' },
303
712
  'Running query (START)'
304
713
  );
305
714
  const service = target === 'remote' ? this.remoteDatabaseService : this.databaseService;
@@ -314,7 +723,7 @@ export class DevToolsService implements StreamUpdateReceiver {
314
723
  time: queryTime,
315
724
  resultType: typeof result,
316
725
  isArray: Array.isArray(result),
317
- Category: 'spooky-client::DevToolsService::runQuery',
726
+ Category: 'sp00ky-client::DevToolsService::runQuery',
318
727
  },
319
728
  'Database returned result'
320
729
  );
@@ -328,7 +737,7 @@ export class DevToolsService implements StreamUpdateReceiver {
328
737
  {
329
738
  serializeTime,
330
739
  serializedLength: JSON.stringify(serialized).length,
331
- Category: 'spooky-client::DevToolsService::runQuery',
740
+ Category: 'sp00ky-client::DevToolsService::runQuery',
332
741
  },
333
742
  'Serialization complete'
334
743
  );
@@ -340,7 +749,7 @@ export class DevToolsService implements StreamUpdateReceiver {
340
749
  };
341
750
  } catch (e: any) {
342
751
  this.logger.error(
343
- { err: e, query, target, Category: 'spooky-client::DevToolsService::runQuery' },
752
+ { err: e, query, target, Category: 'sp00ky-client::DevToolsService::runQuery' },
344
753
  'Query execution failed'
345
754
  );
346
755
  // Ensure we always return a string for error
@@ -353,12 +762,15 @@ export class DevToolsService implements StreamUpdateReceiver {
353
762
 
354
763
  window.postMessage(
355
764
  {
356
- type: 'SPOOKY_DETECTED',
357
- source: 'spooky-devtools-page',
765
+ type: 'SP00KY_DETECTED',
766
+ source: 'sp00ky-devtools-page',
358
767
  data: { version: this.version, detected: true },
359
768
  },
360
769
  '*'
361
770
  );
771
+
772
+ // Dispatch custom event so the devtools page-script can detect late initialization
773
+ window.dispatchEvent(new CustomEvent('sp00ky:init'));
362
774
  }
363
775
  }
364
776
  }