@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.211

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/AGENTS.md +57 -0
  2. package/dist/index.d.ts +2514 -58
  3. package/dist/index.js +12561 -2449
  4. package/dist/otel/index.d.ts +2 -2
  5. package/dist/otel/index.js +6 -6
  6. package/dist/sqlite-open.js +303 -0
  7. package/dist/sqlite-worker.d.ts +1 -0
  8. package/dist/sqlite-worker.js +439 -0
  9. package/dist/tabs-broker-worker.d.ts +8 -0
  10. package/dist/tabs-broker-worker.js +472 -0
  11. package/dist/types.d.ts +751 -11
  12. package/package.json +11 -7
  13. package/scripts/check-broker-bundle.mjs +33 -0
  14. package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
  15. package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
  16. package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
  17. package/src/bucket-blurhash.test.ts +148 -0
  18. package/src/build-globals.d.ts +12 -0
  19. package/src/events/events.test.ts +2 -1
  20. package/src/events/index.ts +3 -0
  21. package/src/index.ts +36 -2
  22. package/src/modules/app-release/index.test.ts +125 -0
  23. package/src/modules/app-release/index.ts +201 -0
  24. package/src/modules/auth/auth.local-first.test.ts +101 -0
  25. package/src/modules/auth/events/index.ts +2 -1
  26. package/src/modules/auth/index.ts +127 -24
  27. package/src/modules/cache/cache.relay.test.ts +95 -0
  28. package/src/modules/cache/index.ts +163 -43
  29. package/src/modules/cache/types.ts +2 -2
  30. package/src/modules/crdt/crdt-field.ts +294 -0
  31. package/src/modules/crdt/crdt-hydration.test.ts +210 -0
  32. package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
  33. package/src/modules/crdt/index.ts +463 -0
  34. package/src/modules/crdt/loro-loader.ts +25 -0
  35. package/src/modules/data/data.hydration.test.ts +142 -0
  36. package/src/modules/data/data.membership.test.ts +523 -0
  37. package/src/modules/data/data.notify-table.test.ts +41 -0
  38. package/src/modules/data/data.pending-ids.test.ts +199 -0
  39. package/src/modules/data/data.rebind.test.ts +170 -0
  40. package/src/modules/data/data.rematerialize.test.ts +114 -0
  41. package/src/modules/data/data.run.test.ts +113 -0
  42. package/src/modules/data/data.settled-writes.test.ts +206 -0
  43. package/src/modules/data/data.status.test.ts +249 -0
  44. package/src/modules/data/id-set-plan.test.ts +122 -0
  45. package/src/modules/data/index.ts +1815 -151
  46. package/src/modules/data/mutation-id.test.ts +25 -0
  47. package/src/modules/data/mutation-id.ts +35 -0
  48. package/src/modules/data/window-query.test.ts +52 -0
  49. package/src/modules/data/window-query.ts +194 -0
  50. package/src/modules/devtools/flags.ts +349 -0
  51. package/src/modules/devtools/index.ts +450 -46
  52. package/src/modules/devtools/notify-throttle.test.ts +154 -0
  53. package/src/modules/devtools/state-shape.test.ts +146 -0
  54. package/src/modules/devtools/storage-info.test.ts +79 -0
  55. package/src/modules/devtools/storage-info.ts +168 -0
  56. package/src/modules/devtools/versions.test.ts +74 -0
  57. package/src/modules/devtools/versions.ts +110 -0
  58. package/src/modules/feature-flag/index.test.ts +251 -0
  59. package/src/modules/feature-flag/index.ts +308 -0
  60. package/src/modules/ref-tables.test.ts +91 -0
  61. package/src/modules/ref-tables.ts +88 -0
  62. package/src/modules/sync/engine.ts +164 -82
  63. package/src/modules/sync/events/index.ts +9 -2
  64. package/src/modules/sync/queue/queue-down.test.ts +180 -0
  65. package/src/modules/sync/queue/queue-down.ts +80 -13
  66. package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
  67. package/src/modules/sync/queue/queue-up.ts +241 -57
  68. package/src/modules/sync/scheduler.pause.test.ts +109 -0
  69. package/src/modules/sync/scheduler.retry.test.ts +237 -0
  70. package/src/modules/sync/scheduler.ts +215 -13
  71. package/src/modules/sync/sync.cleanup.test.ts +116 -0
  72. package/src/modules/sync/sync.health.test.ts +149 -0
  73. package/src/modules/sync/sync.heartbeat.test.ts +80 -0
  74. package/src/modules/sync/sync.live-removal.test.ts +175 -0
  75. package/src/modules/sync/sync.reconnect.test.ts +145 -0
  76. package/src/modules/sync/sync.subquery.test.ts +82 -0
  77. package/src/modules/sync/sync.tabs.test.ts +249 -0
  78. package/src/modules/sync/sync.ts +1726 -99
  79. package/src/modules/sync/utils.test.ts +269 -2
  80. package/src/modules/sync/utils.ts +201 -17
  81. package/src/otel/index.ts +13 -10
  82. package/src/services/blobs/blob-cache.test.ts +359 -0
  83. package/src/services/blobs/blob-cache.ts +603 -0
  84. package/src/services/blobs/blob-manifest.ts +227 -0
  85. package/src/services/blobs/blob-store.test.ts +77 -0
  86. package/src/services/blobs/blob-store.ts +359 -0
  87. package/src/services/blobs/blob.fixture.ts +90 -0
  88. package/src/services/blobs/index.ts +70 -0
  89. package/src/services/database/cache-engine.ts +193 -0
  90. package/src/services/database/connection-supervisor.test.ts +289 -0
  91. package/src/services/database/connection-supervisor.ts +415 -0
  92. package/src/services/database/database.query-timeout.test.ts +83 -0
  93. package/src/services/database/database.ts +41 -12
  94. package/src/services/database/engine-factory.ts +33 -0
  95. package/src/services/database/errors.ts +34 -0
  96. package/src/services/database/events/index.ts +2 -1
  97. package/src/services/database/index.ts +7 -0
  98. package/src/services/database/local-migrator.ts +30 -27
  99. package/src/services/database/local.test.ts +64 -0
  100. package/src/services/database/local.ts +484 -67
  101. package/src/services/database/plan-render.test.ts +159 -0
  102. package/src/services/database/plan-render.ts +108 -0
  103. package/src/services/database/relation-resolver.test.ts +413 -0
  104. package/src/services/database/relation-resolver.ts +0 -0
  105. package/src/services/database/remote.ts +110 -14
  106. package/src/services/database/sqlite-cache-engine.test.ts +616 -0
  107. package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
  108. package/src/services/database/sqlite-cache-engine.ts +1358 -0
  109. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  110. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  111. package/src/services/database/sqlite-lock-verify.test.ts +33 -0
  112. package/src/services/database/sqlite-lock-verify.ts +45 -0
  113. package/src/services/database/sqlite-open.test.ts +150 -0
  114. package/src/services/database/sqlite-open.ts +164 -0
  115. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  116. package/src/services/database/sqlite-plan-sql.ts +138 -0
  117. package/src/services/database/sqlite-projection.test.ts +99 -0
  118. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  119. package/src/services/database/sqlite-select.test.ts +246 -0
  120. package/src/services/database/sqlite-select.ts +131 -0
  121. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  122. package/src/services/database/sqlite-transport.ts +224 -0
  123. package/src/services/database/sqlite-worker.ts +437 -0
  124. package/src/services/database/surql-translate.ts +416 -0
  125. package/src/services/database/surreal-cache-engine.ts +161 -0
  126. package/src/services/logger/index.ts +3 -2
  127. package/src/services/persistence/localstorage.ts +2 -2
  128. package/src/services/persistence/resilient.ts +11 -4
  129. package/src/services/persistence/surrealdb.ts +10 -10
  130. package/src/services/stream-processor/index.ts +796 -84
  131. package/src/services/stream-processor/permissions.test.ts +47 -0
  132. package/src/services/stream-processor/permissions.ts +53 -0
  133. package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
  134. package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
  135. package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
  136. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  137. package/src/services/stream-processor/wasm-types.ts +59 -3
  138. package/src/services/tabs/broker-client.ts +283 -0
  139. package/src/services/tabs/broker.test.ts +327 -0
  140. package/src/services/tabs/coordinator.test.ts +365 -0
  141. package/src/services/tabs/coordinator.ts +633 -0
  142. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  143. package/src/services/tabs/leader-locks.ts +75 -0
  144. package/src/services/tabs/protocol.ts +258 -0
  145. package/src/services/tabs/support.ts +36 -0
  146. package/src/services/tabs/tabs-broker-worker.ts +640 -0
  147. package/src/sp00ky.auth-order.test.ts +92 -0
  148. package/src/sp00ky.init-query.test.ts +183 -0
  149. package/src/sp00ky.local-first.test.ts +60 -0
  150. package/src/sp00ky.ts +1693 -0
  151. package/src/types.ts +528 -13
  152. package/src/utils/blurhash.ts +90 -0
  153. package/src/utils/error-classification.test.ts +44 -0
  154. package/src/utils/error-classification.ts +7 -0
  155. package/src/utils/index.ts +79 -13
  156. package/src/utils/parser.test.ts +49 -120
  157. package/src/utils/parser.ts +32 -2
  158. package/src/utils/semver.test.ts +32 -0
  159. package/src/utils/semver.ts +30 -0
  160. package/src/utils/surql.ts +30 -18
  161. package/src/utils/withRetry.test.ts +1 -1
  162. package/tsdown.config.ts +86 -1
  163. package/src/spooky.ts +0 -395
@@ -1,8 +1,8 @@
1
- import { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
2
- import { Logger } from '../../services/logger/index';
3
- import { SchemaStructure } from '@spooky-sync/query-builder';
1
+ import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
2
+ import type { Logger } from '../../services/logger/index';
3
+ import type { SchemaStructure } from '@spooky-sync/query-builder';
4
4
  import { RecordId } from 'surrealdb';
5
- import { StreamUpdate, StreamUpdateReceiver } from '../../services/stream-processor/index';
5
+ import type { StreamUpdate, StreamUpdateReceiver } from '../../services/stream-processor/index';
6
6
  import { encodeRecordId } from '../../utils/index';
7
7
 
8
8
  // DevTools interfaces (matching extension expectations)
@@ -13,31 +13,173 @@ export interface DevToolsEvent {
13
13
  payload: any;
14
14
  }
15
15
 
16
- import { DataModule } from '../data/index';
17
- import { AuthService } from '../auth/index';
16
+ import type { DataModule } from '../data/index';
17
+ import type { AuthService } from '../auth/index';
18
18
  import { AuthEventTypes } from '../auth/events/index';
19
+ import {
20
+ type BackendInfo,
21
+ emptyBackendInfo,
22
+ parseBackendInfo,
23
+ UNAVAILABLE,
24
+ } from './versions';
25
+ import { walkOpfs, type BlobCacheInfo, type SharedTabsInfo, type StorageInfo } from './storage-info';
26
+ import { FlagsAdminService, type LocalOverrideStore } from './flags';
27
+
28
+ // Real bundled frontend versions, injected at build time by tsdown's
29
+ // version-define plugin (see tsdown.config.ts). The `typeof` guard keeps these
30
+ // from throwing a ReferenceError when a downstream app bundles core from source
31
+ // (where the plugin never runs); in that case they fall back to 'unknown' and
32
+ // DevTools simply reports an unknown frontend version instead of crashing.
33
+ const CORE_VERSION =
34
+ typeof __SP00KY_CORE_VERSION__ !== 'undefined' ? __SP00KY_CORE_VERSION__ : 'unknown';
35
+ const WASM_VERSION =
36
+ typeof __SP00KY_WASM_VERSION__ !== 'undefined' ? __SP00KY_WASM_VERSION__ : 'unknown';
37
+ const SURREAL_VERSION =
38
+ typeof __SP00KY_SURREAL_VERSION__ !== 'undefined' ? __SP00KY_SURREAL_VERSION__ : 'unknown';
19
39
 
20
40
  export class DevToolsService implements StreamUpdateReceiver {
21
41
  private eventsHistory: DevToolsEvent[] = [];
22
42
  private eventIdCounter = 0;
23
- private version = '1.0.0';
43
+ // Real bundled frontend version (injected at build time via tsdown `define`).
44
+ private version = CORE_VERSION;
45
+ // Backend stack info (versions + per-entity status), read via the
46
+ // `fn::spooky::info()` SurrealQL function; empty/'unavailable' until resolved.
47
+ private backendInfo: BackendInfo = emptyBackendInfo();
48
+ // Dormant until a devtools consumer (extension panel or MCP) handshakes via
49
+ // `SP00KY_DEVTOOLS_CONNECT`. While false, `notifyDevTools()`/`addEvent()` do no
50
+ // work, so prod pays zero serialization/postMessage cost for an unwatched panel.
51
+ // `window.__00__.getState()` stays live regardless, so the panel's first paint
52
+ // (the on-demand GET_STATE pull) still works before the push channel turns on.
53
+ private enabled = false;
54
+
55
+ // A state push serializes EVERY active query's full record set (see
56
+ // `getActiveQueries`) and postMessage clones it again, so its cost scales with
57
+ // the whole client dataset — and it is triggered per event, including one per
58
+ // local DB query (`DATABASE_LOCAL_QUERY` → `logEvent`). Unthrottled, a single
59
+ // page load's few hundred local queries turn a handful of MB of rows into GBs
60
+ // of short-lived large-object garbage and OOM the renderer (V8
61
+ // "young object promotion failed"). Coalesce instead: push immediately when
62
+ // idle, then at most once per window, always serializing the LATEST state.
63
+ private static readonly NOTIFY_MIN_INTERVAL_MS = 250;
64
+ private notifyTimer: ReturnType<typeof setTimeout> | null = null;
65
+ private lastNotifyAt = 0;
66
+ /** How many ids a pushed state carries per view; the rest is on demand. */
67
+ private static readonly STATE_IDS_CAP = 200;
68
+ /** devtools numeric hash -> the query's `_00_query` id, for on-demand rows. */
69
+ private hashToQuery = new Map<number, unknown>();
70
+
71
+ /** Shared-tabs snapshot for the panel, wired by Sp00kyClient whenever the
72
+ * feature was REQUESTED (so an inactive/degraded tab still reports why). */
73
+ private tabsInfoProvider: (() => SharedTabsInfo | null) | null = null;
74
+
75
+ setTabsInfoProvider(provider: () => SharedTabsInfo | null): void {
76
+ this.tabsInfoProvider = provider;
77
+ }
78
+
79
+ /** Blob cache counters for the panel, wired by Sp00kyClient. */
80
+ private blobInfoProvider: (() => BlobCacheInfo) | null = null;
81
+
82
+ setBlobInfoProvider(provider: () => BlobCacheInfo): void {
83
+ this.blobInfoProvider = provider;
84
+ }
85
+
86
+ // Full local table list (incl. internal `_00_*`), enumerated from the local DB
87
+ // via our own reliable `service.query` — the DevTools panel's page-eval query
88
+ // bridge is unreliable under load, so the panel relies on this instead.
89
+ private localTables: string[] = [];
90
+ private localTablesFetching = false;
91
+ private localTablesAt = 0;
92
+
93
+ // Feature flag admin, backing the panel's Access tab. The local-override
94
+ // store is injected later (`setFeatureFlagOverrides`) because the
95
+ // FeatureFlagModule is built after this service.
96
+ private featureOverrides: LocalOverrideStore | null = null;
97
+ private readonly flagsAdmin: FlagsAdminService;
24
98
 
25
99
  constructor(
26
- private databaseService: LocalDatabaseService,
100
+ private databaseService: LocalStore,
27
101
  private remoteDatabaseService: RemoteDatabaseService,
28
102
  private logger: Logger,
29
103
  private schema: SchemaStructure,
30
104
  private authService: AuthService<SchemaStructure>,
31
105
  private dataManager?: DataModule<SchemaStructure>
32
106
  ) {
107
+ this.flagsAdmin = new FlagsAdminService({
108
+ remote: this.remoteDatabaseService,
109
+ local: this.databaseService,
110
+ logger: this.logger,
111
+ currentUserId: () => {
112
+ const id = this.authService.currentUser?.id;
113
+ if (!id) return null;
114
+ return id instanceof RecordId ? encodeRecordId(id) : String(id);
115
+ },
116
+ overrides: () => this.featureOverrides,
117
+ });
118
+
33
119
  this.exposeToWindow();
34
120
 
35
- // Subscribe to auth events
121
+ // Stay dormant until a devtools consumer announces itself. The extension's
122
+ // page-script posts this once it detects `window.__00__`; the panel can also
123
+ // disconnect to return us to dormant. Until then we skip all serialization.
124
+ if (typeof window !== 'undefined') {
125
+ window.addEventListener('message', (e) => {
126
+ if (e.source !== window) return;
127
+ const type = (e.data as { type?: string } | undefined)?.type;
128
+ if (type === 'SP00KY_DEVTOOLS_CONNECT') {
129
+ this.enabled = true;
130
+ this.refreshLocalTables();
131
+ this.notifyDevTools();
132
+ } else if (type === 'SP00KY_DEVTOOLS_DISCONNECT') {
133
+ this.enabled = false;
134
+ }
135
+ });
136
+ }
137
+
138
+ // Subscribe to auth events. The initial fire-and-forget version fetch (below)
139
+ // races the remote connection; on the free plan the remote DB (SurrealDB
140
+ // Cloud) has no guest access, so `fn::spooky::info()` is only callable once
141
+ // signed in. Re-fetch when auth resolves — until the versions actually land —
142
+ // instead of leaving them 'unavailable' forever.
36
143
  this.authService.eventSystem.subscribe(AuthEventTypes.AuthStateChanged, () => {
37
- this.notifyDevTools();
144
+ if (this.authService.isAuthenticated && this.backendInfo.versions.ssp === UNAVAILABLE) {
145
+ void this.refreshBackendVersions();
146
+ } else {
147
+ this.notifyDevTools();
148
+ }
38
149
  });
39
150
 
40
- this.logger.debug({ Category: 'spooky-client::DevToolsService::init' }, 'Service initialized');
151
+ // Push state when the local store reports its durability (the open happens
152
+ // during connect, typically before a panel attaches, so this mostly matters
153
+ // for a later bucket switch that loses OPFS).
154
+ this.databaseService.subscribeToStorageHealth?.(() => this.notifyDevTools());
155
+
156
+ // Fire-and-forget backend version discovery; re-push state when it lands.
157
+ void this.refreshBackendVersions();
158
+
159
+ this.logger.debug({ Category: 'sp00ky-client::DevToolsService::init' }, 'Service initialized');
160
+ }
161
+
162
+ /**
163
+ * Re-read backend stack info via the `fn::spooky::info()` SurrealQL function
164
+ * over the open remote connection (no HTTP/CORS), then notify the panel.
165
+ * Never throws: on failure the info stays empty/'unavailable'.
166
+ */
167
+ private async refreshBackendVersions(): Promise<void> {
168
+ try {
169
+ // `RETURN fn::spooky::info()` → one statement result: the /info entity array.
170
+ const result = await this.remoteDatabaseService.query<unknown[]>(
171
+ 'RETURN fn::spooky::info()'
172
+ );
173
+ const first = Array.isArray(result) ? result[0] : result;
174
+ this.backendInfo = parseBackendInfo(first);
175
+ } catch (err) {
176
+ this.logger.debug(
177
+ { err, Category: 'sp00ky-client::DevToolsService::versions' },
178
+ 'fn::spooky::info() unavailable; backend versions stay unavailable'
179
+ );
180
+ this.backendInfo = emptyBackendInfo();
181
+ }
182
+ this.notifyDevTools();
41
183
  }
42
184
 
43
185
  // Get active queries directly from DataManager (single source of truth)
@@ -48,21 +190,51 @@ export class DevToolsService implements StreamUpdateReceiver {
48
190
  const queries = this.dataManager.getActiveQueries();
49
191
  queries.forEach((q) => {
50
192
  const queryHash = this.hashString(encodeRecordId(q.config.id));
193
+ const createdAt =
194
+ q.config.lastActiveAt instanceof Date
195
+ ? q.config.lastActiveAt.getTime()
196
+ : new Date(q.config.lastActiveAt || Date.now()).getTime();
197
+ this.hashToQuery.set(queryHash, q.config.id);
198
+ const localArray = q.config.localArray ?? [];
199
+ const remoteArray = q.config.remoteArray ?? [];
51
200
  result.set(queryHash, {
52
201
  queryHash,
53
202
  status: 'active',
54
- createdAt:
55
- q.config.lastActiveAt instanceof Date
56
- ? q.config.lastActiveAt.getTime()
57
- : new Date(q.config.lastActiveAt || Date.now()).getTime(),
58
- lastUpdate: Date.now(),
203
+ // Runtime fetch status, distinct from the `status: 'active'`
204
+ // registration flag above. `fetchStatus` is 'idle' | 'fetching'.
205
+ fetchStatus: q.status,
206
+ isFetching: q.status === 'fetching',
207
+ createdAt,
208
+ // Real last-update time; before the first update it equals createdAt.
209
+ // (Previously Date.now(), which reset the column on every state push.)
210
+ lastUpdate: q.lastUpdatedAt ?? createdAt,
59
211
  updateCount: q.updateCount,
212
+ ttl: q.config.ttl,
60
213
  query: q.config.surql,
61
214
  variables: q.config.params || {},
62
215
  dataSize: q.records?.length || 0,
63
- data: q.records,
64
- localArray: q.config.localArray,
65
- remoteArray: q.config.remoteArray,
216
+ // Counts and (capped) ids only. The rows themselves used to ride along
217
+ // here: every push then deep-cloned every view's records twice
218
+ // (serialize + postMessage), on a 4k-row client dataset up to four
219
+ // times a second, from inside the ingest call stack. The panel pulls
220
+ // rows on demand through `getQueryRows` instead.
221
+ localCount: localArray.length,
222
+ remoteCount: remoteArray.length,
223
+ localIds: localArray.slice(0, DevToolsService.STATE_IDS_CAP).map(([id]) => id),
224
+ remoteIds: remoteArray.slice(0, DevToolsService.STATE_IDS_CAP).map(([id]) => id),
225
+ idsTruncated:
226
+ localArray.length > DevToolsService.STATE_IDS_CAP ||
227
+ remoteArray.length > DevToolsService.STATE_IDS_CAP,
228
+ // Membership state, so "why is this list empty" is answerable from
229
+ // the panel: is the server's set known, has a non-empty one been seen
230
+ // this session, how many empty reads were ignored.
231
+ membershipKnown: q.config.membershipKnown === true,
232
+ remoteSeen: q.config.remoteSeen === true,
233
+ emptyReads: q.config.emptyReads ?? 0,
234
+ // Detailed per-phase processing-time breakdown (SSP sub-phases, local/
235
+ // remote record fetch, frontend reconcile, registration). Flows to both
236
+ // the DevTools panel and the MCP (which returns activeQueries verbatim).
237
+ timings: this.dataManager.phaseTimings(q),
66
238
  });
67
239
  });
68
240
  return result;
@@ -70,7 +242,7 @@ export class DevToolsService implements StreamUpdateReceiver {
70
242
 
71
243
  public onQueryInitialized(payload: any) {
72
244
  this.logger.debug(
73
- { payload, Category: 'spooky-client::DevToolsService::onQueryInitialized' },
245
+ { payload, Category: 'sp00ky-client::DevToolsService::onQueryInitialized' },
74
246
  'QueryInitialized'
75
247
  );
76
248
  const queryHash = this.hashString(payload.queryId.toString());
@@ -87,7 +259,7 @@ export class DevToolsService implements StreamUpdateReceiver {
87
259
  this.logger.debug(
88
260
  {
89
261
  id: payload.queryId?.toString(),
90
- Category: 'spooky-client::DevToolsService::onQueryUpdated',
262
+ Category: 'sp00ky-client::DevToolsService::onQueryUpdated',
91
263
  },
92
264
  'QueryUpdated'
93
265
  );
@@ -95,18 +267,28 @@ export class DevToolsService implements StreamUpdateReceiver {
95
267
 
96
268
  this.addEvent('QUERY_UPDATED', {
97
269
  queryHash,
98
- data: payload.records,
270
+ recordCount: Array.isArray(payload.records) ? payload.records.length : 0,
99
271
  });
100
272
  this.notifyDevTools();
101
273
  }
102
274
 
103
275
  public onStreamUpdate(update: StreamUpdate) {
276
+ // A synthetic re-materialize is not an ingest (DataModule.scheduleRematerialize).
277
+ if (update.synthetic) return;
104
278
  this.logger.debug(
105
- { update, Category: 'spooky-client::DevToolsService::onStreamUpdate' },
279
+ { queryHash: update.queryHash, Category: 'sp00ky-client::DevToolsService::onStreamUpdate' },
106
280
  'StreamUpdate'
107
281
  );
282
+ // Counts and timings, not the `localArray` itself: that is one [id, version]
283
+ // pair per row of the view, serialized on every single update.
108
284
  this.addEvent('STREAM_UPDATE', {
109
- updates: [update],
285
+ queryHash: update.queryHash,
286
+ op: update.op,
287
+ localCount: update.localArray?.length ?? 0,
288
+ materializationTimeMs: update.materializationTimeMs,
289
+ storeApplyMs: update.storeApplyMs,
290
+ circuitStepMs: update.circuitStepMs,
291
+ transformMs: update.transformMs,
110
292
  });
111
293
  this.notifyDevTools();
112
294
  }
@@ -116,8 +298,10 @@ export class DevToolsService implements StreamUpdateReceiver {
116
298
  payloads.forEach((p) => {
117
299
  this.addEvent('MUTATION_REQUEST_EXECUTION', {
118
300
  mutation: {
119
- type: 'create', // simplifying
120
- data: 'data' in p ? p.data : undefined,
301
+ type: p.type ?? 'create',
302
+ // Field names only; a payload body (a PGN, a document) is not
303
+ // something to clone on every write.
304
+ fields: 'data' in p && p.data && typeof p.data === 'object' ? Object.keys(p.data) : [],
121
305
  selector: encodeRecordId(p.record_id),
122
306
  },
123
307
  });
@@ -142,6 +326,8 @@ export class DevToolsService implements StreamUpdateReceiver {
142
326
  }
143
327
 
144
328
  private addEvent(eventType: string, payload: any) {
329
+ // No consumer attached → skip recording (and the recursive serialize it does).
330
+ if (!this.enabled) return;
145
331
  this.eventsHistory.push({
146
332
  id: this.eventIdCounter++,
147
333
  timestamp: Date.now(),
@@ -151,7 +337,55 @@ export class DevToolsService implements StreamUpdateReceiver {
151
337
  if (this.eventsHistory.length > 100) this.eventsHistory.shift();
152
338
  }
153
339
 
154
- private getState() {
340
+ /** Unwrap a SurrealDB `INFO FOR DB` result to its `{ tables, ... }` object. */
341
+ private unwrapInfo(res: any): any {
342
+ if (!Array.isArray(res) || !res[0]) return null;
343
+ const first = res[0];
344
+ if (first && typeof first === 'object' && 'result' in first) return first.result;
345
+ if (Array.isArray(first)) return first[0];
346
+ return first;
347
+ }
348
+
349
+ /**
350
+ * Refresh the cached full local-table list from `INFO FOR DB`. Fire-and-forget
351
+ * and throttled — called from getState() so the panel gets every table
352
+ * (including internal `_00_*`) without running its own (flaky) queries.
353
+ */
354
+ private refreshLocalTables(): void {
355
+ if (this.localTablesFetching) return;
356
+ const now = Date.now();
357
+ if (now - this.localTablesAt < 30_000) return;
358
+ this.localTablesFetching = true;
359
+ void this.databaseService
360
+ .query<any>('INFO FOR DB')
361
+ .then((res) => {
362
+ const info = this.unwrapInfo(res);
363
+ this.localTablesAt = Date.now();
364
+ if (info && info.tables) {
365
+ // The circuit snapshot table holds a BLOB, not JSON rows; the
366
+ // explorer cannot render it and has nothing to show for it.
367
+ const names = Object.keys(info.tables).filter((n) => n !== '_00_circuit_snapshot');
368
+ const changed =
369
+ names.length !== this.localTables.length ||
370
+ names.some((n, i) => n !== this.localTables[i]);
371
+ this.localTables = names;
372
+ if (changed) this.notifyDevTools();
373
+ }
374
+ })
375
+ .catch(() => {
376
+ // Ignore — fall back to the declared app schema below.
377
+ })
378
+ .finally(() => {
379
+ this.localTablesFetching = false;
380
+ });
381
+ }
382
+
383
+ private getState(opts: { refreshTables?: boolean } = {}) {
384
+ // The local-table list (`INFO FOR DB`, a local round trip) is refreshed on
385
+ // an explicit pull only, never by the push path: pushes follow every sync
386
+ // event, and a local query per push competed with the app's own writes for
387
+ // the single local op queue.
388
+ if (opts.refreshTables) this.refreshLocalTables();
155
389
  return this.serializeForDevTools({
156
390
  eventsHistory: [...this.eventsHistory],
157
391
  activeQueries: Object.fromEntries(this.getActiveQueries()),
@@ -160,24 +394,148 @@ export class DevToolsService implements StreamUpdateReceiver {
160
394
  userId: this.authService.currentUser?.id,
161
395
  },
162
396
  version: this.version,
397
+ versions: {
398
+ frontend: {
399
+ core: CORE_VERSION,
400
+ wasm: WASM_VERSION,
401
+ surrealdb: SURREAL_VERSION,
402
+ },
403
+ backend: this.backendInfo.versions,
404
+ entities: this.backendInfo.entities,
405
+ },
163
406
  database: {
164
- tables: this.schema.tables.map((t) => t.name),
407
+ // Prefer the live local-table list (includes internal `_00_*`); fall
408
+ // back to the declared app schema until the first enumeration lands.
409
+ tables: this.localTables.length
410
+ ? this.localTables
411
+ : this.schema.tables.map((t) => t.name),
165
412
  tableData: {},
413
+ // Which backend answers "Local". The Database explorer labels its source
414
+ // picker with it and explains translation failures against `sqlite`,
415
+ // whose SurrealQL vocabulary is a bounded subset.
416
+ engine: this.databaseService.engineKind ?? 'custom',
417
+ // Durability of the local store. `fallback: true` means persistence was
418
+ // requested but the dataset is actually sitting in RAM.
419
+ storage: this.databaseService.storageHealth ?? { status: 'unknown', fallback: false },
420
+ // Shared-tabs role state (null when the feature is off / fell back).
421
+ tabs: this.tabsInfoProvider?.() ?? null,
166
422
  },
167
423
  });
168
424
  }
169
425
 
170
- private notifyDevTools() {
171
- if (typeof window !== 'undefined') {
172
- window.postMessage(
173
- {
174
- type: 'SPOOKY_STATE_CHANGED',
175
- source: 'spooky-devtools-page',
176
- state: this.getState(),
177
- },
178
- '*'
426
+ /**
427
+ * Full storage diagnostics for the DevTools Storage tab. Every section is
428
+ * gathered independently and failures land in that section's `error` field,
429
+ * so one broken source (a mid-switch worker, a browser without OPFS) never
430
+ * blanks the whole panel.
431
+ */
432
+ public async getStorageInfo(opts?: { tableCounts?: boolean }): Promise<StorageInfo> {
433
+ const nav = typeof navigator !== 'undefined' ? navigator : undefined;
434
+
435
+ const info: StorageInfo = {
436
+ at: Date.now(),
437
+ engine: {
438
+ kind: this.databaseService.engineKind ?? 'custom',
439
+ store: this.databaseService.getConfig()?.store ?? 'memory',
440
+ bucketId: this.databaseService.currentBucketId,
441
+ },
442
+ health: this.databaseService.storageHealth ?? { status: 'unknown', fallback: false },
443
+ tabs: this.tabsInfoProvider?.() ?? null,
444
+ browser: {},
445
+ opfs: { supported: false, entries: [], totalBytes: 0, truncated: false },
446
+ };
447
+
448
+ try {
449
+ if (nav?.storage?.estimate) {
450
+ const est = await nav.storage.estimate();
451
+ info.browser.usage = est.usage;
452
+ info.browser.quota = est.quota;
453
+ // Chrome-only per-storage-system breakdown; absent elsewhere.
454
+ const details = (est as any).usageDetails;
455
+ if (details && typeof details === 'object') info.browser.usageDetails = details;
456
+ }
457
+ if (nav?.storage?.persisted) {
458
+ info.browser.persisted = await nav.storage.persisted();
459
+ }
460
+ } catch (e) {
461
+ info.browser.error = e instanceof Error ? e.message : String(e);
462
+ }
463
+
464
+ info.opfs = await walkOpfs();
465
+
466
+ try {
467
+ info.blobs = this.blobInfoProvider?.();
468
+ } catch (e) {
469
+ this.logger.warn(
470
+ { err: e, Category: 'sp00ky-client::DevToolsService::getStorageInfo' },
471
+ 'Blob cache diagnostics failed'
472
+ );
473
+ }
474
+
475
+ const stats = (globalThis as any).__sqliteStats;
476
+ if (stats && typeof stats === 'object') {
477
+ info.sqliteStats = { ...stats, byType: { ...(stats.byType ?? {}) } };
478
+ }
479
+
480
+ try {
481
+ info.engineDiagnostics = await this.databaseService.getStorageDiagnostics?.(opts);
482
+ } catch (e) {
483
+ this.logger.warn(
484
+ { err: e, Category: 'sp00ky-client::DevToolsService::getStorageInfo' },
485
+ 'Engine storage diagnostics failed'
179
486
  );
180
487
  }
488
+
489
+ return this.serializeForDevTools(info);
490
+ }
491
+
492
+ /** Ask the browser to exempt this origin's storage from eviction. */
493
+ public async requestPersistentStorage(): Promise<{ granted: boolean }> {
494
+ try {
495
+ const granted = (await navigator.storage?.persist?.()) ?? false;
496
+ return { granted };
497
+ } catch {
498
+ return { granted: false };
499
+ }
500
+ }
501
+
502
+ /**
503
+ * Request a state push. Coalesced (see {@link NOTIFY_MIN_INTERVAL_MS}): the
504
+ * first call after an idle period pushes straight away so the panel stays
505
+ * responsive, and any calls during the window collapse into ONE trailing push
506
+ * that serializes the state as of the flush, not as of the request. Callers
507
+ * stay fire-and-forget.
508
+ */
509
+ private notifyDevTools() {
510
+ // No consumer attached → no getState() serialization, no postMessage broadcast.
511
+ if (!this.enabled) return;
512
+ if (typeof window === 'undefined') return;
513
+ // A trailing push is already queued; it will carry this change too.
514
+ if (this.notifyTimer !== null) return;
515
+
516
+ // Always a macrotask, never inline: this is called from inside the ingest
517
+ // and mutation call stacks (onStreamUpdate / onMutation), i.e. inside the
518
+ // `await db.create(...)` the app is waiting on. A push that serializes the
519
+ // state right there charged every write for the panel's refresh.
520
+ const waited = Date.now() - this.lastNotifyAt;
521
+ const delay = Math.max(0, DevToolsService.NOTIFY_MIN_INTERVAL_MS - waited);
522
+ this.notifyTimer = setTimeout(() => {
523
+ this.notifyTimer = null;
524
+ // Still gated on `enabled`: the panel may have disconnected while queued.
525
+ if (this.enabled) this.flushNotify();
526
+ }, delay);
527
+ }
528
+
529
+ private flushNotify() {
530
+ this.lastNotifyAt = Date.now();
531
+ window.postMessage(
532
+ {
533
+ type: 'SP00KY_STATE_CHANGED',
534
+ source: 'sp00ky-devtools-page',
535
+ state: this.getState(),
536
+ },
537
+ '*'
538
+ );
181
539
  }
182
540
 
183
541
  private serializeForDevTools(data: any, seen = new WeakSet<object>()): any {
@@ -218,6 +576,10 @@ export class DevToolsService implements StreamUpdateReceiver {
218
576
  const result: Record<string, any> = {};
219
577
  for (const key in data) {
220
578
  if (Object.prototype.hasOwnProperty.call(data, key)) {
579
+ // Skip absent optional fields: recursing them would emit the STRING
580
+ // 'undefined' (the top-level mapping below), which panels then have
581
+ // to filter back out (see 3d84fe8a).
582
+ if (data[key] === undefined) continue;
221
583
  result[key] = this.serializeForDevTools(data[key], seen);
222
584
  }
223
585
  }
@@ -227,15 +589,54 @@ export class DevToolsService implements StreamUpdateReceiver {
227
589
  return data;
228
590
  }
229
591
 
592
+ /**
593
+ * Hand the FeatureFlagModule to the Access tab so it can read and write local
594
+ * overrides. Called from `Sp00kyClient` once both are constructed; until then
595
+ * the override methods are no-ops that report an empty map.
596
+ */
597
+ public setFeatureFlagOverrides(store: LocalOverrideStore): void {
598
+ this.featureOverrides = store;
599
+ }
600
+
230
601
  private exposeToWindow() {
231
602
  if (typeof window !== 'undefined') {
232
- (window as any).__SPOOKY__ = {
603
+ (window as any).__00__ = {
233
604
  version: this.version,
234
- getState: () => this.getState(),
605
+ getState: () => this.getState({ refreshTables: true }),
606
+ // The rows of ONE view, on demand. The pushed state carries counts and
607
+ // capped ids only (see getActiveQueries); the panel's Data tab and the
608
+ // MCP fetch the rows here when somebody actually looks at them.
609
+ getQueryRows: (queryHash: number) => {
610
+ const id = this.hashToQuery.get(Number(queryHash));
611
+ const q = id !== undefined ? this.dataManager?.getQueryById(id as any) : undefined;
612
+ if (!q) return null;
613
+ return this.serializeForDevTools({
614
+ queryHash: Number(queryHash),
615
+ data: q.records,
616
+ localArray: q.config.localArray,
617
+ remoteArray: q.config.remoteArray,
618
+ });
619
+ },
620
+ // ---- Feature flags (Access tab) --------------------------------
621
+ // Remote reads/writes are admin-gated by SurrealDB, not here: a
622
+ // non-admin gets an empty flag list, and the `fn::feature::*` calls
623
+ // are denied outright. The override methods are purely local and
624
+ // work signed out.
625
+ getFlags: () => this.flagsAdmin.getFlags(),
626
+ setFlagEnabled: (key: string, enabled: boolean) =>
627
+ this.flagsAdmin.setFlagEnabled(key, enabled),
628
+ setFlagUserVariant: (key: string, variant: string, remove: boolean, userId?: string) =>
629
+ this.flagsAdmin.setFlagUserVariant(key, variant, remove, userId),
630
+ setLocalFlagOverride: (key: string, variant: string | null, payload?: unknown) =>
631
+ this.flagsAdmin.setLocalFlagOverride(key, variant, payload),
632
+ clearLocalFlagOverrides: () => this.flagsAdmin.clearLocalFlagOverrides(),
235
633
  clearHistory: () => {
236
634
  this.eventsHistory = [];
237
635
  this.notifyDevTools();
238
636
  },
637
+ refreshVersions: () => this.refreshBackendVersions(),
638
+ getStorageInfo: (opts?: { tableCounts?: boolean }) => this.getStorageInfo(opts),
639
+ requestPersistentStorage: () => this.requestPersistentStorage(),
239
640
  getTableData: async (tableName: string) => {
240
641
  try {
241
642
  // Returns the first statement result as T.
@@ -270,7 +671,7 @@ export class DevToolsService implements StreamUpdateReceiver {
270
671
  return this.serializeForDevTools(records) || [];
271
672
  } catch (e) {
272
673
  this.logger.error(
273
- { err: e, Category: 'spooky-client::DevToolsService::exposeToWindow' },
674
+ { err: e, Category: 'sp00ky-client::DevToolsService::exposeToWindow' },
274
675
  'Failed to get table data'
275
676
  );
276
677
  return [];
@@ -299,7 +700,7 @@ export class DevToolsService implements StreamUpdateReceiver {
299
700
  runQuery: async (query: string, target: 'local' | 'remote' = 'local') => {
300
701
  try {
301
702
  this.logger.debug(
302
- { query, target, Category: 'spooky-client::DevToolsService::runQuery' },
703
+ { query, target, Category: 'sp00ky-client::DevToolsService::runQuery' },
303
704
  'Running query (START)'
304
705
  );
305
706
  const service = target === 'remote' ? this.remoteDatabaseService : this.databaseService;
@@ -314,7 +715,7 @@ export class DevToolsService implements StreamUpdateReceiver {
314
715
  time: queryTime,
315
716
  resultType: typeof result,
316
717
  isArray: Array.isArray(result),
317
- Category: 'spooky-client::DevToolsService::runQuery',
718
+ Category: 'sp00ky-client::DevToolsService::runQuery',
318
719
  },
319
720
  'Database returned result'
320
721
  );
@@ -328,7 +729,7 @@ export class DevToolsService implements StreamUpdateReceiver {
328
729
  {
329
730
  serializeTime,
330
731
  serializedLength: JSON.stringify(serialized).length,
331
- Category: 'spooky-client::DevToolsService::runQuery',
732
+ Category: 'sp00ky-client::DevToolsService::runQuery',
332
733
  },
333
734
  'Serialization complete'
334
735
  );
@@ -340,7 +741,7 @@ export class DevToolsService implements StreamUpdateReceiver {
340
741
  };
341
742
  } catch (e: any) {
342
743
  this.logger.error(
343
- { err: e, query, target, Category: 'spooky-client::DevToolsService::runQuery' },
744
+ { err: e, query, target, Category: 'sp00ky-client::DevToolsService::runQuery' },
344
745
  'Query execution failed'
345
746
  );
346
747
  // Ensure we always return a string for error
@@ -353,12 +754,15 @@ export class DevToolsService implements StreamUpdateReceiver {
353
754
 
354
755
  window.postMessage(
355
756
  {
356
- type: 'SPOOKY_DETECTED',
357
- source: 'spooky-devtools-page',
757
+ type: 'SP00KY_DETECTED',
758
+ source: 'sp00ky-devtools-page',
358
759
  data: { version: this.version, detected: true },
359
760
  },
360
761
  '*'
361
762
  );
763
+
764
+ // Dispatch custom event so the devtools page-script can detect late initialization
765
+ window.dispatchEvent(new CustomEvent('sp00ky:init'));
362
766
  }
363
767
  }
364
768
  }