@spooky-sync/core 0.0.1-canary.27 → 0.0.1-canary.271

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/AGENTS.md +78 -0
  2. package/README.md +56 -16
  3. package/dist/index.d.ts +2279 -81
  4. package/dist/index.js +13490 -2585
  5. package/dist/otel/index.d.ts +2 -2
  6. package/dist/otel/index.js +6 -6
  7. package/dist/sqlite-open.js +331 -0
  8. package/dist/sqlite-worker.d.ts +1 -0
  9. package/dist/sqlite-worker.js +439 -0
  10. package/dist/tabs-broker-worker.d.ts +8 -0
  11. package/dist/tabs-broker-worker.js +472 -0
  12. package/dist/types.d.ts +939 -46
  13. package/package.json +16 -8
  14. package/scripts/check-broker-bundle.mjs +33 -0
  15. package/skills/{spooky-core → sp00ky-core}/SKILL.md +30 -12
  16. package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
  17. package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
  18. package/src/boot/auth-flip.saga.test.ts +82 -0
  19. package/src/boot/auth-flip.saga.ts +62 -0
  20. package/src/boot/boot.saga.test.ts +151 -0
  21. package/src/boot/boot.saga.ts +128 -0
  22. package/src/boot/bucket-switch.saga.test.ts +187 -0
  23. package/src/boot/bucket-switch.saga.ts +131 -0
  24. package/src/boot/preload.saga.test.ts +51 -0
  25. package/src/boot/preload.saga.ts +31 -0
  26. package/src/bucket-blurhash.test.ts +148 -0
  27. package/src/build-globals.d.ts +12 -0
  28. package/src/client/bucket-handle.test.ts +117 -0
  29. package/src/client/bucket-handle.ts +227 -0
  30. package/src/client/router.test.ts +56 -0
  31. package/src/client/router.ts +96 -0
  32. package/src/client/runtime.test.ts +192 -0
  33. package/src/client/runtime.ts +296 -0
  34. package/src/client/services.ts +386 -0
  35. package/src/client/sp00ky-client.test.ts +264 -0
  36. package/src/client/sp00ky-client.ts +528 -0
  37. package/src/events/events.test.ts +2 -1
  38. package/src/events/index.ts +3 -0
  39. package/src/index.ts +37 -2
  40. package/src/kernel/constants.test.ts +19 -0
  41. package/src/kernel/constants.ts +61 -0
  42. package/src/kernel/effects.test.ts +40 -0
  43. package/src/kernel/effects.ts +161 -0
  44. package/src/kernel/events.ts +56 -0
  45. package/src/kernel/interpreter.test.ts +116 -0
  46. package/src/kernel/interpreter.ts +148 -0
  47. package/src/kernel/saga.test.ts +89 -0
  48. package/src/kernel/saga.ts +73 -0
  49. package/src/modules/app-release/index.test.ts +125 -0
  50. package/src/modules/app-release/index.ts +209 -0
  51. package/src/modules/auth/auth.local-first.test.ts +141 -0
  52. package/src/modules/auth/events/index.ts +9 -2
  53. package/src/modules/auth/impersonation-banner.ts +443 -0
  54. package/src/modules/auth/impersonation.test.ts +355 -0
  55. package/src/modules/auth/impersonation.ts +179 -0
  56. package/src/modules/auth/index.ts +344 -27
  57. package/src/modules/crdt/crdt-field.ts +294 -0
  58. package/src/modules/crdt/crdt-hydration.test.ts +210 -0
  59. package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
  60. package/src/modules/crdt/index.ts +463 -0
  61. package/src/modules/crdt/loro-loader.ts +25 -0
  62. package/src/modules/devtools/flags.ts +349 -0
  63. package/src/modules/devtools/index.ts +529 -47
  64. package/src/modules/devtools/notify-throttle.test.ts +154 -0
  65. package/src/modules/devtools/state-shape.test.ts +149 -0
  66. package/src/modules/devtools/storage-info.test.ts +79 -0
  67. package/src/modules/devtools/storage-info.ts +168 -0
  68. package/src/modules/devtools/versions.test.ts +74 -0
  69. package/src/modules/devtools/versions.ts +110 -0
  70. package/src/modules/feature-flag/index.test.ts +251 -0
  71. package/src/modules/feature-flag/index.ts +316 -0
  72. package/src/modules/ref-tables.test.ts +91 -0
  73. package/src/modules/ref-tables.ts +88 -0
  74. package/src/mutation/jobs.test.ts +25 -0
  75. package/src/mutation/jobs.ts +41 -0
  76. package/src/mutation/mutation-id.test.ts +37 -0
  77. package/src/mutation/mutation-id.ts +35 -0
  78. package/src/mutation/push.saga.test.ts +285 -0
  79. package/src/mutation/push.saga.ts +215 -0
  80. package/src/mutation/rows.test.ts +201 -0
  81. package/src/mutation/rows.ts +377 -0
  82. package/src/mutation/tray.saga.test.ts +63 -0
  83. package/src/mutation/tray.saga.ts +48 -0
  84. package/src/mutation/write.saga.test.ts +163 -0
  85. package/src/mutation/write.saga.ts +166 -0
  86. package/src/otel/index.ts +13 -10
  87. package/src/query/env.test.ts +23 -0
  88. package/src/query/env.ts +56 -0
  89. package/src/query/fetch.saga.test.ts +156 -0
  90. package/src/query/fetch.saga.ts +115 -0
  91. package/src/query/hash.test.ts +14 -0
  92. package/src/query/hash.ts +18 -0
  93. package/src/query/id-set-plan.test.ts +132 -0
  94. package/src/query/lifecycle.saga.test.ts +122 -0
  95. package/src/query/lifecycle.saga.ts +101 -0
  96. package/src/query/materialize.saga.test.ts +82 -0
  97. package/src/query/materialize.saga.ts +75 -0
  98. package/src/query/materialize.test.ts +50 -0
  99. package/src/query/materialize.ts +44 -0
  100. package/src/query/membership.saga.test.ts +322 -0
  101. package/src/query/membership.saga.ts +262 -0
  102. package/src/query/membership.test.ts +118 -0
  103. package/src/query/membership.ts +171 -0
  104. package/src/query/register.saga.test.ts +293 -0
  105. package/src/query/register.saga.ts +226 -0
  106. package/src/query/render-set.test.ts +36 -0
  107. package/src/query/render-set.ts +60 -0
  108. package/src/query/sql.test.ts +62 -0
  109. package/src/query/sql.ts +79 -0
  110. package/src/query/window-query.test.ts +70 -0
  111. package/src/query/window-query.ts +195 -0
  112. package/src/services/blobs/blob-cache.test.ts +359 -0
  113. package/src/services/blobs/blob-cache.ts +603 -0
  114. package/src/services/blobs/blob-manifest.ts +227 -0
  115. package/src/services/blobs/blob-store.test.ts +77 -0
  116. package/src/services/blobs/blob-store.ts +359 -0
  117. package/src/services/blobs/blob.fixture.ts +90 -0
  118. package/src/services/blobs/index.ts +70 -0
  119. package/src/services/database/cache-engine.ts +199 -0
  120. package/src/services/database/connection-supervisor.test.ts +318 -0
  121. package/src/services/database/connection-supervisor.ts +422 -0
  122. package/src/services/database/database.query-timeout.test.ts +140 -0
  123. package/src/services/database/database.responses.test.ts +43 -0
  124. package/src/services/database/database.ts +145 -62
  125. package/src/services/database/engine-factory.ts +33 -0
  126. package/src/services/database/errors.ts +34 -0
  127. package/src/services/database/events/index.ts +2 -1
  128. package/src/services/database/index.ts +7 -0
  129. package/src/services/database/local-migrator.ts +30 -27
  130. package/src/services/database/local.test.ts +64 -0
  131. package/src/services/database/local.ts +491 -67
  132. package/src/services/database/plan-render.test.ts +159 -0
  133. package/src/services/database/plan-render.ts +108 -0
  134. package/src/services/database/relation-resolver.test.ts +467 -0
  135. package/src/services/database/relation-resolver.ts +0 -0
  136. package/src/services/database/remote.ts +222 -15
  137. package/src/services/database/sqlite-cache-engine.test.ts +617 -0
  138. package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
  139. package/src/services/database/sqlite-cache-engine.ts +1366 -0
  140. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  141. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  142. package/src/services/database/sqlite-lock-verify.test.ts +33 -0
  143. package/src/services/database/sqlite-lock-verify.ts +45 -0
  144. package/src/services/database/sqlite-open.test.ts +150 -0
  145. package/src/services/database/sqlite-open.ts +164 -0
  146. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  147. package/src/services/database/sqlite-plan-sql.ts +153 -0
  148. package/src/services/database/sqlite-projection.test.ts +121 -0
  149. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  150. package/src/services/database/sqlite-select.test.ts +246 -0
  151. package/src/services/database/sqlite-select.ts +131 -0
  152. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  153. package/src/services/database/sqlite-transport.ts +224 -0
  154. package/src/services/database/sqlite-worker.ts +437 -0
  155. package/src/services/database/surql-translate.ts +416 -0
  156. package/src/services/database/surreal-cache-engine.ts +161 -0
  157. package/src/services/logger/index.ts +3 -2
  158. package/src/services/persistence/localstorage.ts +2 -2
  159. package/src/services/persistence/resilient.ts +11 -4
  160. package/src/services/persistence/surrealdb.ts +10 -10
  161. package/src/services/stream-processor/index.ts +796 -84
  162. package/src/services/stream-processor/permissions.test.ts +47 -0
  163. package/src/services/stream-processor/permissions.ts +53 -0
  164. package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
  165. package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
  166. package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
  167. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  168. package/src/services/stream-processor/wasm-types.ts +59 -3
  169. package/src/services/tabs/broker-client.ts +283 -0
  170. package/src/services/tabs/broker.test.ts +327 -0
  171. package/src/services/tabs/coordinator.test.ts +365 -0
  172. package/src/services/tabs/coordinator.ts +633 -0
  173. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  174. package/src/services/tabs/leader-locks.ts +75 -0
  175. package/src/services/tabs/protocol.ts +262 -0
  176. package/src/services/tabs/support.ts +36 -0
  177. package/src/services/tabs/tabs-broker-worker.ts +640 -0
  178. package/src/sp00ky.ts +5 -0
  179. package/src/state/client-state.ts +191 -0
  180. package/src/state/lifecycle.test.ts +118 -0
  181. package/src/state/lifecycle.ts +86 -0
  182. package/src/state/reducers.test.ts +283 -0
  183. package/src/state/reducers.ts +488 -0
  184. package/src/state/selectors.test.ts +149 -0
  185. package/src/state/selectors.ts +184 -0
  186. package/src/sync/connection.saga.test.ts +86 -0
  187. package/src/sync/connection.saga.ts +85 -0
  188. package/src/sync/live.saga.test.ts +178 -0
  189. package/src/sync/live.saga.ts +107 -0
  190. package/src/sync/policy.health.test.ts +56 -0
  191. package/src/sync/policy.test.ts +659 -0
  192. package/src/sync/policy.ts +502 -0
  193. package/src/sync/poll.saga.test.ts +51 -0
  194. package/src/sync/poll.saga.ts +36 -0
  195. package/src/sync/tabs.saga.test.ts +74 -0
  196. package/src/sync/tabs.saga.ts +93 -0
  197. package/src/testing/adapters.ts +133 -0
  198. package/src/testing/build.ts +63 -0
  199. package/src/testing/fake-services.ts +34 -0
  200. package/src/testing/run-pure.test.ts +92 -0
  201. package/src/testing/run-pure.ts +128 -0
  202. package/src/testing/services.ts +17 -0
  203. package/src/types.ts +603 -14
  204. package/src/utils/blurhash.ts +90 -0
  205. package/src/utils/error-classification.test.ts +54 -0
  206. package/src/utils/error-classification.ts +25 -0
  207. package/src/utils/index.ts +79 -13
  208. package/src/utils/parser.test.ts +49 -120
  209. package/src/utils/parser.ts +32 -2
  210. package/src/utils/semver.test.ts +32 -0
  211. package/src/utils/semver.ts +30 -0
  212. package/src/utils/surql.ts +30 -18
  213. package/src/utils/utils.test.ts +2 -1
  214. package/src/utils/withRetry.test.ts +1 -1
  215. package/tsdown.config.ts +86 -1
  216. package/vitest.config.ts +48 -0
  217. package/src/modules/cache/index.ts +0 -241
  218. package/src/modules/cache/types.ts +0 -19
  219. package/src/modules/data/data.test.ts +0 -58
  220. package/src/modules/data/index.ts +0 -879
  221. package/src/modules/sync/engine.ts +0 -173
  222. package/src/modules/sync/events/index.ts +0 -77
  223. package/src/modules/sync/index.ts +0 -3
  224. package/src/modules/sync/queue/index.ts +0 -2
  225. package/src/modules/sync/queue/queue-down.ts +0 -89
  226. package/src/modules/sync/queue/queue-up.ts +0 -223
  227. package/src/modules/sync/scheduler.ts +0 -84
  228. package/src/modules/sync/sync.ts +0 -421
  229. package/src/modules/sync/utils.test.ts +0 -311
  230. package/src/modules/sync/utils.ts +0 -167
  231. package/src/spooky.ts +0 -395
@@ -0,0 +1,226 @@
1
+ import type { QueryPlan } from '@spooky-sync/query-builder';
2
+ import { RecordId } from 'surrealdb';
3
+ import type { QueryHash, QueryTimeToLive } from '../types';
4
+ import type { Saga } from '../kernel/saga';
5
+ import type { RegisterResult, StatementResult } from '../kernel/effects';
6
+ import { fx } from '../kernel/effects';
7
+ import { AUTH_READY_MAX_ATTEMPTS, AUTH_READY_RETRY_MS, REGISTER_MAX_RETRIES, backoffMs } from '../kernel/constants';
8
+ import type { ClientState, QueryEntry } from '../state/client-state';
9
+ import { emptyTelemetry } from '../state/client-state';
10
+ import { seedLifecycle } from '../state/lifecycle';
11
+ import * as R from '../state/reducers';
12
+ import { desiredRegistrations } from '../state/selectors';
13
+ import { parseDuration } from '../utils/index';
14
+ import type { SagaEnv } from './env';
15
+ import { listRefTable } from './env';
16
+ import { queryHashInput, viewKeyInput } from './hash';
17
+ import { FAILED, isResolvedBefore, parseViewRow, snapshotFromSingle } from './membership';
18
+ import { applyMembership, applySubqueryChildren, markMembershipDirty } from './membership.saga';
19
+ import * as sql from './sql';
20
+
21
+ export interface RegisterInput {
22
+ tableName: string;
23
+ surql: string;
24
+ params: Record<string, unknown>;
25
+ ttl: QueryTimeToLive;
26
+ plan?: QueryPlan;
27
+ }
28
+
29
+ /**
30
+ * Register a query locally: compute its keys, read the durable `_00_view`
31
+ * row, build the SSP local view and publish the entry. Returns as soon as the
32
+ * entry exists (the first paint comes from `materialize`); the remote
33
+ * registration is dispatched, never awaited.
34
+ */
35
+ export function* registerLocal(env: SagaEnv, input: RegisterInput): Saga<QueryHash> {
36
+ const sessionId = (yield fx.state.read((s) => s.sessionId)) as string | null;
37
+ const hash = (yield fx.hash(queryHashInput(input, sessionId))) as string;
38
+ const status = (yield fx.state.read((s) => (s.queries.has(hash) ? 'active' : s.registering.has(hash) ? 'pending' : 'new'))) as
39
+ | 'active'
40
+ | 'pending'
41
+ | 'new';
42
+ if (status === 'active') return hash;
43
+ if (status === 'pending') {
44
+ yield fx.state.wait((s) => !s.registering.has(hash));
45
+ return hash;
46
+ }
47
+ yield fx.state.update(R.beginRegistering(hash));
48
+ try {
49
+ const viewKey = (yield fx.hash(viewKeyInput(input))) as string;
50
+ let viewRow: unknown = null;
51
+ try {
52
+ viewRow = yield fx.local.getById(sql.VIEW_TABLE, sql.viewRecordId(viewKey));
53
+ } catch {
54
+ viewRow = null;
55
+ }
56
+ const view = parseViewRow(viewRow);
57
+ const now = (yield fx.now()) as number;
58
+ const reg = (yield fx.ssp.register({
59
+ queryHash: hash,
60
+ surql: input.surql,
61
+ params: input.params,
62
+ ttl: input.ttl,
63
+ tableName: input.tableName,
64
+ })) as RegisterResult;
65
+ const entry: QueryEntry = {
66
+ def: {
67
+ id: new RecordId('_00_query', hash),
68
+ hash,
69
+ viewKey,
70
+ surql: input.surql,
71
+ params: input.params,
72
+ plan: input.plan,
73
+ ttl: input.ttl,
74
+ ttlMs: parseDuration(input.ttl),
75
+ tableName: input.tableName,
76
+ createdAt: now,
77
+ },
78
+ lifecycle: seedLifecycle(isResolvedBefore(view)),
79
+ remoteArray: view?.ids ?? [],
80
+ localArray: reg.localArray,
81
+ subqueryRemoteArray: [],
82
+ records: [],
83
+ serverState: null,
84
+ subscribers: 0,
85
+ lastSubscriberLeftAt: now,
86
+ lastHeartbeatAt: null,
87
+ lastPolledAt: null,
88
+ registerAttempts: 0,
89
+ viewLostCount: 0,
90
+ viewLostRetryAt: null,
91
+ telemetry: { ...emptyTelemetry(), registrationTimings: reg.timings },
92
+ };
93
+ yield fx.state.update(R.putQuery(entry));
94
+ yield fx.dispatch({ type: 'EnsureRegistered' });
95
+ return hash;
96
+ } finally {
97
+ yield fx.state.update(R.endRegistering(hash));
98
+ }
99
+ }
100
+
101
+ /**
102
+ * Make the remote match the desired set: start a registration for every
103
+ * query that has none. Idempotent. With `requireAuth` (after a reconnect or
104
+ * while degraded) it first waits for `$auth.id` to be visible on the socket:
105
+ * a register issued before that stamps the view with an empty identity.
106
+ */
107
+ export function* ensureRegistered(env: SagaEnv, opts: { requireAuth?: boolean; attempt?: number } = {}): Saga<void> {
108
+ const [hashes, userId] = (yield fx.state.read((s) => [desiredRegistrations(s), s.userId])) as [QueryHash[], string | null];
109
+ if (hashes.length === 0) return;
110
+ if (opts.requireAuth && userId !== null) {
111
+ let authed = false;
112
+ try {
113
+ const res = (yield fx.remote.query('RETURN $auth.id', undefined, env.remoteTimeoutMs)) as StatementResult[];
114
+ authed = res[0]?.status === 'OK' && res[0].result != null;
115
+ } catch {
116
+ authed = false;
117
+ }
118
+ if (!authed) {
119
+ const attempt = (opts.attempt ?? 0) + 1;
120
+ if (attempt >= AUTH_READY_MAX_ATTEMPTS) {
121
+ yield fx.emit({ type: 'log', level: 'warn', message: 'auth identity never became visible; not registering', data: { attempt } });
122
+ return;
123
+ }
124
+ yield fx.timer.set('ensure-registered', AUTH_READY_RETRY_MS, { type: 'EnsureRegistered', requireAuth: true, attempt });
125
+ return;
126
+ }
127
+ }
128
+ for (const hash of hashes) yield fx.dispatch({ type: 'RegisterRemote', hash });
129
+ }
130
+
131
+ const stmt = (results: StatementResult[], i: number): StatementResult | undefined => results[i];
132
+
133
+ /**
134
+ * The SSP answers a registration whose shape is not on the query allowlist
135
+ * with HTTP 403 `{"error":"not_allowlisted"}`. It reaches the client as the
136
+ * `fn::query::register` statement error carrying SurrealDB's http error text
137
+ * (status reason and/or body), and no retry can change the answer.
138
+ */
139
+ export const isNotAllowlisted = (error: unknown): boolean =>
140
+ /not_allowlisted|\b403\b|forbidden/i.test(String((error as { message?: unknown })?.message ?? error));
141
+
142
+ /**
143
+ * One remote registration: `fn::query::register` plus the edge/meta/children
144
+ * read in ONE request, then membership application. Concurrent with every
145
+ * other registration; retried on its own backoff; stops when the entry is
146
+ * gone.
147
+ */
148
+ export function* registerRemote(env: SagaEnv, hash: QueryHash, retry = false): Saga<void> {
149
+ const state = (yield fx.state.read((s) => s)) as ClientState;
150
+ const entry = state.queries.get(hash);
151
+ if (!entry) return;
152
+ const { remote } = entry.lifecycle;
153
+ if (remote === 'registered' || remote === 'failed' || (remote === 'registering' && !retry)) return;
154
+ yield fx.state.update(
155
+ R.compose(R.applyLifecycle(hash, { type: 'remote-registering' }), R.applyLifecycle(hash, { type: 'fetch-begin' }))
156
+ );
157
+ try {
158
+ const table = listRefTable(env, state);
159
+ const results = (yield fx.remote.query(
160
+ sql.registerSelect(table),
161
+ sql.registerVars({ id: entry.def.id, surql: entry.def.surql, params: { ...entry.def.params }, ttl: entry.def.ttl }),
162
+ env.remoteTimeoutMs
163
+ )) as StatementResult[];
164
+ const stillThere = (yield fx.state.read((s) => s.queries.has(hash))) as boolean;
165
+ if (!stillThere) return;
166
+ const register = stmt(results, 0);
167
+ if (!register || register.status === 'ERR') throw new Error(register?.status === 'ERR' ? register.error : 'register returned nothing');
168
+ const answer = (r: StatementResult | undefined): unknown => (r?.status === 'OK' ? r.result : FAILED);
169
+ const snap = snapshotFromSingle(
170
+ answer(stmt(results, 1)) as never,
171
+ answer(stmt(results, 2)) as never,
172
+ answer(stmt(results, 3)) as never
173
+ );
174
+ if (!snap) {
175
+ // Registered, but the read-back did not answer. That says nothing about
176
+ // the row, so read membership again rather than guess: a guess of
177
+ // "gone" re-registered the query, and under load the next read-back
178
+ // failed the same way.
179
+ yield fx.state.update(R.compose(R.applyLifecycle(hash, { type: 'remote-registered' }), R.resetRegisterAttempts(hash)));
180
+ yield fx.emit({ type: 'log', level: 'debug', message: 'registration read-back did not answer; re-reading membership', data: { hash } });
181
+ yield* markMembershipDirty([hash]);
182
+ yield fx.dispatch({ type: 'SyncOutcome', ok: false, error: 'registration read-back failed' });
183
+ return;
184
+ }
185
+ const outcome = yield* applyMembership(hash, snap.primary, snap.meta);
186
+ if (outcome === 'ignored' && snap.meta.present && snap.meta.state === 'materializing') {
187
+ yield fx.state.update(R.compose(R.markMembershipDirty([hash]), R.setMembershipReread(hash, 1)));
188
+ yield fx.timer.set('membership', 150, { type: 'ReadDirtyMembership' });
189
+ }
190
+ yield* applySubqueryChildren(hash, snap.subquery);
191
+ yield fx.state.update(R.compose(R.applyLifecycle(hash, { type: 'remote-registered' }), R.resetRegisterAttempts(hash)));
192
+ yield fx.dispatch({ type: 'SyncOutcome', ok: true });
193
+ } catch (error) {
194
+ const current = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
195
+ if (!current) return;
196
+ if (isNotAllowlisted(error)) {
197
+ yield fx.state.update(R.applyLifecycle(hash, { type: 'remote-failed' }));
198
+ yield fx.emit({
199
+ type: 'log',
200
+ level: 'error',
201
+ message: 'query refused: not allowlisted',
202
+ data: {
203
+ hash,
204
+ table: entry.def.tableName,
205
+ surql: entry.def.surql,
206
+ params: Object.keys(entry.def.params),
207
+ hint: 'Add or regenerate the query allowlist: spky generate, then spky deploy (or restart spky dev).',
208
+ },
209
+ });
210
+ yield fx.dispatch({ type: 'SyncOutcome', ok: false, error });
211
+ return;
212
+ }
213
+ const attempts = current.registerAttempts + 1;
214
+ yield fx.state.update(R.bumpRegisterAttempts(hash));
215
+ yield fx.emit({ type: 'log', level: 'warn', message: 'remote registration failed', data: { hash, attempts, error } });
216
+ yield fx.dispatch({ type: 'SyncOutcome', ok: false, error });
217
+ if (attempts >= REGISTER_MAX_RETRIES) {
218
+ yield fx.state.update(R.applyLifecycle(hash, { type: 'remote-failed' }));
219
+ } else {
220
+ yield fx.timer.set(`register:${hash}`, backoffMs(attempts), { type: 'RegisterRemote', hash, retry: true });
221
+ }
222
+ } finally {
223
+ const stillThere = (yield fx.state.read((s) => s.queries.has(hash))) as boolean;
224
+ if (stillThere) yield fx.state.update(R.applyLifecycle(hash, { type: 'fetch-end' }));
225
+ }
226
+ }
@@ -0,0 +1,36 @@
1
+ import { describe, expect, it } from 'vitest';
2
+ import { buildRenderIds, resolveMembership } from './render-set';
3
+
4
+ const ov = (writes: string[] = [], deletes: string[] = []) => ({ writes: new Set(writes), deletes: new Set(deletes) });
5
+
6
+ describe('resolveMembership', () => {
7
+ it('cold scans, cold window uses the local view, anything else uses membership', () => {
8
+ const remoteArray: [string, number][] = [['t:1', 1]];
9
+ const localArray: [string, number][] = [['t:2', 1]];
10
+ expect(resolveMembership({ phase: 'cold', remoteArray, localArray, isWindow: false })).toBeNull();
11
+ expect(resolveMembership({ phase: 'cold', remoteArray, localArray, isWindow: true })).toBe(localArray);
12
+ expect(resolveMembership({ phase: 'cached', remoteArray, localArray, isWindow: false })).toBe(remoteArray);
13
+ expect(resolveMembership({ phase: 'view-lost', remoteArray, localArray, isWindow: true })).toBe(remoteArray);
14
+ });
15
+ });
16
+
17
+ describe('buildRenderIds', () => {
18
+ const opts = { hasExplicitOrder: false, isWindow: false };
19
+ it('membership minus deletes, sorted', () => {
20
+ expect(buildRenderIds([['t:b', 1], ['t:a', 1], ['t:c', 1]], [], ov([], ['t:c']), opts)).toEqual(['t:a', 't:b']);
21
+ });
22
+ it('adds pending writes the local view admits, never twice, never deleted', () => {
23
+ const ids = buildRenderIds(
24
+ [['t:1', 1]],
25
+ [['t:1', 1], ['t:2', 1], ['t:3', 1], ['t:4', 1]],
26
+ ov(['t:1', 't:2', 't:3', 't:9'], ['t:3']),
27
+ opts
28
+ );
29
+ expect(ids).toEqual(['t:1', 't:2']);
30
+ });
31
+ it('keeps server order for windows and explicit orderBy; dedupes membership', () => {
32
+ const m: [string, number][] = [['t:b', 1], ['t:a', 1], ['t:b', 1]];
33
+ expect(buildRenderIds(m, [], ov(), { hasExplicitOrder: true, isWindow: false })).toEqual(['t:b', 't:a']);
34
+ expect(buildRenderIds(m, [], ov(), { hasExplicitOrder: false, isWindow: true })).toEqual(['t:b', 't:a']);
35
+ });
36
+ });
@@ -0,0 +1,60 @@
1
+ import type { RecordVersionArray } from '../types';
2
+ import type { QueryPhase } from '../state/lifecycle';
3
+ import type { Overlay } from '../state/selectors';
4
+
5
+ export interface RenderInput {
6
+ phase: QueryPhase;
7
+ remoteArray: RecordVersionArray;
8
+ localArray: RecordVersionArray;
9
+ isWindow: boolean;
10
+ }
11
+
12
+ /**
13
+ * The id-set a query renders from, or `null` when it must fall back to a
14
+ * local predicate scan (a cold query that was never resolved on this device).
15
+ * A cold window has no usable scan (re-applying `START m` against the shared
16
+ * store returns the wrong rows), so it renders the SSP's local window instead.
17
+ */
18
+ export function resolveMembership(input: RenderInput): RecordVersionArray | null {
19
+ if (input.phase !== 'cold') return input.remoteArray;
20
+ if (input.isWindow) return input.localArray;
21
+ return null;
22
+ }
23
+
24
+ export interface RenderOptions {
25
+ hasExplicitOrder: boolean;
26
+ isWindow: boolean;
27
+ }
28
+
29
+ /**
30
+ * render = (membership ∪ (writes ∩ localView)) − deletes
31
+ *
32
+ * `writes`/`deletes` are the outbox overlay (pending and acked items). The
33
+ * middle term keeps optimistic writes visible without re-admitting stale rows:
34
+ * `localView` (the SSP's local id-set) says whether the written row matches
35
+ * the query per local truth. Sorted unless the query orders itself or is a
36
+ * window (whose id order is the slice order).
37
+ */
38
+ export function buildRenderIds(
39
+ membership: RecordVersionArray,
40
+ localView: RecordVersionArray,
41
+ overlay: Overlay,
42
+ opts: RenderOptions
43
+ ): string[] {
44
+ const ordered: string[] = [];
45
+ const seen = new Set<string>();
46
+ for (const [id] of membership) {
47
+ if (overlay.deletes.has(id) || seen.has(id)) continue;
48
+ seen.add(id);
49
+ ordered.push(id);
50
+ }
51
+ if (overlay.writes.size > 0) {
52
+ for (const [id] of localView) {
53
+ if (!overlay.writes.has(id) || overlay.deletes.has(id) || seen.has(id)) continue;
54
+ seen.add(id);
55
+ ordered.push(id);
56
+ }
57
+ }
58
+ if (!opts.hasExplicitOrder && !opts.isWindow) ordered.sort();
59
+ return ordered;
60
+ }
@@ -0,0 +1,62 @@
1
+ import { describe, expect, it } from 'vitest';
2
+ import { RecordId } from 'surrealdb';
3
+ import * as sql from './sql';
4
+ import {
5
+ buildListRefBatchSelect,
6
+ buildListRefSelect,
7
+ buildQueryRowCountBatchSelect,
8
+ buildQueryRowCountSelect,
9
+ buildSubqueryListRefSelect,
10
+ } from '../sync/policy';
11
+
12
+ describe('query sql builders (golden against the previous builders)', () => {
13
+ it('edge and meta selects are byte-identical to the old helpers', () => {
14
+ expect(sql.listRefSelect('_00_list_ref_user_x')).toBe(buildListRefSelect('_00_list_ref_user_x'));
15
+ expect(sql.subqueryListRefSelect('t')).toBe(buildSubqueryListRefSelect('t'));
16
+ expect(sql.queryRowCountSelect()).toBe(buildQueryRowCountSelect());
17
+ expect(sql.listRefBatchSelect('t')).toBe(buildListRefBatchSelect('t'));
18
+ expect(sql.queryRowCountBatchSelect()).toBe(buildQueryRowCountBatchSelect());
19
+ });
20
+ it('composes the single, batch and register requests', () => {
21
+ expect(sql.singleSnapshotSelect('t').split(';\n')).toEqual([
22
+ sql.listRefSelect('t'),
23
+ sql.queryRowCountSelect(),
24
+ sql.subqueryListRefSelect('t'),
25
+ ]);
26
+ expect(sql.batchSnapshotSelect('t')).toBe(`${sql.listRefBatchSelect('t')};\n${sql.queryRowCountBatchSelect()}`);
27
+ expect(sql.registerSelect('t').startsWith('fn::query::register($config);\n')).toBe(true);
28
+ const id = new RecordId('_00_query', 'h');
29
+ expect(sql.registerVars({ id, surql: 's', params: { a: 1 }, ttl: '10m' })).toEqual({
30
+ config: { id, surql: 's', params: { a: 1 }, ttl: '10m' },
31
+ in: id,
32
+ });
33
+ });
34
+ it('view rows and ids', () => {
35
+ expect(sql.viewRecordId('k')).toEqual(new RecordId('_00_view', 'k'));
36
+ expect(sql.viewRow([['t:1', 1]], true, 5)).toEqual({ ids: [['t:1', 1]], confirmed: true, updatedAt: 5 });
37
+ expect(sql.readLegacyViewRows()).toBe('SELECT * FROM _00_window');
38
+ expect(sql.countViewRows()).toContain('_00_view');
39
+ expect(sql.bodySelect()).toBe('SELECT * FROM $ids');
40
+ });
41
+ it('heartbeat batch answers per index and detects reclaimed rows', () => {
42
+ const a = new RecordId('_00_query', 'a');
43
+ const b = new RecordId('_00_query', 'b');
44
+ expect(sql.heartbeatBatch([a, b])).toEqual({
45
+ sql: 'fn::query::heartbeat($id0);\nfn::query::heartbeat($id1)',
46
+ vars: { id0: a, id1: b },
47
+ });
48
+ expect(sql.heartbeatRowGone([])).toBe(true);
49
+ expect(sql.heartbeatRowGone([{ id: a }])).toBe(false);
50
+ expect(sql.heartbeatRowGone(undefined)).toBe(false);
51
+ });
52
+ it('upsertBodiesTx MERGEs each body inside one transaction', () => {
53
+ const { query, vars } = sql.upsertBodiesTx([
54
+ { id: 'thing:1', content: { a: 1 } },
55
+ { id: 'thing:2', content: { b: 2 } },
56
+ ]);
57
+ expect(query.sql).toBe(
58
+ 'BEGIN TRANSACTION;\nUPSERT ONLY $id0 MERGE $content0;UPSERT ONLY $id1 MERGE $content1;\nCOMMIT TRANSACTION;'
59
+ );
60
+ expect(vars).toEqual({ id0: 'thing:1', content0: { a: 1 }, id1: 'thing:2', content1: { b: 2 } });
61
+ });
62
+ });
@@ -0,0 +1,79 @@
1
+ import { RecordId } from 'surrealdb';
2
+ import type { RecordVersionArray } from '../types';
3
+ import type { Vars } from '../kernel/effects';
4
+ import { surql } from '../utils/surql';
5
+ import type { SealedQuery } from '../utils/surql';
6
+
7
+ /** Every SurrealQL statement the query side sends, as pure builders. */
8
+
9
+ export const VIEW_TABLE = '_00_view';
10
+ export const LEGACY_VIEW_TABLE = '_00_window';
11
+
12
+ export const viewRecordId = (viewKey: string): RecordId<string> => new RecordId(VIEW_TABLE, viewKey);
13
+
14
+ export const viewRow = (ids: RecordVersionArray, confirmed: boolean, now: number): Record<string, unknown> => ({
15
+ ids,
16
+ confirmed,
17
+ updatedAt: now,
18
+ });
19
+
20
+ /** Copy every legacy `_00_window` row into `_00_view` once. */
21
+ export const readLegacyViewRows = (): string => `SELECT * FROM ${LEGACY_VIEW_TABLE}`;
22
+ export const countViewRows = (): string => `SELECT count() FROM ${VIEW_TABLE} GROUP ALL`;
23
+
24
+ export const listRefSelect = (table: string): string =>
25
+ `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NONE`;
26
+ export const subqueryListRefSelect = (table: string): string =>
27
+ `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NOT NONE`;
28
+ export const queryRowCountSelect = (): string => 'SELECT VALUE { rowCount: rowCount, state: state } FROM ONLY $in';
29
+ export const listRefBatchSelect = (table: string): string =>
30
+ `SELECT in, out, version, parent FROM ${table} WHERE in IN $ins`;
31
+ export const queryRowCountBatchSelect = (): string => 'SELECT VALUE { id: id, rowCount: rowCount, state: state } FROM $ins';
32
+
33
+ /** The single-query read: edges, meta, subquery children in one request. */
34
+ export const singleSnapshotSelect = (table: string): string =>
35
+ `${listRefSelect(table)};\n${queryRowCountSelect()};\n${subqueryListRefSelect(table)}`;
36
+ /** The many-query read: all edges + all metas in one request. */
37
+ export const batchSnapshotSelect = (table: string): string =>
38
+ `${listRefBatchSelect(table)};\n${queryRowCountBatchSelect()}`;
39
+
40
+ export interface RegisterPayload {
41
+ id: RecordId<string>;
42
+ surql: string;
43
+ params: Record<string, unknown>;
44
+ ttl: string;
45
+ }
46
+
47
+ /** Register + read back edges/meta/children in ONE request. */
48
+ export const registerSelect = (table: string): string =>
49
+ `fn::query::register($config);\n${singleSnapshotSelect(table)}`;
50
+ export const registerVars = (payload: RegisterPayload): Vars => ({ config: payload, in: payload.id });
51
+
52
+ /** One heartbeat statement per view id; result index i answers id i. */
53
+ export function heartbeatBatch(ids: ReadonlyArray<RecordId<string>>): { sql: string; vars: Vars } {
54
+ const vars: Vars = {};
55
+ const stmts = ids.map((id, i) => {
56
+ vars[`id${i}`] = id;
57
+ return `fn::query::heartbeat($id${i})`;
58
+ });
59
+ return { sql: stmts.join(';\n'), vars };
60
+ }
61
+
62
+ /** A heartbeat answer whose statement matched no row: the view was reclaimed. */
63
+ export const heartbeatRowGone = (result: unknown): boolean => Array.isArray(result) && result.length === 0;
64
+
65
+ export const bodySelect = (): string => 'SELECT * FROM $ids';
66
+
67
+ /** Local transaction that MERGEs many fetched bodies. */
68
+ export function upsertBodiesTx(records: ReadonlyArray<{ id: unknown; content: Record<string, unknown> }>): {
69
+ query: SealedQuery<unknown>;
70
+ vars: Vars;
71
+ } {
72
+ const vars: Vars = {};
73
+ const stmts = records.map((r, i) => {
74
+ vars[`id${i}`] = r.id;
75
+ vars[`content${i}`] = r.content;
76
+ return surql.upsertMerge(`id${i}`, `content${i}`);
77
+ });
78
+ return { query: surql.seal<void>(surql.tx(stmts)) as SealedQuery<unknown>, vars };
79
+ }
@@ -0,0 +1,70 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import { buildWindowMaterialization } from './window-query';
3
+
4
+ describe('scanner edge cases', () => {
5
+ it('skips escaped quotes inside strings and ignores clauses in parens', () => {
6
+ const q = "SELECT * FROM t WHERE a = 'it\\'s (FROM)' AND b IN (SELECT id FROM u LIMIT 1 START 1) LIMIT 5 START 5";
7
+ expect(buildWindowMaterialization(q)).toEqual({ query: 'SELECT * FROM $__win' });
8
+ });
9
+ it('requires whitespace inside ORDER BY and a number after START', () => {
10
+ expect(buildWindowMaterialization('SELECT * FROM t ORDERBY a LIMIT 5 START 5')).toEqual({ query: 'SELECT * FROM $__win' });
11
+ expect(buildWindowMaterialization('SELECT * FROM t LIMIT 5 START $n')).toBeNull();
12
+ expect(buildWindowMaterialization('SELECT * FROM t LIMIT 5 START')).toBeNull();
13
+ });
14
+ it('matches a keyword at the very start and at the end of the string', () => {
15
+ expect(buildWindowMaterialization('FROM t LIMIT 5 START 5')).toEqual({ query: ' FROM $__win' });
16
+ expect(buildWindowMaterialization('SELECT * FROM t ORDER BY a LIMIT 5 START 5 ORDER')).toEqual({
17
+ query: 'SELECT * FROM $__win ORDER BY a',
18
+ });
19
+ });
20
+ });
21
+
22
+ describe('buildWindowMaterialization', () => {
23
+ it('rewrites the game-list window query (START 30) to select the id-set, keeping ORDER BY', () => {
24
+ const surql =
25
+ 'SELECT * FROM game WHERE database = $database ORDER BY sort_index asc, date desc LIMIT 30 START 30;';
26
+ const r = buildWindowMaterialization(surql);
27
+ expect(r).not.toBeNull();
28
+ expect(r!.query).toBe('SELECT * FROM $__win ORDER BY sort_index asc, date desc');
29
+ });
30
+
31
+ it('returns null for START 0 (offset-free windows still work via the normal re-query)', () => {
32
+ const surql =
33
+ 'SELECT * FROM game WHERE database = $database ORDER BY sort_index asc, date desc LIMIT 30 START 0;';
34
+ expect(buildWindowMaterialization(surql)).toBeNull();
35
+ });
36
+
37
+ it('returns null when there is no START clause', () => {
38
+ expect(
39
+ buildWindowMaterialization('SELECT * FROM game WHERE database = $database LIMIT 30;')
40
+ ).toBeNull();
41
+ expect(buildWindowMaterialization('SELECT * FROM game;')).toBeNull();
42
+ });
43
+
44
+ it('preserves a custom projection', () => {
45
+ const surql = 'SELECT id, white, black FROM game ORDER BY date desc LIMIT 30 START 60;';
46
+ expect(buildWindowMaterialization(surql)!.query).toBe(
47
+ 'SELECT id, white, black FROM $__win ORDER BY date desc'
48
+ );
49
+ });
50
+
51
+ it('omits ORDER BY when the query had none', () => {
52
+ const surql = 'SELECT * FROM game LIMIT 30 START 30;';
53
+ expect(buildWindowMaterialization(surql)!.query).toBe('SELECT * FROM $__win');
54
+ });
55
+
56
+ it('ignores FROM/ORDER BY/LIMIT/START inside subqueries (paren-aware)', () => {
57
+ const surql =
58
+ 'SELECT *, (SELECT * FROM comment WHERE game = $parent.id ORDER BY created_at desc LIMIT 5) AS comments ' +
59
+ 'FROM game ORDER BY sort_index asc LIMIT 30 START 90;';
60
+ expect(buildWindowMaterialization(surql)!.query).toBe(
61
+ 'SELECT *, (SELECT * FROM comment WHERE game = $parent.id ORDER BY created_at desc LIMIT 5) AS comments FROM $__win ORDER BY sort_index asc'
62
+ );
63
+ });
64
+
65
+ it('does not treat a START inside a string literal as the offset', () => {
66
+ const surql = "SELECT * FROM game WHERE note = 'LIMIT 30 START 30' ORDER BY date desc LIMIT 30 START 30;";
67
+ const r = buildWindowMaterialization(surql);
68
+ expect(r!.query).toBe('SELECT * FROM $__win ORDER BY date desc');
69
+ });
70
+ });