@spooky-sync/core 0.0.1-canary.13 → 0.0.1-canary.130
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +1171 -372
- package/dist/index.js +5716 -1278
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +107 -0
- package/dist/types.d.ts +746 -0
- package/package.json +39 -8
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +9 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +77 -32
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +288 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +357 -0
- package/src/modules/data/data.hydration.test.ts +150 -0
- package/src/modules/data/data.rebind.test.ts +147 -0
- package/src/modules/data/data.recurring.test.ts +137 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/index.ts +1228 -127
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +154 -0
- package/src/modules/devtools/index.ts +191 -30
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.test.ts +120 -0
- package/src/modules/feature-flag/index.ts +209 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +101 -37
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.ts +12 -5
- package/src/modules/sync/queue/queue-up.ts +29 -14
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.ts +73 -7
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.ts +1017 -62
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/cache-engine.ts +143 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/engine-factory.ts +32 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +6 -0
- package/src/services/database/local-migrator.ts +28 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +452 -66
- package/src/services/database/plan-render.test.ts +133 -0
- package/src/services/database/plan-render.ts +100 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +13 -13
- package/src/services/database/sqlite-cache-engine.test.ts +85 -0
- package/src/services/database/sqlite-cache-engine.ts +699 -0
- package/src/services/database/sqlite-worker.ts +116 -0
- package/src/services/database/surql-translate.ts +291 -0
- package/src/services/database/surreal-cache-engine.ts +122 -0
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +41 -0
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +295 -38
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +104 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +185 -0
- package/src/sp00ky.ts +1016 -0
- package/src/types.ts +258 -15
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +77 -1
- package/src/spooky.ts +0 -392
package/src/types.ts
CHANGED
|
@@ -1,9 +1,15 @@
|
|
|
1
|
-
import { RecordId, SchemaStructure } from '@spooky-sync/query-builder';
|
|
2
|
-
import { Level } from 'pino';
|
|
3
|
-
import { PushEventOptions } from './events/index';
|
|
4
|
-
import { UpEvent } from './modules/sync/index';
|
|
1
|
+
import type { RecordId, SchemaStructure, QueryPlan } from '@spooky-sync/query-builder';
|
|
2
|
+
import type { Level, LoggerOptions } from 'pino';
|
|
3
|
+
import type { PushEventOptions } from './events/index';
|
|
4
|
+
import type { UpEvent } from './modules/sync/index';
|
|
5
|
+
import type { LocalEngineChoice } from './services/database/cache-engine';
|
|
5
6
|
|
|
6
|
-
export type { Level }
|
|
7
|
+
export type { Level };
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* A pino browser transmit object for forwarding logs to an external sink (e.g. OpenTelemetry).
|
|
11
|
+
*/
|
|
12
|
+
export type PinoTransmit = NonNullable<NonNullable<LoggerOptions['browser']>['transmit']>;
|
|
7
13
|
|
|
8
14
|
/**
|
|
9
15
|
* The type of storage backend to use for the local database.
|
|
@@ -62,25 +68,43 @@ export type QueryTimeToLive =
|
|
|
62
68
|
| '12h'
|
|
63
69
|
| '1d';
|
|
64
70
|
|
|
71
|
+
/**
|
|
72
|
+
* Refresh behavior for `preload` when the data is already cached locally (warm).
|
|
73
|
+
* The FIRST load (cold) always fetches + blocks regardless.
|
|
74
|
+
* - `onUse` (default): do nothing when warm — the data freshens on use, when the
|
|
75
|
+
* real `useQuery` mounts and registers its live view. No network on load.
|
|
76
|
+
* - `background`: return instantly, but kick a one-time silent refetch.
|
|
77
|
+
* - `stale`: like `background`, but only if the cached copy is older than
|
|
78
|
+
* `staleTime`.
|
|
79
|
+
*/
|
|
80
|
+
export type PreloadRefresh = 'onUse' | 'background' | 'stale';
|
|
81
|
+
|
|
82
|
+
export interface PreloadOptions {
|
|
83
|
+
/** How to refresh when the query is already cached locally. Default `onUse`. */
|
|
84
|
+
refresh?: PreloadRefresh;
|
|
85
|
+
/** For `refresh: 'stale'` — max age before a warm copy is refetched. Default `1h`. */
|
|
86
|
+
staleTime?: QueryTimeToLive;
|
|
87
|
+
}
|
|
88
|
+
|
|
65
89
|
/**
|
|
66
90
|
* Result object returned when a query is registered or executed.
|
|
67
91
|
*/
|
|
68
|
-
export interface
|
|
92
|
+
export interface Sp00kyQueryResult {
|
|
69
93
|
/** The unique hash identifier for the query. */
|
|
70
94
|
hash: string;
|
|
71
95
|
}
|
|
72
96
|
|
|
73
|
-
export type
|
|
97
|
+
export type Sp00kyQueryResultPromise = Promise<Sp00kyQueryResult>;
|
|
74
98
|
|
|
75
99
|
export interface EventSubscriptionOptions {
|
|
76
100
|
priority?: number;
|
|
77
101
|
}
|
|
78
102
|
|
|
79
103
|
/**
|
|
80
|
-
* Configuration options for the
|
|
104
|
+
* Configuration options for the Sp00ky client.
|
|
81
105
|
* @template S The schema structure type.
|
|
82
106
|
*/
|
|
83
|
-
export interface
|
|
107
|
+
export interface Sp00kyConfig<S extends SchemaStructure> {
|
|
84
108
|
/** Database connection configuration. */
|
|
85
109
|
database: {
|
|
86
110
|
/** The SurrealDB endpoint URL. */
|
|
@@ -94,8 +118,6 @@ export interface SpookyConfig<S extends SchemaStructure> {
|
|
|
94
118
|
/** Authentication token. */
|
|
95
119
|
token?: string;
|
|
96
120
|
};
|
|
97
|
-
/** Unique client identifier. If not provided, one will be generated. */
|
|
98
|
-
clientId?: string;
|
|
99
121
|
/** The schema definition. */
|
|
100
122
|
schema: S;
|
|
101
123
|
/** The compiled SURQL schema string. */
|
|
@@ -107,13 +129,107 @@ export interface SpookyConfig<S extends SchemaStructure> {
|
|
|
107
129
|
* Can be a custom implementation, 'surrealdb' (default), or 'localstorage'.
|
|
108
130
|
*/
|
|
109
131
|
persistenceClient?: PersistenceClient | 'surrealdb' | 'localstorage';
|
|
110
|
-
/** OpenTelemetry collector endpoint for telemetry data. */
|
|
111
|
-
otelEndpoint?: string;
|
|
112
132
|
/**
|
|
113
|
-
*
|
|
114
|
-
*
|
|
133
|
+
* Local cache engine backend. `'surrealdb'` (default) uses the in-browser
|
|
134
|
+
* SurrealDB-WASM store; `'sqlite'` uses official SQLite-WASM in a Worker with
|
|
135
|
+
* OPFS persistence; or pass a custom {@link LocalCacheEngine}. The local cache
|
|
136
|
+
* is a passive queryable store — reactivity is driven by the remote SSP, not
|
|
137
|
+
* this engine. See `services/database/cache-engine.ts`.
|
|
138
|
+
*/
|
|
139
|
+
localEngine?: LocalEngineChoice;
|
|
140
|
+
/** A pino browser transmit object for forwarding logs (e.g. via @spooky-sync/core/otel). */
|
|
141
|
+
otelTransmit?: PinoTransmit;
|
|
142
|
+
/**
|
|
143
|
+
* Debounce time in milliseconds for stream updates (the client-side SSP
|
|
144
|
+
* aggregation throttle — coalesces the in-browser StreamProcessor's
|
|
145
|
+
* per-record updates per query before notifying readers).
|
|
146
|
+
* Defaults to 50ms.
|
|
115
147
|
*/
|
|
116
148
|
streamDebounceTime?: number;
|
|
149
|
+
/**
|
|
150
|
+
* Debounce time in milliseconds for syncing collaborative (CRDT) field
|
|
151
|
+
* changes to the remote database. Local writes happen immediately on
|
|
152
|
+
* every keystroke (so reload/offline works), but the remote UPSERT is
|
|
153
|
+
* coalesced over this window. Lower = snappier remote propagation +
|
|
154
|
+
* more network traffic; higher = less traffic + more lag for other
|
|
155
|
+
* collaborators. Defaults to 500ms.
|
|
156
|
+
*/
|
|
157
|
+
crdtDebounceMs?: number;
|
|
158
|
+
/**
|
|
159
|
+
* Cadence (ms) for the `_00_list_ref` poll that catches cross-session
|
|
160
|
+
* UPDATEs the SurrealDB v3 LIVE-permission gap drops. Lower = faster
|
|
161
|
+
* convergence + more query load; higher = the inverse. Non-positive
|
|
162
|
+
* values fall back to the default (500ms).
|
|
163
|
+
*/
|
|
164
|
+
refSyncIntervalMs?: number;
|
|
165
|
+
/**
|
|
166
|
+
* Instant-hydrate cold queries: when a query is registered with no server
|
|
167
|
+
* result yet, run its surql directly on the remote (one-shot) and display
|
|
168
|
+
* the result as soon as it lands, while the full realtime registration
|
|
169
|
+
* proceeds in the background. The hydrated rows are ingested with their
|
|
170
|
+
* versions so the registration's `syncRecords` skips re-pulling unchanged
|
|
171
|
+
* bodies. The fetch runs OFF the paint path — `useQuery` resolves and paints
|
|
172
|
+
* from the local cache immediately, and hydrate only fills in what's
|
|
173
|
+
* missing. Skipped entirely when the query was `preload()`ed recently (its
|
|
174
|
+
* rows are already local and fresh; the registration re-syncs them), so a
|
|
175
|
+
* preloaded screen costs zero duplicate fetches. Defaults to `true`.
|
|
176
|
+
*/
|
|
177
|
+
instantHydrate?: boolean;
|
|
178
|
+
/**
|
|
179
|
+
* Enable realtime sync while signed out. When `true`, the client starts its
|
|
180
|
+
* `_00_list_ref` poll (and a LIVE subscription) against the shared
|
|
181
|
+
* `_00_list_ref_anon` table even with no authenticated user, so a logged-out
|
|
182
|
+
* page gets live `useQuery` updates over world-readable tables. Requires the
|
|
183
|
+
* server to be deployed with `anonymousLiveQueries: true` in `sp00ky.yml`
|
|
184
|
+
* (this flag must match it). Defaults to `false`: anonymous clients can read
|
|
185
|
+
* one-shot but never sync live.
|
|
186
|
+
*/
|
|
187
|
+
enableAnonymousLiveQueries?: boolean;
|
|
188
|
+
/**
|
|
189
|
+
* Surface sustained sync failures as a "degraded" health status that the app
|
|
190
|
+
* can observe via `subscribeToSyncHealth` (or the client-solid
|
|
191
|
+
* `useSyncStatus` hook) to render a "can't reach the server" banner.
|
|
192
|
+
*
|
|
193
|
+
* Individual failures — a transient remote 500 on query registration, a
|
|
194
|
+
* dropped WebSocket, etc. — are always swallowed and retried; they never
|
|
195
|
+
* throw at the app. This only controls when a *run* of consecutive failures
|
|
196
|
+
* is reported. Status flips back to `healthy` on the next successful sync
|
|
197
|
+
* round. Defaults to `{ degradeAfterConsecutiveFailures: 3 }`; pass `false`
|
|
198
|
+
* (or `degradeAfterConsecutiveFailures: 0`) to never report degraded.
|
|
199
|
+
*/
|
|
200
|
+
syncHealth?: SyncHealthConfig | false;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Tunables for sync-health reporting. See {@link Sp00kyConfig.syncHealth}. */
|
|
204
|
+
export interface SyncHealthConfig {
|
|
205
|
+
/**
|
|
206
|
+
* Number of consecutive failed sync rounds (up or down) before the status
|
|
207
|
+
* flips from `healthy` to `degraded`. A single transient failure is absorbed
|
|
208
|
+
* by the retry; only a sustained run trips the banner. Defaults to `3`. `0`
|
|
209
|
+
* disables degraded reporting entirely.
|
|
210
|
+
*/
|
|
211
|
+
degradeAfterConsecutiveFailures?: number;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
export type SyncHealthStatus = 'healthy' | 'degraded';
|
|
215
|
+
|
|
216
|
+
/** Snapshot of sync health delivered to `subscribeToSyncHealth` subscribers. */
|
|
217
|
+
export interface SyncHealth {
|
|
218
|
+
/** `'degraded'` once consecutive failures cross the configured threshold. */
|
|
219
|
+
status: SyncHealthStatus;
|
|
220
|
+
/** Consecutive failed sync rounds at the moment of this report. */
|
|
221
|
+
consecutiveFailures: number;
|
|
222
|
+
/** Classification of the most recent failure (only set while `degraded`). */
|
|
223
|
+
kind?: 'network' | 'application';
|
|
224
|
+
/** Message of the most recent failure (only set while `degraded`). */
|
|
225
|
+
error?: string;
|
|
226
|
+
/**
|
|
227
|
+
* `true` once at least one sync round has succeeded this session. Lets a UI
|
|
228
|
+
* distinguish a first-time "connecting" phase (never reached the server yet,
|
|
229
|
+
* so a cold-start failure run is expected) from a real lost connection after
|
|
230
|
+
* a working session. Never resets back to `false` once set.
|
|
231
|
+
*/
|
|
232
|
+
everConnected: boolean;
|
|
117
233
|
}
|
|
118
234
|
|
|
119
235
|
export type QueryHash = string;
|
|
@@ -143,12 +259,27 @@ export interface QueryConfig {
|
|
|
143
259
|
id: RecordId<string>;
|
|
144
260
|
/** The SURQL query string. */
|
|
145
261
|
surql: string;
|
|
262
|
+
/**
|
|
263
|
+
* Engine-neutral plan for `surql` (in-memory only; not persisted to
|
|
264
|
+
* `_00_query`). Present when the query came from the query-builder. Non-
|
|
265
|
+
* SurrealQL local engines (SQLite) materialize via `engine.select(plan)`
|
|
266
|
+
* instead of re-running `surql`, which they cannot parse.
|
|
267
|
+
*/
|
|
268
|
+
plan?: QueryPlan;
|
|
146
269
|
/** Parameters used in the query. */
|
|
147
270
|
params: Record<string, any>;
|
|
148
271
|
/** The version array representing the local state of results. */
|
|
149
272
|
localArray: RecordVersionArray;
|
|
150
273
|
/** The version array representing the remote (server) state of results. */
|
|
151
274
|
remoteArray: RecordVersionArray;
|
|
275
|
+
/**
|
|
276
|
+
* In-memory only (never persisted to `_00_query`): version array of the
|
|
277
|
+
* subquery CHILD rows pulled via `parent IS NOT NONE` edges, so the
|
|
278
|
+
* child-body sync is idempotent across polls. Kept separate from
|
|
279
|
+
* `remoteArray` so related child rows never enter the primary window /
|
|
280
|
+
* `rowCount` / `localArray`.
|
|
281
|
+
*/
|
|
282
|
+
subqueryRemoteArray?: RecordVersionArray;
|
|
152
283
|
/** Time-To-Live for this query. */
|
|
153
284
|
ttl: QueryTimeToLive;
|
|
154
285
|
/** Timestamp when the query was last accessed/active. */
|
|
@@ -159,6 +290,18 @@ export interface QueryConfig {
|
|
|
159
290
|
|
|
160
291
|
export type QueryConfigRecord = QueryConfig & { id: string };
|
|
161
292
|
|
|
293
|
+
/**
|
|
294
|
+
* Runtime fetch status of a live query.
|
|
295
|
+
* - `idle`: registered, initial sync completed, and not currently fetching
|
|
296
|
+
* missing records — the materialized rows are authoritative (a windowed
|
|
297
|
+
* query's short result really is the end of the list).
|
|
298
|
+
* - `fetching`: the query is registering (a query is born `fetching` until its
|
|
299
|
+
* initial remote sync completes) or the sync engine is fetching/ingesting
|
|
300
|
+
* missing records for it. Any pending debounced result is flushed BEFORE the
|
|
301
|
+
* flip back to `idle`, so idle status never races ahead of the rows.
|
|
302
|
+
*/
|
|
303
|
+
export type QueryStatus = 'idle' | 'fetching';
|
|
304
|
+
|
|
162
305
|
/**
|
|
163
306
|
* Internal state of a live query.
|
|
164
307
|
*/
|
|
@@ -167,16 +310,103 @@ export interface QueryState {
|
|
|
167
310
|
config: QueryConfig;
|
|
168
311
|
/** The current cached records for this query. */
|
|
169
312
|
records: Record<string, any>[];
|
|
313
|
+
/** Set once `applyHydration` has run for this query, so the cold instant-hydrate
|
|
314
|
+
* path fires at most once per query (see DataModule.isCold/applyHydration). */
|
|
315
|
+
hydrated?: boolean;
|
|
316
|
+
/** Set once `notifyQuerySynced` has emitted for this registration lifetime.
|
|
317
|
+
* Ephemeral (unlike the persisted `updateCount`), so a re-registered query
|
|
318
|
+
* always emits at least once even when its records are unchanged — otherwise
|
|
319
|
+
* an empty re-registered window would never notify and stay "loading". */
|
|
320
|
+
syncNotified?: boolean;
|
|
170
321
|
/** Timer for TTL expiration. */
|
|
171
322
|
ttlTimer: NodeJS.Timeout | null;
|
|
172
323
|
/** TTL duration in milliseconds. */
|
|
173
324
|
ttlDurationMs: number;
|
|
174
325
|
/** Number of times the query has been updated. */
|
|
175
326
|
updateCount: number;
|
|
327
|
+
/** Timestamp (ms) of the last user-visible update, or null before the first
|
|
328
|
+
* one. Surfaced to DevTools as `lastUpdate` — must NOT be stamped on read. */
|
|
329
|
+
lastUpdatedAt: number | null;
|
|
330
|
+
/**
|
|
331
|
+
* Rolling window of the most recent materialization-step latencies (ms).
|
|
332
|
+
* Capped at MATERIALIZATION_SAMPLE_WINDOW; used to recompute p55/p90/p99
|
|
333
|
+
* before each persist to `_00_query`. Samples themselves are not persisted.
|
|
334
|
+
*/
|
|
335
|
+
materializationSamples: number[];
|
|
336
|
+
/** Most recent end-to-end ingest latency in ms, or null until the first ingest. */
|
|
337
|
+
lastIngestLatencyMs: number | null;
|
|
338
|
+
/** Cumulative count of ingest/materialization errors observed for this query. */
|
|
339
|
+
errorCount: number;
|
|
340
|
+
/**
|
|
341
|
+
* Ephemeral runtime fetch status. Not persisted to `_00_query`; observable
|
|
342
|
+
* via DevTools and the `useQuery` hook. `fetching` while the sync engine is
|
|
343
|
+
* pulling missing records for this query, otherwise `idle`.
|
|
344
|
+
*/
|
|
345
|
+
status: QueryStatus;
|
|
346
|
+
/**
|
|
347
|
+
* Rolling per-phase timing samples (ms), in addition to `materializationSamples`
|
|
348
|
+
* (which holds the SSP whole-ingest wall time). Keyed by `TimingPhase` minus
|
|
349
|
+
* `ssp`. Not persisted — surfaced live to DevTools + MCP via `phaseTimings`.
|
|
350
|
+
*/
|
|
351
|
+
phaseSamples: Record<string, number[]>;
|
|
352
|
+
/** Most recent sample (ms) per phase, or null. */
|
|
353
|
+
phaseLast: Record<string, number | null>;
|
|
354
|
+
/** One-shot SSP registration timings (ms). */
|
|
355
|
+
registrationTimings: RegistrationTimings;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/** Cap on the rolling materialization-sample window kept per query in memory. */
|
|
359
|
+
export const MATERIALIZATION_SAMPLE_WINDOW = 100;
|
|
360
|
+
|
|
361
|
+
/** Timed processing phases surfaced per query. `ssp` is the WASM-ingest wall
|
|
362
|
+
* time; the `ssp*` phases are its internal breakdown from the SSP binding. */
|
|
363
|
+
export type TimingPhase =
|
|
364
|
+
| 'ssp'
|
|
365
|
+
| 'sspStoreApply'
|
|
366
|
+
| 'sspCircuitStep'
|
|
367
|
+
| 'sspTransform'
|
|
368
|
+
| 'localFetch'
|
|
369
|
+
| 'remoteFetch'
|
|
370
|
+
| 'frontend';
|
|
371
|
+
|
|
372
|
+
/** One-shot registration timings (ms), captured once when a query registers. */
|
|
373
|
+
export interface RegistrationTimings {
|
|
374
|
+
/** SSP surql→plan parse + permission injection. */
|
|
375
|
+
parseMs: number | null;
|
|
376
|
+
/** SSP operator-DAG build. */
|
|
377
|
+
planMs: number | null;
|
|
378
|
+
/** SSP initial snapshot evaluation. */
|
|
379
|
+
snapshotMs: number | null;
|
|
380
|
+
/** Wall time of `cache.registerQuery` (register_view round-trip). */
|
|
381
|
+
wallMs: number | null;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** Percentile summary for one timed phase, surfaced to DevTools + MCP. */
|
|
385
|
+
export interface PhaseStat {
|
|
386
|
+
lastMs: number | null;
|
|
387
|
+
p50: number | null;
|
|
388
|
+
p90: number | null;
|
|
389
|
+
p99: number | null;
|
|
390
|
+
count: number;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** Per-query processing-time breakdown surfaced via DevTools panel + MCP. */
|
|
394
|
+
export interface QueryTimings {
|
|
395
|
+
ssp: PhaseStat;
|
|
396
|
+
sspStoreApply: PhaseStat;
|
|
397
|
+
sspCircuitStep: PhaseStat;
|
|
398
|
+
sspTransform: PhaseStat;
|
|
399
|
+
localFetch: PhaseStat;
|
|
400
|
+
remoteFetch: PhaseStat;
|
|
401
|
+
frontend: PhaseStat;
|
|
402
|
+
registration: RegistrationTimings;
|
|
403
|
+
updateCount: number;
|
|
404
|
+
errorCount: number;
|
|
176
405
|
}
|
|
177
406
|
|
|
178
407
|
// Callback types
|
|
179
408
|
export type QueryUpdateCallback = (records: Record<string, any>[]) => void;
|
|
409
|
+
export type QueryStatusCallback = (status: QueryStatus) => void;
|
|
180
410
|
export type MutationCallback = (mutations: UpEvent[]) => void;
|
|
181
411
|
|
|
182
412
|
export type MutationEventType = 'create' | 'update' | 'delete';
|
|
@@ -209,6 +439,19 @@ export interface RunOptions {
|
|
|
209
439
|
assignedTo?: string;
|
|
210
440
|
max_retries?: number;
|
|
211
441
|
retry_strategy?: 'linear' | 'exponential';
|
|
442
|
+
/** Timeout in seconds for the backend HTTP call. Only used if the backend allows timeout override. */
|
|
443
|
+
timeout?: number;
|
|
444
|
+
/**
|
|
445
|
+
* Minimum delay in milliseconds before the job is eligible to run. While
|
|
446
|
+
* delayed the job stays pending (enqueued) and can still be killed.
|
|
447
|
+
*/
|
|
448
|
+
delay?: number;
|
|
449
|
+
/**
|
|
450
|
+
* Interval in milliseconds for a RECURRING job (see `runRecurring`). When set,
|
|
451
|
+
* the job re-runs `interval` ms after each run COMPLETES (drift-free from
|
|
452
|
+
* completion, not wall-clock). Ignored by the plain `run`.
|
|
453
|
+
*/
|
|
454
|
+
interval?: number;
|
|
212
455
|
}
|
|
213
456
|
|
|
214
457
|
/**
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { classifySyncError } from './error-classification';
|
|
3
|
+
|
|
4
|
+
describe('classifySyncError', () => {
|
|
5
|
+
it('classifies surreal ConnectionUnavailableError as network', () => {
|
|
6
|
+
// Thrown synchronously by the WS engine's send() while the socket is down
|
|
7
|
+
// but reconnect hasn't fired. Message contains "connected", not "connection".
|
|
8
|
+
const err = new Error(
|
|
9
|
+
'You must be connected to a SurrealDB instance before performing this operation'
|
|
10
|
+
);
|
|
11
|
+
expect(classifySyncError(err)).toBe('network');
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
it('classifies surreal CallTerminatedError as network', () => {
|
|
15
|
+
const err = new Error(
|
|
16
|
+
'The call has been terminated because the connection was closed'
|
|
17
|
+
);
|
|
18
|
+
expect(classifySyncError(err)).toBe('network');
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it.each([
|
|
22
|
+
'WebSocket connection failed',
|
|
23
|
+
'fetch failed',
|
|
24
|
+
'connect ECONNREFUSED 127.0.0.1:8666',
|
|
25
|
+
'request timed out',
|
|
26
|
+
'The operation was aborted',
|
|
27
|
+
'socket hang up',
|
|
28
|
+
])('classifies %j as network', (message) => {
|
|
29
|
+
expect(classifySyncError(new Error(message))).toBe('network');
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it.each([
|
|
33
|
+
'Permission denied',
|
|
34
|
+
'There was a problem with the database: record already exists',
|
|
35
|
+
'Parse error: unexpected token',
|
|
36
|
+
])('classifies %j as application', (message) => {
|
|
37
|
+
expect(classifySyncError(new Error(message))).toBe('application');
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it('handles non-Error values', () => {
|
|
41
|
+
expect(classifySyncError('connection reset')).toBe('network');
|
|
42
|
+
expect(classifySyncError('boom')).toBe('application');
|
|
43
|
+
});
|
|
44
|
+
});
|
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
const NETWORK_ERROR_PATTERNS = [
|
|
2
2
|
'connection',
|
|
3
|
+
// surreal's ConnectionUnavailableError reads "You must be connected to a
|
|
4
|
+
// SurrealDB instance..." — it contains "connected", not "connection", so it
|
|
5
|
+
// slips past the pattern above. The WS engine throws it synchronously from
|
|
6
|
+
// `send()` while the socket is down but reconnect hasn't fired yet, so it's the
|
|
7
|
+
// canonical error on an idle-dropped socket; classify it as network.
|
|
8
|
+
'must be connected',
|
|
9
|
+
'connectionunavailable',
|
|
3
10
|
'timeout',
|
|
4
11
|
'timed out',
|
|
5
12
|
'websocket',
|
package/src/utils/index.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
|
|
1
|
+
import type { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
|
|
2
2
|
import { Uuid, RecordId, Duration } from 'surrealdb';
|
|
3
|
-
import { Logger } from '../services/logger/index';
|
|
4
|
-
import { QueryTimeToLive } from '../types';
|
|
3
|
+
import type { Logger } from '../services/logger/index';
|
|
4
|
+
import type { QueryTimeToLive } from '../types';
|
|
5
5
|
|
|
6
6
|
export * from './surql';
|
|
7
7
|
export * from './parser';
|
|
@@ -59,7 +59,7 @@ export function generateNewTableId<S extends SchemaStructure, T extends TableNam
|
|
|
59
59
|
|
|
60
60
|
// ==================== SCHEMA ENCODING/DECODING ====================
|
|
61
61
|
|
|
62
|
-
export function
|
|
62
|
+
export function decodeFromSp00ky<S extends SchemaStructure, T extends TableNames<S>>(
|
|
63
63
|
schema: S,
|
|
64
64
|
tableName: T,
|
|
65
65
|
record: TableModel<GetTable<S, T>>
|
|
@@ -74,19 +74,19 @@ export function decodeFromSpooky<S extends SchemaStructure, T extends TableNames
|
|
|
74
74
|
for (const field of Object.keys(table.columns)) {
|
|
75
75
|
const column = table.columns[field] as any;
|
|
76
76
|
const relation = schema.relationships.find((r) => r.from === tableName && r.field === field);
|
|
77
|
-
if ((column.recordId || relation) && encoded[field]
|
|
77
|
+
if ((column.recordId || relation) && encoded[field] !== null && encoded[field] !== undefined) {
|
|
78
78
|
if (encoded[field] instanceof RecordId) {
|
|
79
79
|
encoded[field] = `${encoded[field].table.toString()}:${encoded[field].id}`;
|
|
80
80
|
} else if (
|
|
81
81
|
relation &&
|
|
82
|
-
(encoded[field] instanceof Object || encoded[field]
|
|
82
|
+
(encoded[field] instanceof Object || Array.isArray(encoded[field]))
|
|
83
83
|
) {
|
|
84
84
|
if (Array.isArray(encoded[field])) {
|
|
85
85
|
encoded[field] = encoded[field].map((item) =>
|
|
86
|
-
|
|
86
|
+
decodeFromSp00ky(schema, relation.to, item)
|
|
87
87
|
);
|
|
88
88
|
} else {
|
|
89
|
-
encoded[field] =
|
|
89
|
+
encoded[field] = decodeFromSp00ky(schema, relation.to, encoded[field]);
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
92
|
}
|
|
@@ -97,15 +97,25 @@ export function decodeFromSpooky<S extends SchemaStructure, T extends TableNames
|
|
|
97
97
|
|
|
98
98
|
// ==================== TIME/DURATION UTILITIES ====================
|
|
99
99
|
|
|
100
|
+
/**
|
|
101
|
+
* Read the millisecond count off a surrealdb `Duration`. Different `surrealdb`
|
|
102
|
+
* releases expose it under `milliseconds` (current) or the older private
|
|
103
|
+
* `_milliseconds` field, so read whichever is set; returns 0 when neither is.
|
|
104
|
+
*/
|
|
105
|
+
function durationMillis(duration: Duration): number {
|
|
106
|
+
const d = duration as { milliseconds?: number | bigint; _milliseconds?: number | bigint };
|
|
107
|
+
return Number(d.milliseconds || d._milliseconds || 0);
|
|
108
|
+
}
|
|
109
|
+
|
|
100
110
|
/**
|
|
101
111
|
* Parse duration string or Duration object to milliseconds
|
|
102
112
|
*/
|
|
103
113
|
export function parseDuration(duration: QueryTimeToLive | Duration): number {
|
|
104
114
|
if (duration instanceof Duration) {
|
|
105
|
-
const ms = (duration
|
|
106
|
-
if (ms) return
|
|
115
|
+
const ms = durationMillis(duration);
|
|
116
|
+
if (ms) return ms;
|
|
107
117
|
const str = duration.toString();
|
|
108
|
-
if (str !== '[object Object]') return parseDuration(str as
|
|
118
|
+
if (str !== '[object Object]') return parseDuration(str as QueryTimeToLive);
|
|
109
119
|
return 600000;
|
|
110
120
|
}
|
|
111
121
|
|
|
@@ -117,7 +127,7 @@ export function parseDuration(duration: QueryTimeToLive | Duration): number {
|
|
|
117
127
|
|
|
118
128
|
const match = duration.match(/^(\d+)([smh])$/);
|
|
119
129
|
if (!match) return 600000;
|
|
120
|
-
const val = parseInt(match[1], 10);
|
|
130
|
+
const val = Number.parseInt(match[1], 10);
|
|
121
131
|
const unit = match[2];
|
|
122
132
|
switch (unit) {
|
|
123
133
|
case 's':
|
|
@@ -137,6 +147,18 @@ export async function fileToUint8Array(file: File | Blob): Promise<Uint8Array> {
|
|
|
137
147
|
return new Uint8Array(buffer);
|
|
138
148
|
}
|
|
139
149
|
|
|
150
|
+
// ==================== TEXT UTILITIES ====================
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Convert plain text to simple HTML paragraphs.
|
|
154
|
+
* Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
|
|
155
|
+
*/
|
|
156
|
+
export function textToHtml(text: string): string {
|
|
157
|
+
return text
|
|
158
|
+
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
|
159
|
+
.split('\n').map((l) => `<p>${l || '<br>'}</p>`).join('');
|
|
160
|
+
}
|
|
161
|
+
|
|
140
162
|
// ==================== DATABASE UTILITIES ====================
|
|
141
163
|
|
|
142
164
|
/**
|
|
@@ -165,7 +187,7 @@ export async function withRetry<T>(
|
|
|
165
187
|
attempt: i + 1,
|
|
166
188
|
retries,
|
|
167
189
|
error: msg,
|
|
168
|
-
Category: '
|
|
190
|
+
Category: 'sp00ky-client::utils::withRetry',
|
|
169
191
|
},
|
|
170
192
|
'Retrying DB operation'
|
|
171
193
|
);
|
package/src/utils/parser.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { ColumnSchema
|
|
1
|
+
import type { ColumnSchema} from '@spooky-sync/query-builder';
|
|
2
|
+
import { RecordId } from '@spooky-sync/query-builder';
|
|
2
3
|
import { parseRecordIdString } from './index';
|
|
3
4
|
import { DateTime } from 'surrealdb';
|
|
4
5
|
|
|
@@ -8,7 +9,7 @@ export function cleanRecord(
|
|
|
8
9
|
): Record<string, any> {
|
|
9
10
|
const cleaned: Record<string, any> = {};
|
|
10
11
|
for (const [key, value] of Object.entries(record)) {
|
|
11
|
-
if (key === 'id' || key in tableSchema) {
|
|
12
|
+
if (key === 'id' || key.startsWith('_00_') || key in tableSchema) {
|
|
12
13
|
cleaned[key] = value;
|
|
13
14
|
}
|
|
14
15
|
}
|
package/src/utils/surql.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { MutationEventType } from '../types';
|
|
1
|
+
import type { MutationEventType } from '../types';
|
|
2
2
|
|
|
3
3
|
// ==================== TYPES ====================
|
|
4
4
|
|
|
@@ -34,6 +34,7 @@ export interface SurqlHelper {
|
|
|
34
34
|
keyDataVars: ({ key: string; variable: string } | { statement: string } | string)[]
|
|
35
35
|
): string;
|
|
36
36
|
upsert(idVar: string, dataVar: string): string;
|
|
37
|
+
upsertMerge(idVar: string, dataVar: string): string;
|
|
37
38
|
updateMerge(idVar: string, dataVar: string): string;
|
|
38
39
|
updateSet(
|
|
39
40
|
idVar: string,
|
|
@@ -90,16 +91,16 @@ export const surql: SurqlHelper = {
|
|
|
90
91
|
returnValues: ({ field: string; alias: string } | string)[]
|
|
91
92
|
) {
|
|
92
93
|
return `SELECT ${returnValues
|
|
93
|
-
.map((
|
|
94
|
-
typeof
|
|
95
|
-
?
|
|
96
|
-
: `${
|
|
94
|
+
.map((rv) =>
|
|
95
|
+
typeof rv === 'string'
|
|
96
|
+
? rv
|
|
97
|
+
: `${rv.field} as ${rv.alias}`
|
|
97
98
|
)
|
|
98
99
|
.join(',')} FROM ${table} WHERE ${whereVar
|
|
99
|
-
.map((
|
|
100
|
-
typeof
|
|
101
|
-
? `${
|
|
102
|
-
: `${
|
|
100
|
+
.map((wv) =>
|
|
101
|
+
typeof wv === 'string'
|
|
102
|
+
? `${wv} = $${wv}`
|
|
103
|
+
: `${wv.field} = $${wv.variable}`
|
|
103
104
|
)
|
|
104
105
|
.join(' AND ')}`;
|
|
105
106
|
},
|
|
@@ -127,6 +128,14 @@ export const surql: SurqlHelper = {
|
|
|
127
128
|
return `UPSERT ONLY $${idVar} REPLACE $${dataVar}`;
|
|
128
129
|
},
|
|
129
130
|
|
|
131
|
+
// Use this for sync-down ingestion: the remote payload omits local-only
|
|
132
|
+
// fields injected by the CLI's local-schema setup (`_00_crdt`,
|
|
133
|
+
// `_00_cursor`), and REPLACE would wipe them on every round-trip and
|
|
134
|
+
// break offline reload of CRDT state.
|
|
135
|
+
upsertMerge(idVar: string, dataVar: string) {
|
|
136
|
+
return `UPSERT ONLY $${idVar} MERGE $${dataVar}`;
|
|
137
|
+
},
|
|
138
|
+
|
|
130
139
|
updateMerge(idVar: string, dataVar: string) {
|
|
131
140
|
return `UPDATE ONLY $${idVar} MERGE $${dataVar}`;
|
|
132
141
|
},
|
|
@@ -136,12 +145,12 @@ export const surql: SurqlHelper = {
|
|
|
136
145
|
keyDataVar: ({ key: string; variable: string } | { statement: string } | string)[]
|
|
137
146
|
) {
|
|
138
147
|
return `UPDATE $${idVar} SET ${keyDataVar
|
|
139
|
-
.map((
|
|
140
|
-
typeof
|
|
141
|
-
? `${
|
|
142
|
-
: 'statement' in
|
|
143
|
-
?
|
|
144
|
-
: `${
|
|
148
|
+
.map((kdv) =>
|
|
149
|
+
typeof kdv === 'string'
|
|
150
|
+
? `${kdv} = $${kdv}`
|
|
151
|
+
: 'statement' in kdv
|
|
152
|
+
? kdv.statement
|
|
153
|
+
: `${kdv.key} = $${kdv.variable}`
|
|
145
154
|
)
|
|
146
155
|
.join(', ')}`;
|
|
147
156
|
},
|