@spooky-sync/core 0.0.1-canary.15 → 0.0.1-canary.151
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +1192 -372
- package/dist/index.js +5772 -1270
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/sqlite-plan-sql.js +194 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +202 -0
- package/dist/types.d.ts +755 -0
- package/package.json +40 -8
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +16 -2
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +201 -0
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +77 -32
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +294 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +361 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/data/data.hydration.test.ts +142 -0
- package/src/modules/data/data.rebind.test.ts +147 -0
- package/src/modules/data/data.run.test.ts +113 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/index.ts +1134 -128
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +154 -0
- package/src/modules/devtools/index.ts +191 -30
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.test.ts +120 -0
- package/src/modules/feature-flag/index.ts +209 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +101 -37
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.ts +12 -5
- package/src/modules/sync/queue/queue-up.ts +29 -14
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.ts +73 -7
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.ts +1017 -62
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/cache-engine.ts +143 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/engine-factory.ts +32 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +6 -0
- package/src/services/database/local-migrator.ts +28 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +478 -67
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +13 -13
- package/src/services/database/sqlite-cache-engine.test.ts +215 -0
- package/src/services/database/sqlite-cache-engine.ts +829 -0
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +106 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +279 -0
- package/src/services/database/sqlite-select.ts +113 -0
- package/src/services/database/sqlite-worker.ts +160 -0
- package/src/services/database/surql-translate.ts +291 -0
- package/src/services/database/surreal-cache-engine.ts +139 -0
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +11 -4
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +295 -38
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +104 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +183 -0
- package/src/sp00ky.ts +1000 -0
- package/src/types.ts +267 -15
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +77 -1
- package/src/spooky.ts +0 -395
package/src/types.ts
CHANGED
|
@@ -1,9 +1,15 @@
|
|
|
1
|
-
import { RecordId, SchemaStructure } from '@spooky-sync/query-builder';
|
|
2
|
-
import { Level } from 'pino';
|
|
3
|
-
import { PushEventOptions } from './events/index';
|
|
4
|
-
import { UpEvent } from './modules/sync/index';
|
|
1
|
+
import type { RecordId, SchemaStructure, QueryPlan } from '@spooky-sync/query-builder';
|
|
2
|
+
import type { Level, LoggerOptions } from 'pino';
|
|
3
|
+
import type { PushEventOptions } from './events/index';
|
|
4
|
+
import type { UpEvent } from './modules/sync/index';
|
|
5
|
+
import type { LocalEngineChoice } from './services/database/cache-engine';
|
|
5
6
|
|
|
6
|
-
export type { Level }
|
|
7
|
+
export type { Level };
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* A pino browser transmit object for forwarding logs to an external sink (e.g. OpenTelemetry).
|
|
11
|
+
*/
|
|
12
|
+
export type PinoTransmit = NonNullable<NonNullable<LoggerOptions['browser']>['transmit']>;
|
|
7
13
|
|
|
8
14
|
/**
|
|
9
15
|
* The type of storage backend to use for the local database.
|
|
@@ -62,25 +68,43 @@ export type QueryTimeToLive =
|
|
|
62
68
|
| '12h'
|
|
63
69
|
| '1d';
|
|
64
70
|
|
|
71
|
+
/**
|
|
72
|
+
* Refresh behavior for `preload` when the data is already cached locally (warm).
|
|
73
|
+
* The FIRST load (cold) always fetches + blocks regardless.
|
|
74
|
+
* - `onUse` (default): do nothing when warm — the data freshens on use, when the
|
|
75
|
+
* real `useQuery` mounts and registers its live view. No network on load.
|
|
76
|
+
* - `background`: return instantly, but kick a one-time silent refetch.
|
|
77
|
+
* - `stale`: like `background`, but only if the cached copy is older than
|
|
78
|
+
* `staleTime`.
|
|
79
|
+
*/
|
|
80
|
+
export type PreloadRefresh = 'onUse' | 'background' | 'stale';
|
|
81
|
+
|
|
82
|
+
export interface PreloadOptions {
|
|
83
|
+
/** How to refresh when the query is already cached locally. Default `onUse`. */
|
|
84
|
+
refresh?: PreloadRefresh;
|
|
85
|
+
/** For `refresh: 'stale'` — max age before a warm copy is refetched. Default `1h`. */
|
|
86
|
+
staleTime?: QueryTimeToLive;
|
|
87
|
+
}
|
|
88
|
+
|
|
65
89
|
/**
|
|
66
90
|
* Result object returned when a query is registered or executed.
|
|
67
91
|
*/
|
|
68
|
-
export interface
|
|
92
|
+
export interface Sp00kyQueryResult {
|
|
69
93
|
/** The unique hash identifier for the query. */
|
|
70
94
|
hash: string;
|
|
71
95
|
}
|
|
72
96
|
|
|
73
|
-
export type
|
|
97
|
+
export type Sp00kyQueryResultPromise = Promise<Sp00kyQueryResult>;
|
|
74
98
|
|
|
75
99
|
export interface EventSubscriptionOptions {
|
|
76
100
|
priority?: number;
|
|
77
101
|
}
|
|
78
102
|
|
|
79
103
|
/**
|
|
80
|
-
* Configuration options for the
|
|
104
|
+
* Configuration options for the Sp00ky client.
|
|
81
105
|
* @template S The schema structure type.
|
|
82
106
|
*/
|
|
83
|
-
export interface
|
|
107
|
+
export interface Sp00kyConfig<S extends SchemaStructure> {
|
|
84
108
|
/** Database connection configuration. */
|
|
85
109
|
database: {
|
|
86
110
|
/** The SurrealDB endpoint URL. */
|
|
@@ -93,9 +117,14 @@ export interface SpookyConfig<S extends SchemaStructure> {
|
|
|
93
117
|
store?: StoreType;
|
|
94
118
|
/** Authentication token. */
|
|
95
119
|
token?: string;
|
|
120
|
+
/**
|
|
121
|
+
* SQLite engine only: execute `select` plans (base rows + relation tree +
|
|
122
|
+
* row parsing) inside the worker as ONE round-trip instead of one hop per
|
|
123
|
+
* table/relation level. Defaults to true; set false to force the legacy
|
|
124
|
+
* multi-hop path (escape hatch while the worker-side path beds in).
|
|
125
|
+
*/
|
|
126
|
+
workerSelect?: boolean;
|
|
96
127
|
};
|
|
97
|
-
/** Unique client identifier. If not provided, one will be generated. */
|
|
98
|
-
clientId?: string;
|
|
99
128
|
/** The schema definition. */
|
|
100
129
|
schema: S;
|
|
101
130
|
/** The compiled SURQL schema string. */
|
|
@@ -107,13 +136,115 @@ export interface SpookyConfig<S extends SchemaStructure> {
|
|
|
107
136
|
* Can be a custom implementation, 'surrealdb' (default), or 'localstorage'.
|
|
108
137
|
*/
|
|
109
138
|
persistenceClient?: PersistenceClient | 'surrealdb' | 'localstorage';
|
|
110
|
-
/** OpenTelemetry collector endpoint for telemetry data. */
|
|
111
|
-
otelEndpoint?: string;
|
|
112
139
|
/**
|
|
113
|
-
*
|
|
114
|
-
*
|
|
140
|
+
* Local cache engine backend. `'surrealdb'` (default) uses the in-browser
|
|
141
|
+
* SurrealDB-WASM store; `'sqlite'` uses official SQLite-WASM in a Worker with
|
|
142
|
+
* OPFS persistence; or pass a custom {@link LocalCacheEngine}. The local cache
|
|
143
|
+
* is a passive queryable store — reactivity is driven by the remote SSP, not
|
|
144
|
+
* this engine. See `services/database/cache-engine.ts`.
|
|
145
|
+
*/
|
|
146
|
+
localEngine?: LocalEngineChoice;
|
|
147
|
+
/** A pino browser transmit object for forwarding logs (e.g. via @spooky-sync/core/otel). */
|
|
148
|
+
otelTransmit?: PinoTransmit;
|
|
149
|
+
/**
|
|
150
|
+
* Debounce time in milliseconds for stream updates (the client-side SSP
|
|
151
|
+
* aggregation throttle — coalesces the in-browser StreamProcessor's
|
|
152
|
+
* per-record updates per query before notifying readers).
|
|
153
|
+
* Defaults to 50ms.
|
|
115
154
|
*/
|
|
116
155
|
streamDebounceTime?: number;
|
|
156
|
+
/**
|
|
157
|
+
* Debounce time in milliseconds for syncing collaborative (CRDT) field
|
|
158
|
+
* changes to the remote database. Local writes happen immediately on
|
|
159
|
+
* every keystroke (so reload/offline works), but the remote UPSERT is
|
|
160
|
+
* coalesced over this window. Lower = snappier remote propagation +
|
|
161
|
+
* more network traffic; higher = less traffic + more lag for other
|
|
162
|
+
* collaborators. Defaults to 500ms.
|
|
163
|
+
*/
|
|
164
|
+
crdtDebounceMs?: number;
|
|
165
|
+
/**
|
|
166
|
+
* Enable collaborative CRDT fields. When `true`, the `loro-crdt` engine is
|
|
167
|
+
* preloaded at client startup (fetched as a separate chunk on page load) so
|
|
168
|
+
* the first `openCrdtField` is instant. When omitted/`false`, loro is never
|
|
169
|
+
* loaded unless a CRDT field is explicitly opened — keeping the loro chunk
|
|
170
|
+
* out of apps that don't use collaboration. Defaults to `false`.
|
|
171
|
+
*/
|
|
172
|
+
crdt?: boolean;
|
|
173
|
+
/**
|
|
174
|
+
* Cadence (ms) for the `_00_list_ref` poll that catches cross-session
|
|
175
|
+
* UPDATEs the SurrealDB v3 LIVE-permission gap drops. Lower = faster
|
|
176
|
+
* convergence + more query load; higher = the inverse. Non-positive
|
|
177
|
+
* values fall back to the default (500ms).
|
|
178
|
+
*/
|
|
179
|
+
refSyncIntervalMs?: number;
|
|
180
|
+
/**
|
|
181
|
+
* OPT-IN instant-hydrate for cold queries: when enabled and a query is
|
|
182
|
+
* registered with no server result yet, its surql also runs directly on the
|
|
183
|
+
* remote (one-shot, in the background, OFF the paint path) so rows can land
|
|
184
|
+
* before the full register lifecycle completes. Hydrated rows carry their
|
|
185
|
+
* `_00_rv` versions so the registration's `syncRecords` skips re-pulling
|
|
186
|
+
* unchanged bodies. Regardless of this flag, `useQuery` always resolves and
|
|
187
|
+
* paints from the local cache immediately — however the rows got there
|
|
188
|
+
* (preload, prior sync). Default `false`: the register lifecycle
|
|
189
|
+
* (`fn::query::register` → `_00_list_ref` → record sync) is the single
|
|
190
|
+
* freshness path and no duplicate one-shot fetches are made.
|
|
191
|
+
*/
|
|
192
|
+
instantHydrate?: boolean;
|
|
193
|
+
/**
|
|
194
|
+
* Enable realtime sync while signed out. When `true`, the client starts its
|
|
195
|
+
* `_00_list_ref` poll (and a LIVE subscription) against the shared
|
|
196
|
+
* `_00_list_ref_anon` table even with no authenticated user, so a logged-out
|
|
197
|
+
* page gets live `useQuery` updates over world-readable tables. Requires the
|
|
198
|
+
* server to be deployed with `anonymousLiveQueries: true` in `sp00ky.yml`
|
|
199
|
+
* (this flag must match it). Defaults to `false`: anonymous clients can read
|
|
200
|
+
* one-shot but never sync live.
|
|
201
|
+
*/
|
|
202
|
+
enableAnonymousLiveQueries?: boolean;
|
|
203
|
+
/**
|
|
204
|
+
* Surface sustained sync failures as a "degraded" health status that the app
|
|
205
|
+
* can observe via `subscribeToSyncHealth` (or the client-solid
|
|
206
|
+
* `useSyncStatus` hook) to render a "can't reach the server" banner.
|
|
207
|
+
*
|
|
208
|
+
* Individual failures — a transient remote 500 on query registration, a
|
|
209
|
+
* dropped WebSocket, etc. — are always swallowed and retried; they never
|
|
210
|
+
* throw at the app. This only controls when a *run* of consecutive failures
|
|
211
|
+
* is reported. Status flips back to `healthy` on the next successful sync
|
|
212
|
+
* round. Defaults to `{ degradeAfterConsecutiveFailures: 3 }`; pass `false`
|
|
213
|
+
* (or `degradeAfterConsecutiveFailures: 0`) to never report degraded.
|
|
214
|
+
*/
|
|
215
|
+
syncHealth?: SyncHealthConfig | false;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Tunables for sync-health reporting. See {@link Sp00kyConfig.syncHealth}. */
|
|
219
|
+
export interface SyncHealthConfig {
|
|
220
|
+
/**
|
|
221
|
+
* Number of consecutive failed sync rounds (up or down) before the status
|
|
222
|
+
* flips from `healthy` to `degraded`. A single transient failure is absorbed
|
|
223
|
+
* by the retry; only a sustained run trips the banner. Defaults to `3`. `0`
|
|
224
|
+
* disables degraded reporting entirely.
|
|
225
|
+
*/
|
|
226
|
+
degradeAfterConsecutiveFailures?: number;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export type SyncHealthStatus = 'healthy' | 'degraded';
|
|
230
|
+
|
|
231
|
+
/** Snapshot of sync health delivered to `subscribeToSyncHealth` subscribers. */
|
|
232
|
+
export interface SyncHealth {
|
|
233
|
+
/** `'degraded'` once consecutive failures cross the configured threshold. */
|
|
234
|
+
status: SyncHealthStatus;
|
|
235
|
+
/** Consecutive failed sync rounds at the moment of this report. */
|
|
236
|
+
consecutiveFailures: number;
|
|
237
|
+
/** Classification of the most recent failure (only set while `degraded`). */
|
|
238
|
+
kind?: 'network' | 'application';
|
|
239
|
+
/** Message of the most recent failure (only set while `degraded`). */
|
|
240
|
+
error?: string;
|
|
241
|
+
/**
|
|
242
|
+
* `true` once at least one sync round has succeeded this session. Lets a UI
|
|
243
|
+
* distinguish a first-time "connecting" phase (never reached the server yet,
|
|
244
|
+
* so a cold-start failure run is expected) from a real lost connection after
|
|
245
|
+
* a working session. Never resets back to `false` once set.
|
|
246
|
+
*/
|
|
247
|
+
everConnected: boolean;
|
|
117
248
|
}
|
|
118
249
|
|
|
119
250
|
export type QueryHash = string;
|
|
@@ -143,12 +274,27 @@ export interface QueryConfig {
|
|
|
143
274
|
id: RecordId<string>;
|
|
144
275
|
/** The SURQL query string. */
|
|
145
276
|
surql: string;
|
|
277
|
+
/**
|
|
278
|
+
* Engine-neutral plan for `surql` (in-memory only; not persisted to
|
|
279
|
+
* `_00_query`). Present when the query came from the query-builder. Non-
|
|
280
|
+
* SurrealQL local engines (SQLite) materialize via `engine.select(plan)`
|
|
281
|
+
* instead of re-running `surql`, which they cannot parse.
|
|
282
|
+
*/
|
|
283
|
+
plan?: QueryPlan;
|
|
146
284
|
/** Parameters used in the query. */
|
|
147
285
|
params: Record<string, any>;
|
|
148
286
|
/** The version array representing the local state of results. */
|
|
149
287
|
localArray: RecordVersionArray;
|
|
150
288
|
/** The version array representing the remote (server) state of results. */
|
|
151
289
|
remoteArray: RecordVersionArray;
|
|
290
|
+
/**
|
|
291
|
+
* In-memory only (never persisted to `_00_query`): version array of the
|
|
292
|
+
* subquery CHILD rows pulled via `parent IS NOT NONE` edges, so the
|
|
293
|
+
* child-body sync is idempotent across polls. Kept separate from
|
|
294
|
+
* `remoteArray` so related child rows never enter the primary window /
|
|
295
|
+
* `rowCount` / `localArray`.
|
|
296
|
+
*/
|
|
297
|
+
subqueryRemoteArray?: RecordVersionArray;
|
|
152
298
|
/** Time-To-Live for this query. */
|
|
153
299
|
ttl: QueryTimeToLive;
|
|
154
300
|
/** Timestamp when the query was last accessed/active. */
|
|
@@ -159,6 +305,18 @@ export interface QueryConfig {
|
|
|
159
305
|
|
|
160
306
|
export type QueryConfigRecord = QueryConfig & { id: string };
|
|
161
307
|
|
|
308
|
+
/**
|
|
309
|
+
* Runtime fetch status of a live query.
|
|
310
|
+
* - `idle`: registered, initial sync completed, and not currently fetching
|
|
311
|
+
* missing records — the materialized rows are authoritative (a windowed
|
|
312
|
+
* query's short result really is the end of the list).
|
|
313
|
+
* - `fetching`: the query is registering (a query is born `fetching` until its
|
|
314
|
+
* initial remote sync completes) or the sync engine is fetching/ingesting
|
|
315
|
+
* missing records for it. Any pending debounced result is flushed BEFORE the
|
|
316
|
+
* flip back to `idle`, so idle status never races ahead of the rows.
|
|
317
|
+
*/
|
|
318
|
+
export type QueryStatus = 'idle' | 'fetching';
|
|
319
|
+
|
|
162
320
|
/**
|
|
163
321
|
* Internal state of a live query.
|
|
164
322
|
*/
|
|
@@ -167,16 +325,103 @@ export interface QueryState {
|
|
|
167
325
|
config: QueryConfig;
|
|
168
326
|
/** The current cached records for this query. */
|
|
169
327
|
records: Record<string, any>[];
|
|
328
|
+
/** Set once `applyHydration` has run for this query, so the cold instant-hydrate
|
|
329
|
+
* path fires at most once per query (see DataModule.isCold/applyHydration). */
|
|
330
|
+
hydrated?: boolean;
|
|
331
|
+
/** Set once `notifyQuerySynced` has emitted for this registration lifetime.
|
|
332
|
+
* Ephemeral (unlike the persisted `updateCount`), so a re-registered query
|
|
333
|
+
* always emits at least once even when its records are unchanged — otherwise
|
|
334
|
+
* an empty re-registered window would never notify and stay "loading". */
|
|
335
|
+
syncNotified?: boolean;
|
|
170
336
|
/** Timer for TTL expiration. */
|
|
171
337
|
ttlTimer: NodeJS.Timeout | null;
|
|
172
338
|
/** TTL duration in milliseconds. */
|
|
173
339
|
ttlDurationMs: number;
|
|
174
340
|
/** Number of times the query has been updated. */
|
|
175
341
|
updateCount: number;
|
|
342
|
+
/** Timestamp (ms) of the last user-visible update, or null before the first
|
|
343
|
+
* one. Surfaced to DevTools as `lastUpdate` — must NOT be stamped on read. */
|
|
344
|
+
lastUpdatedAt: number | null;
|
|
345
|
+
/**
|
|
346
|
+
* Rolling window of the most recent materialization-step latencies (ms).
|
|
347
|
+
* Capped at MATERIALIZATION_SAMPLE_WINDOW; used to recompute p55/p90/p99
|
|
348
|
+
* before each persist to `_00_query`. Samples themselves are not persisted.
|
|
349
|
+
*/
|
|
350
|
+
materializationSamples: number[];
|
|
351
|
+
/** Most recent end-to-end ingest latency in ms, or null until the first ingest. */
|
|
352
|
+
lastIngestLatencyMs: number | null;
|
|
353
|
+
/** Cumulative count of ingest/materialization errors observed for this query. */
|
|
354
|
+
errorCount: number;
|
|
355
|
+
/**
|
|
356
|
+
* Ephemeral runtime fetch status. Not persisted to `_00_query`; observable
|
|
357
|
+
* via DevTools and the `useQuery` hook. `fetching` while the sync engine is
|
|
358
|
+
* pulling missing records for this query, otherwise `idle`.
|
|
359
|
+
*/
|
|
360
|
+
status: QueryStatus;
|
|
361
|
+
/**
|
|
362
|
+
* Rolling per-phase timing samples (ms), in addition to `materializationSamples`
|
|
363
|
+
* (which holds the SSP whole-ingest wall time). Keyed by `TimingPhase` minus
|
|
364
|
+
* `ssp`. Not persisted — surfaced live to DevTools + MCP via `phaseTimings`.
|
|
365
|
+
*/
|
|
366
|
+
phaseSamples: Record<string, number[]>;
|
|
367
|
+
/** Most recent sample (ms) per phase, or null. */
|
|
368
|
+
phaseLast: Record<string, number | null>;
|
|
369
|
+
/** One-shot SSP registration timings (ms). */
|
|
370
|
+
registrationTimings: RegistrationTimings;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/** Cap on the rolling materialization-sample window kept per query in memory. */
|
|
374
|
+
export const MATERIALIZATION_SAMPLE_WINDOW = 100;
|
|
375
|
+
|
|
376
|
+
/** Timed processing phases surfaced per query. `ssp` is the WASM-ingest wall
|
|
377
|
+
* time; the `ssp*` phases are its internal breakdown from the SSP binding. */
|
|
378
|
+
export type TimingPhase =
|
|
379
|
+
| 'ssp'
|
|
380
|
+
| 'sspStoreApply'
|
|
381
|
+
| 'sspCircuitStep'
|
|
382
|
+
| 'sspTransform'
|
|
383
|
+
| 'localFetch'
|
|
384
|
+
| 'remoteFetch'
|
|
385
|
+
| 'frontend';
|
|
386
|
+
|
|
387
|
+
/** One-shot registration timings (ms), captured once when a query registers. */
|
|
388
|
+
export interface RegistrationTimings {
|
|
389
|
+
/** SSP surql→plan parse + permission injection. */
|
|
390
|
+
parseMs: number | null;
|
|
391
|
+
/** SSP operator-DAG build. */
|
|
392
|
+
planMs: number | null;
|
|
393
|
+
/** SSP initial snapshot evaluation. */
|
|
394
|
+
snapshotMs: number | null;
|
|
395
|
+
/** Wall time of `cache.registerQuery` (register_view round-trip). */
|
|
396
|
+
wallMs: number | null;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/** Percentile summary for one timed phase, surfaced to DevTools + MCP. */
|
|
400
|
+
export interface PhaseStat {
|
|
401
|
+
lastMs: number | null;
|
|
402
|
+
p50: number | null;
|
|
403
|
+
p90: number | null;
|
|
404
|
+
p99: number | null;
|
|
405
|
+
count: number;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** Per-query processing-time breakdown surfaced via DevTools panel + MCP. */
|
|
409
|
+
export interface QueryTimings {
|
|
410
|
+
ssp: PhaseStat;
|
|
411
|
+
sspStoreApply: PhaseStat;
|
|
412
|
+
sspCircuitStep: PhaseStat;
|
|
413
|
+
sspTransform: PhaseStat;
|
|
414
|
+
localFetch: PhaseStat;
|
|
415
|
+
remoteFetch: PhaseStat;
|
|
416
|
+
frontend: PhaseStat;
|
|
417
|
+
registration: RegistrationTimings;
|
|
418
|
+
updateCount: number;
|
|
419
|
+
errorCount: number;
|
|
176
420
|
}
|
|
177
421
|
|
|
178
422
|
// Callback types
|
|
179
423
|
export type QueryUpdateCallback = (records: Record<string, any>[]) => void;
|
|
424
|
+
export type QueryStatusCallback = (status: QueryStatus) => void;
|
|
180
425
|
export type MutationCallback = (mutations: UpEvent[]) => void;
|
|
181
426
|
|
|
182
427
|
export type MutationEventType = 'create' | 'update' | 'delete';
|
|
@@ -209,6 +454,13 @@ export interface RunOptions {
|
|
|
209
454
|
assignedTo?: string;
|
|
210
455
|
max_retries?: number;
|
|
211
456
|
retry_strategy?: 'linear' | 'exponential';
|
|
457
|
+
/** Timeout in seconds for the backend HTTP call. Only used if the backend allows timeout override. */
|
|
458
|
+
timeout?: number;
|
|
459
|
+
/**
|
|
460
|
+
* Minimum delay in milliseconds before the job is eligible to run. While
|
|
461
|
+
* delayed the job stays pending (enqueued) and can still be killed.
|
|
462
|
+
*/
|
|
463
|
+
delay?: number;
|
|
212
464
|
}
|
|
213
465
|
|
|
214
466
|
/**
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { classifySyncError } from './error-classification';
|
|
3
|
+
|
|
4
|
+
describe('classifySyncError', () => {
|
|
5
|
+
it('classifies surreal ConnectionUnavailableError as network', () => {
|
|
6
|
+
// Thrown synchronously by the WS engine's send() while the socket is down
|
|
7
|
+
// but reconnect hasn't fired. Message contains "connected", not "connection".
|
|
8
|
+
const err = new Error(
|
|
9
|
+
'You must be connected to a SurrealDB instance before performing this operation'
|
|
10
|
+
);
|
|
11
|
+
expect(classifySyncError(err)).toBe('network');
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
it('classifies surreal CallTerminatedError as network', () => {
|
|
15
|
+
const err = new Error(
|
|
16
|
+
'The call has been terminated because the connection was closed'
|
|
17
|
+
);
|
|
18
|
+
expect(classifySyncError(err)).toBe('network');
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it.each([
|
|
22
|
+
'WebSocket connection failed',
|
|
23
|
+
'fetch failed',
|
|
24
|
+
'connect ECONNREFUSED 127.0.0.1:8666',
|
|
25
|
+
'request timed out',
|
|
26
|
+
'The operation was aborted',
|
|
27
|
+
'socket hang up',
|
|
28
|
+
])('classifies %j as network', (message) => {
|
|
29
|
+
expect(classifySyncError(new Error(message))).toBe('network');
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it.each([
|
|
33
|
+
'Permission denied',
|
|
34
|
+
'There was a problem with the database: record already exists',
|
|
35
|
+
'Parse error: unexpected token',
|
|
36
|
+
])('classifies %j as application', (message) => {
|
|
37
|
+
expect(classifySyncError(new Error(message))).toBe('application');
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it('handles non-Error values', () => {
|
|
41
|
+
expect(classifySyncError('connection reset')).toBe('network');
|
|
42
|
+
expect(classifySyncError('boom')).toBe('application');
|
|
43
|
+
});
|
|
44
|
+
});
|
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
const NETWORK_ERROR_PATTERNS = [
|
|
2
2
|
'connection',
|
|
3
|
+
// surreal's ConnectionUnavailableError reads "You must be connected to a
|
|
4
|
+
// SurrealDB instance..." — it contains "connected", not "connection", so it
|
|
5
|
+
// slips past the pattern above. The WS engine throws it synchronously from
|
|
6
|
+
// `send()` while the socket is down but reconnect hasn't fired yet, so it's the
|
|
7
|
+
// canonical error on an idle-dropped socket; classify it as network.
|
|
8
|
+
'must be connected',
|
|
9
|
+
'connectionunavailable',
|
|
3
10
|
'timeout',
|
|
4
11
|
'timed out',
|
|
5
12
|
'websocket',
|
package/src/utils/index.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
|
|
1
|
+
import type { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
|
|
2
2
|
import { Uuid, RecordId, Duration } from 'surrealdb';
|
|
3
|
-
import { Logger } from '../services/logger/index';
|
|
4
|
-
import { QueryTimeToLive } from '../types';
|
|
3
|
+
import type { Logger } from '../services/logger/index';
|
|
4
|
+
import type { QueryTimeToLive } from '../types';
|
|
5
5
|
|
|
6
6
|
export * from './surql';
|
|
7
7
|
export * from './parser';
|
|
@@ -59,7 +59,7 @@ export function generateNewTableId<S extends SchemaStructure, T extends TableNam
|
|
|
59
59
|
|
|
60
60
|
// ==================== SCHEMA ENCODING/DECODING ====================
|
|
61
61
|
|
|
62
|
-
export function
|
|
62
|
+
export function decodeFromSp00ky<S extends SchemaStructure, T extends TableNames<S>>(
|
|
63
63
|
schema: S,
|
|
64
64
|
tableName: T,
|
|
65
65
|
record: TableModel<GetTable<S, T>>
|
|
@@ -74,19 +74,19 @@ export function decodeFromSpooky<S extends SchemaStructure, T extends TableNames
|
|
|
74
74
|
for (const field of Object.keys(table.columns)) {
|
|
75
75
|
const column = table.columns[field] as any;
|
|
76
76
|
const relation = schema.relationships.find((r) => r.from === tableName && r.field === field);
|
|
77
|
-
if ((column.recordId || relation) && encoded[field]
|
|
77
|
+
if ((column.recordId || relation) && encoded[field] !== null && encoded[field] !== undefined) {
|
|
78
78
|
if (encoded[field] instanceof RecordId) {
|
|
79
79
|
encoded[field] = `${encoded[field].table.toString()}:${encoded[field].id}`;
|
|
80
80
|
} else if (
|
|
81
81
|
relation &&
|
|
82
|
-
(encoded[field] instanceof Object || encoded[field]
|
|
82
|
+
(encoded[field] instanceof Object || Array.isArray(encoded[field]))
|
|
83
83
|
) {
|
|
84
84
|
if (Array.isArray(encoded[field])) {
|
|
85
85
|
encoded[field] = encoded[field].map((item) =>
|
|
86
|
-
|
|
86
|
+
decodeFromSp00ky(schema, relation.to, item)
|
|
87
87
|
);
|
|
88
88
|
} else {
|
|
89
|
-
encoded[field] =
|
|
89
|
+
encoded[field] = decodeFromSp00ky(schema, relation.to, encoded[field]);
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
92
|
}
|
|
@@ -97,15 +97,25 @@ export function decodeFromSpooky<S extends SchemaStructure, T extends TableNames
|
|
|
97
97
|
|
|
98
98
|
// ==================== TIME/DURATION UTILITIES ====================
|
|
99
99
|
|
|
100
|
+
/**
|
|
101
|
+
* Read the millisecond count off a surrealdb `Duration`. Different `surrealdb`
|
|
102
|
+
* releases expose it under `milliseconds` (current) or the older private
|
|
103
|
+
* `_milliseconds` field, so read whichever is set; returns 0 when neither is.
|
|
104
|
+
*/
|
|
105
|
+
function durationMillis(duration: Duration): number {
|
|
106
|
+
const d = duration as { milliseconds?: number | bigint; _milliseconds?: number | bigint };
|
|
107
|
+
return Number(d.milliseconds || d._milliseconds || 0);
|
|
108
|
+
}
|
|
109
|
+
|
|
100
110
|
/**
|
|
101
111
|
* Parse duration string or Duration object to milliseconds
|
|
102
112
|
*/
|
|
103
113
|
export function parseDuration(duration: QueryTimeToLive | Duration): number {
|
|
104
114
|
if (duration instanceof Duration) {
|
|
105
|
-
const ms = (duration
|
|
106
|
-
if (ms) return
|
|
115
|
+
const ms = durationMillis(duration);
|
|
116
|
+
if (ms) return ms;
|
|
107
117
|
const str = duration.toString();
|
|
108
|
-
if (str !== '[object Object]') return parseDuration(str as
|
|
118
|
+
if (str !== '[object Object]') return parseDuration(str as QueryTimeToLive);
|
|
109
119
|
return 600000;
|
|
110
120
|
}
|
|
111
121
|
|
|
@@ -117,7 +127,7 @@ export function parseDuration(duration: QueryTimeToLive | Duration): number {
|
|
|
117
127
|
|
|
118
128
|
const match = duration.match(/^(\d+)([smh])$/);
|
|
119
129
|
if (!match) return 600000;
|
|
120
|
-
const val = parseInt(match[1], 10);
|
|
130
|
+
const val = Number.parseInt(match[1], 10);
|
|
121
131
|
const unit = match[2];
|
|
122
132
|
switch (unit) {
|
|
123
133
|
case 's':
|
|
@@ -137,6 +147,18 @@ export async function fileToUint8Array(file: File | Blob): Promise<Uint8Array> {
|
|
|
137
147
|
return new Uint8Array(buffer);
|
|
138
148
|
}
|
|
139
149
|
|
|
150
|
+
// ==================== TEXT UTILITIES ====================
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Convert plain text to simple HTML paragraphs.
|
|
154
|
+
* Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
|
|
155
|
+
*/
|
|
156
|
+
export function textToHtml(text: string): string {
|
|
157
|
+
return text
|
|
158
|
+
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
|
159
|
+
.split('\n').map((l) => `<p>${l || '<br>'}</p>`).join('');
|
|
160
|
+
}
|
|
161
|
+
|
|
140
162
|
// ==================== DATABASE UTILITIES ====================
|
|
141
163
|
|
|
142
164
|
/**
|
|
@@ -165,7 +187,7 @@ export async function withRetry<T>(
|
|
|
165
187
|
attempt: i + 1,
|
|
166
188
|
retries,
|
|
167
189
|
error: msg,
|
|
168
|
-
Category: '
|
|
190
|
+
Category: 'sp00ky-client::utils::withRetry',
|
|
169
191
|
},
|
|
170
192
|
'Retrying DB operation'
|
|
171
193
|
);
|
package/src/utils/parser.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { ColumnSchema
|
|
1
|
+
import type { ColumnSchema} from '@spooky-sync/query-builder';
|
|
2
|
+
import { RecordId } from '@spooky-sync/query-builder';
|
|
2
3
|
import { parseRecordIdString } from './index';
|
|
3
4
|
import { DateTime } from 'surrealdb';
|
|
4
5
|
|
|
@@ -8,7 +9,7 @@ export function cleanRecord(
|
|
|
8
9
|
): Record<string, any> {
|
|
9
10
|
const cleaned: Record<string, any> = {};
|
|
10
11
|
for (const [key, value] of Object.entries(record)) {
|
|
11
|
-
if (key === 'id' || key in tableSchema) {
|
|
12
|
+
if (key === 'id' || key.startsWith('_00_') || key in tableSchema) {
|
|
12
13
|
cleaned[key] = value;
|
|
13
14
|
}
|
|
14
15
|
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { semverGt } from './semver';
|
|
3
|
+
|
|
4
|
+
describe('semverGt', () => {
|
|
5
|
+
it('compares strictly greater', () => {
|
|
6
|
+
expect(semverGt('1.0.1', '1.0.0')).toBe(true);
|
|
7
|
+
expect(semverGt('1.1.0', '1.0.9')).toBe(true);
|
|
8
|
+
expect(semverGt('2.0.0', '1.9.9')).toBe(true);
|
|
9
|
+
expect(semverGt('1.10.0', '1.9.0')).toBe(true); // numeric, not lexicographic
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
it('equal and lower are not greater', () => {
|
|
13
|
+
expect(semverGt('1.0.0', '1.0.0')).toBe(false);
|
|
14
|
+
expect(semverGt('1.0.0', '1.0.1')).toBe(false);
|
|
15
|
+
expect(semverGt('0.9.9', '1.0.0')).toBe(false);
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it('missing parts default to zero', () => {
|
|
19
|
+
expect(semverGt('1.2', '1.2.0')).toBe(false);
|
|
20
|
+
expect(semverGt('1.2.1', '1.2')).toBe(true);
|
|
21
|
+
expect(semverGt('2', '1.9.9')).toBe(true);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
it('junk never compares greater', () => {
|
|
25
|
+
expect(semverGt('abc', '1.0.0')).toBe(false);
|
|
26
|
+
expect(semverGt('1.0.0', 'abc')).toBe(false);
|
|
27
|
+
expect(semverGt('', '0.0.0')).toBe(false);
|
|
28
|
+
expect(semverGt(null, undefined)).toBe(false);
|
|
29
|
+
expect(semverGt('1.0.0-beta', '0.9.0')).toBe(false); // prerelease unsupported
|
|
30
|
+
expect(semverGt('1.2.3.4', '1.0.0')).toBe(false);
|
|
31
|
+
});
|
|
32
|
+
});
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// Minimal semver comparison for app-release version checks. "X.Y.Z" numeric
|
|
2
|
+
// with missing parts read as 0 ("1.2" == "1.2.0"); any malformed input never
|
|
3
|
+
// compares greater, so a bad release row can never nag (or force-reload)
|
|
4
|
+
// every client.
|
|
5
|
+
|
|
6
|
+
function parse(v: unknown): [number, number, number] | null {
|
|
7
|
+
const parts = String(v ?? '')
|
|
8
|
+
.trim()
|
|
9
|
+
.split('.');
|
|
10
|
+
if (parts.length === 0 || parts.length > 3 || parts[0] === '') return null;
|
|
11
|
+
const nums: number[] = [];
|
|
12
|
+
for (let i = 0; i < 3; i++) {
|
|
13
|
+
const raw = parts[i] ?? '0';
|
|
14
|
+
if (!/^\d+$/.test(raw)) return null;
|
|
15
|
+
nums.push(parseInt(raw, 10));
|
|
16
|
+
}
|
|
17
|
+
return nums as [number, number, number];
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** True when `a` is a valid version strictly greater than valid version `b`. */
|
|
21
|
+
export function semverGt(a: unknown, b: unknown): boolean {
|
|
22
|
+
const pa = parse(a);
|
|
23
|
+
const pb = parse(b);
|
|
24
|
+
if (!pa || !pb) return false;
|
|
25
|
+
for (let i = 0; i < 3; i++) {
|
|
26
|
+
if (pa[i] > pb[i]) return true;
|
|
27
|
+
if (pa[i] < pb[i]) return false;
|
|
28
|
+
}
|
|
29
|
+
return false;
|
|
30
|
+
}
|