@spooky-sync/core 0.0.1-canary.4 → 0.0.1-canary.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +18 -367
- package/dist/index.js +268 -287
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/types.d.ts +361 -0
- package/package.json +34 -5
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/events/events.test.ts +2 -1
- package/src/index.ts +1 -1
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +17 -20
- package/src/modules/cache/index.ts +23 -23
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/data/index.ts +61 -43
- package/src/modules/devtools/index.ts +23 -20
- package/src/modules/sync/engine.ts +31 -21
- package/src/modules/sync/events/index.ts +3 -2
- package/src/modules/sync/queue/queue-down.ts +5 -4
- package/src/modules/sync/queue/queue-up.ts +14 -13
- package/src/modules/sync/scheduler.ts +2 -2
- package/src/modules/sync/sync.ts +47 -35
- package/src/modules/sync/utils.test.ts +2 -2
- package/src/modules/sync/utils.ts +15 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/local-migrator.ts +21 -21
- package/src/services/database/local.ts +16 -15
- package/src/services/database/remote.ts +13 -13
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +34 -0
- package/src/services/persistence/surrealdb.ts +9 -9
- package/src/services/stream-processor/index.ts +32 -31
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +2 -2
- package/src/{spooky.ts → sp00ky.ts} +40 -38
- package/src/types.ts +18 -11
- package/src/utils/index.ts +10 -10
- package/src/utils/parser.ts +2 -1
- package/src/utils/surql.ts +15 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { PersistenceClient } from '../../types';
|
|
1
|
+
import type { PersistenceClient } from '../../types';
|
|
2
2
|
import { parseRecordIdString, surql } from '../../utils/index';
|
|
3
|
-
import { Logger } from 'pino';
|
|
4
|
-
import { AbstractDatabaseService } from '../database/database';
|
|
3
|
+
import type { Logger } from 'pino';
|
|
4
|
+
import type { AbstractDatabaseService } from '../database/database';
|
|
5
5
|
|
|
6
6
|
export class SurrealDBPersistenceClient implements PersistenceClient {
|
|
7
7
|
private logger: Logger;
|
|
@@ -15,11 +15,11 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
|
|
|
15
15
|
|
|
16
16
|
async set<T>(key: string, val: T) {
|
|
17
17
|
try {
|
|
18
|
-
const id = parseRecordIdString(`
|
|
18
|
+
const id = parseRecordIdString(`_00_kv:${key}`);
|
|
19
19
|
await this.db.query(surql.seal(surql.upsert('id', 'data')), { id, data: { val } });
|
|
20
20
|
} catch (error) {
|
|
21
21
|
this.logger.error(
|
|
22
|
-
{ error, Category: '
|
|
22
|
+
{ error, Category: 'sp00ky-client::SurrealDBPersistenceClient::set' },
|
|
23
23
|
'Failed to set KV'
|
|
24
24
|
);
|
|
25
25
|
throw error;
|
|
@@ -28,7 +28,7 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
|
|
|
28
28
|
|
|
29
29
|
async get<T>(key: string) {
|
|
30
30
|
try {
|
|
31
|
-
const id = parseRecordIdString(`
|
|
31
|
+
const id = parseRecordIdString(`_00_kv:${key}`);
|
|
32
32
|
const [result] = await this.db.query<[{ val: T }]>(
|
|
33
33
|
surql.seal(surql.selectById('id', ['val'])),
|
|
34
34
|
{
|
|
@@ -41,7 +41,7 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
|
|
|
41
41
|
return result.val;
|
|
42
42
|
} catch (error) {
|
|
43
43
|
this.logger.warn(
|
|
44
|
-
{ error, Category: '
|
|
44
|
+
{ error, Category: 'sp00ky-client::SurrealDBPersistenceClient::get' },
|
|
45
45
|
'Failed to get KV'
|
|
46
46
|
);
|
|
47
47
|
return null;
|
|
@@ -50,11 +50,11 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
|
|
|
50
50
|
|
|
51
51
|
async remove(key: string) {
|
|
52
52
|
try {
|
|
53
|
-
const id = parseRecordIdString(`
|
|
53
|
+
const id = parseRecordIdString(`_00_kv:${key}`);
|
|
54
54
|
await this.db.query(surql.seal(surql.delete('id')), { id });
|
|
55
55
|
} catch (err) {
|
|
56
56
|
this.logger.info(
|
|
57
|
-
{ err, Category: '
|
|
57
|
+
{ err, Category: 'sp00ky-client::SurrealDBPersistenceClient::remove' },
|
|
58
58
|
'Failed to delete KV'
|
|
59
59
|
);
|
|
60
60
|
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
1
|
+
// oxlint-disable-next-line no-named-as-default -- WASM module default export convention
|
|
2
|
+
import init, { Sp00kyProcessor } from '@spooky-sync/ssp-wasm';
|
|
3
|
+
import type { EventDefinition, EventSystem } from '../../events/index';
|
|
4
|
+
import type { Logger } from 'pino';
|
|
5
|
+
import type { LocalDatabaseService } from '../database/index';
|
|
6
|
+
import type { WasmProcessor, WasmStreamUpdate } from './wasm-types';
|
|
7
|
+
import type { Duration } from 'surrealdb';
|
|
8
|
+
import type { PersistenceClient, QueryTimeToLive, RecordVersionArray } from '../../types';
|
|
8
9
|
|
|
9
10
|
// Simple interface for query plan registration (replaces Incantation class)
|
|
10
11
|
interface QueryPlanConfig {
|
|
@@ -80,25 +81,25 @@ export class StreamProcessorService {
|
|
|
80
81
|
if (this.isInitialized) return;
|
|
81
82
|
|
|
82
83
|
this.logger.info(
|
|
83
|
-
{ Category: '
|
|
84
|
+
{ Category: 'sp00ky-client::StreamProcessorService::init' },
|
|
84
85
|
'Initializing WASM...'
|
|
85
86
|
);
|
|
86
87
|
try {
|
|
87
88
|
await init(); // Initialize the WASM module (web target)
|
|
88
|
-
// We cast the generated
|
|
89
|
-
this.processor = new
|
|
89
|
+
// We cast the generated Sp00kyProcessor to our interface which is safer
|
|
90
|
+
this.processor = new Sp00kyProcessor() as unknown as WasmProcessor;
|
|
90
91
|
|
|
91
92
|
// Try to load state
|
|
92
93
|
await this.loadState();
|
|
93
94
|
|
|
94
95
|
this.isInitialized = true;
|
|
95
96
|
this.logger.info(
|
|
96
|
-
{ Category: '
|
|
97
|
+
{ Category: 'sp00ky-client::StreamProcessorService::init' },
|
|
97
98
|
'Initialized successfully'
|
|
98
99
|
);
|
|
99
100
|
} catch (e) {
|
|
100
101
|
this.logger.error(
|
|
101
|
-
{ error: e, Category: '
|
|
102
|
+
{ error: e, Category: 'sp00ky-client::StreamProcessorService::init' },
|
|
102
103
|
'Failed to initialize'
|
|
103
104
|
);
|
|
104
105
|
throw e;
|
|
@@ -108,7 +109,7 @@ export class StreamProcessorService {
|
|
|
108
109
|
async loadState() {
|
|
109
110
|
if (!this.processor) return;
|
|
110
111
|
try {
|
|
111
|
-
const result = await this.persistenceClient.get('
|
|
112
|
+
const result = await this.persistenceClient.get('_00_stream_processor_state');
|
|
112
113
|
|
|
113
114
|
// Check if we have a valid result from the query
|
|
114
115
|
if (
|
|
@@ -122,7 +123,7 @@ export class StreamProcessorService {
|
|
|
122
123
|
this.logger.info(
|
|
123
124
|
{
|
|
124
125
|
stateLength: state.length,
|
|
125
|
-
Category: '
|
|
126
|
+
Category: 'sp00ky-client::StreamProcessorService::loadState',
|
|
126
127
|
},
|
|
127
128
|
'Loading state from DB'
|
|
128
129
|
);
|
|
@@ -132,19 +133,19 @@ export class StreamProcessorService {
|
|
|
132
133
|
(this.processor as any).load_state(state);
|
|
133
134
|
} else {
|
|
134
135
|
this.logger.warn(
|
|
135
|
-
{ Category: '
|
|
136
|
+
{ Category: 'sp00ky-client::StreamProcessorService::loadState' },
|
|
136
137
|
'load_state method not found on processor'
|
|
137
138
|
);
|
|
138
139
|
}
|
|
139
140
|
} else {
|
|
140
141
|
this.logger.info(
|
|
141
|
-
{ Category: '
|
|
142
|
+
{ Category: 'sp00ky-client::StreamProcessorService::loadState' },
|
|
142
143
|
'No saved state found'
|
|
143
144
|
);
|
|
144
145
|
}
|
|
145
146
|
} catch (e) {
|
|
146
147
|
this.logger.error(
|
|
147
|
-
{ error: e, Category: '
|
|
148
|
+
{ error: e, Category: 'sp00ky-client::StreamProcessorService::loadState' },
|
|
148
149
|
'Failed to load state'
|
|
149
150
|
);
|
|
150
151
|
}
|
|
@@ -157,16 +158,16 @@ export class StreamProcessorService {
|
|
|
157
158
|
if (typeof (this.processor as any).save_state === 'function') {
|
|
158
159
|
const state = (this.processor as any).save_state();
|
|
159
160
|
if (state) {
|
|
160
|
-
await this.persistenceClient.set('
|
|
161
|
+
await this.persistenceClient.set('_00_stream_processor_state', state);
|
|
161
162
|
this.logger.trace(
|
|
162
|
-
{ Category: '
|
|
163
|
+
{ Category: 'sp00ky-client::StreamProcessorService::saveState' },
|
|
163
164
|
'State saved'
|
|
164
165
|
);
|
|
165
166
|
}
|
|
166
167
|
}
|
|
167
168
|
} catch (e) {
|
|
168
169
|
this.logger.error(
|
|
169
|
-
{ error: e, Category: '
|
|
170
|
+
{ error: e, Category: 'sp00ky-client::StreamProcessorService::saveState' },
|
|
170
171
|
'Failed to save state'
|
|
171
172
|
);
|
|
172
173
|
}
|
|
@@ -188,14 +189,14 @@ export class StreamProcessorService {
|
|
|
188
189
|
table,
|
|
189
190
|
op,
|
|
190
191
|
id,
|
|
191
|
-
Category: '
|
|
192
|
+
Category: 'sp00ky-client::StreamProcessorService::ingest',
|
|
192
193
|
},
|
|
193
194
|
'Ingesting into ssp'
|
|
194
195
|
);
|
|
195
196
|
|
|
196
197
|
if (!this.processor) {
|
|
197
198
|
this.logger.warn(
|
|
198
|
-
{ Category: '
|
|
199
|
+
{ Category: 'sp00ky-client::StreamProcessorService::ingest' },
|
|
199
200
|
'Not initialized, skipping ingest'
|
|
200
201
|
);
|
|
201
202
|
return [];
|
|
@@ -211,7 +212,7 @@ export class StreamProcessorService {
|
|
|
211
212
|
op,
|
|
212
213
|
id,
|
|
213
214
|
rawUpdates: rawUpdates.length,
|
|
214
|
-
Category: '
|
|
215
|
+
Category: 'sp00ky-client::StreamProcessorService::ingest',
|
|
215
216
|
},
|
|
216
217
|
'Ingesting into ssp done'
|
|
217
218
|
);
|
|
@@ -229,7 +230,7 @@ export class StreamProcessorService {
|
|
|
229
230
|
return rawUpdates;
|
|
230
231
|
} catch (e) {
|
|
231
232
|
this.logger.error(
|
|
232
|
-
{ error: e, Category: '
|
|
233
|
+
{ error: e, Category: 'sp00ky-client::StreamProcessorService::ingest' },
|
|
233
234
|
'Ingesting into ssp failed'
|
|
234
235
|
);
|
|
235
236
|
}
|
|
@@ -243,7 +244,7 @@ export class StreamProcessorService {
|
|
|
243
244
|
registerQueryPlan(queryPlan: QueryPlanConfig) {
|
|
244
245
|
if (!this.processor) {
|
|
245
246
|
this.logger.warn(
|
|
246
|
-
{ Category: '
|
|
247
|
+
{ Category: 'sp00ky-client::StreamProcessorService::registerQueryPlan' },
|
|
247
248
|
'Not initialized, skipping registration'
|
|
248
249
|
);
|
|
249
250
|
return;
|
|
@@ -254,7 +255,7 @@ export class StreamProcessorService {
|
|
|
254
255
|
queryHash: queryPlan.queryHash,
|
|
255
256
|
surql: queryPlan.surql,
|
|
256
257
|
params: queryPlan.params,
|
|
257
|
-
Category: '
|
|
258
|
+
Category: 'sp00ky-client::StreamProcessorService::registerQueryPlan',
|
|
258
259
|
},
|
|
259
260
|
'Registering query plan'
|
|
260
261
|
);
|
|
@@ -272,7 +273,7 @@ export class StreamProcessorService {
|
|
|
272
273
|
});
|
|
273
274
|
|
|
274
275
|
this.logger.debug(
|
|
275
|
-
{ initialUpdate, Category: '
|
|
276
|
+
{ initialUpdate, Category: 'sp00ky-client::StreamProcessorService::registerQueryPlan' },
|
|
276
277
|
'register_view result'
|
|
277
278
|
);
|
|
278
279
|
|
|
@@ -289,14 +290,14 @@ export class StreamProcessorService {
|
|
|
289
290
|
queryHash: queryPlan.queryHash,
|
|
290
291
|
surql: queryPlan.surql,
|
|
291
292
|
params: queryPlan.params,
|
|
292
|
-
Category: '
|
|
293
|
+
Category: 'sp00ky-client::StreamProcessorService::registerQueryPlan',
|
|
293
294
|
},
|
|
294
295
|
'Registered query plan'
|
|
295
296
|
);
|
|
296
297
|
return update;
|
|
297
298
|
} catch (e) {
|
|
298
299
|
this.logger.error(
|
|
299
|
-
{ error: e, Category: '
|
|
300
|
+
{ error: e, Category: 'sp00ky-client::StreamProcessorService::registerQueryPlan' },
|
|
300
301
|
'Error registering query plan'
|
|
301
302
|
);
|
|
302
303
|
throw e;
|
|
@@ -313,7 +314,7 @@ export class StreamProcessorService {
|
|
|
313
314
|
this.saveState();
|
|
314
315
|
} catch (e) {
|
|
315
316
|
this.logger.error(
|
|
316
|
-
{ error: e, Category: '
|
|
317
|
+
{ error: e, Category: 'sp00ky-client::StreamProcessorService::unregisterQueryPlan' },
|
|
317
318
|
'Error unregistering query plan'
|
|
318
319
|
);
|
|
319
320
|
}
|
|
@@ -334,7 +335,7 @@ export class StreamProcessorService {
|
|
|
334
335
|
if (hasTable && hasId && hasToString && isNotPlainObject) {
|
|
335
336
|
const result = value.toString();
|
|
336
337
|
this.logger.trace(
|
|
337
|
-
{ result, Category: '
|
|
338
|
+
{ result, Category: 'sp00ky-client::StreamProcessorService::normalizeValue' },
|
|
338
339
|
'RecordId detected'
|
|
339
340
|
);
|
|
340
341
|
return result;
|
|
@@ -127,7 +127,7 @@ describe('StreamProcessor Ingest Behavior', () => {
|
|
|
127
127
|
const params = { id: new MockRecordId('user', '2dng4ngbicbl0scod87i') };
|
|
128
128
|
const normalizedParams = normalizeValue(params);
|
|
129
129
|
|
|
130
|
-
const
|
|
130
|
+
const _ingestedRecord = {
|
|
131
131
|
id: 'user:2dng4ngbicbl0scod87i',
|
|
132
132
|
username: 'sara',
|
|
133
133
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RecordVersionArray } from '../../types';
|
|
1
|
+
import type { RecordVersionArray } from '../../types';
|
|
2
2
|
|
|
3
3
|
export interface WasmStreamUpdate {
|
|
4
4
|
query_id: string;
|
|
@@ -23,7 +23,7 @@ export interface WasmIngestItem {
|
|
|
23
23
|
version?: number;
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
// Interface matching the
|
|
26
|
+
// Interface matching the Sp00kyProcessor class from WASM
|
|
27
27
|
export interface WasmProcessor {
|
|
28
28
|
ingest(table: string, op: string, id: string, record: any): WasmStreamUpdate[];
|
|
29
29
|
register_view(config: WasmQueryConfig): WasmStreamUpdate | undefined;
|
|
@@ -1,24 +1,21 @@
|
|
|
1
1
|
import { DataModule } from './modules/data/index';
|
|
2
|
-
import {
|
|
3
|
-
|
|
2
|
+
import type {
|
|
3
|
+
Sp00kyConfig,
|
|
4
4
|
QueryTimeToLive,
|
|
5
|
-
|
|
5
|
+
Sp00kyQueryResultPromise,
|
|
6
6
|
PersistenceClient,
|
|
7
|
-
MutationEvent,
|
|
8
7
|
UpdateOptions,
|
|
9
|
-
RunOptions
|
|
10
|
-
} from './types';
|
|
8
|
+
RunOptions} from './types';
|
|
11
9
|
import {
|
|
12
10
|
LocalDatabaseService,
|
|
13
11
|
LocalMigrator,
|
|
14
12
|
RemoteDatabaseService,
|
|
15
13
|
} from './services/database/index';
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
14
|
+
import type { UpEvent } from './modules/sync/index';
|
|
15
|
+
import { Sp00kySync } from './modules/sync/index';
|
|
16
|
+
import type {
|
|
19
17
|
GetTable,
|
|
20
18
|
InnerQuery,
|
|
21
|
-
QueryBuilder,
|
|
22
19
|
QueryOptions,
|
|
23
20
|
SchemaStructure,
|
|
24
21
|
TableModel,
|
|
@@ -26,7 +23,9 @@ import {
|
|
|
26
23
|
BucketNames,
|
|
27
24
|
BackendNames,
|
|
28
25
|
BackendRoutes,
|
|
29
|
-
RoutePayload
|
|
26
|
+
RoutePayload} from '@spooky-sync/query-builder';
|
|
27
|
+
import {
|
|
28
|
+
QueryBuilder
|
|
30
29
|
} from '@spooky-sync/query-builder';
|
|
31
30
|
|
|
32
31
|
import { DevToolsService } from './modules/devtools/index';
|
|
@@ -38,6 +37,7 @@ import { CacheModule } from './modules/cache/index';
|
|
|
38
37
|
import { LocalStoragePersistenceClient } from './services/persistence/localstorage';
|
|
39
38
|
import { generateId, parseParams } from './utils/index';
|
|
40
39
|
import { SurrealDBPersistenceClient } from './services/persistence/surrealdb';
|
|
40
|
+
import { ResilientPersistenceClient } from './services/persistence/resilient';
|
|
41
41
|
|
|
42
42
|
export class BucketHandle {
|
|
43
43
|
constructor(private bucketName: string, private remote: RemoteDatabaseService) {}
|
|
@@ -80,7 +80,7 @@ export class BucketHandle {
|
|
|
80
80
|
}
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
export class
|
|
83
|
+
export class Sp00kyClient<S extends SchemaStructure> {
|
|
84
84
|
private local: LocalDatabaseService;
|
|
85
85
|
private remote: RemoteDatabaseService;
|
|
86
86
|
private persistenceClient: PersistenceClient;
|
|
@@ -88,7 +88,7 @@ export class SpookyClient<S extends SchemaStructure> {
|
|
|
88
88
|
private migrator: LocalMigrator;
|
|
89
89
|
private cache: CacheModule;
|
|
90
90
|
private dataModule: DataModule<S>;
|
|
91
|
-
private sync:
|
|
91
|
+
private sync: Sp00kySync<S>;
|
|
92
92
|
private devTools: DevToolsService;
|
|
93
93
|
|
|
94
94
|
private logger: ReturnType<typeof createLogger>;
|
|
@@ -111,16 +111,16 @@ export class SpookyClient<S extends SchemaStructure> {
|
|
|
111
111
|
return this.sync.subscribeToPendingMutations(cb);
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
-
constructor(private config:
|
|
115
|
-
const logger = createLogger(config.logLevel ?? 'info', config.
|
|
116
|
-
this.logger = logger.child({ service: '
|
|
114
|
+
constructor(private config: Sp00kyConfig<S>) {
|
|
115
|
+
const logger = createLogger(config.logLevel ?? 'info', config.otelTransmit);
|
|
116
|
+
this.logger = logger.child({ service: 'Sp00kyClient' });
|
|
117
117
|
|
|
118
118
|
this.logger.info(
|
|
119
119
|
{
|
|
120
120
|
config: { ...config, schema: '[SchemaStructure]' },
|
|
121
|
-
Category: '
|
|
121
|
+
Category: 'sp00ky-client::Sp00kyClient::constructor',
|
|
122
122
|
},
|
|
123
|
-
'
|
|
123
|
+
'Sp00kyClient initialized'
|
|
124
124
|
);
|
|
125
125
|
|
|
126
126
|
this.local = new LocalDatabaseService(this.config.database, logger);
|
|
@@ -134,6 +134,8 @@ export class SpookyClient<S extends SchemaStructure> {
|
|
|
134
134
|
this.persistenceClient = config.persistenceClient;
|
|
135
135
|
}
|
|
136
136
|
|
|
137
|
+
this.persistenceClient = new ResilientPersistenceClient(this.persistenceClient, logger);
|
|
138
|
+
|
|
137
139
|
this.streamProcessor = new StreamProcessorService(
|
|
138
140
|
new EventSystem(['stream_update']),
|
|
139
141
|
this.local,
|
|
@@ -164,7 +166,7 @@ export class SpookyClient<S extends SchemaStructure> {
|
|
|
164
166
|
this.auth = new AuthService(this.config.schema, this.remote, this.persistenceClient, logger);
|
|
165
167
|
|
|
166
168
|
// Initialize Sync
|
|
167
|
-
this.sync = new
|
|
169
|
+
this.sync = new Sp00kySync(this.local, this.remote, this.cache, this.dataModule, this.config.schema, this.logger);
|
|
168
170
|
|
|
169
171
|
// Initialize DevTools
|
|
170
172
|
this.devTools = new DevToolsService(
|
|
@@ -215,58 +217,58 @@ export class SpookyClient<S extends SchemaStructure> {
|
|
|
215
217
|
|
|
216
218
|
async init() {
|
|
217
219
|
this.logger.info(
|
|
218
|
-
{ Category: '
|
|
219
|
-
'
|
|
220
|
+
{ Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
221
|
+
'Sp00kyClient initialization started'
|
|
220
222
|
);
|
|
221
223
|
try {
|
|
222
224
|
const clientId = this.config.clientId ?? (await this.loadOrGenerateClientId());
|
|
223
225
|
this.persistClientId(clientId);
|
|
224
226
|
this.logger.debug(
|
|
225
|
-
{ clientId, Category: '
|
|
227
|
+
{ clientId, Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
226
228
|
'Client ID loaded'
|
|
227
229
|
);
|
|
228
230
|
|
|
229
231
|
await this.local.connect();
|
|
230
232
|
this.logger.debug(
|
|
231
|
-
{ Category: '
|
|
233
|
+
{ Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
232
234
|
'Local database connected'
|
|
233
235
|
);
|
|
234
236
|
|
|
235
237
|
await this.migrator.provision(this.config.schemaSurql);
|
|
236
|
-
this.logger.debug({ Category: '
|
|
238
|
+
this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Schema provisioned');
|
|
237
239
|
|
|
238
240
|
await this.remote.connect();
|
|
239
241
|
this.logger.debug(
|
|
240
|
-
{ Category: '
|
|
242
|
+
{ Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
241
243
|
'Remote database connected'
|
|
242
244
|
);
|
|
243
245
|
|
|
244
246
|
await this.streamProcessor.init();
|
|
245
247
|
this.logger.debug(
|
|
246
|
-
{ Category: '
|
|
248
|
+
{ Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
247
249
|
'StreamProcessor initialized'
|
|
248
250
|
);
|
|
249
251
|
|
|
250
252
|
await this.auth.init();
|
|
251
|
-
this.logger.debug({ Category: '
|
|
253
|
+
this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Auth initialized');
|
|
252
254
|
|
|
253
255
|
await this.dataModule.init();
|
|
254
256
|
this.logger.debug(
|
|
255
|
-
{ Category: '
|
|
257
|
+
{ Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
256
258
|
'DataModule initialized'
|
|
257
259
|
);
|
|
258
260
|
|
|
259
261
|
await this.sync.init(clientId);
|
|
260
|
-
this.logger.debug({ Category: '
|
|
262
|
+
this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Sync initialized');
|
|
261
263
|
|
|
262
264
|
this.logger.info(
|
|
263
|
-
{ Category: '
|
|
264
|
-
'
|
|
265
|
+
{ Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
266
|
+
'Sp00kyClient initialization completed successfully'
|
|
265
267
|
);
|
|
266
268
|
} catch (e) {
|
|
267
269
|
this.logger.error(
|
|
268
|
-
{ error: e, Category: '
|
|
269
|
-
'
|
|
270
|
+
{ error: e, Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
271
|
+
'Sp00kyClient initialization failed'
|
|
270
272
|
);
|
|
271
273
|
throw e;
|
|
272
274
|
}
|
|
@@ -289,8 +291,8 @@ export class SpookyClient<S extends SchemaStructure> {
|
|
|
289
291
|
table: Table,
|
|
290
292
|
options: QueryOptions<TableModel<GetTable<S, Table>>, false>,
|
|
291
293
|
ttl: QueryTimeToLive = '10m'
|
|
292
|
-
): QueryBuilder<S, Table,
|
|
293
|
-
return new QueryBuilder<S, Table,
|
|
294
|
+
): QueryBuilder<S, Table, Sp00kyQueryResultPromise> {
|
|
295
|
+
return new QueryBuilder<S, Table, Sp00kyQueryResultPromise>(
|
|
294
296
|
this.config.schema,
|
|
295
297
|
table,
|
|
296
298
|
async (q) => ({
|
|
@@ -369,17 +371,17 @@ export class SpookyClient<S extends SchemaStructure> {
|
|
|
369
371
|
|
|
370
372
|
private persistClientId(id: string) {
|
|
371
373
|
try {
|
|
372
|
-
this.persistenceClient.set('
|
|
374
|
+
this.persistenceClient.set('sp00ky_client_id', id);
|
|
373
375
|
} catch (e) {
|
|
374
376
|
this.logger.warn(
|
|
375
|
-
{ error: e, Category: '
|
|
377
|
+
{ error: e, Category: 'sp00ky-client::Sp00kyClient::persistClientId' },
|
|
376
378
|
'Failed to persist client ID'
|
|
377
379
|
);
|
|
378
380
|
}
|
|
379
381
|
}
|
|
380
382
|
|
|
381
383
|
private async loadOrGenerateClientId(): Promise<string> {
|
|
382
|
-
const clientId = await this.persistenceClient.get<string>('
|
|
384
|
+
const clientId = await this.persistenceClient.get<string>('sp00ky_client_id');
|
|
383
385
|
|
|
384
386
|
if (clientId) {
|
|
385
387
|
return clientId;
|
package/src/types.ts
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
|
-
import { RecordId, SchemaStructure } from '@spooky-sync/query-builder';
|
|
2
|
-
import { Level } from 'pino';
|
|
3
|
-
import { PushEventOptions } from './events/index';
|
|
4
|
-
import { UpEvent } from './modules/sync/index';
|
|
1
|
+
import type { RecordId, SchemaStructure } from '@spooky-sync/query-builder';
|
|
2
|
+
import type { Level, LoggerOptions } from 'pino';
|
|
3
|
+
import type { PushEventOptions } from './events/index';
|
|
4
|
+
import type { UpEvent } from './modules/sync/index';
|
|
5
5
|
|
|
6
|
-
export type { Level }
|
|
6
|
+
export type { Level };
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* A pino browser transmit object for forwarding logs to an external sink (e.g. OpenTelemetry).
|
|
10
|
+
*/
|
|
11
|
+
export type PinoTransmit = NonNullable<NonNullable<LoggerOptions['browser']>['transmit']>;
|
|
7
12
|
|
|
8
13
|
/**
|
|
9
14
|
* The type of storage backend to use for the local database.
|
|
@@ -65,22 +70,22 @@ export type QueryTimeToLive =
|
|
|
65
70
|
/**
|
|
66
71
|
* Result object returned when a query is registered or executed.
|
|
67
72
|
*/
|
|
68
|
-
export interface
|
|
73
|
+
export interface Sp00kyQueryResult {
|
|
69
74
|
/** The unique hash identifier for the query. */
|
|
70
75
|
hash: string;
|
|
71
76
|
}
|
|
72
77
|
|
|
73
|
-
export type
|
|
78
|
+
export type Sp00kyQueryResultPromise = Promise<Sp00kyQueryResult>;
|
|
74
79
|
|
|
75
80
|
export interface EventSubscriptionOptions {
|
|
76
81
|
priority?: number;
|
|
77
82
|
}
|
|
78
83
|
|
|
79
84
|
/**
|
|
80
|
-
* Configuration options for the
|
|
85
|
+
* Configuration options for the Sp00ky client.
|
|
81
86
|
* @template S The schema structure type.
|
|
82
87
|
*/
|
|
83
|
-
export interface
|
|
88
|
+
export interface Sp00kyConfig<S extends SchemaStructure> {
|
|
84
89
|
/** Database connection configuration. */
|
|
85
90
|
database: {
|
|
86
91
|
/** The SurrealDB endpoint URL. */
|
|
@@ -107,8 +112,8 @@ export interface SpookyConfig<S extends SchemaStructure> {
|
|
|
107
112
|
* Can be a custom implementation, 'surrealdb' (default), or 'localstorage'.
|
|
108
113
|
*/
|
|
109
114
|
persistenceClient?: PersistenceClient | 'surrealdb' | 'localstorage';
|
|
110
|
-
/**
|
|
111
|
-
|
|
115
|
+
/** A pino browser transmit object for forwarding logs (e.g. via @spooky-sync/core/otel). */
|
|
116
|
+
otelTransmit?: PinoTransmit;
|
|
112
117
|
/**
|
|
113
118
|
* Debounce time in milliseconds for stream updates.
|
|
114
119
|
* Defaults to 100ms.
|
|
@@ -209,6 +214,8 @@ export interface RunOptions {
|
|
|
209
214
|
assignedTo?: string;
|
|
210
215
|
max_retries?: number;
|
|
211
216
|
retry_strategy?: 'linear' | 'exponential';
|
|
217
|
+
/** Timeout in seconds for the backend HTTP call. Only used if the backend allows timeout override. */
|
|
218
|
+
timeout?: number;
|
|
212
219
|
}
|
|
213
220
|
|
|
214
221
|
/**
|
package/src/utils/index.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
|
|
1
|
+
import type { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
|
|
2
2
|
import { Uuid, RecordId, Duration } from 'surrealdb';
|
|
3
|
-
import { Logger } from '../services/logger/index';
|
|
4
|
-
import { QueryTimeToLive } from '../types';
|
|
3
|
+
import type { Logger } from '../services/logger/index';
|
|
4
|
+
import type { QueryTimeToLive } from '../types';
|
|
5
5
|
|
|
6
6
|
export * from './surql';
|
|
7
7
|
export * from './parser';
|
|
@@ -59,7 +59,7 @@ export function generateNewTableId<S extends SchemaStructure, T extends TableNam
|
|
|
59
59
|
|
|
60
60
|
// ==================== SCHEMA ENCODING/DECODING ====================
|
|
61
61
|
|
|
62
|
-
export function
|
|
62
|
+
export function decodeFromSp00ky<S extends SchemaStructure, T extends TableNames<S>>(
|
|
63
63
|
schema: S,
|
|
64
64
|
tableName: T,
|
|
65
65
|
record: TableModel<GetTable<S, T>>
|
|
@@ -74,19 +74,19 @@ export function decodeFromSpooky<S extends SchemaStructure, T extends TableNames
|
|
|
74
74
|
for (const field of Object.keys(table.columns)) {
|
|
75
75
|
const column = table.columns[field] as any;
|
|
76
76
|
const relation = schema.relationships.find((r) => r.from === tableName && r.field === field);
|
|
77
|
-
if ((column.recordId || relation) && encoded[field]
|
|
77
|
+
if ((column.recordId || relation) && encoded[field] !== null && encoded[field] !== undefined) {
|
|
78
78
|
if (encoded[field] instanceof RecordId) {
|
|
79
79
|
encoded[field] = `${encoded[field].table.toString()}:${encoded[field].id}`;
|
|
80
80
|
} else if (
|
|
81
81
|
relation &&
|
|
82
|
-
(encoded[field] instanceof Object || encoded[field]
|
|
82
|
+
(encoded[field] instanceof Object || Array.isArray(encoded[field]))
|
|
83
83
|
) {
|
|
84
84
|
if (Array.isArray(encoded[field])) {
|
|
85
85
|
encoded[field] = encoded[field].map((item) =>
|
|
86
|
-
|
|
86
|
+
decodeFromSp00ky(schema, relation.to, item)
|
|
87
87
|
);
|
|
88
88
|
} else {
|
|
89
|
-
encoded[field] =
|
|
89
|
+
encoded[field] = decodeFromSp00ky(schema, relation.to, encoded[field]);
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
92
|
}
|
|
@@ -117,7 +117,7 @@ export function parseDuration(duration: QueryTimeToLive | Duration): number {
|
|
|
117
117
|
|
|
118
118
|
const match = duration.match(/^(\d+)([smh])$/);
|
|
119
119
|
if (!match) return 600000;
|
|
120
|
-
const val = parseInt(match[1], 10);
|
|
120
|
+
const val = Number.parseInt(match[1], 10);
|
|
121
121
|
const unit = match[2];
|
|
122
122
|
switch (unit) {
|
|
123
123
|
case 's':
|
|
@@ -165,7 +165,7 @@ export async function withRetry<T>(
|
|
|
165
165
|
attempt: i + 1,
|
|
166
166
|
retries,
|
|
167
167
|
error: msg,
|
|
168
|
-
Category: '
|
|
168
|
+
Category: 'sp00ky-client::utils::withRetry',
|
|
169
169
|
},
|
|
170
170
|
'Retrying DB operation'
|
|
171
171
|
);
|
package/src/utils/parser.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { ColumnSchema
|
|
1
|
+
import type { ColumnSchema} from '@spooky-sync/query-builder';
|
|
2
|
+
import { RecordId } from '@spooky-sync/query-builder';
|
|
2
3
|
import { parseRecordIdString } from './index';
|
|
3
4
|
import { DateTime } from 'surrealdb';
|
|
4
5
|
|