@spooky-sync/core 0.0.1-canary.6 → 0.0.1-canary.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +103 -367
- package/dist/index.js +595 -289
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/types.d.ts +361 -0
- package/package.json +35 -5
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/events/events.test.ts +2 -1
- package/src/index.ts +3 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +17 -20
- package/src/modules/cache/index.ts +23 -23
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +189 -0
- package/src/modules/crdt/index.ts +178 -0
- package/src/modules/data/index.ts +64 -43
- package/src/modules/devtools/index.ts +23 -20
- package/src/modules/sync/engine.ts +31 -21
- package/src/modules/sync/events/index.ts +3 -2
- package/src/modules/sync/queue/queue-down.ts +5 -4
- package/src/modules/sync/queue/queue-up.ts +14 -13
- package/src/modules/sync/scheduler.ts +2 -2
- package/src/modules/sync/sync.ts +49 -36
- package/src/modules/sync/utils.test.ts +2 -2
- package/src/modules/sync/utils.ts +15 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/local-migrator.ts +21 -21
- package/src/services/database/local.ts +16 -15
- package/src/services/database/remote.ts +13 -13
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +34 -0
- package/src/services/persistence/surrealdb.ts +9 -9
- package/src/services/stream-processor/index.ts +32 -31
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +2 -2
- package/src/{spooky.ts → sp00ky.ts} +67 -38
- package/src/types.ts +18 -11
- package/src/utils/index.ts +22 -10
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +15 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +1 -1
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
import { RecordId, Duration } from 'surrealdb';
|
|
2
|
-
import {
|
|
2
|
+
import type {
|
|
3
3
|
SchemaStructure,
|
|
4
4
|
TableNames,
|
|
5
5
|
BackendNames,
|
|
6
6
|
BackendRoutes,
|
|
7
7
|
RoutePayload,
|
|
8
8
|
} from '@spooky-sync/query-builder';
|
|
9
|
-
import { LocalDatabaseService } from '../../services/database/index';
|
|
10
|
-
import { CacheModule, RecordWithId } from '../cache/index';
|
|
11
|
-
import { Logger } from '../../services/logger/index';
|
|
12
|
-
import { StreamUpdate } from '../../services/stream-processor/index';
|
|
13
|
-
import {
|
|
14
|
-
MutationEvent,
|
|
9
|
+
import type { LocalDatabaseService } from '../../services/database/index';
|
|
10
|
+
import type { CacheModule, RecordWithId } from '../cache/index';
|
|
11
|
+
import type { Logger } from '../../services/logger/index';
|
|
12
|
+
import type { StreamUpdate } from '../../services/stream-processor/index';
|
|
13
|
+
import type {
|
|
15
14
|
QueryConfig,
|
|
16
15
|
QueryHash,
|
|
17
16
|
QueryState,
|
|
@@ -21,8 +20,7 @@ import {
|
|
|
21
20
|
RecordVersionArray,
|
|
22
21
|
QueryConfigRecord,
|
|
23
22
|
UpdateOptions,
|
|
24
|
-
RunOptions
|
|
25
|
-
} from '../../types';
|
|
23
|
+
RunOptions} from '../../types';
|
|
26
24
|
import {
|
|
27
25
|
parseRecordIdString,
|
|
28
26
|
extractIdPart,
|
|
@@ -34,8 +32,8 @@ import {
|
|
|
34
32
|
extractTablePart,
|
|
35
33
|
generateId,
|
|
36
34
|
} from '../../utils/index';
|
|
37
|
-
import { CreateEvent, DeleteEvent, UpdateEvent } from '../sync/index';
|
|
38
|
-
import { PushEventOptions } from '../../events/index';
|
|
35
|
+
import type { CreateEvent, DeleteEvent, UpdateEvent } from '../sync/index';
|
|
36
|
+
import type { PushEventOptions } from '../../events/index';
|
|
39
37
|
|
|
40
38
|
/**
|
|
41
39
|
* DataModule - Unified query and mutation management
|
|
@@ -62,7 +60,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
62
60
|
}
|
|
63
61
|
|
|
64
62
|
async init(): Promise<void> {
|
|
65
|
-
this.logger.info({ Category: '
|
|
63
|
+
this.logger.info({ Category: 'sp00ky-client::DataModule::init' }, 'DataModule initialized');
|
|
66
64
|
}
|
|
67
65
|
|
|
68
66
|
// ==================== QUERY MANAGEMENT ====================
|
|
@@ -78,15 +76,15 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
78
76
|
): Promise<QueryHash> {
|
|
79
77
|
const hash = await this.calculateHash({ surql: surqlString, params });
|
|
80
78
|
this.logger.debug(
|
|
81
|
-
{ hash, Category: '
|
|
79
|
+
{ hash, Category: 'sp00ky-client::DataModule::query' },
|
|
82
80
|
'Query Initialization: started'
|
|
83
81
|
);
|
|
84
82
|
|
|
85
|
-
const recordId = new RecordId('
|
|
83
|
+
const recordId = new RecordId('_00_query', hash);
|
|
86
84
|
|
|
87
85
|
if (this.activeQueries.has(hash)) {
|
|
88
86
|
this.logger.debug(
|
|
89
|
-
{ hash, Category: '
|
|
87
|
+
{ hash, Category: 'sp00ky-client::DataModule::query' },
|
|
90
88
|
'Query Initialization: exists, returning'
|
|
91
89
|
);
|
|
92
90
|
return hash;
|
|
@@ -95,7 +93,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
95
93
|
// Another call is already creating this query — wait for it
|
|
96
94
|
if (this.pendingQueries.has(hash)) {
|
|
97
95
|
this.logger.debug(
|
|
98
|
-
{ hash, Category: '
|
|
96
|
+
{ hash, Category: 'sp00ky-client::DataModule::query' },
|
|
99
97
|
'Query Initialization: pending, waiting for existing creation'
|
|
100
98
|
);
|
|
101
99
|
await this.pendingQueries.get(hash);
|
|
@@ -103,7 +101,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
103
101
|
}
|
|
104
102
|
|
|
105
103
|
this.logger.debug(
|
|
106
|
-
{ hash, Category: '
|
|
104
|
+
{ hash, Category: 'sp00ky-client::DataModule::query' },
|
|
107
105
|
'Query Initialization: not found, creating new query'
|
|
108
106
|
);
|
|
109
107
|
|
|
@@ -173,6 +171,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
173
171
|
if (op === 'UPDATE') {
|
|
174
172
|
// Clear existing timer if any
|
|
175
173
|
if (this.debounceTimers.has(queryHash)) {
|
|
174
|
+
// oxlint-disable-next-line no-non-null-assertion -- guarded by .has() check above
|
|
176
175
|
clearTimeout(this.debounceTimers.get(queryHash)!);
|
|
177
176
|
}
|
|
178
177
|
|
|
@@ -194,7 +193,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
194
193
|
const queryState = this.activeQueries.get(queryHash);
|
|
195
194
|
if (!queryState) {
|
|
196
195
|
this.logger.warn(
|
|
197
|
-
{ queryHash, Category: '
|
|
196
|
+
{ queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
|
|
198
197
|
'Received update for unknown query. Skipping...'
|
|
199
198
|
);
|
|
200
199
|
return;
|
|
@@ -221,7 +220,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
221
220
|
queryState.records = newRecords;
|
|
222
221
|
if (prevJson === newJson) {
|
|
223
222
|
this.logger.debug(
|
|
224
|
-
{ queryHash, Category: '
|
|
223
|
+
{ queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
|
|
225
224
|
'Query records unchanged, skipping notification'
|
|
226
225
|
);
|
|
227
226
|
return;
|
|
@@ -241,13 +240,13 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
241
240
|
{
|
|
242
241
|
queryHash,
|
|
243
242
|
recordCount: records?.length,
|
|
244
|
-
Category: '
|
|
243
|
+
Category: 'sp00ky-client::DataModule::onStreamUpdate',
|
|
245
244
|
},
|
|
246
245
|
'Query updated from stream'
|
|
247
246
|
);
|
|
248
247
|
} catch (err) {
|
|
249
248
|
this.logger.error(
|
|
250
|
-
{ err, queryHash, Category: '
|
|
249
|
+
{ err, queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
|
|
251
250
|
'Failed to fetch records for stream update'
|
|
252
251
|
);
|
|
253
252
|
}
|
|
@@ -278,7 +277,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
278
277
|
const queryState = this.activeQueries.get(id);
|
|
279
278
|
if (!queryState) {
|
|
280
279
|
this.logger.warn(
|
|
281
|
-
{ id, Category: '
|
|
280
|
+
{ id, Category: 'sp00ky-client::DataModule::updateQueryLocalArray' },
|
|
282
281
|
'Query to update local array not found'
|
|
283
282
|
);
|
|
284
283
|
return;
|
|
@@ -294,7 +293,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
294
293
|
const queryState = this.getQueryByHash(hash);
|
|
295
294
|
if (!queryState) {
|
|
296
295
|
this.logger.warn(
|
|
297
|
-
{ hash, Category: '
|
|
296
|
+
{ hash, Category: 'sp00ky-client::DataModule::updateQueryRemoteArray' },
|
|
298
297
|
'Query to update remote array not found'
|
|
299
298
|
);
|
|
300
299
|
return;
|
|
@@ -370,6 +369,10 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
370
369
|
retry_strategy: options?.retry_strategy ?? 'linear',
|
|
371
370
|
};
|
|
372
371
|
|
|
372
|
+
if (options?.timeout != null) {
|
|
373
|
+
record.timeout = options.timeout;
|
|
374
|
+
}
|
|
375
|
+
|
|
373
376
|
if (options?.assignedTo) {
|
|
374
377
|
record.assigned_to = options.assignedTo;
|
|
375
378
|
}
|
|
@@ -392,7 +395,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
392
395
|
|
|
393
396
|
const rid = parseRecordIdString(id);
|
|
394
397
|
const params = parseParams(tableSchema.columns, data);
|
|
395
|
-
const mutationId = parseRecordIdString(`
|
|
398
|
+
const mutationId = parseRecordIdString(`_00_pending_mutations:${Date.now()}`);
|
|
396
399
|
|
|
397
400
|
const dataKeys = Object.keys(params).map((key) => ({ key, variable: `data_${key}` }));
|
|
398
401
|
const prefixedParams = Object.fromEntries(
|
|
@@ -441,7 +444,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
441
444
|
callback([mutationEvent]);
|
|
442
445
|
}
|
|
443
446
|
|
|
444
|
-
this.logger.debug({ id, Category: '
|
|
447
|
+
this.logger.debug({ id, Category: 'sp00ky-client::DataModule::create' }, 'Record created');
|
|
445
448
|
|
|
446
449
|
return target;
|
|
447
450
|
}
|
|
@@ -463,7 +466,10 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
463
466
|
|
|
464
467
|
const rid = parseRecordIdString(id);
|
|
465
468
|
const params = parseParams(tableSchema.columns, data);
|
|
466
|
-
const mutationId = parseRecordIdString(`
|
|
469
|
+
const mutationId = parseRecordIdString(`_00_pending_mutations:${Date.now()}`);
|
|
470
|
+
|
|
471
|
+
// Note: CRDT state is pushed directly to the _00_crdt table by CrdtField.pushToRemote(),
|
|
472
|
+
// NOT through the record update pipeline. This keeps the record data clean.
|
|
467
473
|
|
|
468
474
|
// Capture current record state before mutation for rollback support
|
|
469
475
|
const [beforeRecord] = await withRetry(this.logger, () =>
|
|
@@ -472,7 +478,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
472
478
|
|
|
473
479
|
const query = surql.seal<{ target: T }>(
|
|
474
480
|
surql.tx([
|
|
475
|
-
surql.updateSet('id', [{ statement: '
|
|
481
|
+
surql.updateSet('id', [{ statement: '_00_rv += 1' }]),
|
|
476
482
|
surql.let('updated', surql.updateMerge('id', 'data')),
|
|
477
483
|
surql.createMutation('update', 'mid', 'id', 'data'),
|
|
478
484
|
surql.returnObject([{ key: 'target', variable: 'updated' }]),
|
|
@@ -496,8 +502,8 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
496
502
|
updatedFields[key] = (target as Record<string, any>)[key];
|
|
497
503
|
}
|
|
498
504
|
}
|
|
499
|
-
if ('
|
|
500
|
-
updatedFields.
|
|
505
|
+
if ('_00_rv' in (target as Record<string, any>)) {
|
|
506
|
+
updatedFields._00_rv = (target as Record<string, any>)._00_rv;
|
|
501
507
|
}
|
|
502
508
|
this.replaceRecordInQueries(updatedFields);
|
|
503
509
|
|
|
@@ -509,7 +515,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
509
515
|
table: table,
|
|
510
516
|
op: 'UPDATE',
|
|
511
517
|
record: parsedRecord,
|
|
512
|
-
version: target.
|
|
518
|
+
version: target._00_rv as number,
|
|
513
519
|
},
|
|
514
520
|
true
|
|
515
521
|
);
|
|
@@ -531,7 +537,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
531
537
|
callback([mutationEvent]);
|
|
532
538
|
}
|
|
533
539
|
|
|
534
|
-
this.logger.debug({ id, Category: '
|
|
540
|
+
this.logger.debug({ id, Category: 'sp00ky-client::DataModule::update' }, 'Record updated');
|
|
535
541
|
|
|
536
542
|
return target;
|
|
537
543
|
}
|
|
@@ -547,14 +553,29 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
547
553
|
}
|
|
548
554
|
|
|
549
555
|
const rid = parseRecordIdString(id);
|
|
550
|
-
const mutationId = parseRecordIdString(`
|
|
556
|
+
const mutationId = parseRecordIdString(`_00_pending_mutations:${Date.now()}`);
|
|
557
|
+
|
|
558
|
+
// Fetch the record before deleting so DBSP can match it against query predicates
|
|
559
|
+
const [beforeRecords] = await this.local.query<[Record<string, any>[]]>(
|
|
560
|
+
'SELECT * FROM ONLY $id',
|
|
561
|
+
{ id: rid }
|
|
562
|
+
);
|
|
563
|
+
const beforeRecord = beforeRecords ?? {};
|
|
551
564
|
|
|
552
565
|
const query = surql.seal<void>(
|
|
553
566
|
surql.tx([surql.delete('id'), surql.createMutation('delete', 'mid', 'id')])
|
|
554
567
|
);
|
|
555
568
|
|
|
556
569
|
await withRetry(this.logger, () => this.local.execute(query, { id: rid, mid: mutationId }));
|
|
557
|
-
await this.cache.delete(table, id, true);
|
|
570
|
+
await this.cache.delete(table, id, true, beforeRecord);
|
|
571
|
+
|
|
572
|
+
// DBSP may not emit view updates for DELETE ops —
|
|
573
|
+
// manually notify all queries that reference this table
|
|
574
|
+
for (const [queryHash, queryState] of this.activeQueries) {
|
|
575
|
+
if (queryState.config.tableName === tableName) {
|
|
576
|
+
await this.notifyQuerySynced(queryHash);
|
|
577
|
+
}
|
|
578
|
+
}
|
|
558
579
|
|
|
559
580
|
// Emit mutation event
|
|
560
581
|
const mutationEvent: DeleteEvent = {
|
|
@@ -567,7 +588,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
567
588
|
callback([mutationEvent]);
|
|
568
589
|
}
|
|
569
590
|
|
|
570
|
-
this.logger.debug({ id, Category: '
|
|
591
|
+
this.logger.debug({ id, Category: 'sp00ky-client::DataModule::delete' }, 'Record deleted');
|
|
571
592
|
}
|
|
572
593
|
|
|
573
594
|
// ==================== ROLLBACK METHODS ====================
|
|
@@ -586,12 +607,12 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
586
607
|
this.removeRecordFromQueries(recordId);
|
|
587
608
|
|
|
588
609
|
this.logger.info(
|
|
589
|
-
{ id, tableName, Category: '
|
|
610
|
+
{ id, tableName, Category: 'sp00ky-client::DataModule::rollbackCreate' },
|
|
590
611
|
'Rolled back optimistic create'
|
|
591
612
|
);
|
|
592
613
|
} catch (err) {
|
|
593
614
|
this.logger.error(
|
|
594
|
-
{ err, id, tableName, Category: '
|
|
615
|
+
{ err, id, tableName, Category: 'sp00ky-client::DataModule::rollbackCreate' },
|
|
595
616
|
'Failed to rollback create'
|
|
596
617
|
);
|
|
597
618
|
}
|
|
@@ -626,7 +647,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
626
647
|
table: tableName,
|
|
627
648
|
op: 'UPDATE',
|
|
628
649
|
record: parsedRecord,
|
|
629
|
-
version: (beforeRecord.
|
|
650
|
+
version: (beforeRecord._00_rv as number) || 1,
|
|
630
651
|
},
|
|
631
652
|
true
|
|
632
653
|
);
|
|
@@ -635,12 +656,12 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
635
656
|
await this.replaceRecordInQueries(beforeRecord);
|
|
636
657
|
|
|
637
658
|
this.logger.info(
|
|
638
|
-
{ id, tableName, Category: '
|
|
659
|
+
{ id, tableName, Category: 'sp00ky-client::DataModule::rollbackUpdate' },
|
|
639
660
|
'Rolled back optimistic update'
|
|
640
661
|
);
|
|
641
662
|
} catch (err) {
|
|
642
663
|
this.logger.error(
|
|
643
|
-
{ err, id, tableName, Category: '
|
|
664
|
+
{ err, id, tableName, Category: 'sp00ky-client::DataModule::rollbackUpdate' },
|
|
644
665
|
'Failed to rollback update'
|
|
645
666
|
);
|
|
646
667
|
}
|
|
@@ -710,7 +731,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
710
731
|
hash,
|
|
711
732
|
tableName,
|
|
712
733
|
recordCount: queryState.records.length,
|
|
713
|
-
Category: '
|
|
734
|
+
Category: 'sp00ky-client::DataModule::query',
|
|
714
735
|
},
|
|
715
736
|
'Query registered'
|
|
716
737
|
);
|
|
@@ -772,7 +793,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
772
793
|
records = result || [];
|
|
773
794
|
} catch (err) {
|
|
774
795
|
this.logger.warn(
|
|
775
|
-
{ err, Category: '
|
|
796
|
+
{ err, Category: 'sp00ky-client::DataModule::createNewQuery' },
|
|
776
797
|
'Failed to load initial cached records'
|
|
777
798
|
);
|
|
778
799
|
}
|
|
@@ -804,7 +825,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
804
825
|
this.logger.debug(
|
|
805
826
|
{
|
|
806
827
|
id: encodeRecordId(queryState.config.id),
|
|
807
|
-
Category: '
|
|
828
|
+
Category: 'sp00ky-client::DataModule::startTTLHeartbeat',
|
|
808
829
|
},
|
|
809
830
|
'TTL heartbeat'
|
|
810
831
|
);
|
|
@@ -851,7 +872,7 @@ export function parseUpdateOptions(
|
|
|
851
872
|
const delay = options.debounced !== true ? (options.debounced?.delay ?? 200) : 200;
|
|
852
873
|
const keyType = options.debounced !== true ? (options.debounced?.key ?? id) : id;
|
|
853
874
|
const key =
|
|
854
|
-
keyType === 'recordId_x_fields' ? `${id}::${Object.keys(data).
|
|
875
|
+
keyType === 'recordId_x_fields' ? `${id}::${Object.keys(data).toSorted().join('#')}` : id;
|
|
855
876
|
|
|
856
877
|
pushEventOptions = {
|
|
857
878
|
debounced: {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
2
|
-
import { Logger } from '../../services/logger/index';
|
|
3
|
-
import { SchemaStructure } from '@spooky-sync/query-builder';
|
|
1
|
+
import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
2
|
+
import type { Logger } from '../../services/logger/index';
|
|
3
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
4
4
|
import { RecordId } from 'surrealdb';
|
|
5
|
-
import { StreamUpdate, StreamUpdateReceiver } from '../../services/stream-processor/index';
|
|
5
|
+
import type { StreamUpdate, StreamUpdateReceiver } from '../../services/stream-processor/index';
|
|
6
6
|
import { encodeRecordId } from '../../utils/index';
|
|
7
7
|
|
|
8
8
|
// DevTools interfaces (matching extension expectations)
|
|
@@ -13,8 +13,8 @@ export interface DevToolsEvent {
|
|
|
13
13
|
payload: any;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
import { DataModule } from '../data/index';
|
|
17
|
-
import { AuthService } from '../auth/index';
|
|
16
|
+
import type { DataModule } from '../data/index';
|
|
17
|
+
import type { AuthService } from '../auth/index';
|
|
18
18
|
import { AuthEventTypes } from '../auth/events/index';
|
|
19
19
|
|
|
20
20
|
export class DevToolsService implements StreamUpdateReceiver {
|
|
@@ -37,7 +37,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
37
37
|
this.notifyDevTools();
|
|
38
38
|
});
|
|
39
39
|
|
|
40
|
-
this.logger.debug({ Category: '
|
|
40
|
+
this.logger.debug({ Category: 'sp00ky-client::DevToolsService::init' }, 'Service initialized');
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
// Get active queries directly from DataManager (single source of truth)
|
|
@@ -70,7 +70,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
70
70
|
|
|
71
71
|
public onQueryInitialized(payload: any) {
|
|
72
72
|
this.logger.debug(
|
|
73
|
-
{ payload, Category: '
|
|
73
|
+
{ payload, Category: 'sp00ky-client::DevToolsService::onQueryInitialized' },
|
|
74
74
|
'QueryInitialized'
|
|
75
75
|
);
|
|
76
76
|
const queryHash = this.hashString(payload.queryId.toString());
|
|
@@ -87,7 +87,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
87
87
|
this.logger.debug(
|
|
88
88
|
{
|
|
89
89
|
id: payload.queryId?.toString(),
|
|
90
|
-
Category: '
|
|
90
|
+
Category: 'sp00ky-client::DevToolsService::onQueryUpdated',
|
|
91
91
|
},
|
|
92
92
|
'QueryUpdated'
|
|
93
93
|
);
|
|
@@ -102,7 +102,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
102
102
|
|
|
103
103
|
public onStreamUpdate(update: StreamUpdate) {
|
|
104
104
|
this.logger.debug(
|
|
105
|
-
{ update, Category: '
|
|
105
|
+
{ update, Category: 'sp00ky-client::DevToolsService::onStreamUpdate' },
|
|
106
106
|
'StreamUpdate'
|
|
107
107
|
);
|
|
108
108
|
this.addEvent('STREAM_UPDATE', {
|
|
@@ -171,8 +171,8 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
171
171
|
if (typeof window !== 'undefined') {
|
|
172
172
|
window.postMessage(
|
|
173
173
|
{
|
|
174
|
-
type: '
|
|
175
|
-
source: '
|
|
174
|
+
type: 'SP00KY_STATE_CHANGED',
|
|
175
|
+
source: 'sp00ky-devtools-page',
|
|
176
176
|
state: this.getState(),
|
|
177
177
|
},
|
|
178
178
|
'*'
|
|
@@ -229,7 +229,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
229
229
|
|
|
230
230
|
private exposeToWindow() {
|
|
231
231
|
if (typeof window !== 'undefined') {
|
|
232
|
-
(window as any).
|
|
232
|
+
(window as any).__00__ = {
|
|
233
233
|
version: this.version,
|
|
234
234
|
getState: () => this.getState(),
|
|
235
235
|
clearHistory: () => {
|
|
@@ -270,7 +270,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
270
270
|
return this.serializeForDevTools(records) || [];
|
|
271
271
|
} catch (e) {
|
|
272
272
|
this.logger.error(
|
|
273
|
-
{ err: e, Category: '
|
|
273
|
+
{ err: e, Category: 'sp00ky-client::DevToolsService::exposeToWindow' },
|
|
274
274
|
'Failed to get table data'
|
|
275
275
|
);
|
|
276
276
|
return [];
|
|
@@ -299,7 +299,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
299
299
|
runQuery: async (query: string, target: 'local' | 'remote' = 'local') => {
|
|
300
300
|
try {
|
|
301
301
|
this.logger.debug(
|
|
302
|
-
{ query, target, Category: '
|
|
302
|
+
{ query, target, Category: 'sp00ky-client::DevToolsService::runQuery' },
|
|
303
303
|
'Running query (START)'
|
|
304
304
|
);
|
|
305
305
|
const service = target === 'remote' ? this.remoteDatabaseService : this.databaseService;
|
|
@@ -314,7 +314,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
314
314
|
time: queryTime,
|
|
315
315
|
resultType: typeof result,
|
|
316
316
|
isArray: Array.isArray(result),
|
|
317
|
-
Category: '
|
|
317
|
+
Category: 'sp00ky-client::DevToolsService::runQuery',
|
|
318
318
|
},
|
|
319
319
|
'Database returned result'
|
|
320
320
|
);
|
|
@@ -328,7 +328,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
328
328
|
{
|
|
329
329
|
serializeTime,
|
|
330
330
|
serializedLength: JSON.stringify(serialized).length,
|
|
331
|
-
Category: '
|
|
331
|
+
Category: 'sp00ky-client::DevToolsService::runQuery',
|
|
332
332
|
},
|
|
333
333
|
'Serialization complete'
|
|
334
334
|
);
|
|
@@ -340,7 +340,7 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
340
340
|
};
|
|
341
341
|
} catch (e: any) {
|
|
342
342
|
this.logger.error(
|
|
343
|
-
{ err: e, query, target, Category: '
|
|
343
|
+
{ err: e, query, target, Category: 'sp00ky-client::DevToolsService::runQuery' },
|
|
344
344
|
'Query execution failed'
|
|
345
345
|
);
|
|
346
346
|
// Ensure we always return a string for error
|
|
@@ -353,12 +353,15 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
353
353
|
|
|
354
354
|
window.postMessage(
|
|
355
355
|
{
|
|
356
|
-
type: '
|
|
357
|
-
source: '
|
|
356
|
+
type: 'SP00KY_DETECTED',
|
|
357
|
+
source: 'sp00ky-devtools-page',
|
|
358
358
|
data: { version: this.version, detected: true },
|
|
359
359
|
},
|
|
360
360
|
'*'
|
|
361
361
|
);
|
|
362
|
+
|
|
363
|
+
// Dispatch custom event so the devtools page-script can detect late initialization
|
|
364
|
+
window.dispatchEvent(new CustomEvent('sp00ky:init'));
|
|
362
365
|
}
|
|
363
366
|
}
|
|
364
367
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { RecordId } from 'surrealdb';
|
|
2
|
-
import { SchemaStructure } from '@spooky-sync/query-builder';
|
|
3
|
-
import { RemoteDatabaseService } from '../../services/database/index';
|
|
4
|
-
import { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
|
|
5
|
-
import { RecordVersionDiff } from '../../types';
|
|
6
|
-
import { Logger } from '../../services/logger/index';
|
|
1
|
+
import type { RecordId } from 'surrealdb';
|
|
2
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
3
|
+
import type { RemoteDatabaseService } from '../../services/database/index';
|
|
4
|
+
import type { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
|
|
5
|
+
import type { RecordVersionDiff } from '../../types';
|
|
6
|
+
import type { Logger } from '../../services/logger/index';
|
|
7
7
|
import { SyncEventTypes, createSyncEventSystem } from './events/index';
|
|
8
8
|
import { encodeRecordId } from '../../utils/index';
|
|
9
9
|
import { cleanRecord } from '../../utils/parser';
|
|
@@ -12,7 +12,7 @@ import { cleanRecord } from '../../utils/parser';
|
|
|
12
12
|
* SyncEngine handles the core sync operations: fetching remote records,
|
|
13
13
|
* caching them locally, and ingesting into DBSP.
|
|
14
14
|
*
|
|
15
|
-
* This is extracted from
|
|
15
|
+
* This is extracted from Sp00kySync to separate "how to sync" from "when to sync".
|
|
16
16
|
*/
|
|
17
17
|
export class SyncEngine {
|
|
18
18
|
private logger: Logger;
|
|
@@ -24,7 +24,7 @@ export class SyncEngine {
|
|
|
24
24
|
private schema: SchemaStructure,
|
|
25
25
|
logger: Logger
|
|
26
26
|
) {
|
|
27
|
-
this.logger = logger.child({ service: '
|
|
27
|
+
this.logger = logger.child({ service: 'Sp00kySync:SyncEngine' });
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
/**
|
|
@@ -40,7 +40,7 @@ export class SyncEngine {
|
|
|
40
40
|
added,
|
|
41
41
|
updated,
|
|
42
42
|
removed,
|
|
43
|
-
Category: '
|
|
43
|
+
Category: 'sp00ky-client::SyncEngine::syncRecords',
|
|
44
44
|
},
|
|
45
45
|
'SyncEngine.syncRecords diff'
|
|
46
46
|
);
|
|
@@ -57,38 +57,48 @@ export class SyncEngine {
|
|
|
57
57
|
return;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
+
// Build a version map from the diff (versions come from _00_list_ref)
|
|
61
|
+
const versionMap = new Map<string, number>();
|
|
62
|
+
for (const item of toFetch) {
|
|
63
|
+
versionMap.set(encodeRecordId(item.id), item.version);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Fetch records from remote — avoid SELECT *, <subquery> FROM $param
|
|
67
|
+
// pattern which drops the * fields in SurrealDB v3 (known bug).
|
|
68
|
+
// Versions are already known from the diff's list_ref data.
|
|
60
69
|
const [remoteResults] = await this.remote.query<[RecordWithId[]]>(
|
|
61
|
-
|
|
70
|
+
'SELECT * FROM $idsToFetch',
|
|
62
71
|
{ idsToFetch }
|
|
63
72
|
);
|
|
64
|
-
|
|
73
|
+
|
|
65
74
|
// Prepare batch for cache (which handles both DB and DBSP)
|
|
66
75
|
const cacheBatch: CacheRecord[] = [];
|
|
67
76
|
|
|
68
|
-
for (const
|
|
77
|
+
for (const record of remoteResults) {
|
|
69
78
|
if (!record?.id) {
|
|
70
79
|
this.logger.warn(
|
|
71
80
|
{
|
|
72
81
|
record,
|
|
73
82
|
idsToFetch,
|
|
74
|
-
Category: '
|
|
83
|
+
Category: 'sp00ky-client::SyncEngine::syncRecords',
|
|
75
84
|
},
|
|
76
|
-
'Remote record has no id. Skipping record'
|
|
85
|
+
'Remote record has no id (possibly deleted). Skipping record'
|
|
77
86
|
);
|
|
78
87
|
continue;
|
|
79
88
|
}
|
|
80
89
|
const fullId = encodeRecordId(record.id);
|
|
81
90
|
const table = record.id.table.toString();
|
|
82
91
|
const isAdded = added.some((item) => encodeRecordId(item.id) === fullId);
|
|
92
|
+
const version = versionMap.get(fullId) ?? 0;
|
|
83
93
|
|
|
84
94
|
const localVersion = this.cache.lookup(fullId);
|
|
85
|
-
if (localVersion &&
|
|
95
|
+
if (localVersion && version <= localVersion) {
|
|
86
96
|
this.logger.info(
|
|
87
97
|
{
|
|
88
98
|
recordId: fullId,
|
|
89
|
-
version
|
|
99
|
+
version,
|
|
90
100
|
localVersion,
|
|
91
|
-
Category: '
|
|
101
|
+
Category: 'sp00ky-client::SyncEngine::syncRecords',
|
|
92
102
|
},
|
|
93
103
|
'Local version is higher than remote version. Skipping record'
|
|
94
104
|
);
|
|
@@ -103,7 +113,7 @@ export class SyncEngine {
|
|
|
103
113
|
table,
|
|
104
114
|
op: isAdded ? 'CREATE' : 'UPDATE',
|
|
105
115
|
record: cleanedRecord as RecordWithId,
|
|
106
|
-
version
|
|
116
|
+
version,
|
|
107
117
|
});
|
|
108
118
|
}
|
|
109
119
|
|
|
@@ -124,7 +134,7 @@ export class SyncEngine {
|
|
|
124
134
|
this.logger.debug(
|
|
125
135
|
{
|
|
126
136
|
removed: removed.map((r) => r.toString()),
|
|
127
|
-
Category: '
|
|
137
|
+
Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
|
|
128
138
|
},
|
|
129
139
|
'Checking removed records'
|
|
130
140
|
);
|
|
@@ -139,7 +149,7 @@ export class SyncEngine {
|
|
|
139
149
|
// If remote check fails (e.g., SurrealDB parameter serialization issue),
|
|
140
150
|
// proceed with deletion — the caller has already determined these should be removed
|
|
141
151
|
this.logger.debug(
|
|
142
|
-
{ Category: '
|
|
152
|
+
{ Category: 'sp00ky-client::SyncEngine::handleRemovedRecords' },
|
|
143
153
|
'Remote existence check failed, proceeding with deletion'
|
|
144
154
|
);
|
|
145
155
|
}
|
|
@@ -150,7 +160,7 @@ export class SyncEngine {
|
|
|
150
160
|
this.logger.debug(
|
|
151
161
|
{
|
|
152
162
|
recordId: recordIdStr,
|
|
153
|
-
Category: '
|
|
163
|
+
Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
|
|
154
164
|
},
|
|
155
165
|
'Deleting confirmed removed record'
|
|
156
166
|
);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import type { EventDefinition, EventSystem } from '../../../events/index';
|
|
2
|
+
import { createEventSystem } from '../../../events/index';
|
|
3
|
+
import type { RecordVersionArray } from '../../../types';
|
|
3
4
|
|
|
4
5
|
export const SyncQueueEventTypes = {
|
|
5
6
|
MutationEnqueued: 'MUTATION_ENQUEUED',
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { LocalDatabaseService } from '../../../services/database/index';
|
|
1
|
+
import type { LocalDatabaseService } from '../../../services/database/index';
|
|
2
|
+
import type {
|
|
3
|
+
SyncQueueEventSystem} from '../events/index';
|
|
2
4
|
import {
|
|
3
5
|
createSyncQueueEventSystem,
|
|
4
|
-
SyncQueueEventSystem,
|
|
5
6
|
SyncQueueEventTypes,
|
|
6
7
|
} from '../events/index';
|
|
7
|
-
import { Logger } from '../../../services/logger/index';
|
|
8
|
+
import type { Logger } from '../../../services/logger/index';
|
|
8
9
|
|
|
9
10
|
export type RegisterEvent = {
|
|
10
11
|
type: 'register';
|
|
@@ -78,7 +79,7 @@ export class DownQueue {
|
|
|
78
79
|
await fn(event);
|
|
79
80
|
} catch (error) {
|
|
80
81
|
this.logger.error(
|
|
81
|
-
{ error, event, Category: '
|
|
82
|
+
{ error, event, Category: 'sp00ky-client::DownQueue::next' },
|
|
82
83
|
'Failed to process query'
|
|
83
84
|
);
|
|
84
85
|
this.queue.unshift(event);
|