@spooky-sync/core 0.0.1-canary.4 → 0.0.1-canary.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +18 -367
- package/dist/index.js +268 -287
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/types.d.ts +361 -0
- package/package.json +34 -5
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/events/events.test.ts +2 -1
- package/src/index.ts +1 -1
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +17 -20
- package/src/modules/cache/index.ts +23 -23
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/data/index.ts +61 -43
- package/src/modules/devtools/index.ts +23 -20
- package/src/modules/sync/engine.ts +31 -21
- package/src/modules/sync/events/index.ts +3 -2
- package/src/modules/sync/queue/queue-down.ts +5 -4
- package/src/modules/sync/queue/queue-up.ts +14 -13
- package/src/modules/sync/scheduler.ts +2 -2
- package/src/modules/sync/sync.ts +47 -35
- package/src/modules/sync/utils.test.ts +2 -2
- package/src/modules/sync/utils.ts +15 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/local-migrator.ts +21 -21
- package/src/services/database/local.ts +16 -15
- package/src/services/database/remote.ts +13 -13
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +34 -0
- package/src/services/persistence/surrealdb.ts +9 -9
- package/src/services/stream-processor/index.ts +32 -31
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +2 -2
- package/src/{spooky.ts → sp00ky.ts} +40 -38
- package/src/types.ts +18 -11
- package/src/utils/index.ts +10 -10
- package/src/utils/parser.ts +2 -1
- package/src/utils/surql.ts +15 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +1 -1
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { RecordId } from 'surrealdb';
|
|
2
|
-
import { SchemaStructure } from '@spooky-sync/query-builder';
|
|
3
|
-
import { RemoteDatabaseService } from '../../services/database/index';
|
|
4
|
-
import { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
|
|
5
|
-
import { RecordVersionDiff } from '../../types';
|
|
6
|
-
import { Logger } from '../../services/logger/index';
|
|
1
|
+
import type { RecordId } from 'surrealdb';
|
|
2
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
3
|
+
import type { RemoteDatabaseService } from '../../services/database/index';
|
|
4
|
+
import type { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
|
|
5
|
+
import type { RecordVersionDiff } from '../../types';
|
|
6
|
+
import type { Logger } from '../../services/logger/index';
|
|
7
7
|
import { SyncEventTypes, createSyncEventSystem } from './events/index';
|
|
8
8
|
import { encodeRecordId } from '../../utils/index';
|
|
9
9
|
import { cleanRecord } from '../../utils/parser';
|
|
@@ -12,7 +12,7 @@ import { cleanRecord } from '../../utils/parser';
|
|
|
12
12
|
* SyncEngine handles the core sync operations: fetching remote records,
|
|
13
13
|
* caching them locally, and ingesting into DBSP.
|
|
14
14
|
*
|
|
15
|
-
* This is extracted from
|
|
15
|
+
* This is extracted from Sp00kySync to separate "how to sync" from "when to sync".
|
|
16
16
|
*/
|
|
17
17
|
export class SyncEngine {
|
|
18
18
|
private logger: Logger;
|
|
@@ -24,7 +24,7 @@ export class SyncEngine {
|
|
|
24
24
|
private schema: SchemaStructure,
|
|
25
25
|
logger: Logger
|
|
26
26
|
) {
|
|
27
|
-
this.logger = logger.child({ service: '
|
|
27
|
+
this.logger = logger.child({ service: 'Sp00kySync:SyncEngine' });
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
/**
|
|
@@ -40,7 +40,7 @@ export class SyncEngine {
|
|
|
40
40
|
added,
|
|
41
41
|
updated,
|
|
42
42
|
removed,
|
|
43
|
-
Category: '
|
|
43
|
+
Category: 'sp00ky-client::SyncEngine::syncRecords',
|
|
44
44
|
},
|
|
45
45
|
'SyncEngine.syncRecords diff'
|
|
46
46
|
);
|
|
@@ -57,38 +57,48 @@ export class SyncEngine {
|
|
|
57
57
|
return;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
+
// Build a version map from the diff (versions come from _00_list_ref)
|
|
61
|
+
const versionMap = new Map<string, number>();
|
|
62
|
+
for (const item of toFetch) {
|
|
63
|
+
versionMap.set(encodeRecordId(item.id), item.version);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Fetch records from remote — avoid SELECT *, <subquery> FROM $param
|
|
67
|
+
// pattern which drops the * fields in SurrealDB v3 (known bug).
|
|
68
|
+
// Versions are already known from the diff's list_ref data.
|
|
60
69
|
const [remoteResults] = await this.remote.query<[RecordWithId[]]>(
|
|
61
|
-
|
|
70
|
+
'SELECT * FROM $idsToFetch',
|
|
62
71
|
{ idsToFetch }
|
|
63
72
|
);
|
|
64
|
-
|
|
73
|
+
|
|
65
74
|
// Prepare batch for cache (which handles both DB and DBSP)
|
|
66
75
|
const cacheBatch: CacheRecord[] = [];
|
|
67
76
|
|
|
68
|
-
for (const
|
|
77
|
+
for (const record of remoteResults) {
|
|
69
78
|
if (!record?.id) {
|
|
70
79
|
this.logger.warn(
|
|
71
80
|
{
|
|
72
81
|
record,
|
|
73
82
|
idsToFetch,
|
|
74
|
-
Category: '
|
|
83
|
+
Category: 'sp00ky-client::SyncEngine::syncRecords',
|
|
75
84
|
},
|
|
76
|
-
'Remote record has no id. Skipping record'
|
|
85
|
+
'Remote record has no id (possibly deleted). Skipping record'
|
|
77
86
|
);
|
|
78
87
|
continue;
|
|
79
88
|
}
|
|
80
89
|
const fullId = encodeRecordId(record.id);
|
|
81
90
|
const table = record.id.table.toString();
|
|
82
91
|
const isAdded = added.some((item) => encodeRecordId(item.id) === fullId);
|
|
92
|
+
const version = versionMap.get(fullId) ?? 0;
|
|
83
93
|
|
|
84
94
|
const localVersion = this.cache.lookup(fullId);
|
|
85
|
-
if (localVersion &&
|
|
95
|
+
if (localVersion && version <= localVersion) {
|
|
86
96
|
this.logger.info(
|
|
87
97
|
{
|
|
88
98
|
recordId: fullId,
|
|
89
|
-
version
|
|
99
|
+
version,
|
|
90
100
|
localVersion,
|
|
91
|
-
Category: '
|
|
101
|
+
Category: 'sp00ky-client::SyncEngine::syncRecords',
|
|
92
102
|
},
|
|
93
103
|
'Local version is higher than remote version. Skipping record'
|
|
94
104
|
);
|
|
@@ -103,7 +113,7 @@ export class SyncEngine {
|
|
|
103
113
|
table,
|
|
104
114
|
op: isAdded ? 'CREATE' : 'UPDATE',
|
|
105
115
|
record: cleanedRecord as RecordWithId,
|
|
106
|
-
version
|
|
116
|
+
version,
|
|
107
117
|
});
|
|
108
118
|
}
|
|
109
119
|
|
|
@@ -124,7 +134,7 @@ export class SyncEngine {
|
|
|
124
134
|
this.logger.debug(
|
|
125
135
|
{
|
|
126
136
|
removed: removed.map((r) => r.toString()),
|
|
127
|
-
Category: '
|
|
137
|
+
Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
|
|
128
138
|
},
|
|
129
139
|
'Checking removed records'
|
|
130
140
|
);
|
|
@@ -139,7 +149,7 @@ export class SyncEngine {
|
|
|
139
149
|
// If remote check fails (e.g., SurrealDB parameter serialization issue),
|
|
140
150
|
// proceed with deletion — the caller has already determined these should be removed
|
|
141
151
|
this.logger.debug(
|
|
142
|
-
{ Category: '
|
|
152
|
+
{ Category: 'sp00ky-client::SyncEngine::handleRemovedRecords' },
|
|
143
153
|
'Remote existence check failed, proceeding with deletion'
|
|
144
154
|
);
|
|
145
155
|
}
|
|
@@ -150,7 +160,7 @@ export class SyncEngine {
|
|
|
150
160
|
this.logger.debug(
|
|
151
161
|
{
|
|
152
162
|
recordId: recordIdStr,
|
|
153
|
-
Category: '
|
|
163
|
+
Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
|
|
154
164
|
},
|
|
155
165
|
'Deleting confirmed removed record'
|
|
156
166
|
);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import type { EventDefinition, EventSystem } from '../../../events/index';
|
|
2
|
+
import { createEventSystem } from '../../../events/index';
|
|
3
|
+
import type { RecordVersionArray } from '../../../types';
|
|
3
4
|
|
|
4
5
|
export const SyncQueueEventTypes = {
|
|
5
6
|
MutationEnqueued: 'MUTATION_ENQUEUED',
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { LocalDatabaseService } from '../../../services/database/index';
|
|
1
|
+
import type { LocalDatabaseService } from '../../../services/database/index';
|
|
2
|
+
import type {
|
|
3
|
+
SyncQueueEventSystem} from '../events/index';
|
|
2
4
|
import {
|
|
3
5
|
createSyncQueueEventSystem,
|
|
4
|
-
SyncQueueEventSystem,
|
|
5
6
|
SyncQueueEventTypes,
|
|
6
7
|
} from '../events/index';
|
|
7
|
-
import { Logger } from '../../../services/logger/index';
|
|
8
|
+
import type { Logger } from '../../../services/logger/index';
|
|
8
9
|
|
|
9
10
|
export type RegisterEvent = {
|
|
10
11
|
type: 'register';
|
|
@@ -78,7 +79,7 @@ export class DownQueue {
|
|
|
78
79
|
await fn(event);
|
|
79
80
|
} catch (error) {
|
|
80
81
|
this.logger.error(
|
|
81
|
-
{ error, event, Category: '
|
|
82
|
+
{ error, event, Category: 'sp00ky-client::DownQueue::next' },
|
|
82
83
|
'Failed to process query'
|
|
83
84
|
);
|
|
84
85
|
this.queue.unshift(event);
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { RecordId } from 'surrealdb';
|
|
2
|
-
import { LocalDatabaseService } from '../../../services/database/index';
|
|
1
|
+
import type { RecordId } from 'surrealdb';
|
|
2
|
+
import type { LocalDatabaseService } from '../../../services/database/index';
|
|
3
|
+
import type {
|
|
4
|
+
SyncQueueEventSystem} from '../events/index';
|
|
3
5
|
import {
|
|
4
6
|
createSyncQueueEventSystem,
|
|
5
|
-
SyncQueueEventSystem,
|
|
6
7
|
SyncQueueEventTypes,
|
|
7
8
|
} from '../events/index';
|
|
8
9
|
import { parseRecordIdString, extractTablePart, classifySyncError } from '../../../utils/index';
|
|
9
|
-
import { Logger } from '../../../services/logger/index';
|
|
10
|
-
import { PushEventOptions } from '../../../events/index';
|
|
10
|
+
import type { Logger } from '../../../services/logger/index';
|
|
11
|
+
import type { PushEventOptions } from '../../../events/index';
|
|
11
12
|
|
|
12
13
|
export type CreateEvent = {
|
|
13
14
|
type: 'create';
|
|
@@ -114,7 +115,7 @@ export class UpQueue {
|
|
|
114
115
|
|
|
115
116
|
if (errorType === 'network') {
|
|
116
117
|
this.logger.error(
|
|
117
|
-
{ error, event, Category: '
|
|
118
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
118
119
|
'Network error processing mutation, re-queuing'
|
|
119
120
|
);
|
|
120
121
|
this.queue.unshift(event);
|
|
@@ -123,14 +124,14 @@ export class UpQueue {
|
|
|
123
124
|
|
|
124
125
|
// Application error — rollback instead of re-queuing
|
|
125
126
|
this.logger.error(
|
|
126
|
-
{ error, event, Category: '
|
|
127
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
127
128
|
'Application error processing mutation, rolling back'
|
|
128
129
|
);
|
|
129
130
|
try {
|
|
130
131
|
await this.removeEventFromDatabase(event.mutation_id);
|
|
131
132
|
} catch (removeError) {
|
|
132
133
|
this.logger.error(
|
|
133
|
-
{ error: removeError, event, Category: '
|
|
134
|
+
{ error: removeError, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
134
135
|
'Failed to remove rolled-back mutation from database'
|
|
135
136
|
);
|
|
136
137
|
}
|
|
@@ -139,7 +140,7 @@ export class UpQueue {
|
|
|
139
140
|
await onRollback(event, error instanceof Error ? error : new Error(String(error)));
|
|
140
141
|
} catch (rollbackError) {
|
|
141
142
|
this.logger.error(
|
|
142
|
-
{ error: rollbackError, event, Category: '
|
|
143
|
+
{ error: rollbackError, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
143
144
|
'Rollback handler failed'
|
|
144
145
|
);
|
|
145
146
|
}
|
|
@@ -154,7 +155,7 @@ export class UpQueue {
|
|
|
154
155
|
await this.removeEventFromDatabase(event.mutation_id);
|
|
155
156
|
} catch (error) {
|
|
156
157
|
this.logger.error(
|
|
157
|
-
{ error, event, Category: '
|
|
158
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
158
159
|
'Failed to remove mutation from database after successful processing'
|
|
159
160
|
);
|
|
160
161
|
}
|
|
@@ -172,7 +173,7 @@ export class UpQueue {
|
|
|
172
173
|
async loadFromDatabase() {
|
|
173
174
|
try {
|
|
174
175
|
const [records] = await this.local.query<any>(
|
|
175
|
-
`SELECT * FROM
|
|
176
|
+
`SELECT * FROM _00_pending_mutations ORDER BY created_at ASC`
|
|
176
177
|
);
|
|
177
178
|
|
|
178
179
|
this.queue = records
|
|
@@ -205,7 +206,7 @@ export class UpQueue {
|
|
|
205
206
|
{
|
|
206
207
|
mutationType: r.mutationType,
|
|
207
208
|
record: r,
|
|
208
|
-
Category: '
|
|
209
|
+
Category: 'sp00ky-client::UpQueue::loadFromDatabase',
|
|
209
210
|
},
|
|
210
211
|
'Unknown mutation type'
|
|
211
212
|
);
|
|
@@ -215,7 +216,7 @@ export class UpQueue {
|
|
|
215
216
|
.filter((e: UpEvent | null): e is UpEvent => e !== null);
|
|
216
217
|
} catch (error) {
|
|
217
218
|
this.logger.error(
|
|
218
|
-
{ error, Category: '
|
|
219
|
+
{ error, Category: 'sp00ky-client::UpQueue::loadFromDatabase' },
|
|
219
220
|
'Failed to load pending mutations from database'
|
|
220
221
|
);
|
|
221
222
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Logger } from '../../services/logger/index';
|
|
2
|
-
import { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
|
|
1
|
+
import type { Logger } from '../../services/logger/index';
|
|
2
|
+
import type { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
|
|
3
3
|
import { SyncQueueEventTypes } from './events/index';
|
|
4
4
|
|
|
5
5
|
/**
|
package/src/modules/sync/sync.ts
CHANGED
|
@@ -1,24 +1,25 @@
|
|
|
1
|
-
import { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
2
|
-
import {
|
|
1
|
+
import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
2
|
+
import type { RecordVersionArray } from '../../types';
|
|
3
3
|
import { createSyncEventSystem, SyncEventTypes, SyncQueueEventTypes } from './events/index';
|
|
4
|
-
import { Logger } from '../../services/logger/index';
|
|
5
|
-
import { DownEvent,
|
|
6
|
-
import {
|
|
4
|
+
import type { Logger } from '../../services/logger/index';
|
|
5
|
+
import type { DownEvent, UpEvent} from './queue/index';
|
|
6
|
+
import { DownQueue, UpQueue } from './queue/index';
|
|
7
|
+
import type { RecordId, Uuid } from 'surrealdb';
|
|
7
8
|
import { ArraySyncer, createDiffFromDbOp } from './utils';
|
|
8
9
|
import { SyncEngine } from './engine';
|
|
9
10
|
import { SyncScheduler } from './scheduler';
|
|
10
|
-
import { SchemaStructure } from '@spooky-sync/query-builder';
|
|
11
|
-
import { CacheModule } from '../cache/index';
|
|
12
|
-
import { DataModule } from '../data/index';
|
|
13
|
-
import { encodeRecordId, extractTablePart,
|
|
11
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
12
|
+
import type { CacheModule } from '../cache/index';
|
|
13
|
+
import type { DataModule } from '../data/index';
|
|
14
|
+
import { encodeRecordId, extractTablePart, surql } from '../../utils/index';
|
|
14
15
|
|
|
15
16
|
/**
|
|
16
|
-
* The main synchronization engine for
|
|
17
|
+
* The main synchronization engine for Sp00ky.
|
|
17
18
|
* Handles the bidirectional synchronization between the local database and the remote backend.
|
|
18
19
|
* Uses a queue-based architecture with 'up' (local to remote) and 'down' (remote to local) queues.
|
|
19
20
|
* @template S The schema structure type.
|
|
20
21
|
*/
|
|
21
|
-
export class
|
|
22
|
+
export class Sp00kySync<S extends SchemaStructure> {
|
|
22
23
|
private clientId: string = '';
|
|
23
24
|
private upQueue: UpQueue;
|
|
24
25
|
private downQueue: DownQueue;
|
|
@@ -59,7 +60,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
59
60
|
private schema: S,
|
|
60
61
|
logger: Logger
|
|
61
62
|
) {
|
|
62
|
-
this.logger = logger.child({ service: '
|
|
63
|
+
this.logger = logger.child({ service: 'Sp00kySync' });
|
|
63
64
|
this.upQueue = new UpQueue(this.local, this.logger);
|
|
64
65
|
this.downQueue = new DownQueue(this.local, this.logger);
|
|
65
66
|
this.syncEngine = new SyncEngine(this.remote, this.cache, this.schema, this.logger);
|
|
@@ -80,7 +81,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
80
81
|
* @throws Error if already initialized.
|
|
81
82
|
*/
|
|
82
83
|
public async init(clientId: string) {
|
|
83
|
-
if (this.isInit) throw new Error('
|
|
84
|
+
if (this.isInit) throw new Error('Sp00kySync is already initialized');
|
|
84
85
|
this.clientId = clientId;
|
|
85
86
|
this.isInit = true;
|
|
86
87
|
await this.scheduler.init();
|
|
@@ -92,17 +93,17 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
92
93
|
|
|
93
94
|
private async startRefLiveQueries() {
|
|
94
95
|
this.logger.debug(
|
|
95
|
-
{ clientId: this.clientId, Category: '
|
|
96
|
+
{ clientId: this.clientId, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
96
97
|
'Starting ref live queries'
|
|
97
98
|
);
|
|
98
99
|
|
|
99
100
|
const [queryUuid] = await this.remote.query<[Uuid]>(
|
|
100
|
-
'LIVE SELECT * FROM
|
|
101
|
+
'LIVE SELECT * FROM _00_list_ref'
|
|
101
102
|
);
|
|
102
103
|
|
|
103
104
|
(await this.remote.getClient().liveOf(queryUuid)).subscribe((message) => {
|
|
104
105
|
this.logger.debug(
|
|
105
|
-
{ message, Category: '
|
|
106
|
+
{ message, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
106
107
|
'Live update received'
|
|
107
108
|
);
|
|
108
109
|
if (message.action === 'KILLED') return;
|
|
@@ -113,7 +114,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
113
114
|
message.value.version as number
|
|
114
115
|
).catch((err) => {
|
|
115
116
|
this.logger.error(
|
|
116
|
-
{ err, Category: '
|
|
117
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
117
118
|
'Error handling remote list ref change'
|
|
118
119
|
);
|
|
119
120
|
});
|
|
@@ -126,13 +127,25 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
126
127
|
recordId: RecordId,
|
|
127
128
|
version: number
|
|
128
129
|
) {
|
|
130
|
+
if (action === 'DELETE') {
|
|
131
|
+
this.logger.debug(
|
|
132
|
+
{
|
|
133
|
+
queryId: queryId.toString(),
|
|
134
|
+
recordId: recordId.toString(),
|
|
135
|
+
Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
|
|
136
|
+
},
|
|
137
|
+
'Ignoring DELETE on list_ref — should not happen'
|
|
138
|
+
);
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
|
|
129
142
|
const existing = this.dataModule.getQueryById(queryId);
|
|
130
143
|
|
|
131
144
|
if (!existing) {
|
|
132
145
|
this.logger.warn(
|
|
133
146
|
{
|
|
134
147
|
queryId: queryId.toString(),
|
|
135
|
-
Category: '
|
|
148
|
+
Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
|
|
136
149
|
},
|
|
137
150
|
'Received remote update for unknown local query'
|
|
138
151
|
);
|
|
@@ -148,7 +161,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
148
161
|
recordId,
|
|
149
162
|
version,
|
|
150
163
|
localArray,
|
|
151
|
-
Category: '
|
|
164
|
+
Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
|
|
152
165
|
},
|
|
153
166
|
'Live update is being processed'
|
|
154
167
|
);
|
|
@@ -166,10 +179,9 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
166
179
|
|
|
167
180
|
private async processUpEvent(event: UpEvent) {
|
|
168
181
|
this.logger.debug(
|
|
169
|
-
{ event, Category: '
|
|
182
|
+
{ event, Category: 'sp00ky-client::Sp00kySync::processUpEvent' },
|
|
170
183
|
'Processing up event'
|
|
171
184
|
);
|
|
172
|
-
console.log('xx1', event);
|
|
173
185
|
switch (event.type) {
|
|
174
186
|
case 'create':
|
|
175
187
|
const dataKeys = Object.keys(event.data).map((key) => ({ key, variable: `data_${key}` }));
|
|
@@ -195,7 +207,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
195
207
|
break;
|
|
196
208
|
default:
|
|
197
209
|
this.logger.error(
|
|
198
|
-
{ event, Category: '
|
|
210
|
+
{ event, Category: 'sp00ky-client::Sp00kySync::processUpEvent' },
|
|
199
211
|
'processUpEvent unknown event type'
|
|
200
212
|
);
|
|
201
213
|
return;
|
|
@@ -215,7 +227,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
215
227
|
recordId,
|
|
216
228
|
tableName,
|
|
217
229
|
error: error.message,
|
|
218
|
-
Category: '
|
|
230
|
+
Category: 'sp00ky-client::Sp00kySync::handleRollback',
|
|
219
231
|
},
|
|
220
232
|
'Rolling back failed mutation'
|
|
221
233
|
);
|
|
@@ -231,7 +243,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
231
243
|
this.logger.warn(
|
|
232
244
|
{
|
|
233
245
|
recordId,
|
|
234
|
-
Category: '
|
|
246
|
+
Category: 'sp00ky-client::Sp00kySync::handleRollback',
|
|
235
247
|
},
|
|
236
248
|
'Cannot rollback update: no beforeRecord available. Down-sync will reconcile.'
|
|
237
249
|
);
|
|
@@ -241,7 +253,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
241
253
|
this.logger.warn(
|
|
242
254
|
{
|
|
243
255
|
recordId,
|
|
244
|
-
Category: '
|
|
256
|
+
Category: 'sp00ky-client::Sp00kySync::handleRollback',
|
|
245
257
|
},
|
|
246
258
|
'Delete rollback not implemented. Down-sync will reconcile.'
|
|
247
259
|
);
|
|
@@ -257,7 +269,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
257
269
|
|
|
258
270
|
private async processDownEvent(event: DownEvent) {
|
|
259
271
|
this.logger.debug(
|
|
260
|
-
{ event, Category: '
|
|
272
|
+
{ event, Category: 'sp00ky-client::Sp00kySync::processDownEvent' },
|
|
261
273
|
'Processing down event'
|
|
262
274
|
);
|
|
263
275
|
switch (event.type) {
|
|
@@ -281,7 +293,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
281
293
|
const queryState = this.dataModule.getQueryByHash(hash);
|
|
282
294
|
if (!queryState) {
|
|
283
295
|
this.logger.warn(
|
|
284
|
-
{ hash, Category: '
|
|
296
|
+
{ hash, Category: 'sp00ky-client::Sp00kySync::syncQuery' },
|
|
285
297
|
'Query not found'
|
|
286
298
|
);
|
|
287
299
|
return;
|
|
@@ -309,7 +321,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
309
321
|
private async registerQuery(queryHash: string) {
|
|
310
322
|
try {
|
|
311
323
|
this.logger.debug(
|
|
312
|
-
{ queryHash, Category: '
|
|
324
|
+
{ queryHash, Category: 'sp00ky-client::Sp00kySync::registerQuery' },
|
|
313
325
|
'Register Query state'
|
|
314
326
|
);
|
|
315
327
|
await this.createRemoteQuery(queryHash);
|
|
@@ -319,7 +331,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
319
331
|
await this.dataModule.notifyQuerySynced(queryHash);
|
|
320
332
|
} catch (e) {
|
|
321
333
|
this.logger.error(
|
|
322
|
-
{ err: e, Category: '
|
|
334
|
+
{ err: e, Category: 'sp00ky-client::Sp00kySync::registerQuery' },
|
|
323
335
|
'registerQuery error'
|
|
324
336
|
);
|
|
325
337
|
throw e;
|
|
@@ -331,7 +343,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
331
343
|
|
|
332
344
|
if (!queryState) {
|
|
333
345
|
this.logger.warn(
|
|
334
|
-
{ queryHash, Category: '
|
|
346
|
+
{ queryHash, Category: 'sp00ky-client::Sp00kySync::createRemoteQuery' },
|
|
335
347
|
'Query to register not found'
|
|
336
348
|
);
|
|
337
349
|
throw new Error('Query to register not found');
|
|
@@ -348,7 +360,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
348
360
|
});
|
|
349
361
|
|
|
350
362
|
const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
|
|
351
|
-
surql.selectByFieldsAnd('
|
|
363
|
+
surql.selectByFieldsAnd('_00_list_ref', ['in'], ['out', 'version']),
|
|
352
364
|
{
|
|
353
365
|
in: queryState.config.id,
|
|
354
366
|
}
|
|
@@ -358,7 +370,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
358
370
|
{
|
|
359
371
|
queryId: encodeRecordId(queryState.config.id),
|
|
360
372
|
items,
|
|
361
|
-
Category: '
|
|
373
|
+
Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
|
|
362
374
|
},
|
|
363
375
|
'Got query record version array from remote'
|
|
364
376
|
);
|
|
@@ -369,7 +381,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
369
381
|
{
|
|
370
382
|
queryId: encodeRecordId(queryState.config.id),
|
|
371
383
|
array,
|
|
372
|
-
Category: '
|
|
384
|
+
Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
|
|
373
385
|
},
|
|
374
386
|
'createdRemoteQuery'
|
|
375
387
|
);
|
|
@@ -384,7 +396,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
384
396
|
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
385
397
|
if (!queryState) {
|
|
386
398
|
this.logger.warn(
|
|
387
|
-
{ queryHash, Category: '
|
|
399
|
+
{ queryHash, Category: 'sp00ky-client::Sp00kySync::heartbeatQuery' },
|
|
388
400
|
'Query to register not found'
|
|
389
401
|
);
|
|
390
402
|
throw new Error('Query to register not found');
|
|
@@ -398,7 +410,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
398
410
|
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
399
411
|
if (!queryState) {
|
|
400
412
|
this.logger.warn(
|
|
401
|
-
{ queryHash, Category: '
|
|
413
|
+
{ queryHash, Category: 'sp00ky-client::Sp00kySync::cleanupQuery' },
|
|
402
414
|
'Query to register not found'
|
|
403
415
|
);
|
|
404
416
|
throw new Error('Query to register not found');
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
createDiffFromDbOp,
|
|
7
7
|
ArraySyncer,
|
|
8
8
|
} from './utils';
|
|
9
|
-
import { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
9
|
+
import type { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
10
10
|
import { encodeRecordId } from '../../utils/index';
|
|
11
11
|
|
|
12
12
|
function rid(table: string, id: string): RecordId<string> {
|
|
@@ -305,7 +305,7 @@ describe('ArraySyncer', () => {
|
|
|
305
305
|
expect(diff!.removed).toHaveLength(3);
|
|
306
306
|
// Check they come in sorted order
|
|
307
307
|
const removedIds = diff!.removed.map((r) => encodeRecordId(r));
|
|
308
|
-
const sorted = [...removedIds].
|
|
308
|
+
const sorted = [...removedIds].toSorted();
|
|
309
309
|
expect(removedIds).toEqual(sorted);
|
|
310
310
|
});
|
|
311
311
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { RecordId } from 'surrealdb';
|
|
2
|
-
import { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
1
|
+
import type { RecordId } from 'surrealdb';
|
|
2
|
+
import type { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
3
3
|
import { parseRecordIdString, encodeRecordId } from '../../utils/index';
|
|
4
4
|
|
|
5
5
|
export class ArraySyncer {
|
|
@@ -8,8 +8,8 @@ export class ArraySyncer {
|
|
|
8
8
|
private needsSort = false;
|
|
9
9
|
|
|
10
10
|
constructor(localArray: RecordVersionArray, remoteArray: RecordVersionArray) {
|
|
11
|
-
this.remoteArray = remoteArray.
|
|
12
|
-
this.localArray = localArray.
|
|
11
|
+
this.remoteArray = remoteArray.toSorted((a, b) => a[0].localeCompare(b[0]));
|
|
12
|
+
this.localArray = localArray.toSorted((a, b) => a[0].localeCompare(b[0]));
|
|
13
13
|
}
|
|
14
14
|
|
|
15
15
|
/**
|
|
@@ -49,7 +49,6 @@ export class ArraySyncer {
|
|
|
49
49
|
this.localArray.sort((a, b) => a[0].localeCompare(b[0]));
|
|
50
50
|
this.needsSort = false;
|
|
51
51
|
}
|
|
52
|
-
console.log('xxxx555', this.localArray, this.remoteArray);
|
|
53
52
|
const diff = diffRecordVersionArray(this.localArray, this.remoteArray);
|
|
54
53
|
return diff;
|
|
55
54
|
}
|
|
@@ -93,10 +92,12 @@ export function diffRecordVersionArray(
|
|
|
93
92
|
return {
|
|
94
93
|
added: added.map((id) => ({
|
|
95
94
|
id: parseRecordIdString(id),
|
|
95
|
+
// oxlint-disable-next-line no-non-null-assertion
|
|
96
96
|
version: remoteMap.get(id)!,
|
|
97
97
|
})),
|
|
98
98
|
updated: updated.map((id) => ({
|
|
99
99
|
id: parseRecordIdString(id),
|
|
100
|
+
// oxlint-disable-next-line no-non-null-assertion
|
|
100
101
|
version: remoteMap.get(id)!,
|
|
101
102
|
})),
|
|
102
103
|
removed: removed.map(parseRecordIdString),
|
|
@@ -127,7 +128,7 @@ export function applyRecordVersionDiff(
|
|
|
127
128
|
currentMap.set(encodeRecordId(item.id), item.version);
|
|
128
129
|
}
|
|
129
130
|
|
|
130
|
-
return Array.from(currentMap).
|
|
131
|
+
return Array.from(currentMap).toSorted((a, b) => a[0].localeCompare(b[0]));
|
|
131
132
|
}
|
|
132
133
|
|
|
133
134
|
export function createDiffFromDbOp(
|
|
@@ -136,17 +137,14 @@ export function createDiffFromDbOp(
|
|
|
136
137
|
version: number,
|
|
137
138
|
versions?: RecordVersionArray
|
|
138
139
|
): RecordVersionDiff {
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
removed: [],
|
|
148
|
-
};
|
|
149
|
-
}
|
|
140
|
+
const old = versions?.find((record) => record[0] === encodeRecordId(recordId));
|
|
141
|
+
|
|
142
|
+
if (old && old[1] >= version) {
|
|
143
|
+
return {
|
|
144
|
+
added: [],
|
|
145
|
+
updated: [],
|
|
146
|
+
removed: [],
|
|
147
|
+
};
|
|
150
148
|
}
|
|
151
149
|
|
|
152
150
|
if (op === 'CREATE') {
|