@spooky-sync/core 0.0.1-canary.6 → 0.0.1-canary.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +103 -367
- package/dist/index.js +595 -289
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/types.d.ts +361 -0
- package/package.json +35 -5
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/events/events.test.ts +2 -1
- package/src/index.ts +3 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +17 -20
- package/src/modules/cache/index.ts +23 -23
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +189 -0
- package/src/modules/crdt/index.ts +178 -0
- package/src/modules/data/index.ts +64 -43
- package/src/modules/devtools/index.ts +23 -20
- package/src/modules/sync/engine.ts +31 -21
- package/src/modules/sync/events/index.ts +3 -2
- package/src/modules/sync/queue/queue-down.ts +5 -4
- package/src/modules/sync/queue/queue-up.ts +14 -13
- package/src/modules/sync/scheduler.ts +2 -2
- package/src/modules/sync/sync.ts +49 -36
- package/src/modules/sync/utils.test.ts +2 -2
- package/src/modules/sync/utils.ts +15 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/local-migrator.ts +21 -21
- package/src/services/database/local.ts +16 -15
- package/src/services/database/remote.ts +13 -13
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +34 -0
- package/src/services/persistence/surrealdb.ts +9 -9
- package/src/services/stream-processor/index.ts +32 -31
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +2 -2
- package/src/{spooky.ts → sp00ky.ts} +67 -38
- package/src/types.ts +18 -11
- package/src/utils/index.ts +22 -10
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +15 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +1 -1
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { RecordId } from 'surrealdb';
|
|
2
|
-
import { LocalDatabaseService } from '../../../services/database/index';
|
|
1
|
+
import type { RecordId } from 'surrealdb';
|
|
2
|
+
import type { LocalDatabaseService } from '../../../services/database/index';
|
|
3
|
+
import type {
|
|
4
|
+
SyncQueueEventSystem} from '../events/index';
|
|
3
5
|
import {
|
|
4
6
|
createSyncQueueEventSystem,
|
|
5
|
-
SyncQueueEventSystem,
|
|
6
7
|
SyncQueueEventTypes,
|
|
7
8
|
} from '../events/index';
|
|
8
9
|
import { parseRecordIdString, extractTablePart, classifySyncError } from '../../../utils/index';
|
|
9
|
-
import { Logger } from '../../../services/logger/index';
|
|
10
|
-
import { PushEventOptions } from '../../../events/index';
|
|
10
|
+
import type { Logger } from '../../../services/logger/index';
|
|
11
|
+
import type { PushEventOptions } from '../../../events/index';
|
|
11
12
|
|
|
12
13
|
export type CreateEvent = {
|
|
13
14
|
type: 'create';
|
|
@@ -114,7 +115,7 @@ export class UpQueue {
|
|
|
114
115
|
|
|
115
116
|
if (errorType === 'network') {
|
|
116
117
|
this.logger.error(
|
|
117
|
-
{ error, event, Category: '
|
|
118
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
118
119
|
'Network error processing mutation, re-queuing'
|
|
119
120
|
);
|
|
120
121
|
this.queue.unshift(event);
|
|
@@ -123,14 +124,14 @@ export class UpQueue {
|
|
|
123
124
|
|
|
124
125
|
// Application error — rollback instead of re-queuing
|
|
125
126
|
this.logger.error(
|
|
126
|
-
{ error, event, Category: '
|
|
127
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
127
128
|
'Application error processing mutation, rolling back'
|
|
128
129
|
);
|
|
129
130
|
try {
|
|
130
131
|
await this.removeEventFromDatabase(event.mutation_id);
|
|
131
132
|
} catch (removeError) {
|
|
132
133
|
this.logger.error(
|
|
133
|
-
{ error: removeError, event, Category: '
|
|
134
|
+
{ error: removeError, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
134
135
|
'Failed to remove rolled-back mutation from database'
|
|
135
136
|
);
|
|
136
137
|
}
|
|
@@ -139,7 +140,7 @@ export class UpQueue {
|
|
|
139
140
|
await onRollback(event, error instanceof Error ? error : new Error(String(error)));
|
|
140
141
|
} catch (rollbackError) {
|
|
141
142
|
this.logger.error(
|
|
142
|
-
{ error: rollbackError, event, Category: '
|
|
143
|
+
{ error: rollbackError, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
143
144
|
'Rollback handler failed'
|
|
144
145
|
);
|
|
145
146
|
}
|
|
@@ -154,7 +155,7 @@ export class UpQueue {
|
|
|
154
155
|
await this.removeEventFromDatabase(event.mutation_id);
|
|
155
156
|
} catch (error) {
|
|
156
157
|
this.logger.error(
|
|
157
|
-
{ error, event, Category: '
|
|
158
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
158
159
|
'Failed to remove mutation from database after successful processing'
|
|
159
160
|
);
|
|
160
161
|
}
|
|
@@ -172,7 +173,7 @@ export class UpQueue {
|
|
|
172
173
|
async loadFromDatabase() {
|
|
173
174
|
try {
|
|
174
175
|
const [records] = await this.local.query<any>(
|
|
175
|
-
`SELECT * FROM
|
|
176
|
+
`SELECT * FROM _00_pending_mutations ORDER BY created_at ASC`
|
|
176
177
|
);
|
|
177
178
|
|
|
178
179
|
this.queue = records
|
|
@@ -205,7 +206,7 @@ export class UpQueue {
|
|
|
205
206
|
{
|
|
206
207
|
mutationType: r.mutationType,
|
|
207
208
|
record: r,
|
|
208
|
-
Category: '
|
|
209
|
+
Category: 'sp00ky-client::UpQueue::loadFromDatabase',
|
|
209
210
|
},
|
|
210
211
|
'Unknown mutation type'
|
|
211
212
|
);
|
|
@@ -215,7 +216,7 @@ export class UpQueue {
|
|
|
215
216
|
.filter((e: UpEvent | null): e is UpEvent => e !== null);
|
|
216
217
|
} catch (error) {
|
|
217
218
|
this.logger.error(
|
|
218
|
-
{ error, Category: '
|
|
219
|
+
{ error, Category: 'sp00ky-client::UpQueue::loadFromDatabase' },
|
|
219
220
|
'Failed to load pending mutations from database'
|
|
220
221
|
);
|
|
221
222
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Logger } from '../../services/logger/index';
|
|
2
|
-
import { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
|
|
1
|
+
import type { Logger } from '../../services/logger/index';
|
|
2
|
+
import type { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
|
|
3
3
|
import { SyncQueueEventTypes } from './events/index';
|
|
4
4
|
|
|
5
5
|
/**
|
package/src/modules/sync/sync.ts
CHANGED
|
@@ -1,24 +1,25 @@
|
|
|
1
|
-
import { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
2
|
-
import {
|
|
1
|
+
import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
2
|
+
import type { RecordVersionArray } from '../../types';
|
|
3
3
|
import { createSyncEventSystem, SyncEventTypes, SyncQueueEventTypes } from './events/index';
|
|
4
|
-
import { Logger } from '../../services/logger/index';
|
|
5
|
-
import { DownEvent,
|
|
6
|
-
import {
|
|
4
|
+
import type { Logger } from '../../services/logger/index';
|
|
5
|
+
import type { DownEvent, UpEvent} from './queue/index';
|
|
6
|
+
import { DownQueue, UpQueue } from './queue/index';
|
|
7
|
+
import type { RecordId, Uuid } from 'surrealdb';
|
|
7
8
|
import { ArraySyncer, createDiffFromDbOp } from './utils';
|
|
8
9
|
import { SyncEngine } from './engine';
|
|
9
10
|
import { SyncScheduler } from './scheduler';
|
|
10
|
-
import { SchemaStructure } from '@spooky-sync/query-builder';
|
|
11
|
-
import { CacheModule } from '../cache/index';
|
|
12
|
-
import { DataModule } from '../data/index';
|
|
13
|
-
import { encodeRecordId, extractTablePart,
|
|
11
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
12
|
+
import type { CacheModule } from '../cache/index';
|
|
13
|
+
import type { DataModule } from '../data/index';
|
|
14
|
+
import { encodeRecordId, extractTablePart, surql } from '../../utils/index';
|
|
14
15
|
|
|
15
16
|
/**
|
|
16
|
-
* The main synchronization engine for
|
|
17
|
+
* The main synchronization engine for Sp00ky.
|
|
17
18
|
* Handles the bidirectional synchronization between the local database and the remote backend.
|
|
18
19
|
* Uses a queue-based architecture with 'up' (local to remote) and 'down' (remote to local) queues.
|
|
19
20
|
* @template S The schema structure type.
|
|
20
21
|
*/
|
|
21
|
-
export class
|
|
22
|
+
export class Sp00kySync<S extends SchemaStructure> {
|
|
22
23
|
private clientId: string = '';
|
|
23
24
|
private upQueue: UpQueue;
|
|
24
25
|
private downQueue: DownQueue;
|
|
@@ -59,7 +60,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
59
60
|
private schema: S,
|
|
60
61
|
logger: Logger
|
|
61
62
|
) {
|
|
62
|
-
this.logger = logger.child({ service: '
|
|
63
|
+
this.logger = logger.child({ service: 'Sp00kySync' });
|
|
63
64
|
this.upQueue = new UpQueue(this.local, this.logger);
|
|
64
65
|
this.downQueue = new DownQueue(this.local, this.logger);
|
|
65
66
|
this.syncEngine = new SyncEngine(this.remote, this.cache, this.schema, this.logger);
|
|
@@ -80,7 +81,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
80
81
|
* @throws Error if already initialized.
|
|
81
82
|
*/
|
|
82
83
|
public async init(clientId: string) {
|
|
83
|
-
if (this.isInit) throw new Error('
|
|
84
|
+
if (this.isInit) throw new Error('Sp00kySync is already initialized');
|
|
84
85
|
this.clientId = clientId;
|
|
85
86
|
this.isInit = true;
|
|
86
87
|
await this.scheduler.init();
|
|
@@ -92,17 +93,17 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
92
93
|
|
|
93
94
|
private async startRefLiveQueries() {
|
|
94
95
|
this.logger.debug(
|
|
95
|
-
{ clientId: this.clientId, Category: '
|
|
96
|
+
{ clientId: this.clientId, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
96
97
|
'Starting ref live queries'
|
|
97
98
|
);
|
|
98
99
|
|
|
99
100
|
const [queryUuid] = await this.remote.query<[Uuid]>(
|
|
100
|
-
'LIVE SELECT * FROM
|
|
101
|
+
'LIVE SELECT * FROM _00_list_ref'
|
|
101
102
|
);
|
|
102
103
|
|
|
103
104
|
(await this.remote.getClient().liveOf(queryUuid)).subscribe((message) => {
|
|
104
105
|
this.logger.debug(
|
|
105
|
-
{ message, Category: '
|
|
106
|
+
{ message, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
106
107
|
'Live update received'
|
|
107
108
|
);
|
|
108
109
|
if (message.action === 'KILLED') return;
|
|
@@ -113,7 +114,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
113
114
|
message.value.version as number
|
|
114
115
|
).catch((err) => {
|
|
115
116
|
this.logger.error(
|
|
116
|
-
{ err, Category: '
|
|
117
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
|
|
117
118
|
'Error handling remote list ref change'
|
|
118
119
|
);
|
|
119
120
|
});
|
|
@@ -126,13 +127,25 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
126
127
|
recordId: RecordId,
|
|
127
128
|
version: number
|
|
128
129
|
) {
|
|
130
|
+
if (action === 'DELETE') {
|
|
131
|
+
this.logger.debug(
|
|
132
|
+
{
|
|
133
|
+
queryId: queryId.toString(),
|
|
134
|
+
recordId: recordId.toString(),
|
|
135
|
+
Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
|
|
136
|
+
},
|
|
137
|
+
'Ignoring DELETE on list_ref — should not happen'
|
|
138
|
+
);
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
|
|
129
142
|
const existing = this.dataModule.getQueryById(queryId);
|
|
130
143
|
|
|
131
144
|
if (!existing) {
|
|
132
145
|
this.logger.warn(
|
|
133
146
|
{
|
|
134
147
|
queryId: queryId.toString(),
|
|
135
|
-
Category: '
|
|
148
|
+
Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
|
|
136
149
|
},
|
|
137
150
|
'Received remote update for unknown local query'
|
|
138
151
|
);
|
|
@@ -148,7 +161,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
148
161
|
recordId,
|
|
149
162
|
version,
|
|
150
163
|
localArray,
|
|
151
|
-
Category: '
|
|
164
|
+
Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
|
|
152
165
|
},
|
|
153
166
|
'Live update is being processed'
|
|
154
167
|
);
|
|
@@ -166,12 +179,11 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
166
179
|
|
|
167
180
|
private async processUpEvent(event: UpEvent) {
|
|
168
181
|
this.logger.debug(
|
|
169
|
-
{ event, Category: '
|
|
182
|
+
{ event, Category: 'sp00ky-client::Sp00kySync::processUpEvent' },
|
|
170
183
|
'Processing up event'
|
|
171
184
|
);
|
|
172
|
-
console.log('xx1', event);
|
|
173
185
|
switch (event.type) {
|
|
174
|
-
case 'create':
|
|
186
|
+
case 'create': {
|
|
175
187
|
const dataKeys = Object.keys(event.data).map((key) => ({ key, variable: `data_${key}` }));
|
|
176
188
|
const prefixedParams = Object.fromEntries(
|
|
177
189
|
dataKeys.map(({ key, variable }) => [variable, event.data[key]])
|
|
@@ -182,6 +194,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
182
194
|
...prefixedParams,
|
|
183
195
|
});
|
|
184
196
|
break;
|
|
197
|
+
}
|
|
185
198
|
case 'update':
|
|
186
199
|
await this.remote.query(`UPDATE $id MERGE $data`, {
|
|
187
200
|
id: event.record_id,
|
|
@@ -195,7 +208,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
195
208
|
break;
|
|
196
209
|
default:
|
|
197
210
|
this.logger.error(
|
|
198
|
-
{ event, Category: '
|
|
211
|
+
{ event, Category: 'sp00ky-client::Sp00kySync::processUpEvent' },
|
|
199
212
|
'processUpEvent unknown event type'
|
|
200
213
|
);
|
|
201
214
|
return;
|
|
@@ -215,7 +228,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
215
228
|
recordId,
|
|
216
229
|
tableName,
|
|
217
230
|
error: error.message,
|
|
218
|
-
Category: '
|
|
231
|
+
Category: 'sp00ky-client::Sp00kySync::handleRollback',
|
|
219
232
|
},
|
|
220
233
|
'Rolling back failed mutation'
|
|
221
234
|
);
|
|
@@ -231,7 +244,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
231
244
|
this.logger.warn(
|
|
232
245
|
{
|
|
233
246
|
recordId,
|
|
234
|
-
Category: '
|
|
247
|
+
Category: 'sp00ky-client::Sp00kySync::handleRollback',
|
|
235
248
|
},
|
|
236
249
|
'Cannot rollback update: no beforeRecord available. Down-sync will reconcile.'
|
|
237
250
|
);
|
|
@@ -241,7 +254,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
241
254
|
this.logger.warn(
|
|
242
255
|
{
|
|
243
256
|
recordId,
|
|
244
|
-
Category: '
|
|
257
|
+
Category: 'sp00ky-client::Sp00kySync::handleRollback',
|
|
245
258
|
},
|
|
246
259
|
'Delete rollback not implemented. Down-sync will reconcile.'
|
|
247
260
|
);
|
|
@@ -257,7 +270,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
257
270
|
|
|
258
271
|
private async processDownEvent(event: DownEvent) {
|
|
259
272
|
this.logger.debug(
|
|
260
|
-
{ event, Category: '
|
|
273
|
+
{ event, Category: 'sp00ky-client::Sp00kySync::processDownEvent' },
|
|
261
274
|
'Processing down event'
|
|
262
275
|
);
|
|
263
276
|
switch (event.type) {
|
|
@@ -281,7 +294,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
281
294
|
const queryState = this.dataModule.getQueryByHash(hash);
|
|
282
295
|
if (!queryState) {
|
|
283
296
|
this.logger.warn(
|
|
284
|
-
{ hash, Category: '
|
|
297
|
+
{ hash, Category: 'sp00ky-client::Sp00kySync::syncQuery' },
|
|
285
298
|
'Query not found'
|
|
286
299
|
);
|
|
287
300
|
return;
|
|
@@ -309,7 +322,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
309
322
|
private async registerQuery(queryHash: string) {
|
|
310
323
|
try {
|
|
311
324
|
this.logger.debug(
|
|
312
|
-
{ queryHash, Category: '
|
|
325
|
+
{ queryHash, Category: 'sp00ky-client::Sp00kySync::registerQuery' },
|
|
313
326
|
'Register Query state'
|
|
314
327
|
);
|
|
315
328
|
await this.createRemoteQuery(queryHash);
|
|
@@ -319,7 +332,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
319
332
|
await this.dataModule.notifyQuerySynced(queryHash);
|
|
320
333
|
} catch (e) {
|
|
321
334
|
this.logger.error(
|
|
322
|
-
{ err: e, Category: '
|
|
335
|
+
{ err: e, Category: 'sp00ky-client::Sp00kySync::registerQuery' },
|
|
323
336
|
'registerQuery error'
|
|
324
337
|
);
|
|
325
338
|
throw e;
|
|
@@ -331,7 +344,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
331
344
|
|
|
332
345
|
if (!queryState) {
|
|
333
346
|
this.logger.warn(
|
|
334
|
-
{ queryHash, Category: '
|
|
347
|
+
{ queryHash, Category: 'sp00ky-client::Sp00kySync::createRemoteQuery' },
|
|
335
348
|
'Query to register not found'
|
|
336
349
|
);
|
|
337
350
|
throw new Error('Query to register not found');
|
|
@@ -348,7 +361,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
348
361
|
});
|
|
349
362
|
|
|
350
363
|
const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
|
|
351
|
-
surql.selectByFieldsAnd('
|
|
364
|
+
surql.selectByFieldsAnd('_00_list_ref', ['in'], ['out', 'version']),
|
|
352
365
|
{
|
|
353
366
|
in: queryState.config.id,
|
|
354
367
|
}
|
|
@@ -358,7 +371,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
358
371
|
{
|
|
359
372
|
queryId: encodeRecordId(queryState.config.id),
|
|
360
373
|
items,
|
|
361
|
-
Category: '
|
|
374
|
+
Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
|
|
362
375
|
},
|
|
363
376
|
'Got query record version array from remote'
|
|
364
377
|
);
|
|
@@ -369,7 +382,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
369
382
|
{
|
|
370
383
|
queryId: encodeRecordId(queryState.config.id),
|
|
371
384
|
array,
|
|
372
|
-
Category: '
|
|
385
|
+
Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
|
|
373
386
|
},
|
|
374
387
|
'createdRemoteQuery'
|
|
375
388
|
);
|
|
@@ -384,7 +397,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
384
397
|
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
385
398
|
if (!queryState) {
|
|
386
399
|
this.logger.warn(
|
|
387
|
-
{ queryHash, Category: '
|
|
400
|
+
{ queryHash, Category: 'sp00ky-client::Sp00kySync::heartbeatQuery' },
|
|
388
401
|
'Query to register not found'
|
|
389
402
|
);
|
|
390
403
|
throw new Error('Query to register not found');
|
|
@@ -398,7 +411,7 @@ export class SpookySync<S extends SchemaStructure> {
|
|
|
398
411
|
const queryState = this.dataModule.getQueryByHash(queryHash);
|
|
399
412
|
if (!queryState) {
|
|
400
413
|
this.logger.warn(
|
|
401
|
-
{ queryHash, Category: '
|
|
414
|
+
{ queryHash, Category: 'sp00ky-client::Sp00kySync::cleanupQuery' },
|
|
402
415
|
'Query to register not found'
|
|
403
416
|
);
|
|
404
417
|
throw new Error('Query to register not found');
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
createDiffFromDbOp,
|
|
7
7
|
ArraySyncer,
|
|
8
8
|
} from './utils';
|
|
9
|
-
import { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
9
|
+
import type { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
10
10
|
import { encodeRecordId } from '../../utils/index';
|
|
11
11
|
|
|
12
12
|
function rid(table: string, id: string): RecordId<string> {
|
|
@@ -305,7 +305,7 @@ describe('ArraySyncer', () => {
|
|
|
305
305
|
expect(diff!.removed).toHaveLength(3);
|
|
306
306
|
// Check they come in sorted order
|
|
307
307
|
const removedIds = diff!.removed.map((r) => encodeRecordId(r));
|
|
308
|
-
const sorted = [...removedIds].
|
|
308
|
+
const sorted = [...removedIds].toSorted();
|
|
309
309
|
expect(removedIds).toEqual(sorted);
|
|
310
310
|
});
|
|
311
311
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { RecordId } from 'surrealdb';
|
|
2
|
-
import { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
1
|
+
import type { RecordId } from 'surrealdb';
|
|
2
|
+
import type { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
3
3
|
import { parseRecordIdString, encodeRecordId } from '../../utils/index';
|
|
4
4
|
|
|
5
5
|
export class ArraySyncer {
|
|
@@ -8,8 +8,8 @@ export class ArraySyncer {
|
|
|
8
8
|
private needsSort = false;
|
|
9
9
|
|
|
10
10
|
constructor(localArray: RecordVersionArray, remoteArray: RecordVersionArray) {
|
|
11
|
-
this.remoteArray = remoteArray.
|
|
12
|
-
this.localArray = localArray.
|
|
11
|
+
this.remoteArray = remoteArray.toSorted((a, b) => a[0].localeCompare(b[0]));
|
|
12
|
+
this.localArray = localArray.toSorted((a, b) => a[0].localeCompare(b[0]));
|
|
13
13
|
}
|
|
14
14
|
|
|
15
15
|
/**
|
|
@@ -49,7 +49,6 @@ export class ArraySyncer {
|
|
|
49
49
|
this.localArray.sort((a, b) => a[0].localeCompare(b[0]));
|
|
50
50
|
this.needsSort = false;
|
|
51
51
|
}
|
|
52
|
-
console.log('xxxx555', this.localArray, this.remoteArray);
|
|
53
52
|
const diff = diffRecordVersionArray(this.localArray, this.remoteArray);
|
|
54
53
|
return diff;
|
|
55
54
|
}
|
|
@@ -93,10 +92,12 @@ export function diffRecordVersionArray(
|
|
|
93
92
|
return {
|
|
94
93
|
added: added.map((id) => ({
|
|
95
94
|
id: parseRecordIdString(id),
|
|
95
|
+
// oxlint-disable-next-line no-non-null-assertion
|
|
96
96
|
version: remoteMap.get(id)!,
|
|
97
97
|
})),
|
|
98
98
|
updated: updated.map((id) => ({
|
|
99
99
|
id: parseRecordIdString(id),
|
|
100
|
+
// oxlint-disable-next-line no-non-null-assertion
|
|
100
101
|
version: remoteMap.get(id)!,
|
|
101
102
|
})),
|
|
102
103
|
removed: removed.map(parseRecordIdString),
|
|
@@ -127,7 +128,7 @@ export function applyRecordVersionDiff(
|
|
|
127
128
|
currentMap.set(encodeRecordId(item.id), item.version);
|
|
128
129
|
}
|
|
129
130
|
|
|
130
|
-
return Array.from(currentMap).
|
|
131
|
+
return Array.from(currentMap).toSorted((a, b) => a[0].localeCompare(b[0]));
|
|
131
132
|
}
|
|
132
133
|
|
|
133
134
|
export function createDiffFromDbOp(
|
|
@@ -136,17 +137,14 @@ export function createDiffFromDbOp(
|
|
|
136
137
|
version: number,
|
|
137
138
|
versions?: RecordVersionArray
|
|
138
139
|
): RecordVersionDiff {
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
removed: [],
|
|
148
|
-
};
|
|
149
|
-
}
|
|
140
|
+
const old = versions?.find((record) => record[0] === encodeRecordId(recordId));
|
|
141
|
+
|
|
142
|
+
if (old && old[1] >= version) {
|
|
143
|
+
return {
|
|
144
|
+
added: [],
|
|
145
|
+
updated: [],
|
|
146
|
+
removed: [],
|
|
147
|
+
};
|
|
150
148
|
}
|
|
151
149
|
|
|
152
150
|
if (op === 'CREATE') {
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import type { Level } from 'pino';
|
|
2
|
+
import type { PinoTransmit } from '../types';
|
|
3
|
+
|
|
4
|
+
// Map pino levels to OTEL severity numbers
|
|
5
|
+
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#severity-fields
|
|
6
|
+
function mapLevelToSeverityNumber(level: string): number {
|
|
7
|
+
switch (level) {
|
|
8
|
+
case 'trace':
|
|
9
|
+
return 1;
|
|
10
|
+
case 'debug':
|
|
11
|
+
return 5;
|
|
12
|
+
case 'info':
|
|
13
|
+
return 9;
|
|
14
|
+
case 'warn':
|
|
15
|
+
return 13;
|
|
16
|
+
case 'error':
|
|
17
|
+
return 17;
|
|
18
|
+
case 'fatal':
|
|
19
|
+
return 21;
|
|
20
|
+
default:
|
|
21
|
+
return 9;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async function loadOtelModules(otelEndpoint: string) {
|
|
26
|
+
const [{ LoggerProvider, BatchLogRecordProcessor }, { OTLPLogExporter }, { resourceFromAttributes }, { ATTR_SERVICE_NAME }] =
|
|
27
|
+
await Promise.all([
|
|
28
|
+
import('@opentelemetry/sdk-logs'),
|
|
29
|
+
import('@opentelemetry/exporter-logs-otlp-proto'),
|
|
30
|
+
import('@opentelemetry/resources'),
|
|
31
|
+
import('@opentelemetry/semantic-conventions'),
|
|
32
|
+
]);
|
|
33
|
+
|
|
34
|
+
const resource = resourceFromAttributes({
|
|
35
|
+
[ATTR_SERVICE_NAME]: 'sp00ky-client',
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const exporter = new OTLPLogExporter({
|
|
39
|
+
url: otelEndpoint,
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
const loggerProvider = new LoggerProvider({
|
|
43
|
+
resource,
|
|
44
|
+
processors: [new BatchLogRecordProcessor(exporter)],
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
const otelLoggerCache: Record<string, ReturnType<typeof loggerProvider.getLogger>> = {};
|
|
48
|
+
|
|
49
|
+
return (category: string) => {
|
|
50
|
+
if (!otelLoggerCache[category]) {
|
|
51
|
+
otelLoggerCache[category] = loggerProvider.getLogger(category);
|
|
52
|
+
}
|
|
53
|
+
return otelLoggerCache[category];
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Creates a pino browser transmit object that forwards logs to an OpenTelemetry collector.
|
|
59
|
+
*
|
|
60
|
+
* @example
|
|
61
|
+
* ```ts
|
|
62
|
+
* import { createOtelTransmit } from '@spooky-sync/core/otel';
|
|
63
|
+
*
|
|
64
|
+
* new Sp00kyClient({
|
|
65
|
+
* // ...
|
|
66
|
+
* otelTransmit: createOtelTransmit('http://localhost:4318/v1/logs'),
|
|
67
|
+
* });
|
|
68
|
+
* ```
|
|
69
|
+
*/
|
|
70
|
+
export function createOtelTransmit(endpoint: string, level: Level = 'info'): PinoTransmit {
|
|
71
|
+
// Start loading OTel modules eagerly (don't await — we're synchronous)
|
|
72
|
+
const otelReady = loadOtelModules(endpoint);
|
|
73
|
+
|
|
74
|
+
return {
|
|
75
|
+
level: level,
|
|
76
|
+
send: (levelLabel: string, logEvent: any) => {
|
|
77
|
+
// oxlint-disable-next-line promise/always-return
|
|
78
|
+
otelReady.then((getOtelLogger) => {
|
|
79
|
+
try {
|
|
80
|
+
const messages = [...logEvent.messages];
|
|
81
|
+
const severityNumber = mapLevelToSeverityNumber(levelLabel);
|
|
82
|
+
|
|
83
|
+
// Construct the message body
|
|
84
|
+
let body = '';
|
|
85
|
+
const lastMsg = messages.pop();
|
|
86
|
+
|
|
87
|
+
if (typeof lastMsg === 'string') {
|
|
88
|
+
body = lastMsg;
|
|
89
|
+
} else if (lastMsg) {
|
|
90
|
+
body = JSON.stringify(lastMsg);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
let category = 'sp00ky-client::unknown';
|
|
94
|
+
|
|
95
|
+
const attributes = {};
|
|
96
|
+
for (const msg of messages) {
|
|
97
|
+
if (typeof msg === 'object') {
|
|
98
|
+
if (msg.Category) {
|
|
99
|
+
category = msg.Category;
|
|
100
|
+
delete msg.Category;
|
|
101
|
+
}
|
|
102
|
+
Object.assign(attributes, msg);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Emit to OTEL SDK
|
|
107
|
+
getOtelLogger(category).emit({
|
|
108
|
+
severityNumber: severityNumber,
|
|
109
|
+
severityText: levelLabel.toUpperCase(),
|
|
110
|
+
body: body,
|
|
111
|
+
attributes: {
|
|
112
|
+
...logEvent.bindings[0],
|
|
113
|
+
...attributes,
|
|
114
|
+
},
|
|
115
|
+
timestamp: new Date(logEvent.ts),
|
|
116
|
+
});
|
|
117
|
+
} catch (e) {
|
|
118
|
+
// oxlint-disable-next-line no-console
|
|
119
|
+
console.warn('Failed to transmit log to OTEL endpoint', e);
|
|
120
|
+
}
|
|
121
|
+
}).catch((e) => {
|
|
122
|
+
// oxlint-disable-next-line no-console
|
|
123
|
+
console.warn('Failed to load OpenTelemetry modules', e);
|
|
124
|
+
});
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
}
|
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import { Surreal, SurrealTransaction } from 'surrealdb';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import type { Surreal, SurrealTransaction } from 'surrealdb';
|
|
2
|
+
import type { Logger } from '../logger/index';
|
|
3
|
+
import type {
|
|
4
4
|
DatabaseEventSystem,
|
|
5
|
-
DatabaseEventTypes
|
|
6
|
-
|
|
7
|
-
} from './events/index';
|
|
8
|
-
import { SealedQuery } from '../../utils/surql';
|
|
5
|
+
DatabaseEventTypes} from './events/index';
|
|
6
|
+
import type { SealedQuery } from '../../utils/surql';
|
|
9
7
|
|
|
10
8
|
export abstract class AbstractDatabaseService {
|
|
11
9
|
protected client: Surreal;
|
|
@@ -43,11 +41,12 @@ export abstract class AbstractDatabaseService {
|
|
|
43
41
|
async query<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T> {
|
|
44
42
|
return new Promise((resolve, reject) => {
|
|
45
43
|
this.queryQueue = this.queryQueue
|
|
44
|
+
// oxlint-disable-next-line promise/always-return
|
|
46
45
|
.then(async () => {
|
|
47
46
|
const startTime = performance.now();
|
|
48
47
|
try {
|
|
49
48
|
this.logger.debug(
|
|
50
|
-
{ query, vars, Category: '
|
|
49
|
+
{ query, vars, Category: 'sp00ky-client::Database::query' },
|
|
51
50
|
'Executing query'
|
|
52
51
|
);
|
|
53
52
|
const pending = this.client.query(query, vars);
|
|
@@ -67,7 +66,7 @@ export abstract class AbstractDatabaseService {
|
|
|
67
66
|
|
|
68
67
|
resolve(result);
|
|
69
68
|
this.logger.trace(
|
|
70
|
-
{ query, result, Category: '
|
|
69
|
+
{ query, result, Category: 'sp00ky-client::Database::query' },
|
|
71
70
|
'Query executed successfully'
|
|
72
71
|
);
|
|
73
72
|
} catch (err) {
|
|
@@ -84,9 +83,10 @@ export abstract class AbstractDatabaseService {
|
|
|
84
83
|
});
|
|
85
84
|
|
|
86
85
|
this.logger.error(
|
|
87
|
-
{ query, vars, err, Category: '
|
|
86
|
+
{ query, vars, err, Category: 'sp00ky-client::Database::query' },
|
|
88
87
|
'Query execution failed'
|
|
89
88
|
);
|
|
89
|
+
// oxlint-disable-next-line no-multiple-resolved -- resolve/reject are in try/catch, mutually exclusive
|
|
90
90
|
reject(err);
|
|
91
91
|
}
|
|
92
92
|
})
|
|
@@ -102,7 +102,7 @@ export abstract class AbstractDatabaseService {
|
|
|
102
102
|
}
|
|
103
103
|
|
|
104
104
|
async close(): Promise<void> {
|
|
105
|
-
this.logger.info({ Category: '
|
|
105
|
+
this.logger.info({ Category: 'sp00ky-client::Database::close' }, 'Closing database connection');
|
|
106
106
|
await this.client.close();
|
|
107
107
|
}
|
|
108
108
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { EventDefinition, EventSystem } from '../../../events/index';
|
|
2
|
+
import { createEventSystem } from '../../../events/index';
|
|
2
3
|
|
|
3
4
|
export const DatabaseEventTypes = {
|
|
4
5
|
LocalQuery: 'DATABASE_LOCAL_QUERY',
|