@spooky-sync/core 0.0.1-canary.9 → 0.0.1-canary.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +907 -339
- package/dist/index.js +2837 -402
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/types.d.ts +543 -0
- package/package.json +39 -9
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +9 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +41 -29
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +281 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +352 -0
- package/src/modules/data/data.status.test.ts +108 -0
- package/src/modules/data/index.ts +732 -109
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +130 -0
- package/src/modules/devtools/index.ts +115 -21
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.ts +166 -0
- package/src/modules/ref-tables.test.ts +66 -0
- package/src/modules/ref-tables.ts +69 -0
- package/src/modules/sync/engine.ts +101 -37
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.ts +5 -4
- package/src/modules/sync/queue/queue-up.ts +14 -13
- package/src/modules/sync/scheduler.ts +40 -3
- package/src/modules/sync/sync.ts +893 -59
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/local-migrator.ts +21 -21
- package/src/services/database/local.test.ts +33 -0
- package/src/services/database/local.ts +192 -32
- package/src/services/database/remote.ts +13 -13
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +41 -0
- package/src/services/persistence/surrealdb.ts +9 -9
- package/src/services/stream-processor/index.ts +248 -37
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +65 -0
- package/src/sp00ky.ts +648 -0
- package/src/types.ts +192 -15
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +55 -1
- package/src/spooky.ts +0 -392
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { RecordId } from 'surrealdb';
|
|
2
|
-
import { LocalDatabaseService } from '../../../services/database/index';
|
|
1
|
+
import type { RecordId } from 'surrealdb';
|
|
2
|
+
import type { LocalDatabaseService } from '../../../services/database/index';
|
|
3
|
+
import type {
|
|
4
|
+
SyncQueueEventSystem} from '../events/index';
|
|
3
5
|
import {
|
|
4
6
|
createSyncQueueEventSystem,
|
|
5
|
-
SyncQueueEventSystem,
|
|
6
7
|
SyncQueueEventTypes,
|
|
7
8
|
} from '../events/index';
|
|
8
9
|
import { parseRecordIdString, extractTablePart, classifySyncError } from '../../../utils/index';
|
|
9
|
-
import { Logger } from '../../../services/logger/index';
|
|
10
|
-
import { PushEventOptions } from '../../../events/index';
|
|
10
|
+
import type { Logger } from '../../../services/logger/index';
|
|
11
|
+
import type { PushEventOptions } from '../../../events/index';
|
|
11
12
|
|
|
12
13
|
export type CreateEvent = {
|
|
13
14
|
type: 'create';
|
|
@@ -114,7 +115,7 @@ export class UpQueue {
|
|
|
114
115
|
|
|
115
116
|
if (errorType === 'network') {
|
|
116
117
|
this.logger.error(
|
|
117
|
-
{ error, event, Category: '
|
|
118
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
118
119
|
'Network error processing mutation, re-queuing'
|
|
119
120
|
);
|
|
120
121
|
this.queue.unshift(event);
|
|
@@ -123,14 +124,14 @@ export class UpQueue {
|
|
|
123
124
|
|
|
124
125
|
// Application error — rollback instead of re-queuing
|
|
125
126
|
this.logger.error(
|
|
126
|
-
{ error, event, Category: '
|
|
127
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
127
128
|
'Application error processing mutation, rolling back'
|
|
128
129
|
);
|
|
129
130
|
try {
|
|
130
131
|
await this.removeEventFromDatabase(event.mutation_id);
|
|
131
132
|
} catch (removeError) {
|
|
132
133
|
this.logger.error(
|
|
133
|
-
{ error: removeError, event, Category: '
|
|
134
|
+
{ error: removeError, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
134
135
|
'Failed to remove rolled-back mutation from database'
|
|
135
136
|
);
|
|
136
137
|
}
|
|
@@ -139,7 +140,7 @@ export class UpQueue {
|
|
|
139
140
|
await onRollback(event, error instanceof Error ? error : new Error(String(error)));
|
|
140
141
|
} catch (rollbackError) {
|
|
141
142
|
this.logger.error(
|
|
142
|
-
{ error: rollbackError, event, Category: '
|
|
143
|
+
{ error: rollbackError, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
143
144
|
'Rollback handler failed'
|
|
144
145
|
);
|
|
145
146
|
}
|
|
@@ -154,7 +155,7 @@ export class UpQueue {
|
|
|
154
155
|
await this.removeEventFromDatabase(event.mutation_id);
|
|
155
156
|
} catch (error) {
|
|
156
157
|
this.logger.error(
|
|
157
|
-
{ error, event, Category: '
|
|
158
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
158
159
|
'Failed to remove mutation from database after successful processing'
|
|
159
160
|
);
|
|
160
161
|
}
|
|
@@ -172,7 +173,7 @@ export class UpQueue {
|
|
|
172
173
|
async loadFromDatabase() {
|
|
173
174
|
try {
|
|
174
175
|
const [records] = await this.local.query<any>(
|
|
175
|
-
`SELECT * FROM
|
|
176
|
+
`SELECT * FROM _00_pending_mutations ORDER BY created_at ASC`
|
|
176
177
|
);
|
|
177
178
|
|
|
178
179
|
this.queue = records
|
|
@@ -205,7 +206,7 @@ export class UpQueue {
|
|
|
205
206
|
{
|
|
206
207
|
mutationType: r.mutationType,
|
|
207
208
|
record: r,
|
|
208
|
-
Category: '
|
|
209
|
+
Category: 'sp00ky-client::UpQueue::loadFromDatabase',
|
|
209
210
|
},
|
|
210
211
|
'Unknown mutation type'
|
|
211
212
|
);
|
|
@@ -215,7 +216,7 @@ export class UpQueue {
|
|
|
215
216
|
.filter((e: UpEvent | null): e is UpEvent => e !== null);
|
|
216
217
|
} catch (error) {
|
|
217
218
|
this.logger.error(
|
|
218
|
-
{ error, Category: '
|
|
219
|
+
{ error, Category: 'sp00ky-client::UpQueue::loadFromDatabase' },
|
|
219
220
|
'Failed to load pending mutations from database'
|
|
220
221
|
);
|
|
221
222
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Logger } from '../../services/logger/index';
|
|
2
|
-
import { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
|
|
1
|
+
import type { Logger } from '../../services/logger/index';
|
|
2
|
+
import type { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
|
|
3
3
|
import { SyncQueueEventTypes } from './events/index';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -16,7 +16,12 @@ export class SyncScheduler {
|
|
|
16
16
|
private onProcessUp: (event: UpEvent) => Promise<void>,
|
|
17
17
|
private onProcessDown: (event: DownEvent) => Promise<void>,
|
|
18
18
|
private logger: Logger,
|
|
19
|
-
private onRollback?: RollbackCallback
|
|
19
|
+
private onRollback?: RollbackCallback,
|
|
20
|
+
// Reports the outcome of each drained sync round (one syncUp/syncDown pass
|
|
21
|
+
// that actually processed ≥1 item): `ok=true` on a clean drain, `ok=false`
|
|
22
|
+
// with the error when the round halted on a failure. Drives the consumer's
|
|
23
|
+
// sync-health tracking; empty/no-op rounds report nothing.
|
|
24
|
+
private onSyncOutcome?: (ok: boolean, error?: unknown) => void
|
|
20
25
|
) {}
|
|
21
26
|
|
|
22
27
|
async init() {
|
|
@@ -50,10 +55,26 @@ export class SyncScheduler {
|
|
|
50
55
|
async syncUp() {
|
|
51
56
|
if (this.isSyncingUp) return;
|
|
52
57
|
this.isSyncingUp = true;
|
|
58
|
+
let processedAny = false;
|
|
53
59
|
try {
|
|
54
60
|
while (this.upQueue.size > 0) {
|
|
55
61
|
await this.upQueue.next(this.onProcessUp, this.onRollback);
|
|
62
|
+
processedAny = true;
|
|
56
63
|
}
|
|
64
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
65
|
+
} catch (error) {
|
|
66
|
+
this.onSyncOutcome?.(false, error);
|
|
67
|
+
// syncUp runs fire-and-forget — it's wired to the MutationEnqueued event
|
|
68
|
+
// (broadcast synchronously, return value dropped) and is also kicked off
|
|
69
|
+
// via `void this.syncDown()` below. A rejection escaping here therefore
|
|
70
|
+
// surfaces as an *unhandled promise rejection* in the console rather than
|
|
71
|
+
// anything a caller can catch. UpQueue.next already logs the failing item
|
|
72
|
+
// (and re-queues it for retry on the next trigger), so swallow here to
|
|
73
|
+
// keep the failure contained instead of leaking it globally.
|
|
74
|
+
this.logger.debug(
|
|
75
|
+
{ error, Category: 'sp00ky-client::SyncScheduler::syncUp' },
|
|
76
|
+
'syncUp halted on a queue error; item re-queued, will retry on next trigger'
|
|
77
|
+
);
|
|
57
78
|
} finally {
|
|
58
79
|
this.isSyncingUp = false;
|
|
59
80
|
void this.syncDown();
|
|
@@ -68,11 +89,27 @@ export class SyncScheduler {
|
|
|
68
89
|
if (this.upQueue.size > 0) return;
|
|
69
90
|
|
|
70
91
|
this.isSyncingDown = true;
|
|
92
|
+
let processedAny = false;
|
|
71
93
|
try {
|
|
72
94
|
while (this.downQueue.size > 0) {
|
|
73
95
|
if (this.upQueue.size > 0) break;
|
|
74
96
|
await this.downQueue.next(this.onProcessDown);
|
|
97
|
+
processedAny = true;
|
|
75
98
|
}
|
|
99
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
100
|
+
} catch (error) {
|
|
101
|
+
this.onSyncOutcome?.(false, error);
|
|
102
|
+
// Same fire-and-forget story as syncUp: this is the QueryItemEnqueued
|
|
103
|
+
// subscriber (and is also called via `void this.syncDown()`), so a thrown
|
|
104
|
+
// error here becomes an unhandled rejection. The canonical case is a
|
|
105
|
+
// transient remote 500 on `fn::query::register` — DownQueue.next logs it
|
|
106
|
+
// and re-queues the event at the head; we just stop draining this pass and
|
|
107
|
+
// let the next enqueue retry, without spamming the console with an
|
|
108
|
+
// "Uncaught (in promise) ... 500 Internal Server Error".
|
|
109
|
+
this.logger.debug(
|
|
110
|
+
{ error, Category: 'sp00ky-client::SyncScheduler::syncDown' },
|
|
111
|
+
'syncDown halted on a queue error; item re-queued, will retry on next trigger'
|
|
112
|
+
);
|
|
76
113
|
} finally {
|
|
77
114
|
this.isSyncingDown = false;
|
|
78
115
|
}
|