@spooky-sync/core 0.0.1-canary.8 → 0.0.1-canary.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +854 -339
- package/dist/index.js +2633 -396
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/types.d.ts +494 -0
- package/package.json +39 -9
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +9 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +41 -29
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +281 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +352 -0
- package/src/modules/data/data.status.test.ts +108 -0
- package/src/modules/data/index.ts +726 -108
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +130 -0
- package/src/modules/devtools/index.ts +89 -21
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.ts +166 -0
- package/src/modules/ref-tables.test.ts +56 -0
- package/src/modules/ref-tables.ts +57 -0
- package/src/modules/sync/engine.ts +97 -37
- package/src/modules/sync/events/index.ts +3 -2
- package/src/modules/sync/queue/queue-down.ts +5 -4
- package/src/modules/sync/queue/queue-up.ts +14 -13
- package/src/modules/sync/scheduler.ts +2 -2
- package/src/modules/sync/sync.ts +676 -58
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/local-migrator.ts +21 -21
- package/src/services/database/local.test.ts +33 -0
- package/src/services/database/local.ts +192 -32
- package/src/services/database/remote.ts +13 -13
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +41 -0
- package/src/services/persistence/surrealdb.ts +9 -9
- package/src/services/stream-processor/index.ts +248 -37
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +65 -0
- package/src/sp00ky.ts +625 -0
- package/src/types.ts +140 -13
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +55 -1
- package/src/spooky.ts +0 -392
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
2
|
+
import type { DataModule } from '../data/index';
|
|
3
|
+
import type { Sp00kySync } from '../sync/index';
|
|
4
|
+
import type { AuthService } from '../auth/index';
|
|
5
|
+
import type { Logger } from '../../services/logger/index';
|
|
6
|
+
import type { QueryTimeToLive } from '../../types';
|
|
7
|
+
|
|
8
|
+
const FEATURE_QUERY =
|
|
9
|
+
'SELECT key, variant, payload FROM _00_user_feature WHERE key = $key';
|
|
10
|
+
|
|
11
|
+
export interface FeatureFlagSnapshot {
|
|
12
|
+
variant: string | undefined;
|
|
13
|
+
payload: unknown | undefined;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface FeatureFlagOptions {
|
|
17
|
+
fallback?: string;
|
|
18
|
+
ttl?: QueryTimeToLive;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export class FeatureFlagHandle {
|
|
22
|
+
private latest: FeatureFlagSnapshot = { variant: undefined, payload: undefined };
|
|
23
|
+
private listeners = new Set<(s: FeatureFlagSnapshot) => void>();
|
|
24
|
+
private unsubscribeFn: (() => void) | null = null;
|
|
25
|
+
private onCloseFn: (() => void) | null = null;
|
|
26
|
+
private closed = false;
|
|
27
|
+
|
|
28
|
+
constructor(
|
|
29
|
+
public readonly key: string,
|
|
30
|
+
public readonly fallback: string | undefined,
|
|
31
|
+
) {}
|
|
32
|
+
|
|
33
|
+
attach(unsubscribe: () => void): void {
|
|
34
|
+
this.unsubscribeFn?.();
|
|
35
|
+
this.unsubscribeFn = unsubscribe;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
detach(): void {
|
|
39
|
+
this.unsubscribeFn?.();
|
|
40
|
+
this.unsubscribeFn = null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
set(snapshot: FeatureFlagSnapshot): void {
|
|
44
|
+
if (this.closed) return;
|
|
45
|
+
this.latest = snapshot;
|
|
46
|
+
for (const cb of this.listeners) cb(snapshot);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
variant(): string | undefined {
|
|
50
|
+
return this.latest.variant ?? this.fallback;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
payload<T = unknown>(): T | undefined {
|
|
54
|
+
return this.latest.payload as T | undefined;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
enabled(): boolean {
|
|
58
|
+
const v = this.variant();
|
|
59
|
+
return v !== undefined && v !== 'off';
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
subscribe(cb: (s: FeatureFlagSnapshot) => void): () => void {
|
|
63
|
+
this.listeners.add(cb);
|
|
64
|
+
cb({ variant: this.variant(), payload: this.latest.payload });
|
|
65
|
+
return () => {
|
|
66
|
+
this.listeners.delete(cb);
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
onClose(cb: () => void): void {
|
|
71
|
+
this.onCloseFn = cb;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
close(): void {
|
|
75
|
+
if (this.closed) return;
|
|
76
|
+
this.closed = true;
|
|
77
|
+
this.listeners.clear();
|
|
78
|
+
this.detach();
|
|
79
|
+
this.onCloseFn?.();
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export interface FeatureFlagModuleDeps<S extends SchemaStructure> {
|
|
84
|
+
dataModule: DataModule<S>;
|
|
85
|
+
sync: Sp00kySync<S>;
|
|
86
|
+
auth: AuthService<S>;
|
|
87
|
+
logger: Logger;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
interface ActiveHandle {
|
|
91
|
+
handle: FeatureFlagHandle;
|
|
92
|
+
ttl: QueryTimeToLive;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export class FeatureFlagModule<S extends SchemaStructure> {
|
|
96
|
+
private logger: Logger;
|
|
97
|
+
private active = new Set<ActiveHandle>();
|
|
98
|
+
private authUnsubscribe: (() => void) | null = null;
|
|
99
|
+
private lastUserId: string | null = null;
|
|
100
|
+
|
|
101
|
+
constructor(private deps: FeatureFlagModuleDeps<S>) {
|
|
102
|
+
this.logger = deps.logger.child({ service: 'FeatureFlagModule' });
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
init(): void {
|
|
106
|
+
if (this.authUnsubscribe) return;
|
|
107
|
+
this.authUnsubscribe = this.deps.auth.subscribe((userId) => {
|
|
108
|
+
if (userId === this.lastUserId) return;
|
|
109
|
+
this.lastUserId = userId;
|
|
110
|
+
void this.refreshAll();
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
feature(key: string, options: FeatureFlagOptions = {}): FeatureFlagHandle {
|
|
115
|
+
const handle = new FeatureFlagHandle(key, options.fallback);
|
|
116
|
+
const entry: ActiveHandle = { handle, ttl: options.ttl ?? '10m' };
|
|
117
|
+
this.active.add(entry);
|
|
118
|
+
handle.onClose(() => this.active.delete(entry));
|
|
119
|
+
void this.register(entry);
|
|
120
|
+
return handle;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async closeAll(): Promise<void> {
|
|
124
|
+
this.authUnsubscribe?.();
|
|
125
|
+
this.authUnsubscribe = null;
|
|
126
|
+
for (const entry of [...this.active]) entry.handle.close();
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
private async refreshAll(): Promise<void> {
|
|
130
|
+
for (const entry of this.active) {
|
|
131
|
+
entry.handle.detach();
|
|
132
|
+
entry.handle.set({ variant: undefined, payload: undefined });
|
|
133
|
+
await this.register(entry);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
private async register(entry: ActiveHandle): Promise<void> {
|
|
138
|
+
const { handle, ttl } = entry;
|
|
139
|
+
try {
|
|
140
|
+
const hash = await this.deps.dataModule.query(
|
|
141
|
+
'_00_user_feature' as any,
|
|
142
|
+
FEATURE_QUERY,
|
|
143
|
+
{ key: handle.key },
|
|
144
|
+
ttl,
|
|
145
|
+
);
|
|
146
|
+
|
|
147
|
+
this.deps.sync.enqueueDownEvent({ type: 'register', payload: { hash } });
|
|
148
|
+
|
|
149
|
+
const unsub = this.deps.dataModule.subscribe(
|
|
150
|
+
hash,
|
|
151
|
+
(records) => {
|
|
152
|
+
const row = records[0] as { variant?: string; payload?: unknown } | undefined;
|
|
153
|
+
handle.set({ variant: row?.variant, payload: row?.payload });
|
|
154
|
+
},
|
|
155
|
+
{ immediate: true },
|
|
156
|
+
);
|
|
157
|
+
|
|
158
|
+
handle.attach(unsub);
|
|
159
|
+
} catch (err) {
|
|
160
|
+
this.logger.warn(
|
|
161
|
+
{ err, key: handle.key, Category: 'sp00ky-client::FeatureFlagModule::register' },
|
|
162
|
+
'Failed to register feature flag query',
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { listRefTableFor, sanitizeUserId } from './ref-tables';
|
|
4
|
+
|
|
5
|
+
describe('listRefTableFor', () => {
|
|
6
|
+
it('returns global table in single mode regardless of user', () => {
|
|
7
|
+
expect(listRefTableFor('single', 'user:abc')).toBe('_00_list_ref');
|
|
8
|
+
expect(listRefTableFor('single', null)).toBe('_00_list_ref');
|
|
9
|
+
expect(listRefTableFor('single', undefined)).toBe('_00_list_ref');
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
it('returns per-user table in dedicated mode with valid user id', () => {
|
|
13
|
+
expect(listRefTableFor('dedicated', 'user:abc')).toBe(
|
|
14
|
+
'_00_list_ref_user_abc'
|
|
15
|
+
);
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it('accepts a RecordId object in dedicated mode', () => {
|
|
19
|
+
const rid = new RecordId('user', 'def');
|
|
20
|
+
expect(listRefTableFor('dedicated', rid)).toBe('_00_list_ref_user_def');
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
it('falls back to global in dedicated mode when user id is missing', () => {
|
|
24
|
+
expect(listRefTableFor('dedicated', null)).toBe('_00_list_ref');
|
|
25
|
+
expect(listRefTableFor('dedicated', undefined)).toBe('_00_list_ref');
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('falls back to global in dedicated mode when user id has invalid chars', () => {
|
|
29
|
+
// SurrealDB table identifiers only accept alphanumerics + underscore.
|
|
30
|
+
expect(listRefTableFor('dedicated', 'user:abc-with-dash')).toBe(
|
|
31
|
+
'_00_list_ref'
|
|
32
|
+
);
|
|
33
|
+
expect(listRefTableFor('dedicated', 'user:abc.dot')).toBe('_00_list_ref');
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
describe('sanitizeUserId', () => {
|
|
38
|
+
it('strips the user: prefix', () => {
|
|
39
|
+
expect(sanitizeUserId('user:abc123')).toBe('abc123');
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it('accepts plain ids without the user: prefix', () => {
|
|
43
|
+
expect(sanitizeUserId('abc123')).toBe('abc123');
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('accepts RecordId objects', () => {
|
|
47
|
+
expect(sanitizeUserId(new RecordId('user', 'xyz'))).toBe('xyz');
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it('returns null for invalid id shapes', () => {
|
|
51
|
+
expect(sanitizeUserId(null)).toBeNull();
|
|
52
|
+
expect(sanitizeUserId(undefined)).toBeNull();
|
|
53
|
+
expect(sanitizeUserId('user:')).toBeNull();
|
|
54
|
+
expect(sanitizeUserId('user:has-dash')).toBeNull();
|
|
55
|
+
});
|
|
56
|
+
});
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// Client-side mirror of `packages/ssp-protocol/src/lib.rs`'s
|
|
2
|
+
// `query_table_for` / `list_ref_table_for`. Same naming convention so
|
|
3
|
+
// the LIVE subscription, the initial-fetch read, and the SSP's writes
|
|
4
|
+
// all land on the same table.
|
|
5
|
+
//
|
|
6
|
+
// The mode is currently hardcoded to `dedicated` because that's the
|
|
7
|
+
// only mode the e2e suite exercises and threading the value through
|
|
8
|
+
// codegen wasn't necessary to land the cross-session fix. If single
|
|
9
|
+
// mode ever needs to be exposed from the TS client too, the SSP server
|
|
10
|
+
// already reads it from `SPKY_SSP_REF_MODE`; add a matching codegen
|
|
11
|
+
// export then.
|
|
12
|
+
|
|
13
|
+
export type RefMode = 'single' | 'dedicated';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Default ref-storage mode for this client build. Mirrors the SSP's
|
|
17
|
+
* default (`RefMode::Dedicated`) so cross-session sync works out of the
|
|
18
|
+
* box.
|
|
19
|
+
*/
|
|
20
|
+
export const DEFAULT_REF_MODE: RefMode = 'dedicated';
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Sanitize a user record id (e.g. `"user:abc"`) into the segment that
|
|
24
|
+
* goes into a dedicated table name (e.g. `"abc"`). Returns `null` if
|
|
25
|
+
* the id is missing the `user:` prefix or contains characters that
|
|
26
|
+
* aren't valid in a SurrealDB table identifier — the server-side
|
|
27
|
+
* `ssp_protocol::sanitize_user_id` uses the same predicate.
|
|
28
|
+
*
|
|
29
|
+
* Accepts both string ids (`"user:abc"`) and SurrealDB `RecordId`
|
|
30
|
+
* objects (which only stringify cleanly via `.toString()`), since
|
|
31
|
+
* `AuthService` passes the record-id object as-is to its subscribers.
|
|
32
|
+
*/
|
|
33
|
+
export function sanitizeUserId(userId: unknown): string | null {
|
|
34
|
+
if (userId === null || userId === undefined) return null;
|
|
35
|
+
const asString =
|
|
36
|
+
typeof userId === 'string'
|
|
37
|
+
? userId
|
|
38
|
+
: typeof (userId as { toString?: unknown }).toString === 'function'
|
|
39
|
+
? (userId as { toString: () => string }).toString()
|
|
40
|
+
: null;
|
|
41
|
+
if (!asString) return null;
|
|
42
|
+
const raw = asString.startsWith('user:') ? asString.slice('user:'.length) : asString;
|
|
43
|
+
if (raw.length === 0) return null;
|
|
44
|
+
if (!/^[A-Za-z0-9_]+$/.test(raw)) return null;
|
|
45
|
+
return raw;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Returns the `_00_list_ref` table name for `(mode, userId)`. Falls
|
|
50
|
+
* back to the global `_00_list_ref` when sanitization fails or in
|
|
51
|
+
* single mode.
|
|
52
|
+
*/
|
|
53
|
+
export function listRefTableFor(mode: RefMode, userId: unknown): string {
|
|
54
|
+
if (mode === 'single') return '_00_list_ref';
|
|
55
|
+
const uid = sanitizeUserId(userId);
|
|
56
|
+
return uid ? `_00_list_ref_user_${uid}` : '_00_list_ref';
|
|
57
|
+
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { RecordId } from 'surrealdb';
|
|
2
|
-
import { SchemaStructure } from '@spooky-sync/query-builder';
|
|
3
|
-
import { RemoteDatabaseService } from '../../services/database/index';
|
|
4
|
-
import { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
|
|
5
|
-
import { RecordVersionDiff } from '../../types';
|
|
6
|
-
import { Logger } from '../../services/logger/index';
|
|
1
|
+
import type { RecordId } from 'surrealdb';
|
|
2
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
3
|
+
import type { RemoteDatabaseService } from '../../services/database/index';
|
|
4
|
+
import type { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
|
|
5
|
+
import type { RecordVersionDiff } from '../../types';
|
|
6
|
+
import type { Logger } from '../../services/logger/index';
|
|
7
7
|
import { SyncEventTypes, createSyncEventSystem } from './events/index';
|
|
8
8
|
import { encodeRecordId } from '../../utils/index';
|
|
9
9
|
import { cleanRecord } from '../../utils/parser';
|
|
@@ -12,7 +12,7 @@ import { cleanRecord } from '../../utils/parser';
|
|
|
12
12
|
* SyncEngine handles the core sync operations: fetching remote records,
|
|
13
13
|
* caching them locally, and ingesting into DBSP.
|
|
14
14
|
*
|
|
15
|
-
* This is extracted from
|
|
15
|
+
* This is extracted from Sp00kySync to separate "how to sync" from "when to sync".
|
|
16
16
|
*/
|
|
17
17
|
export class SyncEngine {
|
|
18
18
|
private logger: Logger;
|
|
@@ -24,7 +24,7 @@ export class SyncEngine {
|
|
|
24
24
|
private schema: SchemaStructure,
|
|
25
25
|
logger: Logger
|
|
26
26
|
) {
|
|
27
|
-
this.logger = logger.child({ service: '
|
|
27
|
+
this.logger = logger.child({ service: 'Sp00kySync:SyncEngine' });
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
/**
|
|
@@ -32,7 +32,9 @@ export class SyncEngine {
|
|
|
32
32
|
* Main entry point for sync operations.
|
|
33
33
|
* Uses batch processing to minimize events emitted.
|
|
34
34
|
*/
|
|
35
|
-
async syncRecords(
|
|
35
|
+
async syncRecords(
|
|
36
|
+
diff: RecordVersionDiff
|
|
37
|
+
): Promise<{ remoteFetchMs: number; stillRemoteIds: string[] }> {
|
|
36
38
|
const { added, updated, removed } = diff;
|
|
37
39
|
|
|
38
40
|
this.logger.debug(
|
|
@@ -40,55 +42,70 @@ export class SyncEngine {
|
|
|
40
42
|
added,
|
|
41
43
|
updated,
|
|
42
44
|
removed,
|
|
43
|
-
Category: '
|
|
45
|
+
Category: 'sp00ky-client::SyncEngine::syncRecords',
|
|
44
46
|
},
|
|
45
47
|
'SyncEngine.syncRecords diff'
|
|
46
48
|
);
|
|
47
49
|
|
|
48
|
-
// Handle removed records: verify they don't exist remotely before deleting
|
|
50
|
+
// Handle removed records: verify they don't exist remotely before deleting
|
|
51
|
+
// locally. Returns ids that LEFT the view's list_ref but still exist upstream
|
|
52
|
+
// (so they weren't deleted) — the caller converges localArray to drop them.
|
|
53
|
+
let stillRemoteIds: string[] = [];
|
|
49
54
|
if (removed.length > 0) {
|
|
50
|
-
await this.handleRemovedRecords(removed);
|
|
55
|
+
stillRemoteIds = await this.handleRemovedRecords(removed);
|
|
51
56
|
}
|
|
52
57
|
|
|
53
58
|
// Fetch added/updated records from remote
|
|
54
59
|
const toFetch = [...added, ...updated];
|
|
55
60
|
const idsToFetch = toFetch.map((x) => x.id);
|
|
56
61
|
if (idsToFetch.length === 0) {
|
|
57
|
-
return;
|
|
62
|
+
return { remoteFetchMs: 0, stillRemoteIds };
|
|
58
63
|
}
|
|
59
64
|
|
|
65
|
+
// Build a version map from the diff (versions come from _00_list_ref)
|
|
66
|
+
const versionMap = new Map<string, number>();
|
|
67
|
+
for (const item of toFetch) {
|
|
68
|
+
versionMap.set(encodeRecordId(item.id), item.version);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Fetch records from remote — avoid SELECT *, <subquery> FROM $param
|
|
72
|
+
// pattern which drops the * fields in SurrealDB v3 (known bug).
|
|
73
|
+
// Versions are already known from the diff's list_ref data.
|
|
74
|
+
const remoteFetchStart = performance.now();
|
|
60
75
|
const [remoteResults] = await this.remote.query<[RecordWithId[]]>(
|
|
61
|
-
|
|
76
|
+
'SELECT * FROM $idsToFetch',
|
|
62
77
|
{ idsToFetch }
|
|
63
78
|
);
|
|
64
|
-
|
|
79
|
+
const remoteFetchMs = performance.now() - remoteFetchStart;
|
|
80
|
+
|
|
65
81
|
// Prepare batch for cache (which handles both DB and DBSP)
|
|
66
82
|
const cacheBatch: CacheRecord[] = [];
|
|
67
83
|
|
|
68
|
-
for (const
|
|
84
|
+
for (const record of remoteResults) {
|
|
69
85
|
if (!record?.id) {
|
|
70
86
|
this.logger.warn(
|
|
71
87
|
{
|
|
72
88
|
record,
|
|
73
89
|
idsToFetch,
|
|
74
|
-
Category: '
|
|
90
|
+
Category: 'sp00ky-client::SyncEngine::syncRecords',
|
|
75
91
|
},
|
|
76
|
-
'Remote record has no id. Skipping record'
|
|
92
|
+
'Remote record has no id (possibly deleted). Skipping record'
|
|
77
93
|
);
|
|
78
94
|
continue;
|
|
79
95
|
}
|
|
80
96
|
const fullId = encodeRecordId(record.id);
|
|
81
97
|
const table = record.id.table.toString();
|
|
82
98
|
const isAdded = added.some((item) => encodeRecordId(item.id) === fullId);
|
|
99
|
+
const version = versionMap.get(fullId) ?? 0;
|
|
83
100
|
|
|
84
101
|
const localVersion = this.cache.lookup(fullId);
|
|
85
|
-
if (localVersion &&
|
|
102
|
+
if (localVersion && version <= localVersion) {
|
|
86
103
|
this.logger.info(
|
|
87
104
|
{
|
|
88
105
|
recordId: fullId,
|
|
89
|
-
version
|
|
106
|
+
version,
|
|
90
107
|
localVersion,
|
|
91
|
-
Category: '
|
|
108
|
+
Category: 'sp00ky-client::SyncEngine::syncRecords',
|
|
92
109
|
},
|
|
93
110
|
'Local version is higher than remote version. Skipping record'
|
|
94
111
|
);
|
|
@@ -103,7 +120,7 @@ export class SyncEngine {
|
|
|
103
120
|
table,
|
|
104
121
|
op: isAdded ? 'CREATE' : 'UPDATE',
|
|
105
122
|
record: cleanedRecord as RecordWithId,
|
|
106
|
-
version
|
|
123
|
+
version,
|
|
107
124
|
});
|
|
108
125
|
}
|
|
109
126
|
|
|
@@ -115,49 +132,92 @@ export class SyncEngine {
|
|
|
115
132
|
this.events.emit(SyncEventTypes.RemoteDataIngested, {
|
|
116
133
|
records: remoteResults,
|
|
117
134
|
});
|
|
135
|
+
|
|
136
|
+
return { remoteFetchMs, stillRemoteIds };
|
|
118
137
|
}
|
|
119
138
|
|
|
120
139
|
/**
|
|
121
140
|
* Handle records that exist locally but not in remote array.
|
|
141
|
+
*
|
|
142
|
+
* "Removed" here is a derived signal: the SSP's `_00_list_ref` array no
|
|
143
|
+
* longer references a record that exists locally. That can mean the row
|
|
144
|
+
* was genuinely deleted upstream — but it can also be a benign race
|
|
145
|
+
* (e.g. a record we just created hasn't propagated into the SSP's
|
|
146
|
+
* incantation list yet). Before deleting locally we verify against
|
|
147
|
+
* upstream SurrealDB: if the row still exists there, skip the delete.
|
|
148
|
+
*
|
|
149
|
+
* On verification failure we skip deletion too. Losing a stale local
|
|
150
|
+
* row to a later sync round is recoverable; deleting a fresh row that
|
|
151
|
+
* upstream still has is not.
|
|
122
152
|
*/
|
|
123
|
-
private async handleRemovedRecords(removed: RecordId[]): Promise<
|
|
153
|
+
private async handleRemovedRecords(removed: RecordId[]): Promise<string[]> {
|
|
124
154
|
this.logger.debug(
|
|
125
155
|
{
|
|
126
156
|
removed: removed.map((r) => r.toString()),
|
|
127
|
-
Category: '
|
|
157
|
+
Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
|
|
128
158
|
},
|
|
129
159
|
'Checking removed records'
|
|
130
160
|
);
|
|
131
161
|
|
|
132
|
-
|
|
162
|
+
// Confirm which of the "removed" ids still exist remotely by selecting the
|
|
163
|
+
// records directly: `SELECT id FROM $ids` (the records to fetch ARE the
|
|
164
|
+
// FROM target).
|
|
165
|
+
//
|
|
166
|
+
// We must NOT use `WHERE id IN $ids` here: on SurrealDB v3.1.x, record-id
|
|
167
|
+
// matching with `IN` is broken — `SELECT id FROM <table> WHERE id IN
|
|
168
|
+
// [<recordid>]` returns NO rows even for an existing record (plain-field
|
|
169
|
+
// `IN` works; record-id `IN` does not). That made this check report EVERY
|
|
170
|
+
// removed id as gone and DELETE live local records (e.g. a freshly-created
|
|
171
|
+
// collection vanished mid-session). `SELECT id FROM $ids` matches correctly.
|
|
172
|
+
// (The old comment claimed `FROM $ids` returned Internal/0 on v3.0; it works
|
|
173
|
+
// on v3.1, and even if it ever errored the catch below skips deletion —
|
|
174
|
+
// strictly safer than the silent empty-result the `IN` form produced.)
|
|
175
|
+
let existingRemoteIds: Set<string>;
|
|
133
176
|
try {
|
|
134
|
-
const [
|
|
135
|
-
ids
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
// proceed with deletion — the caller has already determined these should be removed
|
|
141
|
-
this.logger.debug(
|
|
142
|
-
{ Category: 'spooky-client::SyncEngine::handleRemovedRecords' },
|
|
143
|
-
'Remote existence check failed, proceeding with deletion'
|
|
177
|
+
const [existing] = await this.remote.query<[{ id: RecordId }[]]>(
|
|
178
|
+
'SELECT id FROM $ids',
|
|
179
|
+
{ ids: removed }
|
|
180
|
+
);
|
|
181
|
+
existingRemoteIds = new Set(
|
|
182
|
+
(existing ?? []).map((row) => encodeRecordId(row.id))
|
|
144
183
|
);
|
|
184
|
+
} catch (err) {
|
|
185
|
+
// Verification failed. Skip deletion entirely — the next sync
|
|
186
|
+
// round re-derives the diff and we get another shot. The
|
|
187
|
+
// alternative (delete on uncertainty) destroys freshly-created
|
|
188
|
+
// rows when the SSP hasn't yet refreshed `_00_list_ref`.
|
|
189
|
+
this.logger.warn(
|
|
190
|
+
{
|
|
191
|
+
err,
|
|
192
|
+
removed: removed.map((r) => r.toString()),
|
|
193
|
+
Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
|
|
194
|
+
},
|
|
195
|
+
'Remote existence check failed, skipping deletion to avoid clobbering fresh data'
|
|
196
|
+
);
|
|
197
|
+
return [];
|
|
145
198
|
}
|
|
146
199
|
|
|
200
|
+
// Ids that left the view's list_ref but STILL exist upstream — not deletions,
|
|
201
|
+
// just a view-membership change (e.g. a record whose field changed so it no
|
|
202
|
+
// longer matches the query). The caller drops these from `localArray` so the
|
|
203
|
+
// poll's diff stops re-flagging them every tick (the `job:` churn).
|
|
204
|
+
const stillRemoteIds: string[] = [];
|
|
147
205
|
for (const recordId of removed) {
|
|
148
206
|
const recordIdStr = encodeRecordId(recordId);
|
|
149
207
|
if (!existingRemoteIds.has(recordIdStr)) {
|
|
150
208
|
this.logger.debug(
|
|
151
209
|
{
|
|
152
210
|
recordId: recordIdStr,
|
|
153
|
-
Category: '
|
|
211
|
+
Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
|
|
154
212
|
},
|
|
155
213
|
'Deleting confirmed removed record'
|
|
156
214
|
);
|
|
157
|
-
|
|
158
215
|
// Use CacheModule to handle both local DB and DBSP deletion
|
|
159
216
|
await this.cache.delete(recordId.table.name, recordIdStr);
|
|
217
|
+
} else {
|
|
218
|
+
stillRemoteIds.push(recordIdStr);
|
|
160
219
|
}
|
|
161
220
|
}
|
|
221
|
+
return stillRemoteIds;
|
|
162
222
|
}
|
|
163
223
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import type { EventDefinition, EventSystem } from '../../../events/index';
|
|
2
|
+
import { createEventSystem } from '../../../events/index';
|
|
3
|
+
import type { RecordVersionArray } from '../../../types';
|
|
3
4
|
|
|
4
5
|
export const SyncQueueEventTypes = {
|
|
5
6
|
MutationEnqueued: 'MUTATION_ENQUEUED',
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { LocalDatabaseService } from '../../../services/database/index';
|
|
1
|
+
import type { LocalDatabaseService } from '../../../services/database/index';
|
|
2
|
+
import type {
|
|
3
|
+
SyncQueueEventSystem} from '../events/index';
|
|
2
4
|
import {
|
|
3
5
|
createSyncQueueEventSystem,
|
|
4
|
-
SyncQueueEventSystem,
|
|
5
6
|
SyncQueueEventTypes,
|
|
6
7
|
} from '../events/index';
|
|
7
|
-
import { Logger } from '../../../services/logger/index';
|
|
8
|
+
import type { Logger } from '../../../services/logger/index';
|
|
8
9
|
|
|
9
10
|
export type RegisterEvent = {
|
|
10
11
|
type: 'register';
|
|
@@ -78,7 +79,7 @@ export class DownQueue {
|
|
|
78
79
|
await fn(event);
|
|
79
80
|
} catch (error) {
|
|
80
81
|
this.logger.error(
|
|
81
|
-
{ error, event, Category: '
|
|
82
|
+
{ error, event, Category: 'sp00ky-client::DownQueue::next' },
|
|
82
83
|
'Failed to process query'
|
|
83
84
|
);
|
|
84
85
|
this.queue.unshift(event);
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { RecordId } from 'surrealdb';
|
|
2
|
-
import { LocalDatabaseService } from '../../../services/database/index';
|
|
1
|
+
import type { RecordId } from 'surrealdb';
|
|
2
|
+
import type { LocalDatabaseService } from '../../../services/database/index';
|
|
3
|
+
import type {
|
|
4
|
+
SyncQueueEventSystem} from '../events/index';
|
|
3
5
|
import {
|
|
4
6
|
createSyncQueueEventSystem,
|
|
5
|
-
SyncQueueEventSystem,
|
|
6
7
|
SyncQueueEventTypes,
|
|
7
8
|
} from '../events/index';
|
|
8
9
|
import { parseRecordIdString, extractTablePart, classifySyncError } from '../../../utils/index';
|
|
9
|
-
import { Logger } from '../../../services/logger/index';
|
|
10
|
-
import { PushEventOptions } from '../../../events/index';
|
|
10
|
+
import type { Logger } from '../../../services/logger/index';
|
|
11
|
+
import type { PushEventOptions } from '../../../events/index';
|
|
11
12
|
|
|
12
13
|
export type CreateEvent = {
|
|
13
14
|
type: 'create';
|
|
@@ -114,7 +115,7 @@ export class UpQueue {
|
|
|
114
115
|
|
|
115
116
|
if (errorType === 'network') {
|
|
116
117
|
this.logger.error(
|
|
117
|
-
{ error, event, Category: '
|
|
118
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
118
119
|
'Network error processing mutation, re-queuing'
|
|
119
120
|
);
|
|
120
121
|
this.queue.unshift(event);
|
|
@@ -123,14 +124,14 @@ export class UpQueue {
|
|
|
123
124
|
|
|
124
125
|
// Application error — rollback instead of re-queuing
|
|
125
126
|
this.logger.error(
|
|
126
|
-
{ error, event, Category: '
|
|
127
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
127
128
|
'Application error processing mutation, rolling back'
|
|
128
129
|
);
|
|
129
130
|
try {
|
|
130
131
|
await this.removeEventFromDatabase(event.mutation_id);
|
|
131
132
|
} catch (removeError) {
|
|
132
133
|
this.logger.error(
|
|
133
|
-
{ error: removeError, event, Category: '
|
|
134
|
+
{ error: removeError, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
134
135
|
'Failed to remove rolled-back mutation from database'
|
|
135
136
|
);
|
|
136
137
|
}
|
|
@@ -139,7 +140,7 @@ export class UpQueue {
|
|
|
139
140
|
await onRollback(event, error instanceof Error ? error : new Error(String(error)));
|
|
140
141
|
} catch (rollbackError) {
|
|
141
142
|
this.logger.error(
|
|
142
|
-
{ error: rollbackError, event, Category: '
|
|
143
|
+
{ error: rollbackError, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
143
144
|
'Rollback handler failed'
|
|
144
145
|
);
|
|
145
146
|
}
|
|
@@ -154,7 +155,7 @@ export class UpQueue {
|
|
|
154
155
|
await this.removeEventFromDatabase(event.mutation_id);
|
|
155
156
|
} catch (error) {
|
|
156
157
|
this.logger.error(
|
|
157
|
-
{ error, event, Category: '
|
|
158
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
158
159
|
'Failed to remove mutation from database after successful processing'
|
|
159
160
|
);
|
|
160
161
|
}
|
|
@@ -172,7 +173,7 @@ export class UpQueue {
|
|
|
172
173
|
async loadFromDatabase() {
|
|
173
174
|
try {
|
|
174
175
|
const [records] = await this.local.query<any>(
|
|
175
|
-
`SELECT * FROM
|
|
176
|
+
`SELECT * FROM _00_pending_mutations ORDER BY created_at ASC`
|
|
176
177
|
);
|
|
177
178
|
|
|
178
179
|
this.queue = records
|
|
@@ -205,7 +206,7 @@ export class UpQueue {
|
|
|
205
206
|
{
|
|
206
207
|
mutationType: r.mutationType,
|
|
207
208
|
record: r,
|
|
208
|
-
Category: '
|
|
209
|
+
Category: 'sp00ky-client::UpQueue::loadFromDatabase',
|
|
209
210
|
},
|
|
210
211
|
'Unknown mutation type'
|
|
211
212
|
);
|
|
@@ -215,7 +216,7 @@ export class UpQueue {
|
|
|
215
216
|
.filter((e: UpEvent | null): e is UpEvent => e !== null);
|
|
216
217
|
} catch (error) {
|
|
217
218
|
this.logger.error(
|
|
218
|
-
{ error, Category: '
|
|
219
|
+
{ error, Category: 'sp00ky-client::UpQueue::loadFromDatabase' },
|
|
219
220
|
'Failed to load pending mutations from database'
|
|
220
221
|
);
|
|
221
222
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Logger } from '../../services/logger/index';
|
|
2
|
-
import { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
|
|
1
|
+
import type { Logger } from '../../services/logger/index';
|
|
2
|
+
import type { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
|
|
3
3
|
import { SyncQueueEventTypes } from './events/index';
|
|
4
4
|
|
|
5
5
|
/**
|