local-lite 0.0.0-stage → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +361 -2
- package/dist/adapters/local/better-sqlite3.d.ts +23 -0
- package/dist/adapters/local/better-sqlite3.js +46 -0
- package/dist/adapters/local/expo-sqlite.d.ts +8 -0
- package/dist/adapters/local/expo-sqlite.js +30 -0
- package/dist/adapters/local/index.d.ts +13 -0
- package/dist/adapters/local/index.js +22 -0
- package/dist/adapters/local/libsql.d.ts +24 -0
- package/dist/adapters/local/libsql.js +87 -0
- package/dist/adapters/local/node-sqlite.d.ts +24 -0
- package/dist/adapters/local/node-sqlite.js +44 -0
- package/dist/adapters/local/op-sqlite.d.ts +26 -0
- package/dist/adapters/local/op-sqlite.js +59 -0
- package/dist/adapters/local/sql-js.d.ts +26 -0
- package/dist/adapters/local/sql-js.js +63 -0
- package/dist/adapters/network/index.d.ts +11 -0
- package/dist/adapters/network/index.js +53 -0
- package/dist/adapters/remote/convex.d.ts +71 -0
- package/dist/adapters/remote/convex.js +185 -0
- package/dist/adapters/remote/demo.d.ts +21 -0
- package/dist/adapters/remote/demo.js +46 -0
- package/dist/adapters/remote/firestore.d.ts +64 -0
- package/dist/adapters/remote/firestore.js +302 -0
- package/dist/adapters/remote/index.d.ts +10 -0
- package/dist/adapters/remote/index.js +20 -0
- package/dist/adapters/remote/supabase.d.ts +97 -0
- package/dist/adapters/remote/supabase.js +276 -0
- package/dist/config/index.d.ts +24 -0
- package/dist/config/index.js +101 -0
- package/dist/config/resolve-local.d.ts +7 -0
- package/dist/config/resolve-local.js +68 -0
- package/dist/config/resolve-remote.d.ts +7 -0
- package/dist/config/resolve-remote.js +81 -0
- package/dist/config/types.d.ts +159 -0
- package/dist/config/types.js +2 -0
- package/dist/core/client.d.ts +153 -0
- package/dist/core/client.js +445 -0
- package/dist/core/conflict.d.ts +5 -0
- package/dist/core/conflict.js +38 -0
- package/dist/core/meta-store.d.ts +28 -0
- package/dist/core/meta-store.js +64 -0
- package/dist/core/outbox-store.d.ts +76 -0
- package/dist/core/outbox-store.js +265 -0
- package/dist/core/query.d.ts +13 -0
- package/dist/core/query.js +150 -0
- package/dist/core/schema-manager.d.ts +39 -0
- package/dist/core/schema-manager.js +150 -0
- package/dist/core/sync-worker.d.ts +89 -0
- package/dist/core/sync-worker.js +473 -0
- package/dist/core/table-manager.d.ts +19 -0
- package/dist/core/table-manager.js +74 -0
- package/dist/core/types.d.ts +156 -0
- package/dist/core/types.js +2 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.js +34 -0
- package/dist/react/index.d.ts +50 -0
- package/dist/react/index.js +174 -0
- package/dist/reactive/event-emitter.d.ts +7 -0
- package/dist/reactive/event-emitter.js +34 -0
- package/dist/utils/logger.d.ts +51 -0
- package/dist/utils/logger.js +169 -0
- package/dist/utils/sql.d.ts +4 -0
- package/dist/utils/sql.js +9 -0
- package/dist/utils/uuid.d.ts +7 -0
- package/dist/utils/uuid.js +27 -0
- package/package.json +131 -4
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { LocalDBAdapter } from '../adapters/local/index.js';
|
|
2
|
+
import { RemoteDBAdapter } from '../adapters/remote/index.js';
|
|
3
|
+
import { EventEmitter } from '../reactive/event-emitter.js';
|
|
4
|
+
import { ConflictResolver, ConflictStrategy, DeltaChanges, NetworkStatusProvider, RetryConfig, SyncMetrics } from './types.js';
|
|
5
|
+
import { SchemaManager } from './schema-manager.js';
|
|
6
|
+
import { SyncLogger, CustomLogger, LogLevel } from '../utils/logger.js';
|
|
7
|
+
export interface SyncWorkerOptions {
|
|
8
|
+
pushIntervalMs?: number;
|
|
9
|
+
pullIntervalMs?: number;
|
|
10
|
+
maxRetries?: number;
|
|
11
|
+
retry?: number | RetryConfig;
|
|
12
|
+
logger?: SyncLogger | CustomLogger | boolean;
|
|
13
|
+
logLevel?: LogLevel;
|
|
14
|
+
conflictResolver?: ConflictStrategy | ConflictResolver;
|
|
15
|
+
network?: NetworkStatusProvider;
|
|
16
|
+
primaryKey?: string;
|
|
17
|
+
autoSchema?: boolean;
|
|
18
|
+
schemaManager?: SchemaManager;
|
|
19
|
+
}
|
|
20
|
+
export declare class SyncWorker {
|
|
21
|
+
private db;
|
|
22
|
+
private remote;
|
|
23
|
+
private events;
|
|
24
|
+
private network;
|
|
25
|
+
private conflictResolver;
|
|
26
|
+
private primaryKey;
|
|
27
|
+
private autoSchema;
|
|
28
|
+
private schemaManager?;
|
|
29
|
+
logger: SyncLogger;
|
|
30
|
+
retryConfig: RetryConfig;
|
|
31
|
+
private pushIntervalMs;
|
|
32
|
+
private pullIntervalMs;
|
|
33
|
+
private maxRetries;
|
|
34
|
+
private pushIntervalTimer?;
|
|
35
|
+
private pullIntervalTimer?;
|
|
36
|
+
private pushDebounceTimer?;
|
|
37
|
+
private networkUnsubscribe?;
|
|
38
|
+
private isPushing;
|
|
39
|
+
private isPulling;
|
|
40
|
+
private lastPulledTimestamp;
|
|
41
|
+
private cursorLoaded;
|
|
42
|
+
private deltaQueue;
|
|
43
|
+
private totalPushedOperations;
|
|
44
|
+
private totalPulledDeltas;
|
|
45
|
+
private totalPushCycles;
|
|
46
|
+
private totalPullCycles;
|
|
47
|
+
private lastPushDurationMs;
|
|
48
|
+
private lastPullDurationMs;
|
|
49
|
+
private lastPushTimestamp;
|
|
50
|
+
private lastPullTimestamp;
|
|
51
|
+
private lastPushError;
|
|
52
|
+
private lastPullError;
|
|
53
|
+
constructor(db: LocalDBAdapter, remote: RemoteDBAdapter, events: EventEmitter, options?: SyncWorkerOptions);
|
|
54
|
+
/**
|
|
55
|
+
* Starts background interval push/pull loops and listens for network & realtime updates.
|
|
56
|
+
*/
|
|
57
|
+
start(): void;
|
|
58
|
+
/**
|
|
59
|
+
* Stops all active intervals, timers, and listeners.
|
|
60
|
+
*/
|
|
61
|
+
stop(): void;
|
|
62
|
+
/**
|
|
63
|
+
* Schedules an immediate push with a short debounce.
|
|
64
|
+
*/
|
|
65
|
+
schedulePush(delayMs?: number): void;
|
|
66
|
+
private isOnline;
|
|
67
|
+
private emitStatus;
|
|
68
|
+
/**
|
|
69
|
+
* Pushes pending outbox records to the remote database.
|
|
70
|
+
*/
|
|
71
|
+
pushLocalChanges(): Promise<void>;
|
|
72
|
+
private handlePushFailure;
|
|
73
|
+
/**
|
|
74
|
+
* Pulls delta changes from the remote backend since lastPulledTimestamp.
|
|
75
|
+
*/
|
|
76
|
+
pullRemoteChanges(): Promise<void>;
|
|
77
|
+
/**
|
|
78
|
+
* Applies incoming server delta changes to local tables with conflict resolution.
|
|
79
|
+
* Serializes execution in a queue to prevent concurrent SQLite transactions.
|
|
80
|
+
*/
|
|
81
|
+
applyDeltasLocally(deltas: DeltaChanges): Promise<void>;
|
|
82
|
+
private processDeltasLocally;
|
|
83
|
+
getLastPulledTimestamp(): Promise<number>;
|
|
84
|
+
resetSyncCursor(): Promise<void>;
|
|
85
|
+
/**
|
|
86
|
+
* Collects real-time engine telemetry snapshot.
|
|
87
|
+
*/
|
|
88
|
+
getMetrics(): Promise<SyncMetrics>;
|
|
89
|
+
}
|
|
@@ -0,0 +1,473 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SyncWorker = void 0;
|
|
4
|
+
const conflict_js_1 = require("./conflict.js");
|
|
5
|
+
const index_js_1 = require("../adapters/network/index.js");
|
|
6
|
+
const outbox_store_js_1 = require("./outbox-store.js");
|
|
7
|
+
const meta_store_js_1 = require("./meta-store.js");
|
|
8
|
+
const table_manager_js_1 = require("./table-manager.js");
|
|
9
|
+
const logger_js_1 = require("../utils/logger.js");
|
|
10
|
+
class SyncWorker {
|
|
11
|
+
db;
|
|
12
|
+
remote;
|
|
13
|
+
events;
|
|
14
|
+
network;
|
|
15
|
+
conflictResolver;
|
|
16
|
+
primaryKey;
|
|
17
|
+
autoSchema;
|
|
18
|
+
schemaManager;
|
|
19
|
+
logger;
|
|
20
|
+
retryConfig;
|
|
21
|
+
pushIntervalMs;
|
|
22
|
+
pullIntervalMs;
|
|
23
|
+
maxRetries;
|
|
24
|
+
pushIntervalTimer;
|
|
25
|
+
pullIntervalTimer;
|
|
26
|
+
pushDebounceTimer;
|
|
27
|
+
networkUnsubscribe;
|
|
28
|
+
isPushing = false;
|
|
29
|
+
isPulling = false;
|
|
30
|
+
lastPulledTimestamp = 0;
|
|
31
|
+
cursorLoaded = false;
|
|
32
|
+
deltaQueue = Promise.resolve();
|
|
33
|
+
// Telemetry metrics
|
|
34
|
+
totalPushedOperations = 0;
|
|
35
|
+
totalPulledDeltas = 0;
|
|
36
|
+
totalPushCycles = 0;
|
|
37
|
+
totalPullCycles = 0;
|
|
38
|
+
lastPushDurationMs = 0;
|
|
39
|
+
lastPullDurationMs = 0;
|
|
40
|
+
lastPushTimestamp = 0;
|
|
41
|
+
lastPullTimestamp = 0;
|
|
42
|
+
lastPushError = null;
|
|
43
|
+
lastPullError = null;
|
|
44
|
+
constructor(db, remote, events, options = {}) {
|
|
45
|
+
this.db = db;
|
|
46
|
+
this.remote = remote;
|
|
47
|
+
this.events = events;
|
|
48
|
+
this.network = options.network ?? new index_js_1.DefaultNetworkStatusProvider();
|
|
49
|
+
this.conflictResolver = (0, conflict_js_1.getConflictResolver)(options.conflictResolver ?? 'last-write-wins');
|
|
50
|
+
this.primaryKey = options.primaryKey ?? 'id';
|
|
51
|
+
this.autoSchema = options.autoSchema ?? false;
|
|
52
|
+
this.schemaManager = options.schemaManager;
|
|
53
|
+
this.pushIntervalMs = options.pushIntervalMs ?? 5000;
|
|
54
|
+
this.pullIntervalMs = options.pullIntervalMs ?? 10000;
|
|
55
|
+
// Configure logger
|
|
56
|
+
if (options.logger instanceof logger_js_1.SyncLogger) {
|
|
57
|
+
this.logger = options.logger;
|
|
58
|
+
}
|
|
59
|
+
else if (typeof options.logger === 'boolean') {
|
|
60
|
+
this.logger = new logger_js_1.SyncLogger({ enabled: options.logger, level: options.logLevel });
|
|
61
|
+
}
|
|
62
|
+
else if (options.logger && typeof options.logger === 'object') {
|
|
63
|
+
this.logger = new logger_js_1.SyncLogger({ customLogger: options.logger, level: options.logLevel });
|
|
64
|
+
}
|
|
65
|
+
else {
|
|
66
|
+
this.logger = options.logLevel ? new logger_js_1.SyncLogger({ level: options.logLevel }) : logger_js_1.defaultLogger;
|
|
67
|
+
}
|
|
68
|
+
// Configure retries & backoff
|
|
69
|
+
const r = options.retry;
|
|
70
|
+
if (typeof r === 'number') {
|
|
71
|
+
this.retryConfig = {
|
|
72
|
+
maxRetries: r,
|
|
73
|
+
initialDelayMs: 1000,
|
|
74
|
+
backoffMultiplier: 1.5,
|
|
75
|
+
maxDelayMs: 30000,
|
|
76
|
+
jitter: true,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
else if (typeof r === 'object' && r !== null) {
|
|
80
|
+
this.retryConfig = {
|
|
81
|
+
maxRetries: r.maxRetries ?? options.maxRetries ?? 5,
|
|
82
|
+
initialDelayMs: r.initialDelayMs ?? 1000,
|
|
83
|
+
backoffMultiplier: r.backoffMultiplier ?? 1.5,
|
|
84
|
+
maxDelayMs: r.maxDelayMs ?? 30000,
|
|
85
|
+
jitter: r.jitter ?? true,
|
|
86
|
+
shouldRetry: r.shouldRetry,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
this.retryConfig = {
|
|
91
|
+
maxRetries: options.maxRetries ?? 5,
|
|
92
|
+
initialDelayMs: 1000,
|
|
93
|
+
backoffMultiplier: 1.5,
|
|
94
|
+
maxDelayMs: 30000,
|
|
95
|
+
jitter: true,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
this.maxRetries = this.retryConfig.maxRetries ?? 5;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Starts background interval push/pull loops and listens for network & realtime updates.
|
|
102
|
+
*/
|
|
103
|
+
start() {
|
|
104
|
+
this.stop();
|
|
105
|
+
this.pushIntervalTimer = setInterval(() => this.pushLocalChanges(), this.pushIntervalMs);
|
|
106
|
+
this.pullIntervalTimer = setInterval(() => this.pullRemoteChanges(), this.pullIntervalMs);
|
|
107
|
+
// Subscribe to live remote changes
|
|
108
|
+
this.remote.subscribeToLiveChanges((deltas) => {
|
|
109
|
+
this.applyDeltasLocally(deltas).catch((err) => {
|
|
110
|
+
this.logger.error('Error applying live remote deltas:', err);
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
// Subscribe to network connectivity changes
|
|
114
|
+
this.networkUnsubscribe = this.network.subscribe((online) => {
|
|
115
|
+
this.emitStatus();
|
|
116
|
+
if (online) {
|
|
117
|
+
this.schedulePush(0);
|
|
118
|
+
this.pullRemoteChanges().catch((err) => {
|
|
119
|
+
this.logger.error('Error pulling changes on reconnect:', err);
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
// Initial pull
|
|
124
|
+
this.pullRemoteChanges().catch((err) => {
|
|
125
|
+
this.logger.error('Initial pull failed:', err);
|
|
126
|
+
});
|
|
127
|
+
this.emitStatus();
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Stops all active intervals, timers, and listeners.
|
|
131
|
+
*/
|
|
132
|
+
stop() {
|
|
133
|
+
if (this.pushIntervalTimer) {
|
|
134
|
+
clearInterval(this.pushIntervalTimer);
|
|
135
|
+
this.pushIntervalTimer = undefined;
|
|
136
|
+
}
|
|
137
|
+
if (this.pullIntervalTimer) {
|
|
138
|
+
clearInterval(this.pullIntervalTimer);
|
|
139
|
+
this.pullIntervalTimer = undefined;
|
|
140
|
+
}
|
|
141
|
+
if (this.pushDebounceTimer) {
|
|
142
|
+
clearTimeout(this.pushDebounceTimer);
|
|
143
|
+
this.pushDebounceTimer = undefined;
|
|
144
|
+
}
|
|
145
|
+
if (this.networkUnsubscribe) {
|
|
146
|
+
this.networkUnsubscribe();
|
|
147
|
+
this.networkUnsubscribe = undefined;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Schedules an immediate push with a short debounce.
|
|
152
|
+
*/
|
|
153
|
+
schedulePush(delayMs = 50) {
|
|
154
|
+
if (this.pushDebounceTimer) {
|
|
155
|
+
clearTimeout(this.pushDebounceTimer);
|
|
156
|
+
}
|
|
157
|
+
this.pushDebounceTimer = setTimeout(() => {
|
|
158
|
+
this.pushDebounceTimer = undefined;
|
|
159
|
+
this.pushLocalChanges().catch((err) => {
|
|
160
|
+
this.logger.error('Debounced push failed:', err);
|
|
161
|
+
});
|
|
162
|
+
}, delayMs);
|
|
163
|
+
}
|
|
164
|
+
async isOnline() {
|
|
165
|
+
try {
|
|
166
|
+
return await Promise.resolve(this.network.isOnline());
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
return true;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
async emitStatus() {
|
|
173
|
+
try {
|
|
174
|
+
const online = await this.isOnline();
|
|
175
|
+
const counts = await outbox_store_js_1.OutboxStore.getCounts(this.db);
|
|
176
|
+
const statusEvent = {
|
|
177
|
+
isOnline: online,
|
|
178
|
+
isPushing: this.isPushing,
|
|
179
|
+
isPulling: this.isPulling,
|
|
180
|
+
pendingCount: counts.pending,
|
|
181
|
+
};
|
|
182
|
+
this.events.emit('sync:status', statusEvent);
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
// Ignored
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Pushes pending outbox records to the remote database.
|
|
190
|
+
*/
|
|
191
|
+
async pushLocalChanges() {
|
|
192
|
+
if (this.isPushing)
|
|
193
|
+
return;
|
|
194
|
+
if (!(await this.isOnline()))
|
|
195
|
+
return;
|
|
196
|
+
this.isPushing = true;
|
|
197
|
+
this.emitStatus();
|
|
198
|
+
const startTime = Date.now();
|
|
199
|
+
let pendingRecords = [];
|
|
200
|
+
try {
|
|
201
|
+
this.totalPushCycles++;
|
|
202
|
+
pendingRecords = await outbox_store_js_1.OutboxStore.getPending(this.db, this.maxRetries);
|
|
203
|
+
if (pendingRecords.length === 0) {
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
const operations = pendingRecords.map((r) => ({
|
|
207
|
+
id: r.id,
|
|
208
|
+
tableName: r.tableName,
|
|
209
|
+
rowId: r.rowId,
|
|
210
|
+
operation: r.operation,
|
|
211
|
+
payload: r.payload,
|
|
212
|
+
timestamp: r.timestamp,
|
|
213
|
+
}));
|
|
214
|
+
const recordIds = operations.map((o) => o.id);
|
|
215
|
+
// Mark records as PROCESSING
|
|
216
|
+
await outbox_store_js_1.OutboxStore.markProcessing(this.db, recordIds);
|
|
217
|
+
const success = await this.remote.pushChanges(operations);
|
|
218
|
+
const durationMs = Date.now() - startTime;
|
|
219
|
+
this.lastPushDurationMs = durationMs;
|
|
220
|
+
this.lastPushTimestamp = Date.now();
|
|
221
|
+
if (success) {
|
|
222
|
+
// Successful push: delete rows from outbox
|
|
223
|
+
await outbox_store_js_1.OutboxStore.deleteCompleted(this.db, recordIds);
|
|
224
|
+
this.totalPushedOperations += operations.length;
|
|
225
|
+
this.lastPushError = null;
|
|
226
|
+
const pushEvent = {
|
|
227
|
+
success: true,
|
|
228
|
+
count: operations.length,
|
|
229
|
+
durationMs,
|
|
230
|
+
error: null,
|
|
231
|
+
};
|
|
232
|
+
this.events.emit('sync:push', pushEvent);
|
|
233
|
+
this.events.emit('sync_outbox', { operation: 'DELETE', rowId: 'batch' });
|
|
234
|
+
}
|
|
235
|
+
else {
|
|
236
|
+
const errMsg = 'Remote push returned false';
|
|
237
|
+
this.lastPushError = errMsg;
|
|
238
|
+
const pushEvent = {
|
|
239
|
+
success: false,
|
|
240
|
+
count: operations.length,
|
|
241
|
+
durationMs,
|
|
242
|
+
error: errMsg,
|
|
243
|
+
};
|
|
244
|
+
this.events.emit('sync:push', pushEvent);
|
|
245
|
+
await this.handlePushFailure(pendingRecords, errMsg);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
catch (error) {
|
|
249
|
+
const durationMs = Date.now() - startTime;
|
|
250
|
+
this.lastPushDurationMs = durationMs;
|
|
251
|
+
this.lastPushTimestamp = Date.now();
|
|
252
|
+
const errMsg = error?.message || String(error);
|
|
253
|
+
this.lastPushError = errMsg;
|
|
254
|
+
this.logger.error('Sync Engine Outbox Push Error:', error);
|
|
255
|
+
const pushEvent = {
|
|
256
|
+
success: false,
|
|
257
|
+
count: pendingRecords.length,
|
|
258
|
+
durationMs,
|
|
259
|
+
error: errMsg,
|
|
260
|
+
};
|
|
261
|
+
this.events.emit('sync:push', pushEvent);
|
|
262
|
+
if (pendingRecords.length > 0) {
|
|
263
|
+
await this.handlePushFailure(pendingRecords, errMsg, error);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
finally {
|
|
267
|
+
this.isPushing = false;
|
|
268
|
+
this.emitStatus();
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
async handlePushFailure(records, errorMessage, errorObj) {
|
|
272
|
+
try {
|
|
273
|
+
const recordIds = records.map((r) => r.id);
|
|
274
|
+
await outbox_store_js_1.OutboxStore.markFailed(this.db, recordIds, errorMessage);
|
|
275
|
+
this.events.emit('sync_outbox', { operation: 'UPDATE', rowId: 'batch' });
|
|
276
|
+
// If custom shouldRetry filter specifies an operation should NOT be retried, mark it FAILED permanently
|
|
277
|
+
if (this.retryConfig.shouldRetry && errorObj) {
|
|
278
|
+
for (const rec of records) {
|
|
279
|
+
if (!this.retryConfig.shouldRetry(errorObj, rec)) {
|
|
280
|
+
await outbox_store_js_1.OutboxStore.markFailed(this.db, [rec.id], `Non-retryable error: ${errorMessage}`);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// Schedule next retry with exponential backoff & optional jitter
|
|
285
|
+
const currentRetry = Math.max(...records.map((r) => r.retryCount || 0));
|
|
286
|
+
const isInfinite = !Number.isFinite(this.maxRetries) || this.maxRetries === -1;
|
|
287
|
+
if (isInfinite || currentRetry < this.maxRetries) {
|
|
288
|
+
let backoffDelay = Math.min((this.retryConfig.initialDelayMs || 1000) *
|
|
289
|
+
Math.pow(this.retryConfig.backoffMultiplier || 1.5, currentRetry), this.retryConfig.maxDelayMs || 30000);
|
|
290
|
+
// Apply ±15% jitter if enabled to avoid synchronized retry stampedes
|
|
291
|
+
if (this.retryConfig.jitter !== false) {
|
|
292
|
+
const jitterFactor = 0.85 + Math.random() * 0.3;
|
|
293
|
+
backoffDelay = Math.round(backoffDelay * jitterFactor);
|
|
294
|
+
}
|
|
295
|
+
this.schedulePush(backoffDelay);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
catch (err) {
|
|
299
|
+
this.logger.error('Failed to update outbox failure status:', err);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
/**
|
|
303
|
+
* Pulls delta changes from the remote backend since lastPulledTimestamp.
|
|
304
|
+
*/
|
|
305
|
+
async pullRemoteChanges() {
|
|
306
|
+
if (this.isPulling)
|
|
307
|
+
return;
|
|
308
|
+
if (!(await this.isOnline()))
|
|
309
|
+
return;
|
|
310
|
+
this.isPulling = true;
|
|
311
|
+
this.emitStatus();
|
|
312
|
+
const startTime = Date.now();
|
|
313
|
+
try {
|
|
314
|
+
this.totalPullCycles++;
|
|
315
|
+
if (!this.cursorLoaded) {
|
|
316
|
+
this.lastPulledTimestamp = await meta_store_js_1.MetaStore.getLastSyncedAt(this.db);
|
|
317
|
+
this.cursorLoaded = true;
|
|
318
|
+
}
|
|
319
|
+
const deltas = await this.remote.fetchChanges(this.lastPulledTimestamp);
|
|
320
|
+
const durationMs = Date.now() - startTime;
|
|
321
|
+
this.lastPullDurationMs = durationMs;
|
|
322
|
+
this.lastPullTimestamp = Date.now();
|
|
323
|
+
this.lastPullError = null;
|
|
324
|
+
const changesCount = deltas?.changes?.length || 0;
|
|
325
|
+
if (deltas) {
|
|
326
|
+
this.totalPulledDeltas += changesCount;
|
|
327
|
+
await this.applyDeltasLocally(deltas);
|
|
328
|
+
}
|
|
329
|
+
const pullEvent = {
|
|
330
|
+
success: true,
|
|
331
|
+
changesCount,
|
|
332
|
+
durationMs,
|
|
333
|
+
timestamp: this.lastPulledTimestamp,
|
|
334
|
+
error: null,
|
|
335
|
+
};
|
|
336
|
+
this.events.emit('sync:pull', pullEvent);
|
|
337
|
+
}
|
|
338
|
+
catch (error) {
|
|
339
|
+
const durationMs = Date.now() - startTime;
|
|
340
|
+
this.lastPullDurationMs = durationMs;
|
|
341
|
+
this.lastPullTimestamp = Date.now();
|
|
342
|
+
const errMsg = error?.message || String(error);
|
|
343
|
+
this.lastPullError = errMsg;
|
|
344
|
+
this.logger.error('Sync Engine Delta Pull Error:', error);
|
|
345
|
+
const pullEvent = {
|
|
346
|
+
success: false,
|
|
347
|
+
changesCount: 0,
|
|
348
|
+
durationMs,
|
|
349
|
+
timestamp: this.lastPulledTimestamp,
|
|
350
|
+
error: errMsg,
|
|
351
|
+
};
|
|
352
|
+
this.events.emit('sync:pull', pullEvent);
|
|
353
|
+
}
|
|
354
|
+
finally {
|
|
355
|
+
this.isPulling = false;
|
|
356
|
+
this.emitStatus();
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Applies incoming server delta changes to local tables with conflict resolution.
|
|
361
|
+
* Serializes execution in a queue to prevent concurrent SQLite transactions.
|
|
362
|
+
*/
|
|
363
|
+
async applyDeltasLocally(deltas) {
|
|
364
|
+
const task = async () => {
|
|
365
|
+
await this.processDeltasLocally(deltas);
|
|
366
|
+
};
|
|
367
|
+
const next = this.deltaQueue.then(task, task);
|
|
368
|
+
this.deltaQueue = next.catch(() => { });
|
|
369
|
+
return await next;
|
|
370
|
+
}
|
|
371
|
+
async processDeltasLocally(deltas) {
|
|
372
|
+
if (!deltas.changes || deltas.changes.length === 0) {
|
|
373
|
+
if (deltas.timestamp && deltas.timestamp > this.lastPulledTimestamp) {
|
|
374
|
+
this.lastPulledTimestamp = deltas.timestamp;
|
|
375
|
+
await meta_store_js_1.MetaStore.setLastSyncedAt(this.db, this.lastPulledTimestamp);
|
|
376
|
+
}
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
await this.db.transaction(async (tx) => {
|
|
380
|
+
for (const change of deltas.changes) {
|
|
381
|
+
// 1. Check for pending local mutations for this (table_name, row_id)
|
|
382
|
+
const localPendingRecord = await outbox_store_js_1.OutboxStore.findPendingByRow(tx, change.tableName, change.rowId);
|
|
383
|
+
// 2. Query current local row if it exists
|
|
384
|
+
const localRow = await table_manager_js_1.TableManager.fetchRow(tx, change.tableName, change.rowId, this.primaryKey, { autoSchema: this.autoSchema, schemaManager: this.schemaManager });
|
|
385
|
+
// 3. Conflict resolution if there is a pending local change
|
|
386
|
+
if (localPendingRecord) {
|
|
387
|
+
const resolution = await Promise.resolve(this.conflictResolver({
|
|
388
|
+
tableName: change.tableName,
|
|
389
|
+
rowId: change.rowId,
|
|
390
|
+
remoteChange: change,
|
|
391
|
+
localOutboxRecord: localPendingRecord,
|
|
392
|
+
localRow,
|
|
393
|
+
}));
|
|
394
|
+
const resolutionStr = typeof resolution === 'string' ? resolution : 'merge';
|
|
395
|
+
const conflictEvent = {
|
|
396
|
+
tableName: change.tableName,
|
|
397
|
+
rowId: change.rowId,
|
|
398
|
+
resolution: resolutionStr,
|
|
399
|
+
timestamp: Date.now(),
|
|
400
|
+
};
|
|
401
|
+
this.events.emit('sync:conflict', conflictEvent);
|
|
402
|
+
if (resolution === 'keep-local') {
|
|
403
|
+
// Local offline change takes precedence
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
if (resolution === 'apply-remote') {
|
|
407
|
+
// Remote change wins: remove the superseded local pending outbox mutation
|
|
408
|
+
await outbox_store_js_1.OutboxStore.deleteRecord(tx, localPendingRecord.id);
|
|
409
|
+
}
|
|
410
|
+
else if (typeof resolution === 'object' && resolution.mergePayload) {
|
|
411
|
+
// Merge scenario
|
|
412
|
+
change.payload = resolution.mergePayload;
|
|
413
|
+
await outbox_store_js_1.OutboxStore.updatePayload(tx, localPendingRecord.id, resolution.mergePayload);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
// 4. Apply remote change to local database
|
|
417
|
+
await table_manager_js_1.TableManager.applyMutation(tx, change.tableName, change.rowId, change.operation, change.payload, this.primaryKey, { autoSchema: this.autoSchema, schemaManager: this.schemaManager });
|
|
418
|
+
// 5. Emit table change event to notify reactive listeners
|
|
419
|
+
this.events.emit(change.tableName, {
|
|
420
|
+
operation: change.operation,
|
|
421
|
+
rowId: change.rowId,
|
|
422
|
+
payload: change.payload,
|
|
423
|
+
source: 'remote',
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
// Persist sync cursor atomically within the same transaction as delta writes
|
|
427
|
+
if (deltas.timestamp && deltas.timestamp > this.lastPulledTimestamp) {
|
|
428
|
+
await meta_store_js_1.MetaStore.setLastSyncedAt(tx, deltas.timestamp);
|
|
429
|
+
}
|
|
430
|
+
});
|
|
431
|
+
if (deltas.timestamp) {
|
|
432
|
+
this.lastPulledTimestamp = Math.max(this.lastPulledTimestamp, deltas.timestamp);
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
async getLastPulledTimestamp() {
|
|
436
|
+
if (!this.cursorLoaded) {
|
|
437
|
+
this.lastPulledTimestamp = await meta_store_js_1.MetaStore.getLastSyncedAt(this.db);
|
|
438
|
+
this.cursorLoaded = true;
|
|
439
|
+
}
|
|
440
|
+
return this.lastPulledTimestamp;
|
|
441
|
+
}
|
|
442
|
+
async resetSyncCursor() {
|
|
443
|
+
await meta_store_js_1.MetaStore.clearLastSyncedAt(this.db);
|
|
444
|
+
this.lastPulledTimestamp = 0;
|
|
445
|
+
this.cursorLoaded = true;
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Collects real-time engine telemetry snapshot.
|
|
449
|
+
*/
|
|
450
|
+
async getMetrics() {
|
|
451
|
+
const counts = await outbox_store_js_1.OutboxStore.getCounts(this.db);
|
|
452
|
+
const online = await this.isOnline();
|
|
453
|
+
return {
|
|
454
|
+
outboxPendingCount: counts.pending,
|
|
455
|
+
outboxFailedCount: counts.failed,
|
|
456
|
+
outboxProcessingCount: counts.processing,
|
|
457
|
+
totalPushedOperations: this.totalPushedOperations,
|
|
458
|
+
totalPulledDeltas: this.totalPulledDeltas,
|
|
459
|
+
totalPushCycles: this.totalPushCycles,
|
|
460
|
+
totalPullCycles: this.totalPullCycles,
|
|
461
|
+
lastPushDurationMs: this.lastPushDurationMs,
|
|
462
|
+
lastPullDurationMs: this.lastPullDurationMs,
|
|
463
|
+
lastPushTimestamp: this.lastPushTimestamp,
|
|
464
|
+
lastPullTimestamp: this.lastPullTimestamp,
|
|
465
|
+
lastPushError: this.lastPushError,
|
|
466
|
+
lastPullError: this.lastPullError,
|
|
467
|
+
isOnline: online,
|
|
468
|
+
isPushing: this.isPushing,
|
|
469
|
+
isPulling: this.isPulling,
|
|
470
|
+
};
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
exports.SyncWorker = SyncWorker;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { LocalTransaction } from '../adapters/local/index.js';
|
|
2
|
+
import { SyncOp } from './types.js';
|
|
3
|
+
import { SchemaManager } from './schema-manager.js';
|
|
4
|
+
export interface TableMutationOptions {
|
|
5
|
+
autoSchema?: boolean;
|
|
6
|
+
schemaManager?: SchemaManager;
|
|
7
|
+
}
|
|
8
|
+
export declare class TableManager {
|
|
9
|
+
/**
|
|
10
|
+
* Applies an insert, update, or delete mutation on a local data table.
|
|
11
|
+
* If an INSERT is specified but the row already exists, it gracefully executes an UPDATE.
|
|
12
|
+
* If an UPDATE is specified but the row does not exist, it inserts it.
|
|
13
|
+
*/
|
|
14
|
+
static applyMutation(tx: LocalTransaction, tableName: string, rowId: string, operation: SyncOp, payload: Record<string, any> | null, primaryKey?: string, options?: TableMutationOptions): Promise<void>;
|
|
15
|
+
/**
|
|
16
|
+
* Fetches an existing row from a local table by primary key.
|
|
17
|
+
*/
|
|
18
|
+
static fetchRow(tx: LocalTransaction, tableName: string, rowId: string, primaryKey?: string, options?: TableMutationOptions): Promise<Record<string, any> | null>;
|
|
19
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TableManager = void 0;
|
|
4
|
+
const sql_js_1 = require("../utils/sql.js");
|
|
5
|
+
const schema_manager_js_1 = require("./schema-manager.js");
|
|
6
|
+
const defaultSchemaManager = new schema_manager_js_1.SchemaManager();
|
|
7
|
+
class TableManager {
|
|
8
|
+
/**
|
|
9
|
+
* Applies an insert, update, or delete mutation on a local data table.
|
|
10
|
+
* If an INSERT is specified but the row already exists, it gracefully executes an UPDATE.
|
|
11
|
+
* If an UPDATE is specified but the row does not exist, it inserts it.
|
|
12
|
+
*/
|
|
13
|
+
static async applyMutation(tx, tableName, rowId, operation, payload, primaryKey = 'id', options) {
|
|
14
|
+
const safeTable = (0, sql_js_1.escapeIdentifier)(tableName);
|
|
15
|
+
const safePk = (0, sql_js_1.escapeIdentifier)(primaryKey);
|
|
16
|
+
const sm = options?.schemaManager ?? defaultSchemaManager;
|
|
17
|
+
if (options?.autoSchema) {
|
|
18
|
+
await sm.ensureTable(tx, tableName, primaryKey);
|
|
19
|
+
}
|
|
20
|
+
if (operation === 'DELETE') {
|
|
21
|
+
await tx.execute(`DELETE FROM ${safeTable} WHERE ${safePk} = ?`, [rowId]);
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
let rowData;
|
|
25
|
+
if (options?.autoSchema) {
|
|
26
|
+
const formatted = await sm.ensureColumns(tx, tableName, payload, primaryKey);
|
|
27
|
+
rowData = { ...formatted, [primaryKey]: rowId };
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
const formatted = {};
|
|
31
|
+
if (payload && typeof payload === 'object') {
|
|
32
|
+
for (const k of Object.keys(payload)) {
|
|
33
|
+
formatted[k] = (0, schema_manager_js_1.formatValueForStorage)(payload[k]);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
rowData = { ...formatted, [primaryKey]: rowId };
|
|
37
|
+
}
|
|
38
|
+
const keys = Object.keys(rowData);
|
|
39
|
+
if (keys.length === 0)
|
|
40
|
+
return;
|
|
41
|
+
const existing = await tx.execute(`SELECT 1 FROM ${safeTable} WHERE ${safePk} = ? LIMIT 1`, [rowId]);
|
|
42
|
+
if (existing && existing.length > 0) {
|
|
43
|
+
// Row exists -> perform UPDATE
|
|
44
|
+
const updateKeys = keys.filter((k) => k !== primaryKey);
|
|
45
|
+
if (updateKeys.length > 0) {
|
|
46
|
+
const setClause = updateKeys.map((k) => `${(0, sql_js_1.escapeIdentifier)(k)} = ?`).join(', ');
|
|
47
|
+
const values = updateKeys.map((k) => rowData[k]);
|
|
48
|
+
values.push(rowId);
|
|
49
|
+
await tx.execute(`UPDATE ${safeTable} SET ${setClause} WHERE ${safePk} = ?`, values);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
// Row does not exist -> perform INSERT
|
|
54
|
+
const cols = keys.map((k) => (0, sql_js_1.escapeIdentifier)(k)).join(', ');
|
|
55
|
+
const placeholders = keys.map(() => '?').join(', ');
|
|
56
|
+
const values = keys.map((k) => rowData[k]);
|
|
57
|
+
await tx.execute(`INSERT INTO ${safeTable} (${cols}) VALUES (${placeholders})`, values);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Fetches an existing row from a local table by primary key.
|
|
62
|
+
*/
|
|
63
|
+
static async fetchRow(tx, tableName, rowId, primaryKey = 'id', options) {
|
|
64
|
+
const safeTable = (0, sql_js_1.escapeIdentifier)(tableName);
|
|
65
|
+
const safePk = (0, sql_js_1.escapeIdentifier)(primaryKey);
|
|
66
|
+
const sm = options?.schemaManager ?? defaultSchemaManager;
|
|
67
|
+
if (options?.autoSchema) {
|
|
68
|
+
await sm.ensureTable(tx, tableName, primaryKey);
|
|
69
|
+
}
|
|
70
|
+
const rows = await tx.execute(`SELECT * FROM ${safeTable} WHERE ${safePk} = ? LIMIT 1`, [rowId]);
|
|
71
|
+
return rows && rows.length > 0 ? rows[0] : null;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
exports.TableManager = TableManager;
|