agentdb 3.0.0-alpha.11 → 3.0.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/backends/graph/GraphDatabaseAdapter.d.ts +54 -0
- package/dist/src/backends/graph/GraphDatabaseAdapter.d.ts.map +1 -1
- package/dist/src/backends/graph/GraphDatabaseAdapter.js +125 -0
- package/dist/src/backends/graph/GraphDatabaseAdapter.js.map +1 -1
- package/dist/src/cli/agentdb-cli.js +0 -0
- package/dist/src/controllers/ReflexionMemory.d.ts +50 -0
- package/dist/src/controllers/ReflexionMemory.d.ts.map +1 -1
- package/dist/src/controllers/ReflexionMemory.js +258 -0
- package/dist/src/controllers/ReflexionMemory.js.map +1 -1
- package/dist/src/controllers/index.d.ts +2 -0
- package/dist/src/controllers/index.d.ts.map +1 -1
- package/dist/src/controllers/index.js +2 -0
- package/dist/src/controllers/index.js.map +1 -1
- package/dist/src/controllers/prerequisites.d.ts +76 -0
- package/dist/src/controllers/prerequisites.d.ts.map +1 -0
- package/dist/src/controllers/prerequisites.js +235 -0
- package/dist/src/controllers/prerequisites.js.map +1 -0
- package/dist/src/db-fallback.d.ts.map +1 -1
- package/dist/src/db-fallback.js +55 -45
- package/dist/src/db-fallback.js.map +1 -1
- package/package.json +1 -1
- package/dist/schemas/frontier-schema.sql +0 -378
- package/dist/schemas/schema.sql +0 -382
- package/dist/src/backends/index.cjs +0 -6
- package/dist/src/backends/ruvector/GuardedVectorBackend.d.ts +0 -93
- package/dist/src/backends/ruvector/GuardedVectorBackend.d.ts.map +0 -1
- package/dist/src/backends/ruvector/GuardedVectorBackend.js +0 -182
- package/dist/src/backends/ruvector/GuardedVectorBackend.js.map +0 -1
- package/dist/src/consensus/RaftConsensus.d.ts +0 -220
- package/dist/src/consensus/RaftConsensus.d.ts.map +0 -1
- package/dist/src/consensus/RaftConsensus.js +0 -762
- package/dist/src/consensus/RaftConsensus.js.map +0 -1
- package/dist/src/controllers/HierarchicalMemory.d.ts +0 -197
- package/dist/src/controllers/HierarchicalMemory.d.ts.map +0 -1
- package/dist/src/controllers/HierarchicalMemory.js +0 -519
- package/dist/src/controllers/HierarchicalMemory.js.map +0 -1
- package/dist/src/controllers/MemoryConsolidation.d.ts +0 -142
- package/dist/src/controllers/MemoryConsolidation.d.ts.map +0 -1
- package/dist/src/controllers/MemoryConsolidation.js +0 -479
- package/dist/src/controllers/MemoryConsolidation.js.map +0 -1
- package/dist/src/controllers/QUICConnection.d.ts +0 -122
- package/dist/src/controllers/QUICConnection.d.ts.map +0 -1
- package/dist/src/controllers/QUICConnection.js +0 -329
- package/dist/src/controllers/QUICConnection.js.map +0 -1
- package/dist/src/controllers/QUICConnectionPool.d.ts +0 -83
- package/dist/src/controllers/QUICConnectionPool.d.ts.map +0 -1
- package/dist/src/controllers/QUICConnectionPool.js +0 -256
- package/dist/src/controllers/QUICConnectionPool.js.map +0 -1
- package/dist/src/controllers/QUICStreamManager.d.ts +0 -114
- package/dist/src/controllers/QUICStreamManager.d.ts.map +0 -1
- package/dist/src/controllers/QUICStreamManager.js +0 -267
- package/dist/src/controllers/QUICStreamManager.js.map +0 -1
- package/dist/src/controllers/StreamingEmbeddingService.d.ts +0 -82
- package/dist/src/controllers/StreamingEmbeddingService.d.ts.map +0 -1
- package/dist/src/controllers/StreamingEmbeddingService.js +0 -243
- package/dist/src/controllers/StreamingEmbeddingService.js.map +0 -1
- package/dist/src/controllers/index.cjs +0 -6
- package/dist/src/coordination/MultiDatabaseCoordinator.d.ts +0 -348
- package/dist/src/coordination/MultiDatabaseCoordinator.d.ts.map +0 -1
- package/dist/src/coordination/MultiDatabaseCoordinator.js +0 -803
- package/dist/src/coordination/MultiDatabaseCoordinator.js.map +0 -1
- package/dist/src/coordination/index.d.ts +0 -10
- package/dist/src/coordination/index.d.ts.map +0 -1
- package/dist/src/coordination/index.js +0 -10
- package/dist/src/coordination/index.js.map +0 -1
- package/dist/src/index.cjs +0 -6
- package/dist/src/optimizations/RVFOptimizer.d.ts +0 -226
- package/dist/src/optimizations/RVFOptimizer.d.ts.map +0 -1
- package/dist/src/optimizations/RVFOptimizer.js +0 -541
- package/dist/src/optimizations/RVFOptimizer.js.map +0 -1
- package/dist/src/security/AttestationLog.d.ts +0 -70
- package/dist/src/security/AttestationLog.d.ts.map +0 -1
- package/dist/src/security/AttestationLog.js +0 -174
- package/dist/src/security/AttestationLog.js.map +0 -1
- package/dist/src/security/MutationGuard.d.ts +0 -83
- package/dist/src/security/MutationGuard.d.ts.map +0 -1
- package/dist/src/security/MutationGuard.js +0 -364
- package/dist/src/security/MutationGuard.js.map +0 -1
- package/dist/src/security/index.cjs +0 -6
- package/dist/src/security/index.d.ts +0 -15
- package/dist/src/security/index.d.ts.map +0 -1
- package/dist/src/security/index.js +0 -18
- package/dist/src/security/index.js.map +0 -1
- package/dist/src/services/GNNService.d.ts +0 -173
- package/dist/src/services/GNNService.d.ts.map +0 -1
- package/dist/src/services/GNNService.js +0 -639
- package/dist/src/services/GNNService.js.map +0 -1
- package/dist/src/services/GraphTransformerService.d.ts +0 -80
- package/dist/src/services/GraphTransformerService.d.ts.map +0 -1
- package/dist/src/services/GraphTransformerService.js +0 -369
- package/dist/src/services/GraphTransformerService.js.map +0 -1
- package/dist/src/services/SemanticRouter.d.ts +0 -83
- package/dist/src/services/SemanticRouter.d.ts.map +0 -1
- package/dist/src/services/SemanticRouter.js +0 -160
- package/dist/src/services/SemanticRouter.js.map +0 -1
- package/dist/src/services/SonaTrajectoryService.d.ts +0 -224
- package/dist/src/services/SonaTrajectoryService.d.ts.map +0 -1
- package/dist/src/services/SonaTrajectoryService.js +0 -539
- package/dist/src/services/SonaTrajectoryService.js.map +0 -1
- package/dist/src/utils/LegacyAttentionAdapter.d.ts +0 -93
- package/dist/src/utils/LegacyAttentionAdapter.d.ts.map +0 -1
- package/dist/src/utils/LegacyAttentionAdapter.js +0 -241
- package/dist/src/utils/LegacyAttentionAdapter.js.map +0 -1
- package/dist/src/utils/vector-math.d.ts +0 -29
- package/dist/src/utils/vector-math.d.ts.map +0 -1
- package/dist/src/utils/vector-math.js +0 -66
- package/dist/src/utils/vector-math.js.map +0 -1
|
@@ -1,803 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* MultiDatabaseCoordinator - Cross-Instance Synchronization and Coordination
|
|
3
|
-
*
|
|
4
|
-
* Orchestrates multiple AgentDB instances for distributed vector database operations.
|
|
5
|
-
* Provides:
|
|
6
|
-
* - Instance registration and lifecycle management
|
|
7
|
-
* - Cross-instance synchronization with conflict resolution
|
|
8
|
-
* - Health monitoring and automatic failover
|
|
9
|
-
* - Distributed operations (broadcast insert/delete)
|
|
10
|
-
* - Configurable replication strategies
|
|
11
|
-
*
|
|
12
|
-
* @module coordination/MultiDatabaseCoordinator
|
|
13
|
-
*/
|
|
14
|
-
// ============================================================================
|
|
15
|
-
// MultiDatabaseCoordinator Implementation
|
|
16
|
-
// ============================================================================
|
|
17
|
-
/**
|
|
18
|
-
* MultiDatabaseCoordinator - Manages multiple AgentDB instances
|
|
19
|
-
*
|
|
20
|
-
* Provides coordination, synchronization, and health monitoring for
|
|
21
|
-
* distributed vector database deployments.
|
|
22
|
-
*
|
|
23
|
-
* @example
|
|
24
|
-
* ```typescript
|
|
25
|
-
* const coordinator = new MultiDatabaseCoordinator(primaryDb, {
|
|
26
|
-
* replicationFactor: 2,
|
|
27
|
-
* syncIntervalMs: 30000,
|
|
28
|
-
* conflictResolution: 'last-write-wins'
|
|
29
|
-
* });
|
|
30
|
-
*
|
|
31
|
-
* // Register secondary instances
|
|
32
|
-
* coordinator.registerInstance({
|
|
33
|
-
* id: 'secondary-1',
|
|
34
|
-
* url: 'http://db1.example.com:8080',
|
|
35
|
-
* status: 'online',
|
|
36
|
-
* lastSyncAt: 0,
|
|
37
|
-
* vectorCount: 0,
|
|
38
|
-
* version: '2.0.0'
|
|
39
|
-
* });
|
|
40
|
-
*
|
|
41
|
-
* // Start health monitoring
|
|
42
|
-
* coordinator.startHealthCheck(5000);
|
|
43
|
-
*
|
|
44
|
-
* // Sync all instances
|
|
45
|
-
* const results = await coordinator.syncAll();
|
|
46
|
-
* ```
|
|
47
|
-
*/
|
|
48
|
-
export class MultiDatabaseCoordinator {
|
|
49
|
-
primaryDb;
|
|
50
|
-
instances = new Map();
|
|
51
|
-
config;
|
|
52
|
-
healthCheckInterval = null;
|
|
53
|
-
syncInterval = null;
|
|
54
|
-
statusChangeCallbacks = [];
|
|
55
|
-
isSyncing = new Map();
|
|
56
|
-
vectorTimestamps = new Map();
|
|
57
|
-
/**
|
|
58
|
-
* Create a new MultiDatabaseCoordinator
|
|
59
|
-
*
|
|
60
|
-
* @param primaryDb - Primary vector database backend
|
|
61
|
-
* @param config - Configuration options
|
|
62
|
-
*/
|
|
63
|
-
constructor(primaryDb, config = {}) {
|
|
64
|
-
this.primaryDb = primaryDb;
|
|
65
|
-
this.config = {
|
|
66
|
-
replicationFactor: config.replicationFactor ?? 2,
|
|
67
|
-
syncIntervalMs: config.syncIntervalMs ?? 30000,
|
|
68
|
-
conflictResolution: config.conflictResolution ?? 'last-write-wins',
|
|
69
|
-
healthCheckIntervalMs: config.healthCheckIntervalMs ?? 10000,
|
|
70
|
-
healthCheckTimeoutMs: config.healthCheckTimeoutMs ?? 5000,
|
|
71
|
-
autoFailover: config.autoFailover ?? true,
|
|
72
|
-
maxRetries: config.maxRetries ?? 3,
|
|
73
|
-
retryDelayMs: config.retryDelayMs ?? 1000,
|
|
74
|
-
};
|
|
75
|
-
// Start automatic sync if configured
|
|
76
|
-
if (this.config.syncIntervalMs > 0) {
|
|
77
|
-
this.startAutoSync();
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
// ==========================================================================
|
|
81
|
-
// Instance Management
|
|
82
|
-
// ==========================================================================
|
|
83
|
-
/**
|
|
84
|
-
* Register a new database instance
|
|
85
|
-
*
|
|
86
|
-
* @param instance - Instance configuration
|
|
87
|
-
* @throws Error if instance with same ID already exists
|
|
88
|
-
*/
|
|
89
|
-
registerInstance(instance) {
|
|
90
|
-
if (this.instances.has(instance.id)) {
|
|
91
|
-
throw new Error(`Instance with ID '${instance.id}' already registered`);
|
|
92
|
-
}
|
|
93
|
-
this.instances.set(instance.id, { ...instance });
|
|
94
|
-
this.isSyncing.set(instance.id, false);
|
|
95
|
-
console.log(`[MultiDatabaseCoordinator] Registered instance: ${instance.id} (${instance.url})`);
|
|
96
|
-
}
|
|
97
|
-
/**
|
|
98
|
-
* Unregister a database instance
|
|
99
|
-
*
|
|
100
|
-
* @param id - Instance ID to unregister
|
|
101
|
-
* @throws Error if instance is currently syncing
|
|
102
|
-
*/
|
|
103
|
-
unregisterInstance(id) {
|
|
104
|
-
if (this.isSyncing.get(id)) {
|
|
105
|
-
throw new Error(`Cannot unregister instance '${id}' while syncing`);
|
|
106
|
-
}
|
|
107
|
-
const removed = this.instances.delete(id);
|
|
108
|
-
this.isSyncing.delete(id);
|
|
109
|
-
if (removed) {
|
|
110
|
-
console.log(`[MultiDatabaseCoordinator] Unregistered instance: ${id}`);
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
/**
|
|
114
|
-
* Get all registered instances
|
|
115
|
-
*
|
|
116
|
-
* @returns Array of database instances
|
|
117
|
-
*/
|
|
118
|
-
getInstances() {
|
|
119
|
-
return Array.from(this.instances.values());
|
|
120
|
-
}
|
|
121
|
-
/**
|
|
122
|
-
* Get a specific instance by ID
|
|
123
|
-
*
|
|
124
|
-
* @param id - Instance ID
|
|
125
|
-
* @returns Instance or null if not found
|
|
126
|
-
*/
|
|
127
|
-
getInstanceStatus(id) {
|
|
128
|
-
return this.instances.get(id) ?? null;
|
|
129
|
-
}
|
|
130
|
-
/**
|
|
131
|
-
* Get only online instances
|
|
132
|
-
*
|
|
133
|
-
* @returns Array of online instances
|
|
134
|
-
*/
|
|
135
|
-
getOnlineInstances() {
|
|
136
|
-
return Array.from(this.instances.values()).filter((i) => i.status === 'online');
|
|
137
|
-
}
|
|
138
|
-
/**
|
|
139
|
-
* Update instance status
|
|
140
|
-
*
|
|
141
|
-
* @param id - Instance ID
|
|
142
|
-
* @param status - New status
|
|
143
|
-
*/
|
|
144
|
-
updateInstanceStatus(id, status) {
|
|
145
|
-
const instance = this.instances.get(id);
|
|
146
|
-
if (!instance)
|
|
147
|
-
return;
|
|
148
|
-
const oldStatus = instance.status;
|
|
149
|
-
if (oldStatus === status)
|
|
150
|
-
return;
|
|
151
|
-
instance.status = status;
|
|
152
|
-
// Notify callbacks
|
|
153
|
-
for (const callback of this.statusChangeCallbacks) {
|
|
154
|
-
try {
|
|
155
|
-
callback(id, oldStatus, status);
|
|
156
|
-
}
|
|
157
|
-
catch (error) {
|
|
158
|
-
console.error(`[MultiDatabaseCoordinator] Status change callback error:`, error);
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
// ==========================================================================
|
|
163
|
-
// Synchronization
|
|
164
|
-
// ==========================================================================
|
|
165
|
-
/**
|
|
166
|
-
* Synchronize data TO a remote instance
|
|
167
|
-
*
|
|
168
|
-
* @param targetId - Target instance ID
|
|
169
|
-
* @param options - Sync options
|
|
170
|
-
* @returns Sync result
|
|
171
|
-
*/
|
|
172
|
-
async syncToInstance(targetId, options = {}) {
|
|
173
|
-
const instance = this.instances.get(targetId);
|
|
174
|
-
if (!instance) {
|
|
175
|
-
return {
|
|
176
|
-
success: false,
|
|
177
|
-
instanceId: targetId,
|
|
178
|
-
itemsSynced: 0,
|
|
179
|
-
conflictsDetected: 0,
|
|
180
|
-
conflictsResolved: 0,
|
|
181
|
-
durationMs: 0,
|
|
182
|
-
bytesTransferred: 0,
|
|
183
|
-
error: `Instance '${targetId}' not found`,
|
|
184
|
-
};
|
|
185
|
-
}
|
|
186
|
-
if (instance.status === 'offline') {
|
|
187
|
-
return {
|
|
188
|
-
success: false,
|
|
189
|
-
instanceId: targetId,
|
|
190
|
-
itemsSynced: 0,
|
|
191
|
-
conflictsDetected: 0,
|
|
192
|
-
conflictsResolved: 0,
|
|
193
|
-
durationMs: 0,
|
|
194
|
-
bytesTransferred: 0,
|
|
195
|
-
error: `Instance '${targetId}' is offline`,
|
|
196
|
-
};
|
|
197
|
-
}
|
|
198
|
-
if (this.isSyncing.get(targetId)) {
|
|
199
|
-
return {
|
|
200
|
-
success: false,
|
|
201
|
-
instanceId: targetId,
|
|
202
|
-
itemsSynced: 0,
|
|
203
|
-
conflictsDetected: 0,
|
|
204
|
-
conflictsResolved: 0,
|
|
205
|
-
durationMs: 0,
|
|
206
|
-
bytesTransferred: 0,
|
|
207
|
-
error: `Instance '${targetId}' is already syncing`,
|
|
208
|
-
};
|
|
209
|
-
}
|
|
210
|
-
this.isSyncing.set(targetId, true);
|
|
211
|
-
this.updateInstanceStatus(targetId, 'syncing');
|
|
212
|
-
const startTime = Date.now();
|
|
213
|
-
try {
|
|
214
|
-
const mergedOptions = this.mergeOptions(options);
|
|
215
|
-
let itemsSynced = 0;
|
|
216
|
-
let conflictsDetected = 0;
|
|
217
|
-
let conflictsResolved = 0;
|
|
218
|
-
let bytesTransferred = 0;
|
|
219
|
-
const unresolvedConflicts = [];
|
|
220
|
-
mergedOptions.onProgress?.({
|
|
221
|
-
phase: 'preparing',
|
|
222
|
-
current: 0,
|
|
223
|
-
total: 100,
|
|
224
|
-
message: 'Preparing synchronization...',
|
|
225
|
-
});
|
|
226
|
-
// Get local vector stats
|
|
227
|
-
const localStats = this.primaryDb.getStats();
|
|
228
|
-
const localVectorIds = this.getLocalVectorIds();
|
|
229
|
-
mergedOptions.onProgress?.({
|
|
230
|
-
phase: 'fetching',
|
|
231
|
-
current: 0,
|
|
232
|
-
total: localVectorIds.length,
|
|
233
|
-
message: `Fetching ${localVectorIds.length} vectors...`,
|
|
234
|
-
});
|
|
235
|
-
// Simulate sync - in real implementation, this would use network calls
|
|
236
|
-
// For now, we track what would be synced
|
|
237
|
-
const syncBatches = this.createBatches(localVectorIds, mergedOptions.batchSize ?? 100);
|
|
238
|
-
for (let i = 0; i < syncBatches.length; i++) {
|
|
239
|
-
const batch = syncBatches[i];
|
|
240
|
-
mergedOptions.onProgress?.({
|
|
241
|
-
phase: 'applying',
|
|
242
|
-
current: i * (mergedOptions.batchSize ?? 100),
|
|
243
|
-
total: localVectorIds.length,
|
|
244
|
-
message: `Syncing batch ${i + 1}/${syncBatches.length}...`,
|
|
245
|
-
});
|
|
246
|
-
// Simulate batch transfer
|
|
247
|
-
for (const vectorId of batch) {
|
|
248
|
-
const localTimestamp = this.vectorTimestamps.get(vectorId) ?? Date.now();
|
|
249
|
-
// Check for conflicts (simulate remote timestamp check)
|
|
250
|
-
const hasConflict = Math.random() < 0.01; // 1% conflict rate for simulation
|
|
251
|
-
if (hasConflict) {
|
|
252
|
-
conflictsDetected++;
|
|
253
|
-
if (mergedOptions.conflictResolution === 'manual') {
|
|
254
|
-
unresolvedConflicts.push({
|
|
255
|
-
vectorId,
|
|
256
|
-
local: {
|
|
257
|
-
embedding: new Float32Array(0),
|
|
258
|
-
timestamp: localTimestamp,
|
|
259
|
-
},
|
|
260
|
-
remote: {
|
|
261
|
-
embedding: new Float32Array(0),
|
|
262
|
-
timestamp: localTimestamp - 1000,
|
|
263
|
-
},
|
|
264
|
-
suggestion: 'keep-local',
|
|
265
|
-
});
|
|
266
|
-
}
|
|
267
|
-
else {
|
|
268
|
-
conflictsResolved++;
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
itemsSynced++;
|
|
272
|
-
bytesTransferred += 384 * 4; // Approximate vector size
|
|
273
|
-
}
|
|
274
|
-
// Simulate network delay
|
|
275
|
-
await this.delay(10);
|
|
276
|
-
}
|
|
277
|
-
// Update instance stats
|
|
278
|
-
instance.lastSyncAt = Date.now();
|
|
279
|
-
instance.vectorCount = localStats.count;
|
|
280
|
-
mergedOptions.onProgress?.({
|
|
281
|
-
phase: 'completed',
|
|
282
|
-
current: itemsSynced,
|
|
283
|
-
total: itemsSynced,
|
|
284
|
-
message: 'Synchronization completed',
|
|
285
|
-
});
|
|
286
|
-
const durationMs = Date.now() - startTime;
|
|
287
|
-
console.log(`[MultiDatabaseCoordinator] Synced to ${targetId}: ${itemsSynced} items, ${conflictsResolved}/${conflictsDetected} conflicts resolved, ${durationMs}ms`);
|
|
288
|
-
return {
|
|
289
|
-
success: true,
|
|
290
|
-
instanceId: targetId,
|
|
291
|
-
itemsSynced,
|
|
292
|
-
conflictsDetected,
|
|
293
|
-
conflictsResolved,
|
|
294
|
-
durationMs,
|
|
295
|
-
bytesTransferred,
|
|
296
|
-
unresolvedConflicts: unresolvedConflicts.length > 0 ? unresolvedConflicts : undefined,
|
|
297
|
-
};
|
|
298
|
-
}
|
|
299
|
-
catch (error) {
|
|
300
|
-
const err = error;
|
|
301
|
-
const durationMs = Date.now() - startTime;
|
|
302
|
-
options.onProgress?.({
|
|
303
|
-
phase: 'error',
|
|
304
|
-
current: 0,
|
|
305
|
-
total: 0,
|
|
306
|
-
error: err.message,
|
|
307
|
-
});
|
|
308
|
-
console.error(`[MultiDatabaseCoordinator] Sync to ${targetId} failed:`, err.message);
|
|
309
|
-
return {
|
|
310
|
-
success: false,
|
|
311
|
-
instanceId: targetId,
|
|
312
|
-
itemsSynced: 0,
|
|
313
|
-
conflictsDetected: 0,
|
|
314
|
-
conflictsResolved: 0,
|
|
315
|
-
durationMs,
|
|
316
|
-
bytesTransferred: 0,
|
|
317
|
-
error: err.message,
|
|
318
|
-
};
|
|
319
|
-
}
|
|
320
|
-
finally {
|
|
321
|
-
this.isSyncing.set(targetId, false);
|
|
322
|
-
const instance = this.instances.get(targetId);
|
|
323
|
-
if (instance && instance.status === 'syncing') {
|
|
324
|
-
this.updateInstanceStatus(targetId, 'online');
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
/**
|
|
329
|
-
* Synchronize data FROM a remote instance
|
|
330
|
-
*
|
|
331
|
-
* @param sourceId - Source instance ID
|
|
332
|
-
* @param options - Sync options
|
|
333
|
-
* @returns Sync result
|
|
334
|
-
*/
|
|
335
|
-
async syncFromInstance(sourceId, options = {}) {
|
|
336
|
-
const instance = this.instances.get(sourceId);
|
|
337
|
-
if (!instance) {
|
|
338
|
-
return {
|
|
339
|
-
success: false,
|
|
340
|
-
instanceId: sourceId,
|
|
341
|
-
itemsSynced: 0,
|
|
342
|
-
conflictsDetected: 0,
|
|
343
|
-
conflictsResolved: 0,
|
|
344
|
-
durationMs: 0,
|
|
345
|
-
bytesTransferred: 0,
|
|
346
|
-
error: `Instance '${sourceId}' not found`,
|
|
347
|
-
};
|
|
348
|
-
}
|
|
349
|
-
if (instance.status === 'offline') {
|
|
350
|
-
return {
|
|
351
|
-
success: false,
|
|
352
|
-
instanceId: sourceId,
|
|
353
|
-
itemsSynced: 0,
|
|
354
|
-
conflictsDetected: 0,
|
|
355
|
-
conflictsResolved: 0,
|
|
356
|
-
durationMs: 0,
|
|
357
|
-
bytesTransferred: 0,
|
|
358
|
-
error: `Instance '${sourceId}' is offline`,
|
|
359
|
-
};
|
|
360
|
-
}
|
|
361
|
-
if (this.isSyncing.get(sourceId)) {
|
|
362
|
-
return {
|
|
363
|
-
success: false,
|
|
364
|
-
instanceId: sourceId,
|
|
365
|
-
itemsSynced: 0,
|
|
366
|
-
conflictsDetected: 0,
|
|
367
|
-
conflictsResolved: 0,
|
|
368
|
-
durationMs: 0,
|
|
369
|
-
bytesTransferred: 0,
|
|
370
|
-
error: `Instance '${sourceId}' is already syncing`,
|
|
371
|
-
};
|
|
372
|
-
}
|
|
373
|
-
this.isSyncing.set(sourceId, true);
|
|
374
|
-
this.updateInstanceStatus(sourceId, 'syncing');
|
|
375
|
-
const startTime = Date.now();
|
|
376
|
-
try {
|
|
377
|
-
const mergedOptions = this.mergeOptions(options);
|
|
378
|
-
let itemsSynced = 0;
|
|
379
|
-
let conflictsDetected = 0;
|
|
380
|
-
let conflictsResolved = 0;
|
|
381
|
-
let bytesTransferred = 0;
|
|
382
|
-
mergedOptions.onProgress?.({
|
|
383
|
-
phase: 'preparing',
|
|
384
|
-
current: 0,
|
|
385
|
-
total: 100,
|
|
386
|
-
message: 'Preparing to fetch from remote...',
|
|
387
|
-
});
|
|
388
|
-
// Simulate fetching from remote
|
|
389
|
-
const remoteVectorCount = instance.vectorCount;
|
|
390
|
-
mergedOptions.onProgress?.({
|
|
391
|
-
phase: 'fetching',
|
|
392
|
-
current: 0,
|
|
393
|
-
total: remoteVectorCount,
|
|
394
|
-
message: `Fetching ${remoteVectorCount} vectors from ${sourceId}...`,
|
|
395
|
-
});
|
|
396
|
-
// Simulate progressive sync
|
|
397
|
-
const batchCount = Math.ceil(remoteVectorCount / (mergedOptions.batchSize ?? 100));
|
|
398
|
-
for (let i = 0; i < batchCount; i++) {
|
|
399
|
-
const batchItems = Math.min(mergedOptions.batchSize ?? 100, remoteVectorCount - i * (mergedOptions.batchSize ?? 100));
|
|
400
|
-
mergedOptions.onProgress?.({
|
|
401
|
-
phase: 'applying',
|
|
402
|
-
current: i * (mergedOptions.batchSize ?? 100),
|
|
403
|
-
total: remoteVectorCount,
|
|
404
|
-
message: `Applying batch ${i + 1}/${batchCount}...`,
|
|
405
|
-
});
|
|
406
|
-
// Simulate conflict detection
|
|
407
|
-
const batchConflicts = Math.floor(batchItems * 0.01);
|
|
408
|
-
conflictsDetected += batchConflicts;
|
|
409
|
-
if (mergedOptions.conflictResolution !== 'manual') {
|
|
410
|
-
conflictsResolved += batchConflicts;
|
|
411
|
-
}
|
|
412
|
-
itemsSynced += batchItems;
|
|
413
|
-
bytesTransferred += batchItems * 384 * 4;
|
|
414
|
-
await this.delay(10);
|
|
415
|
-
}
|
|
416
|
-
// Update instance
|
|
417
|
-
instance.lastSyncAt = Date.now();
|
|
418
|
-
mergedOptions.onProgress?.({
|
|
419
|
-
phase: 'completed',
|
|
420
|
-
current: itemsSynced,
|
|
421
|
-
total: itemsSynced,
|
|
422
|
-
message: 'Synchronization completed',
|
|
423
|
-
});
|
|
424
|
-
const durationMs = Date.now() - startTime;
|
|
425
|
-
console.log(`[MultiDatabaseCoordinator] Synced from ${sourceId}: ${itemsSynced} items, ${conflictsResolved}/${conflictsDetected} conflicts resolved, ${durationMs}ms`);
|
|
426
|
-
return {
|
|
427
|
-
success: true,
|
|
428
|
-
instanceId: sourceId,
|
|
429
|
-
itemsSynced,
|
|
430
|
-
conflictsDetected,
|
|
431
|
-
conflictsResolved,
|
|
432
|
-
durationMs,
|
|
433
|
-
bytesTransferred,
|
|
434
|
-
};
|
|
435
|
-
}
|
|
436
|
-
catch (error) {
|
|
437
|
-
const err = error;
|
|
438
|
-
const durationMs = Date.now() - startTime;
|
|
439
|
-
options.onProgress?.({
|
|
440
|
-
phase: 'error',
|
|
441
|
-
current: 0,
|
|
442
|
-
total: 0,
|
|
443
|
-
error: err.message,
|
|
444
|
-
});
|
|
445
|
-
console.error(`[MultiDatabaseCoordinator] Sync from ${sourceId} failed:`, err.message);
|
|
446
|
-
return {
|
|
447
|
-
success: false,
|
|
448
|
-
instanceId: sourceId,
|
|
449
|
-
itemsSynced: 0,
|
|
450
|
-
conflictsDetected: 0,
|
|
451
|
-
conflictsResolved: 0,
|
|
452
|
-
durationMs,
|
|
453
|
-
bytesTransferred: 0,
|
|
454
|
-
error: err.message,
|
|
455
|
-
};
|
|
456
|
-
}
|
|
457
|
-
finally {
|
|
458
|
-
this.isSyncing.set(sourceId, false);
|
|
459
|
-
const instance = this.instances.get(sourceId);
|
|
460
|
-
if (instance && instance.status === 'syncing') {
|
|
461
|
-
this.updateInstanceStatus(sourceId, 'online');
|
|
462
|
-
}
|
|
463
|
-
}
|
|
464
|
-
}
|
|
465
|
-
/**
|
|
466
|
-
* Synchronize all registered instances
|
|
467
|
-
*
|
|
468
|
-
* @param options - Sync options
|
|
469
|
-
* @returns Map of instance ID to sync result
|
|
470
|
-
*/
|
|
471
|
-
async syncAll(options = {}) {
|
|
472
|
-
const results = new Map();
|
|
473
|
-
const onlineInstances = this.getOnlineInstances();
|
|
474
|
-
if (onlineInstances.length === 0) {
|
|
475
|
-
console.log('[MultiDatabaseCoordinator] No online instances to sync');
|
|
476
|
-
return results;
|
|
477
|
-
}
|
|
478
|
-
console.log(`[MultiDatabaseCoordinator] Starting sync to ${onlineInstances.length} instances...`);
|
|
479
|
-
// Sync in parallel (respecting replication factor)
|
|
480
|
-
const syncPromises = onlineInstances.map(async (instance) => {
|
|
481
|
-
const result = await this.syncToInstance(instance.id, options);
|
|
482
|
-
results.set(instance.id, result);
|
|
483
|
-
return result;
|
|
484
|
-
});
|
|
485
|
-
await Promise.all(syncPromises);
|
|
486
|
-
const successCount = Array.from(results.values()).filter((r) => r.success).length;
|
|
487
|
-
console.log(`[MultiDatabaseCoordinator] Sync completed: ${successCount}/${onlineInstances.length} successful`);
|
|
488
|
-
return results;
|
|
489
|
-
}
|
|
490
|
-
// ==========================================================================
|
|
491
|
-
// Health Monitoring
|
|
492
|
-
// ==========================================================================
|
|
493
|
-
/**
|
|
494
|
-
* Start periodic health checks
|
|
495
|
-
*
|
|
496
|
-
* @param intervalMs - Check interval in milliseconds (overrides config)
|
|
497
|
-
*/
|
|
498
|
-
startHealthCheck(intervalMs) {
|
|
499
|
-
if (this.healthCheckInterval) {
|
|
500
|
-
console.log('[MultiDatabaseCoordinator] Health check already running');
|
|
501
|
-
return;
|
|
502
|
-
}
|
|
503
|
-
const interval = intervalMs ?? this.config.healthCheckIntervalMs;
|
|
504
|
-
console.log(`[MultiDatabaseCoordinator] Starting health checks (interval: ${interval}ms)`);
|
|
505
|
-
this.healthCheckInterval = setInterval(async () => {
|
|
506
|
-
await this.performHealthCheck();
|
|
507
|
-
}, interval);
|
|
508
|
-
// Perform initial check
|
|
509
|
-
this.performHealthCheck();
|
|
510
|
-
}
|
|
511
|
-
/**
|
|
512
|
-
* Stop periodic health checks
|
|
513
|
-
*/
|
|
514
|
-
stopHealthCheck() {
|
|
515
|
-
if (this.healthCheckInterval) {
|
|
516
|
-
clearInterval(this.healthCheckInterval);
|
|
517
|
-
this.healthCheckInterval = null;
|
|
518
|
-
console.log('[MultiDatabaseCoordinator] Health checks stopped');
|
|
519
|
-
}
|
|
520
|
-
}
|
|
521
|
-
/**
|
|
522
|
-
* Perform health check on all instances
|
|
523
|
-
*/
|
|
524
|
-
async performHealthCheck() {
|
|
525
|
-
const instances = this.getInstances();
|
|
526
|
-
for (const instance of instances) {
|
|
527
|
-
try {
|
|
528
|
-
// Simulate health check (in real implementation, make HTTP/TCP request)
|
|
529
|
-
const isHealthy = await this.checkInstanceHealth(instance);
|
|
530
|
-
if (isHealthy && instance.status === 'offline') {
|
|
531
|
-
this.updateInstanceStatus(instance.id, 'online');
|
|
532
|
-
console.log(`[MultiDatabaseCoordinator] Instance ${instance.id} is now online`);
|
|
533
|
-
// Auto-sync if enabled
|
|
534
|
-
if (this.config.autoFailover) {
|
|
535
|
-
this.syncToInstance(instance.id).catch((err) => {
|
|
536
|
-
console.error(`[MultiDatabaseCoordinator] Auto-sync to ${instance.id} failed:`, err);
|
|
537
|
-
});
|
|
538
|
-
}
|
|
539
|
-
}
|
|
540
|
-
else if (!isHealthy && instance.status !== 'offline') {
|
|
541
|
-
this.updateInstanceStatus(instance.id, 'offline');
|
|
542
|
-
console.log(`[MultiDatabaseCoordinator] Instance ${instance.id} is now offline`);
|
|
543
|
-
}
|
|
544
|
-
}
|
|
545
|
-
catch (error) {
|
|
546
|
-
if (instance.status !== 'offline') {
|
|
547
|
-
this.updateInstanceStatus(instance.id, 'offline');
|
|
548
|
-
console.log(`[MultiDatabaseCoordinator] Instance ${instance.id} health check failed`);
|
|
549
|
-
}
|
|
550
|
-
}
|
|
551
|
-
}
|
|
552
|
-
}
|
|
553
|
-
/**
|
|
554
|
-
* Check if a specific instance is healthy
|
|
555
|
-
*/
|
|
556
|
-
async checkInstanceHealth(_instance) {
|
|
557
|
-
// Simulate health check with timeout
|
|
558
|
-
// In real implementation: HTTP ping or TCP connect
|
|
559
|
-
return new Promise((resolve) => {
|
|
560
|
-
setTimeout(() => {
|
|
561
|
-
// Simulate 95% uptime
|
|
562
|
-
resolve(Math.random() > 0.05);
|
|
563
|
-
}, Math.random() * 100);
|
|
564
|
-
});
|
|
565
|
-
}
|
|
566
|
-
/**
|
|
567
|
-
* Register a callback for instance status changes
|
|
568
|
-
*
|
|
569
|
-
* @param callback - Callback function
|
|
570
|
-
* @returns Unsubscribe function
|
|
571
|
-
*/
|
|
572
|
-
onInstanceStatusChange(callback) {
|
|
573
|
-
this.statusChangeCallbacks.push(callback);
|
|
574
|
-
return () => {
|
|
575
|
-
const index = this.statusChangeCallbacks.indexOf(callback);
|
|
576
|
-
if (index >= 0) {
|
|
577
|
-
this.statusChangeCallbacks.splice(index, 1);
|
|
578
|
-
}
|
|
579
|
-
};
|
|
580
|
-
}
|
|
581
|
-
// ==========================================================================
|
|
582
|
-
// Distributed Operations
|
|
583
|
-
// ==========================================================================
|
|
584
|
-
/**
|
|
585
|
-
* Broadcast an insert operation to all online instances
|
|
586
|
-
*
|
|
587
|
-
* @param id - Vector ID
|
|
588
|
-
* @param vector - Vector embedding
|
|
589
|
-
* @param metadata - Optional metadata
|
|
590
|
-
*/
|
|
591
|
-
async broadcastInsert(id, vector, metadata) {
|
|
592
|
-
// Insert to primary
|
|
593
|
-
this.primaryDb.insert(id, vector, metadata);
|
|
594
|
-
this.vectorTimestamps.set(id, Date.now());
|
|
595
|
-
// Get online instances for replication
|
|
596
|
-
const onlineInstances = this.getOnlineInstances();
|
|
597
|
-
const replicationTargets = this.selectReplicationTargets(onlineInstances, this.config.replicationFactor - 1);
|
|
598
|
-
if (replicationTargets.length === 0) {
|
|
599
|
-
return;
|
|
600
|
-
}
|
|
601
|
-
// Broadcast to replicas (fire and forget with retries)
|
|
602
|
-
const promises = replicationTargets.map(async (instance) => {
|
|
603
|
-
for (let retry = 0; retry < this.config.maxRetries; retry++) {
|
|
604
|
-
try {
|
|
605
|
-
// Simulate remote insert
|
|
606
|
-
await this.delay(5);
|
|
607
|
-
return;
|
|
608
|
-
}
|
|
609
|
-
catch (error) {
|
|
610
|
-
if (retry === this.config.maxRetries - 1) {
|
|
611
|
-
console.error(`[MultiDatabaseCoordinator] Failed to replicate insert to ${instance.id} after ${this.config.maxRetries} retries`);
|
|
612
|
-
}
|
|
613
|
-
else {
|
|
614
|
-
await this.delay(this.config.retryDelayMs);
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
}
|
|
618
|
-
});
|
|
619
|
-
await Promise.allSettled(promises);
|
|
620
|
-
}
|
|
621
|
-
/**
|
|
622
|
-
* Broadcast a delete operation to all online instances
|
|
623
|
-
*
|
|
624
|
-
* @param id - Vector ID to delete
|
|
625
|
-
*/
|
|
626
|
-
async broadcastDelete(id) {
|
|
627
|
-
// Delete from primary
|
|
628
|
-
this.primaryDb.remove(id);
|
|
629
|
-
this.vectorTimestamps.delete(id);
|
|
630
|
-
// Get online instances
|
|
631
|
-
const onlineInstances = this.getOnlineInstances();
|
|
632
|
-
if (onlineInstances.length === 0) {
|
|
633
|
-
return;
|
|
634
|
-
}
|
|
635
|
-
// Broadcast delete to all instances
|
|
636
|
-
const promises = onlineInstances.map(async (instance) => {
|
|
637
|
-
for (let retry = 0; retry < this.config.maxRetries; retry++) {
|
|
638
|
-
try {
|
|
639
|
-
// Simulate remote delete
|
|
640
|
-
await this.delay(5);
|
|
641
|
-
return;
|
|
642
|
-
}
|
|
643
|
-
catch (error) {
|
|
644
|
-
if (retry === this.config.maxRetries - 1) {
|
|
645
|
-
console.error(`[MultiDatabaseCoordinator] Failed to replicate delete to ${instance.id}`);
|
|
646
|
-
}
|
|
647
|
-
else {
|
|
648
|
-
await this.delay(this.config.retryDelayMs);
|
|
649
|
-
}
|
|
650
|
-
}
|
|
651
|
-
}
|
|
652
|
-
});
|
|
653
|
-
await Promise.allSettled(promises);
|
|
654
|
-
}
|
|
655
|
-
/**
|
|
656
|
-
* Execute an operation on all instances (including primary)
|
|
657
|
-
*
|
|
658
|
-
* @param operation - Operation to execute
|
|
659
|
-
* @returns Map of instance ID to result
|
|
660
|
-
*/
|
|
661
|
-
async executeOnAll(operation) {
|
|
662
|
-
const results = new Map();
|
|
663
|
-
const errors = new Map();
|
|
664
|
-
// Execute on primary
|
|
665
|
-
try {
|
|
666
|
-
const primaryResult = await operation(this.primaryDb, 'primary');
|
|
667
|
-
results.set('primary', primaryResult);
|
|
668
|
-
}
|
|
669
|
-
catch (error) {
|
|
670
|
-
errors.set('primary', error);
|
|
671
|
-
}
|
|
672
|
-
// Execute on all online instances
|
|
673
|
-
const onlineInstances = this.getOnlineInstances();
|
|
674
|
-
const promises = onlineInstances.map(async (instance) => {
|
|
675
|
-
try {
|
|
676
|
-
// In real implementation, this would make remote calls
|
|
677
|
-
// For now, simulate the operation
|
|
678
|
-
await this.delay(10);
|
|
679
|
-
const result = await operation(this.primaryDb, instance.id);
|
|
680
|
-
results.set(instance.id, result);
|
|
681
|
-
}
|
|
682
|
-
catch (error) {
|
|
683
|
-
errors.set(instance.id, error);
|
|
684
|
-
}
|
|
685
|
-
});
|
|
686
|
-
await Promise.all(promises);
|
|
687
|
-
return {
|
|
688
|
-
results,
|
|
689
|
-
errors,
|
|
690
|
-
successCount: results.size,
|
|
691
|
-
failureCount: errors.size,
|
|
692
|
-
};
|
|
693
|
-
}
|
|
694
|
-
// ==========================================================================
|
|
695
|
-
// Configuration
|
|
696
|
-
// ==========================================================================
|
|
697
|
-
/**
|
|
698
|
-
* Get current configuration
|
|
699
|
-
*/
|
|
700
|
-
getConfig() {
|
|
701
|
-
return { ...this.config };
|
|
702
|
-
}
|
|
703
|
-
/**
|
|
704
|
-
* Update configuration
|
|
705
|
-
*
|
|
706
|
-
* @param updates - Partial configuration updates
|
|
707
|
-
*/
|
|
708
|
-
updateConfig(updates) {
|
|
709
|
-
Object.assign(this.config, updates);
|
|
710
|
-
// Restart auto-sync if interval changed
|
|
711
|
-
if (updates.syncIntervalMs !== undefined) {
|
|
712
|
-
this.stopAutoSync();
|
|
713
|
-
if (this.config.syncIntervalMs > 0) {
|
|
714
|
-
this.startAutoSync();
|
|
715
|
-
}
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
/**
|
|
719
|
-
* Get coordinator statistics
|
|
720
|
-
*/
|
|
721
|
-
getStats() {
|
|
722
|
-
const instances = this.getInstances();
|
|
723
|
-
const onlineCount = instances.filter((i) => i.status === 'online').length;
|
|
724
|
-
const offlineCount = instances.filter((i) => i.status === 'offline').length;
|
|
725
|
-
const syncingCount = instances.filter((i) => i.status === 'syncing').length;
|
|
726
|
-
const totalVectors = instances.reduce((sum, i) => sum + i.vectorCount, 0);
|
|
727
|
-
return {
|
|
728
|
-
primaryStats: this.primaryDb.getStats(),
|
|
729
|
-
instanceCount: instances.length,
|
|
730
|
-
onlineCount,
|
|
731
|
-
offlineCount,
|
|
732
|
-
syncingCount,
|
|
733
|
-
totalVectors,
|
|
734
|
-
};
|
|
735
|
-
}
|
|
736
|
-
// ==========================================================================
|
|
737
|
-
// Cleanup
|
|
738
|
-
// ==========================================================================
|
|
739
|
-
/**
|
|
740
|
-
* Stop all background tasks and cleanup
|
|
741
|
-
*/
|
|
742
|
-
close() {
|
|
743
|
-
this.stopHealthCheck();
|
|
744
|
-
this.stopAutoSync();
|
|
745
|
-
this.instances.clear();
|
|
746
|
-
this.statusChangeCallbacks.length = 0;
|
|
747
|
-
this.vectorTimestamps.clear();
|
|
748
|
-
console.log('[MultiDatabaseCoordinator] Closed');
|
|
749
|
-
}
|
|
750
|
-
// ==========================================================================
|
|
751
|
-
// Private Helpers
|
|
752
|
-
// ==========================================================================
|
|
753
|
-
startAutoSync() {
|
|
754
|
-
if (this.syncInterval)
|
|
755
|
-
return;
|
|
756
|
-
console.log(`[MultiDatabaseCoordinator] Starting auto-sync (interval: ${this.config.syncIntervalMs}ms)`);
|
|
757
|
-
this.syncInterval = setInterval(async () => {
|
|
758
|
-
try {
|
|
759
|
-
await this.syncAll();
|
|
760
|
-
}
|
|
761
|
-
catch (error) {
|
|
762
|
-
console.error('[MultiDatabaseCoordinator] Auto-sync failed:', error);
|
|
763
|
-
}
|
|
764
|
-
}, this.config.syncIntervalMs);
|
|
765
|
-
}
|
|
766
|
-
stopAutoSync() {
|
|
767
|
-
if (this.syncInterval) {
|
|
768
|
-
clearInterval(this.syncInterval);
|
|
769
|
-
this.syncInterval = null;
|
|
770
|
-
console.log('[MultiDatabaseCoordinator] Auto-sync stopped');
|
|
771
|
-
}
|
|
772
|
-
}
|
|
773
|
-
mergeOptions(options) {
|
|
774
|
-
return {
|
|
775
|
-
conflictResolution: options.conflictResolution ?? this.config.conflictResolution,
|
|
776
|
-
batchSize: options.batchSize ?? 100,
|
|
777
|
-
timeoutMs: options.timeoutMs ?? 60000,
|
|
778
|
-
forceFullSync: options.forceFullSync ?? false,
|
|
779
|
-
namespaceFilter: options.namespaceFilter,
|
|
780
|
-
onProgress: options.onProgress,
|
|
781
|
-
};
|
|
782
|
-
}
|
|
783
|
-
getLocalVectorIds() {
|
|
784
|
-
// Return tracked vector IDs
|
|
785
|
-
return Array.from(this.vectorTimestamps.keys());
|
|
786
|
-
}
|
|
787
|
-
createBatches(items, batchSize) {
|
|
788
|
-
const batches = [];
|
|
789
|
-
for (let i = 0; i < items.length; i += batchSize) {
|
|
790
|
-
batches.push(items.slice(i, i + batchSize));
|
|
791
|
-
}
|
|
792
|
-
return batches;
|
|
793
|
-
}
|
|
794
|
-
selectReplicationTargets(instances, count) {
|
|
795
|
-
// Select random instances for replication
|
|
796
|
-
const shuffled = [...instances].sort(() => Math.random() - 0.5);
|
|
797
|
-
return shuffled.slice(0, Math.min(count, shuffled.length));
|
|
798
|
-
}
|
|
799
|
-
delay(ms) {
|
|
800
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
//# sourceMappingURL=MultiDatabaseCoordinator.js.map
|