agentdb 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +38 -0
- package/LICENSE-APACHE +190 -0
- package/LICENSE-MIT +21 -0
- package/README.md +953 -0
- package/bin/agentdb.js +485 -0
- package/bin/plugin-cli-wrapper.mjs +21 -0
- package/dist/cache/query-cache.d.ts +105 -0
- package/dist/cache/query-cache.d.ts.map +1 -0
- package/dist/cache/query-cache.js +224 -0
- package/dist/cache/query-cache.js.map +1 -0
- package/dist/cache/query-cache.mjs +219 -0
- package/dist/cli/cache/query-cache.d.ts +104 -0
- package/dist/cli/cache/query-cache.js +244 -0
- package/dist/cli/cli/db-commands.d.ts +48 -0
- package/dist/cli/cli/db-commands.js +613 -0
- package/dist/cli/commands.d.ts +7 -0
- package/dist/cli/commands.d.ts.map +1 -0
- package/dist/cli/commands.js +113 -0
- package/dist/cli/commands.js.map +1 -0
- package/dist/cli/commands.mjs +104 -0
- package/dist/cli/core/backend-interface.d.ts +70 -0
- package/dist/cli/core/backend-interface.js +15 -0
- package/dist/cli/core/native-backend.d.ts +140 -0
- package/dist/cli/core/native-backend.js +432 -0
- package/dist/cli/core/vector-db.d.ts +126 -0
- package/dist/cli/core/vector-db.js +338 -0
- package/dist/cli/core/wasm-backend.d.ts +95 -0
- package/dist/cli/core/wasm-backend.js +418 -0
- package/dist/cli/db-commands.d.ts +49 -0
- package/dist/cli/db-commands.d.ts.map +1 -0
- package/dist/cli/db-commands.js +533 -0
- package/dist/cli/db-commands.js.map +1 -0
- package/dist/cli/db-commands.mjs +522 -0
- package/dist/cli/generator.d.ts +11 -0
- package/dist/cli/generator.d.ts.map +1 -0
- package/dist/cli/generator.js +567 -0
- package/dist/cli/generator.js.map +1 -0
- package/dist/cli/generator.mjs +527 -0
- package/dist/cli/help.d.ts +18 -0
- package/dist/cli/help.d.ts.map +1 -0
- package/dist/cli/help.js +676 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/help.mjs +667 -0
- package/dist/cli/index/hnsw.d.ts +164 -0
- package/dist/cli/index/hnsw.js +558 -0
- package/dist/cli/plugin-cli.d.ts +7 -0
- package/dist/cli/plugin-cli.d.ts.map +1 -0
- package/dist/cli/plugin-cli.js +295 -0
- package/dist/cli/plugin-cli.js.map +1 -0
- package/dist/cli/plugin-cli.mjs +289 -0
- package/dist/cli/quantization/product-quantization.d.ts +108 -0
- package/dist/cli/quantization/product-quantization.js +350 -0
- package/dist/cli/query/query-builder.d.ts +322 -0
- package/dist/cli/query/query-builder.js +600 -0
- package/dist/cli/templates.d.ts +14 -0
- package/dist/cli/templates.d.ts.map +1 -0
- package/dist/cli/templates.js +182 -0
- package/dist/cli/templates.js.map +1 -0
- package/dist/cli/templates.mjs +176 -0
- package/dist/cli/types/index.d.ts +116 -0
- package/dist/cli/types/index.js +5 -0
- package/dist/cli/types.d.ts +91 -0
- package/dist/cli/types.d.ts.map +1 -0
- package/dist/cli/types.js +6 -0
- package/dist/cli/types.js.map +1 -0
- package/dist/cli/types.mjs +4 -0
- package/dist/cli/wizard/index.d.ts +6 -0
- package/dist/cli/wizard/index.d.ts.map +1 -0
- package/dist/cli/wizard/index.js +138 -0
- package/dist/cli/wizard/index.js.map +1 -0
- package/dist/cli/wizard/index.mjs +131 -0
- package/dist/cli/wizard/prompts.d.ts +11 -0
- package/dist/cli/wizard/prompts.d.ts.map +1 -0
- package/dist/cli/wizard/prompts.js +482 -0
- package/dist/cli/wizard/prompts.js.map +1 -0
- package/dist/cli/wizard/prompts.mjs +470 -0
- package/dist/cli/wizard/validator.d.ts +13 -0
- package/dist/cli/wizard/validator.d.ts.map +1 -0
- package/dist/cli/wizard/validator.js +234 -0
- package/dist/cli/wizard/validator.js.map +1 -0
- package/dist/cli/wizard/validator.mjs +224 -0
- package/dist/core/backend-interface.d.ts +71 -0
- package/dist/core/backend-interface.d.ts.map +1 -0
- package/dist/core/backend-interface.js +16 -0
- package/dist/core/backend-interface.js.map +1 -0
- package/dist/core/backend-interface.mjs +12 -0
- package/dist/core/native-backend.d.ts +141 -0
- package/dist/core/native-backend.d.ts.map +1 -0
- package/dist/core/native-backend.js +457 -0
- package/dist/core/native-backend.js.map +1 -0
- package/dist/core/native-backend.mjs +449 -0
- package/dist/core/vector-db.d.ts +127 -0
- package/dist/core/vector-db.d.ts.map +1 -0
- package/dist/core/vector-db.js +266 -0
- package/dist/core/vector-db.js.map +1 -0
- package/dist/core/vector-db.mjs +261 -0
- package/dist/core/wasm-backend.d.ts +96 -0
- package/dist/core/wasm-backend.d.ts.map +1 -0
- package/dist/core/wasm-backend.js +393 -0
- package/dist/core/wasm-backend.js.map +1 -0
- package/dist/core/wasm-backend.mjs +385 -0
- package/dist/index/hnsw-optimized.d.ts +75 -0
- package/dist/index/hnsw-optimized.d.ts.map +1 -0
- package/dist/index/hnsw-optimized.js +412 -0
- package/dist/index/hnsw-optimized.js.map +1 -0
- package/dist/index/hnsw-optimized.mjs +407 -0
- package/dist/index/hnsw.d.ts +165 -0
- package/dist/index/hnsw.d.ts.map +1 -0
- package/dist/index/hnsw.js +521 -0
- package/dist/index/hnsw.js.map +1 -0
- package/dist/index/hnsw.mjs +516 -0
- package/dist/index.d.ts +57 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +82 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +63 -0
- package/dist/mcp-server.d.ts +27 -0
- package/dist/mcp-server.d.ts.map +1 -0
- package/dist/mcp-server.js +789 -0
- package/dist/mcp-server.js.map +1 -0
- package/dist/mcp-server.mjs +784 -0
- package/dist/plugins/base-plugin.d.ts +114 -0
- package/dist/plugins/base-plugin.d.ts.map +1 -0
- package/dist/plugins/base-plugin.js +313 -0
- package/dist/plugins/base-plugin.js.map +1 -0
- package/dist/plugins/base-plugin.mjs +275 -0
- package/dist/plugins/implementations/active-learning.d.ts +135 -0
- package/dist/plugins/implementations/active-learning.d.ts.map +1 -0
- package/dist/plugins/implementations/active-learning.js +372 -0
- package/dist/plugins/implementations/active-learning.js.map +1 -0
- package/dist/plugins/implementations/active-learning.mjs +367 -0
- package/dist/plugins/implementations/actor-critic.d.ts +64 -0
- package/dist/plugins/implementations/actor-critic.d.ts.map +1 -0
- package/dist/plugins/implementations/actor-critic.js +363 -0
- package/dist/plugins/implementations/actor-critic.js.map +1 -0
- package/dist/plugins/implementations/actor-critic.mjs +358 -0
- package/dist/plugins/implementations/adversarial-training.d.ts +133 -0
- package/dist/plugins/implementations/adversarial-training.d.ts.map +1 -0
- package/dist/plugins/implementations/adversarial-training.js +409 -0
- package/dist/plugins/implementations/adversarial-training.js.map +1 -0
- package/dist/plugins/implementations/adversarial-training.mjs +404 -0
- package/dist/plugins/implementations/curriculum-learning.d.ts +132 -0
- package/dist/plugins/implementations/curriculum-learning.d.ts.map +1 -0
- package/dist/plugins/implementations/curriculum-learning.js +354 -0
- package/dist/plugins/implementations/curriculum-learning.js.map +1 -0
- package/dist/plugins/implementations/curriculum-learning.mjs +349 -0
- package/dist/plugins/implementations/decision-transformer.d.ts +77 -0
- package/dist/plugins/implementations/decision-transformer.d.ts.map +1 -0
- package/dist/plugins/implementations/decision-transformer.js +422 -0
- package/dist/plugins/implementations/decision-transformer.js.map +1 -0
- package/dist/plugins/implementations/decision-transformer.mjs +417 -0
- package/dist/plugins/implementations/federated-learning.d.ts +126 -0
- package/dist/plugins/implementations/federated-learning.d.ts.map +1 -0
- package/dist/plugins/implementations/federated-learning.js +436 -0
- package/dist/plugins/implementations/federated-learning.js.map +1 -0
- package/dist/plugins/implementations/federated-learning.mjs +431 -0
- package/dist/plugins/implementations/index.d.ts +30 -0
- package/dist/plugins/implementations/index.d.ts.map +1 -0
- package/dist/plugins/implementations/index.js +45 -0
- package/dist/plugins/implementations/index.js.map +1 -0
- package/dist/plugins/implementations/index.mjs +31 -0
- package/dist/plugins/implementations/multi-task-learning.d.ts +115 -0
- package/dist/plugins/implementations/multi-task-learning.d.ts.map +1 -0
- package/dist/plugins/implementations/multi-task-learning.js +369 -0
- package/dist/plugins/implementations/multi-task-learning.js.map +1 -0
- package/dist/plugins/implementations/multi-task-learning.mjs +364 -0
- package/dist/plugins/implementations/neural-architecture-search.d.ts +148 -0
- package/dist/plugins/implementations/neural-architecture-search.d.ts.map +1 -0
- package/dist/plugins/implementations/neural-architecture-search.js +379 -0
- package/dist/plugins/implementations/neural-architecture-search.js.map +1 -0
- package/dist/plugins/implementations/neural-architecture-search.mjs +374 -0
- package/dist/plugins/implementations/q-learning.d.ts +98 -0
- package/dist/plugins/implementations/q-learning.d.ts.map +1 -0
- package/dist/plugins/implementations/q-learning.js +435 -0
- package/dist/plugins/implementations/q-learning.js.map +1 -0
- package/dist/plugins/implementations/q-learning.mjs +430 -0
- package/dist/plugins/implementations/sarsa.d.ts +103 -0
- package/dist/plugins/implementations/sarsa.d.ts.map +1 -0
- package/dist/plugins/implementations/sarsa.js +347 -0
- package/dist/plugins/implementations/sarsa.js.map +1 -0
- package/dist/plugins/implementations/sarsa.mjs +342 -0
- package/dist/plugins/index.d.ts +107 -0
- package/dist/plugins/index.d.ts.map +1 -0
- package/dist/plugins/index.js +179 -0
- package/dist/plugins/index.js.map +1 -0
- package/dist/plugins/index.mjs +168 -0
- package/dist/plugins/interface.d.ts +439 -0
- package/dist/plugins/interface.d.ts.map +1 -0
- package/dist/plugins/interface.js +12 -0
- package/dist/plugins/interface.js.map +1 -0
- package/dist/plugins/interface.mjs +10 -0
- package/dist/plugins/learning-plugin.interface.d.ts +257 -0
- package/dist/plugins/learning-plugin.interface.d.ts.map +1 -0
- package/dist/plugins/learning-plugin.interface.js +7 -0
- package/dist/plugins/learning-plugin.interface.js.map +1 -0
- package/dist/plugins/learning-plugin.interface.mjs +5 -0
- package/dist/plugins/plugin-exports.d.ts +71 -0
- package/dist/plugins/plugin-exports.d.ts.map +1 -0
- package/dist/plugins/plugin-exports.js +78 -0
- package/dist/plugins/plugin-exports.js.map +1 -0
- package/dist/plugins/plugin-exports.mjs +69 -0
- package/dist/plugins/registry.d.ts +206 -0
- package/dist/plugins/registry.d.ts.map +1 -0
- package/dist/plugins/registry.js +365 -0
- package/dist/plugins/registry.js.map +1 -0
- package/dist/plugins/registry.mjs +356 -0
- package/dist/plugins/validator.d.ts +63 -0
- package/dist/plugins/validator.d.ts.map +1 -0
- package/dist/plugins/validator.js +464 -0
- package/dist/plugins/validator.js.map +1 -0
- package/dist/plugins/validator.mjs +458 -0
- package/dist/quantization/binary-quantization.d.ts +104 -0
- package/dist/quantization/binary-quantization.d.ts.map +1 -0
- package/dist/quantization/binary-quantization.js +246 -0
- package/dist/quantization/binary-quantization.js.map +1 -0
- package/dist/quantization/binary-quantization.mjs +240 -0
- package/dist/quantization/optimized-pq.d.ts +138 -0
- package/dist/quantization/optimized-pq.d.ts.map +1 -0
- package/dist/quantization/optimized-pq.js +320 -0
- package/dist/quantization/optimized-pq.js.map +1 -0
- package/dist/quantization/optimized-pq.mjs +313 -0
- package/dist/quantization/product-quantization.d.ts +109 -0
- package/dist/quantization/product-quantization.d.ts.map +1 -0
- package/dist/quantization/product-quantization.js +287 -0
- package/dist/quantization/product-quantization.js.map +1 -0
- package/dist/quantization/product-quantization.mjs +282 -0
- package/dist/quantization/scalar-quantization.d.ts +100 -0
- package/dist/quantization/scalar-quantization.d.ts.map +1 -0
- package/dist/quantization/scalar-quantization.js +324 -0
- package/dist/quantization/scalar-quantization.js.map +1 -0
- package/dist/quantization/scalar-quantization.mjs +319 -0
- package/dist/query/index.d.ts +6 -0
- package/dist/query/index.d.ts.map +1 -0
- package/dist/query/index.js +9 -0
- package/dist/query/index.js.map +1 -0
- package/dist/query/index.mjs +4 -0
- package/dist/query/query-builder.d.ts +323 -0
- package/dist/query/query-builder.d.ts.map +1 -0
- package/dist/query/query-builder.js +524 -0
- package/dist/query/query-builder.js.map +1 -0
- package/dist/query/query-builder.mjs +519 -0
- package/dist/reasoning/context-synthesizer.d.ts +57 -0
- package/dist/reasoning/context-synthesizer.d.ts.map +1 -0
- package/dist/reasoning/context-synthesizer.js +224 -0
- package/dist/reasoning/context-synthesizer.js.map +1 -0
- package/dist/reasoning/context-synthesizer.mjs +219 -0
- package/dist/reasoning/experience-curator.d.ts +66 -0
- package/dist/reasoning/experience-curator.d.ts.map +1 -0
- package/dist/reasoning/experience-curator.js +288 -0
- package/dist/reasoning/experience-curator.js.map +1 -0
- package/dist/reasoning/experience-curator.mjs +283 -0
- package/dist/reasoning/memory-optimizer.d.ts +69 -0
- package/dist/reasoning/memory-optimizer.d.ts.map +1 -0
- package/dist/reasoning/memory-optimizer.js +331 -0
- package/dist/reasoning/memory-optimizer.js.map +1 -0
- package/dist/reasoning/memory-optimizer.mjs +326 -0
- package/dist/reasoning/pattern-matcher.d.ts +59 -0
- package/dist/reasoning/pattern-matcher.d.ts.map +1 -0
- package/dist/reasoning/pattern-matcher.js +229 -0
- package/dist/reasoning/pattern-matcher.js.map +1 -0
- package/dist/reasoning/pattern-matcher.mjs +224 -0
- package/dist/reasoningbank/adapter/agentdb-adapter.d.ts +118 -0
- package/dist/reasoningbank/adapter/agentdb-adapter.d.ts.map +1 -0
- package/dist/reasoningbank/adapter/agentdb-adapter.js +477 -0
- package/dist/reasoningbank/adapter/agentdb-adapter.js.map +1 -0
- package/dist/reasoningbank/adapter/types.d.ts +113 -0
- package/dist/reasoningbank/adapter/types.d.ts.map +1 -0
- package/dist/reasoningbank/adapter/types.js +9 -0
- package/dist/reasoningbank/adapter/types.js.map +1 -0
- package/dist/reasoningbank/cli/commands.d.ts +16 -0
- package/dist/reasoningbank/cli/commands.d.ts.map +1 -0
- package/dist/reasoningbank/cli/commands.js +272 -0
- package/dist/reasoningbank/cli/commands.js.map +1 -0
- package/dist/reasoningbank/mcp/agentdb-tools.d.ts +319 -0
- package/dist/reasoningbank/mcp/agentdb-tools.d.ts.map +1 -0
- package/dist/reasoningbank/mcp/agentdb-tools.js +301 -0
- package/dist/reasoningbank/mcp/agentdb-tools.js.map +1 -0
- package/dist/reasoningbank/migration/migrate.d.ts +25 -0
- package/dist/reasoningbank/migration/migrate.d.ts.map +1 -0
- package/dist/reasoningbank/migration/migrate.js +178 -0
- package/dist/reasoningbank/migration/migrate.js.map +1 -0
- package/dist/reasoningbank/reasoning/context-synthesizer.d.ts +37 -0
- package/dist/reasoningbank/reasoning/context-synthesizer.d.ts.map +1 -0
- package/dist/reasoningbank/reasoning/context-synthesizer.js +114 -0
- package/dist/reasoningbank/reasoning/context-synthesizer.js.map +1 -0
- package/dist/reasoningbank/reasoning/experience-curator.d.ts +39 -0
- package/dist/reasoningbank/reasoning/experience-curator.d.ts.map +1 -0
- package/dist/reasoningbank/reasoning/experience-curator.js +98 -0
- package/dist/reasoningbank/reasoning/experience-curator.js.map +1 -0
- package/dist/reasoningbank/reasoning/memory-optimizer.d.ts +44 -0
- package/dist/reasoningbank/reasoning/memory-optimizer.d.ts.map +1 -0
- package/dist/reasoningbank/reasoning/memory-optimizer.js +184 -0
- package/dist/reasoningbank/reasoning/memory-optimizer.js.map +1 -0
- package/dist/reasoningbank/reasoning/pattern-matcher.d.ts +40 -0
- package/dist/reasoningbank/reasoning/pattern-matcher.d.ts.map +1 -0
- package/dist/reasoningbank/reasoning/pattern-matcher.js +87 -0
- package/dist/reasoningbank/reasoning/pattern-matcher.js.map +1 -0
- package/dist/reasoningbank/sync/quic-sync.d.ts +77 -0
- package/dist/reasoningbank/sync/quic-sync.d.ts.map +1 -0
- package/dist/reasoningbank/sync/quic-sync.js +165 -0
- package/dist/reasoningbank/sync/quic-sync.js.map +1 -0
- package/dist/sync/conflict.d.ts +78 -0
- package/dist/sync/conflict.d.ts.map +1 -0
- package/dist/sync/conflict.js +202 -0
- package/dist/sync/conflict.js.map +1 -0
- package/dist/sync/conflict.mjs +196 -0
- package/dist/sync/coordinator.d.ts +111 -0
- package/dist/sync/coordinator.d.ts.map +1 -0
- package/dist/sync/coordinator.js +256 -0
- package/dist/sync/coordinator.js.map +1 -0
- package/dist/sync/coordinator.mjs +250 -0
- package/dist/sync/delta.d.ts +81 -0
- package/dist/sync/delta.d.ts.map +1 -0
- package/dist/sync/delta.js +245 -0
- package/dist/sync/delta.js.map +1 -0
- package/dist/sync/delta.mjs +238 -0
- package/dist/sync/index.d.ts +11 -0
- package/dist/sync/index.d.ts.map +1 -0
- package/dist/sync/index.js +22 -0
- package/dist/sync/index.js.map +1 -0
- package/dist/sync/index.mjs +9 -0
- package/dist/sync/quic-sync.d.ts +81 -0
- package/dist/sync/quic-sync.d.ts.map +1 -0
- package/dist/sync/quic-sync.js +329 -0
- package/dist/sync/quic-sync.js.map +1 -0
- package/dist/sync/quic-sync.mjs +323 -0
- package/dist/sync/types.d.ts +168 -0
- package/dist/sync/types.d.ts.map +1 -0
- package/dist/sync/types.js +8 -0
- package/dist/sync/types.js.map +1 -0
- package/dist/sync/types.mjs +6 -0
- package/dist/types/index.d.ts +117 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +6 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/index.mjs +4 -0
- package/dist/wasm-loader.d.ts +32 -0
- package/dist/wasm-loader.d.ts.map +1 -0
- package/dist/wasm-loader.js +75 -0
- package/dist/wasm-loader.js.map +1 -0
- package/dist/wasm-loader.mjs +64 -0
- package/examples/adaptive-learning.ts +284 -0
- package/examples/browser/README.md +732 -0
- package/examples/browser/adaptive-recommendations/index.html +427 -0
- package/examples/browser/collaborative-filtering/index.html +310 -0
- package/examples/browser/continual-learning/index.html +736 -0
- package/examples/browser/experience-replay/index.html +616 -0
- package/examples/browser/index.html +369 -0
- package/examples/browser/meta-learning/index.html +789 -0
- package/examples/browser/neuro-symbolic/index.html +692 -0
- package/examples/browser/pattern-learning/index.html +620 -0
- package/examples/browser/quantum-inspired/index.html +728 -0
- package/examples/browser/rag/index.html +624 -0
- package/examples/browser/swarm-intelligence/index.html +811 -0
- package/examples/browser-basic.html +170 -0
- package/examples/hnsw-example.ts +148 -0
- package/examples/node-basic.js +70 -0
- package/examples/quic-sync-example.ts +310 -0
- package/examples/quick-start.js +68 -0
- package/examples/wasm-example.ts +222 -0
- package/package.json +118 -0
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLiteVector QUIC Sync - Multi-Shard Coordinator
|
|
3
|
+
*
|
|
4
|
+
* Orchestrates synchronization across multiple shards and peers
|
|
5
|
+
*/
|
|
6
|
+
import { VectorQuicSync } from './quic-sync.mjs';
|
|
7
|
+
/**
|
|
8
|
+
* Multi-shard synchronization coordinator
|
|
9
|
+
*/
|
|
10
|
+
export class ShardCoordinator {
|
|
11
|
+
constructor(sync, maxConcurrentSyncs = 5) {
|
|
12
|
+
this.peers = new Map();
|
|
13
|
+
this.syncTasks = [];
|
|
14
|
+
this.activeSyncs = new Set();
|
|
15
|
+
this.sync = sync;
|
|
16
|
+
this.maxConcurrentSyncs = maxConcurrentSyncs;
|
|
17
|
+
this.stats = {
|
|
18
|
+
totalShards: 0,
|
|
19
|
+
activeSyncs: 0,
|
|
20
|
+
totalSyncs: 0,
|
|
21
|
+
totalConflicts: 0,
|
|
22
|
+
avgSyncDurationMs: 0,
|
|
23
|
+
bytesSent: 0,
|
|
24
|
+
bytesReceived: 0
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Register peer for synchronization
|
|
29
|
+
*/
|
|
30
|
+
registerPeer(peer) {
|
|
31
|
+
this.peers.set(peer.nodeId, peer);
|
|
32
|
+
this.stats.totalShards += peer.shards.length;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Unregister peer
|
|
36
|
+
*/
|
|
37
|
+
unregisterPeer(nodeId) {
|
|
38
|
+
const peer = this.peers.get(nodeId);
|
|
39
|
+
if (peer) {
|
|
40
|
+
this.stats.totalShards -= peer.shards.length;
|
|
41
|
+
this.peers.delete(nodeId);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Get registered peers
|
|
46
|
+
*/
|
|
47
|
+
getPeers() {
|
|
48
|
+
return Array.from(this.peers.values());
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Schedule shard sync with peer
|
|
52
|
+
*/
|
|
53
|
+
scheduleSyncTask(shardId, peerAddress, priority = 0) {
|
|
54
|
+
const task = {
|
|
55
|
+
shardId,
|
|
56
|
+
peerAddress,
|
|
57
|
+
priority,
|
|
58
|
+
scheduledAt: Date.now(),
|
|
59
|
+
retries: 0
|
|
60
|
+
};
|
|
61
|
+
this.syncTasks.push(task);
|
|
62
|
+
this.syncTasks.sort((a, b) => b.priority - a.priority);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Execute scheduled sync tasks
|
|
66
|
+
*/
|
|
67
|
+
async executeTasks() {
|
|
68
|
+
const results = [];
|
|
69
|
+
while (this.syncTasks.length > 0 && this.activeSyncs.size < this.maxConcurrentSyncs) {
|
|
70
|
+
const task = this.syncTasks.shift();
|
|
71
|
+
const taskKey = `${task.shardId}:${task.peerAddress}`;
|
|
72
|
+
// Skip if already syncing
|
|
73
|
+
if (this.activeSyncs.has(taskKey)) {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
// Execute sync
|
|
77
|
+
this.activeSyncs.add(taskKey);
|
|
78
|
+
this.stats.activeSyncs = this.activeSyncs.size;
|
|
79
|
+
const result = await this.executeTask(task);
|
|
80
|
+
results.push(result);
|
|
81
|
+
this.activeSyncs.delete(taskKey);
|
|
82
|
+
this.stats.activeSyncs = this.activeSyncs.size;
|
|
83
|
+
// Update stats
|
|
84
|
+
this.updateStats(result);
|
|
85
|
+
// Retry on failure
|
|
86
|
+
if (!result.success && task.retries < 3) {
|
|
87
|
+
task.retries++;
|
|
88
|
+
task.priority -= 1;
|
|
89
|
+
this.syncTasks.push(task);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return results;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Execute single sync task
|
|
96
|
+
*/
|
|
97
|
+
async executeTask(task) {
|
|
98
|
+
try {
|
|
99
|
+
const result = await this.sync.sync(task.shardId, task.peerAddress);
|
|
100
|
+
return result;
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
return {
|
|
104
|
+
shardId: task.shardId,
|
|
105
|
+
changesApplied: 0,
|
|
106
|
+
conflictsDetected: 0,
|
|
107
|
+
conflictsResolved: 0,
|
|
108
|
+
conflictsUnresolved: [],
|
|
109
|
+
durationMs: 0,
|
|
110
|
+
success: false,
|
|
111
|
+
error: error instanceof Error ? error.message : String(error)
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Sync all shards with all peers
|
|
117
|
+
*/
|
|
118
|
+
async syncAll() {
|
|
119
|
+
const results = [];
|
|
120
|
+
for (const peer of this.peers.values()) {
|
|
121
|
+
for (const shardId of peer.shards) {
|
|
122
|
+
this.scheduleSyncTask(shardId, peer.address, 1);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return this.executeTasks();
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Sync specific shard with all peers that have it
|
|
129
|
+
*/
|
|
130
|
+
async syncShard(shardId) {
|
|
131
|
+
const results = [];
|
|
132
|
+
for (const peer of this.peers.values()) {
|
|
133
|
+
if (peer.shards.includes(shardId)) {
|
|
134
|
+
this.scheduleSyncTask(shardId, peer.address, 5);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return this.executeTasks();
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Sync all shards with specific peer
|
|
141
|
+
*/
|
|
142
|
+
async syncWithPeer(peerAddress) {
|
|
143
|
+
const peer = Array.from(this.peers.values()).find(p => p.address === peerAddress);
|
|
144
|
+
if (!peer) {
|
|
145
|
+
throw new Error(`Peer not found: ${peerAddress}`);
|
|
146
|
+
}
|
|
147
|
+
for (const shardId of peer.shards) {
|
|
148
|
+
this.scheduleSyncTask(shardId, peer.address, 2);
|
|
149
|
+
}
|
|
150
|
+
return this.executeTasks();
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Get state of all managed shards
|
|
154
|
+
*/
|
|
155
|
+
async getAllShardStates() {
|
|
156
|
+
const states = [];
|
|
157
|
+
const shardIds = new Set();
|
|
158
|
+
// Collect all unique shard IDs
|
|
159
|
+
for (const peer of this.peers.values()) {
|
|
160
|
+
for (const shardId of peer.shards) {
|
|
161
|
+
shardIds.add(shardId);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
// Get state for each shard
|
|
165
|
+
for (const shardId of shardIds) {
|
|
166
|
+
const state = await this.sync.getShardState(shardId);
|
|
167
|
+
states.push(state);
|
|
168
|
+
}
|
|
169
|
+
return states;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Start automatic sync for all shards
|
|
173
|
+
*/
|
|
174
|
+
startAutoSyncAll(intervalMs) {
|
|
175
|
+
for (const peer of this.peers.values()) {
|
|
176
|
+
for (const shardId of peer.shards) {
|
|
177
|
+
this.sync.startAutoSync(shardId, peer.address);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Stop all automatic syncs
|
|
183
|
+
*/
|
|
184
|
+
stopAutoSyncAll() {
|
|
185
|
+
this.sync.stopAllAutoSyncs();
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Get coordinator statistics
|
|
189
|
+
*/
|
|
190
|
+
getStats() {
|
|
191
|
+
return { ...this.stats };
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Update statistics after sync
|
|
195
|
+
*/
|
|
196
|
+
updateStats(result) {
|
|
197
|
+
this.stats.totalSyncs++;
|
|
198
|
+
this.stats.totalConflicts += result.conflictsDetected;
|
|
199
|
+
// Update moving average for sync duration
|
|
200
|
+
const alpha = 0.3; // Smoothing factor
|
|
201
|
+
this.stats.avgSyncDurationMs =
|
|
202
|
+
this.stats.avgSyncDurationMs * (1 - alpha) + result.durationMs * alpha;
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Get pending sync tasks count
|
|
206
|
+
*/
|
|
207
|
+
getPendingTasksCount() {
|
|
208
|
+
return this.syncTasks.length;
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Get active syncs count
|
|
212
|
+
*/
|
|
213
|
+
getActiveSyncsCount() {
|
|
214
|
+
return this.activeSyncs.size;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Clear all pending tasks
|
|
218
|
+
*/
|
|
219
|
+
clearPendingTasks() {
|
|
220
|
+
this.syncTasks = [];
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Health check for all peers
|
|
224
|
+
*/
|
|
225
|
+
async healthCheck() {
|
|
226
|
+
const health = new Map();
|
|
227
|
+
for (const peer of this.peers.values()) {
|
|
228
|
+
try {
|
|
229
|
+
// Attempt sync with minimal data to check connectivity
|
|
230
|
+
const result = await this.sync.sync(peer.shards[0] || 'health-check', peer.address);
|
|
231
|
+
health.set(peer.nodeId, result.success);
|
|
232
|
+
peer.status = result.success ? 'online' : 'offline';
|
|
233
|
+
peer.lastContact = Date.now();
|
|
234
|
+
}
|
|
235
|
+
catch {
|
|
236
|
+
health.set(peer.nodeId, false);
|
|
237
|
+
peer.status = 'offline';
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
return health;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Create coordinator with convenience wrapper
|
|
245
|
+
*/
|
|
246
|
+
export async function createShardCoordinator(db, nodeId, config, maxConcurrentSyncs) {
|
|
247
|
+
const sync = new VectorQuicSync(db, nodeId, config);
|
|
248
|
+
await sync.initialize();
|
|
249
|
+
return new ShardCoordinator(sync, maxConcurrentSyncs);
|
|
250
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLiteVector QUIC Sync - Delta Computation
|
|
3
|
+
*
|
|
4
|
+
* Efficient delta computation from SQLite changelog for sync
|
|
5
|
+
*/
|
|
6
|
+
import type { VectorChange, VectorDelta } from './types';
|
|
7
|
+
/**
|
|
8
|
+
* Delta encoder for vector changes
|
|
9
|
+
*/
|
|
10
|
+
export declare class DeltaEncoder {
|
|
11
|
+
/**
|
|
12
|
+
* Encode changes into compressed delta package
|
|
13
|
+
*/
|
|
14
|
+
static encode(shardId: string, changes: VectorChange[], compression?: 'none' | 'msgpack' | 'lz4'): VectorDelta;
|
|
15
|
+
/**
|
|
16
|
+
* Decode delta package back to changes
|
|
17
|
+
*/
|
|
18
|
+
static decode(delta: VectorDelta): VectorChange[];
|
|
19
|
+
/**
|
|
20
|
+
* Serialize delta to bytes for QUIC transmission
|
|
21
|
+
*/
|
|
22
|
+
static serialize(delta: VectorDelta): Uint8Array;
|
|
23
|
+
/**
|
|
24
|
+
* Deserialize bytes back to delta
|
|
25
|
+
*/
|
|
26
|
+
static deserialize(bytes: Uint8Array): VectorDelta;
|
|
27
|
+
/**
|
|
28
|
+
* Calculate checksum for changes
|
|
29
|
+
*/
|
|
30
|
+
private static calculateChecksum;
|
|
31
|
+
/**
|
|
32
|
+
* Split large change sets into batches
|
|
33
|
+
*/
|
|
34
|
+
static batch(changes: VectorChange[], batchSize: number): VectorChange[][];
|
|
35
|
+
/**
|
|
36
|
+
* Optimize delta by removing redundant operations
|
|
37
|
+
*/
|
|
38
|
+
static optimize(changes: VectorChange[]): VectorChange[];
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* SQLite changelog reader
|
|
42
|
+
*/
|
|
43
|
+
export declare class ChangelogReader {
|
|
44
|
+
private db;
|
|
45
|
+
constructor(db: any);
|
|
46
|
+
/**
|
|
47
|
+
* Read changes from SQLite changelog
|
|
48
|
+
*/
|
|
49
|
+
readChanges(shardId: string, fromChangeId: number, toChangeId?: number): Promise<VectorChange[]>;
|
|
50
|
+
/**
|
|
51
|
+
* Get latest change ID for shard
|
|
52
|
+
*/
|
|
53
|
+
getLatestChangeId(shardId: string): Promise<number>;
|
|
54
|
+
/**
|
|
55
|
+
* Get version vector for shard
|
|
56
|
+
*/
|
|
57
|
+
getVersionVector(shardId: string): Promise<Map<string, number>>;
|
|
58
|
+
/**
|
|
59
|
+
* Update version vector after sync
|
|
60
|
+
*/
|
|
61
|
+
updateVersionVector(shardId: string, versionVector: Map<string, number>): Promise<void>;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Version vector utilities for causal ordering
|
|
65
|
+
*/
|
|
66
|
+
export declare class VersionVector {
|
|
67
|
+
/**
|
|
68
|
+
* Compare two version vectors
|
|
69
|
+
* Returns: -1 (a < b), 0 (concurrent), 1 (a > b)
|
|
70
|
+
*/
|
|
71
|
+
static compare(a: Map<string, number>, b: Map<string, number>): -1 | 0 | 1;
|
|
72
|
+
/**
|
|
73
|
+
* Merge two version vectors (element-wise max)
|
|
74
|
+
*/
|
|
75
|
+
static merge(a: Map<string, number>, b: Map<string, number>): Map<string, number>;
|
|
76
|
+
/**
|
|
77
|
+
* Increment version for node
|
|
78
|
+
*/
|
|
79
|
+
static increment(vv: Map<string, number>, nodeId: string): Map<string, number>;
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=delta.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delta.d.ts","sourceRoot":"","sources":["../../src/sync/delta.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAmB,MAAM,SAAS,CAAC;AAE1E;;GAEG;AACH,qBAAa,YAAY;IACvB;;OAEG;IACH,MAAM,CAAC,MAAM,CACX,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,YAAY,EAAE,EACvB,WAAW,GAAE,MAAM,GAAG,SAAS,GAAG,KAAiB,GAClD,WAAW;IAkCd;;OAEG;IACH,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,YAAY,EAAE;IAqBjD;;OAEG;IACH,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,UAAU;IAKhD;;OAEG;IACH,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW;IAalD;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAgBhC;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,YAAY,EAAE,EAAE;IAU1E;;OAEG;IACH,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE;CAczD;AAED;;GAEG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,EAAE,CAAM;gBAEJ,EAAE,EAAE,GAAG;IAInB;;OAEG;IACG,WAAW,CACf,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,EACpB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,YAAY,EAAE,CAAC;IAmC1B;;OAEG;IACG,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAQzD;;OAEG;IACG,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAYrE;;OAEG;IACG,mBAAmB,CACvB,OAAO,EAAE,MAAM,EACf,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACjC,OAAO,CAAC,IAAI,CAAC;CAQjB;AAED;;GAEG;AACH,qBAAa,aAAa;IACxB;;;OAGG;IACH,MAAM,CAAC,OAAO,CACZ,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACtB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACrB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IAuBb;;OAEG;IACH,MAAM,CAAC,KAAK,CACV,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACtB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACrB,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAWtB;;OAEG;IACH,MAAM,CAAC,SAAS,CACd,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACvB,MAAM,EAAE,MAAM,GACb,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;CAKvB"}
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* SQLiteVector QUIC Sync - Delta Computation
|
|
4
|
+
*
|
|
5
|
+
* Efficient delta computation from SQLite changelog for sync
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.VersionVector = exports.ChangelogReader = exports.DeltaEncoder = void 0;
|
|
9
|
+
const crypto_1 = require("crypto");
|
|
10
|
+
const msgpackr_1 = require("msgpackr");
|
|
11
|
+
/**
|
|
12
|
+
* Delta encoder for vector changes
|
|
13
|
+
*/
|
|
14
|
+
class DeltaEncoder {
|
|
15
|
+
/**
|
|
16
|
+
* Encode changes into compressed delta package
|
|
17
|
+
*/
|
|
18
|
+
static encode(shardId, changes, compression = 'msgpack') {
|
|
19
|
+
if (changes.length === 0) {
|
|
20
|
+
throw new Error('Cannot encode empty changes');
|
|
21
|
+
}
|
|
22
|
+
const fromChangeId = Math.min(...changes.map(c => c.id));
|
|
23
|
+
const toChangeId = Math.max(...changes.map(c => c.id));
|
|
24
|
+
// Serialize changes based on compression
|
|
25
|
+
let processedChanges = changes;
|
|
26
|
+
if (compression === 'msgpack') {
|
|
27
|
+
// MessagePack handles compression automatically
|
|
28
|
+
processedChanges = changes.map(c => ({
|
|
29
|
+
...c,
|
|
30
|
+
// Convert Map to Object for serialization
|
|
31
|
+
versionVector: Object.fromEntries(c.versionVector),
|
|
32
|
+
// Convert Float32Array to regular array for msgpack
|
|
33
|
+
vector: c.vector ? Array.from(c.vector) : undefined
|
|
34
|
+
}));
|
|
35
|
+
}
|
|
36
|
+
// Calculate checksum
|
|
37
|
+
const checksum = this.calculateChecksum(changes);
|
|
38
|
+
return {
|
|
39
|
+
shardId,
|
|
40
|
+
fromChangeId,
|
|
41
|
+
toChangeId,
|
|
42
|
+
changes: processedChanges,
|
|
43
|
+
checksum,
|
|
44
|
+
compression
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Decode delta package back to changes
|
|
49
|
+
*/
|
|
50
|
+
static decode(delta) {
|
|
51
|
+
// Verify checksum
|
|
52
|
+
const calculatedChecksum = this.calculateChecksum(delta.changes);
|
|
53
|
+
if (calculatedChecksum !== delta.checksum) {
|
|
54
|
+
throw new Error('Delta checksum mismatch - data corruption detected');
|
|
55
|
+
}
|
|
56
|
+
// Deserialize based on compression
|
|
57
|
+
if (delta.compression === 'msgpack') {
|
|
58
|
+
return delta.changes.map(c => ({
|
|
59
|
+
...c,
|
|
60
|
+
// Convert Object back to Map
|
|
61
|
+
versionVector: new Map(Object.entries(c.versionVector)),
|
|
62
|
+
// Convert array back to Float32Array
|
|
63
|
+
vector: c.vector ? new Float32Array(c.vector) : undefined
|
|
64
|
+
}));
|
|
65
|
+
}
|
|
66
|
+
return delta.changes;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Serialize delta to bytes for QUIC transmission
|
|
70
|
+
*/
|
|
71
|
+
static serialize(delta) {
|
|
72
|
+
const packed = (0, msgpackr_1.pack)(delta);
|
|
73
|
+
return packed;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Deserialize bytes back to delta
|
|
77
|
+
*/
|
|
78
|
+
static deserialize(bytes) {
|
|
79
|
+
const delta = (0, msgpackr_1.unpack)(bytes);
|
|
80
|
+
// Reconstruct Maps and typed arrays
|
|
81
|
+
delta.changes = delta.changes.map(c => ({
|
|
82
|
+
...c,
|
|
83
|
+
versionVector: new Map(Object.entries(c.versionVector || {})),
|
|
84
|
+
vector: c.vector ? new Float32Array(c.vector) : undefined
|
|
85
|
+
}));
|
|
86
|
+
return delta;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Calculate checksum for changes
|
|
90
|
+
*/
|
|
91
|
+
static calculateChecksum(changes) {
|
|
92
|
+
const hash = (0, crypto_1.createHash)('sha256');
|
|
93
|
+
// Sort by ID for deterministic hash
|
|
94
|
+
const sorted = [...changes].sort((a, b) => a.id - b.id);
|
|
95
|
+
for (const change of sorted) {
|
|
96
|
+
hash.update(change.id.toString());
|
|
97
|
+
hash.update(change.operation);
|
|
98
|
+
hash.update(change.vectorId);
|
|
99
|
+
hash.update(change.timestamp.toString());
|
|
100
|
+
}
|
|
101
|
+
return hash.digest('hex');
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Split large change sets into batches
|
|
105
|
+
*/
|
|
106
|
+
static batch(changes, batchSize) {
|
|
107
|
+
const batches = [];
|
|
108
|
+
for (let i = 0; i < changes.length; i += batchSize) {
|
|
109
|
+
batches.push(changes.slice(i, i + batchSize));
|
|
110
|
+
}
|
|
111
|
+
return batches;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Optimize delta by removing redundant operations
|
|
115
|
+
*/
|
|
116
|
+
static optimize(changes) {
|
|
117
|
+
const latestByVector = new Map();
|
|
118
|
+
// Keep only latest change per vector
|
|
119
|
+
for (const change of changes) {
|
|
120
|
+
const existing = latestByVector.get(change.vectorId);
|
|
121
|
+
if (!existing || change.timestamp > existing.timestamp) {
|
|
122
|
+
latestByVector.set(change.vectorId, change);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return Array.from(latestByVector.values()).sort((a, b) => a.id - b.id);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
exports.DeltaEncoder = DeltaEncoder;
|
|
129
|
+
/**
|
|
130
|
+
* SQLite changelog reader
|
|
131
|
+
*/
|
|
132
|
+
class ChangelogReader {
|
|
133
|
+
constructor(db) {
|
|
134
|
+
this.db = db;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Read changes from SQLite changelog
|
|
138
|
+
*/
|
|
139
|
+
async readChanges(shardId, fromChangeId, toChangeId) {
|
|
140
|
+
const changes = [];
|
|
141
|
+
// Query changelog table
|
|
142
|
+
const query = toChangeId
|
|
143
|
+
? `SELECT * FROM vector_changelog
|
|
144
|
+
WHERE shard_id = ? AND change_id > ? AND change_id <= ?
|
|
145
|
+
ORDER BY change_id ASC`
|
|
146
|
+
: `SELECT * FROM vector_changelog
|
|
147
|
+
WHERE shard_id = ? AND change_id > ?
|
|
148
|
+
ORDER BY change_id ASC`;
|
|
149
|
+
const params = toChangeId
|
|
150
|
+
? [shardId, fromChangeId, toChangeId]
|
|
151
|
+
: [shardId, fromChangeId];
|
|
152
|
+
const rows = this.db.prepare(query).all(...params);
|
|
153
|
+
for (const row of rows) {
|
|
154
|
+
changes.push({
|
|
155
|
+
id: row.change_id,
|
|
156
|
+
operation: row.operation,
|
|
157
|
+
shardId: row.shard_id,
|
|
158
|
+
vectorId: row.vector_id,
|
|
159
|
+
vector: row.vector_data ? new Float32Array(JSON.parse(row.vector_data)) : undefined,
|
|
160
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
|
|
161
|
+
timestamp: row.timestamp,
|
|
162
|
+
sourceNode: row.source_node,
|
|
163
|
+
versionVector: new Map(JSON.parse(row.version_vector || '{}'))
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
return changes;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Get latest change ID for shard
|
|
170
|
+
*/
|
|
171
|
+
async getLatestChangeId(shardId) {
|
|
172
|
+
const row = this.db.prepare(`SELECT MAX(change_id) as max_id FROM vector_changelog WHERE shard_id = ?`).get(shardId);
|
|
173
|
+
return row?.max_id || 0;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Get version vector for shard
|
|
177
|
+
*/
|
|
178
|
+
async getVersionVector(shardId) {
|
|
179
|
+
const row = this.db.prepare(`SELECT version_vector FROM shard_metadata WHERE shard_id = ?`).get(shardId);
|
|
180
|
+
if (!row) {
|
|
181
|
+
return new Map();
|
|
182
|
+
}
|
|
183
|
+
return new Map(Object.entries(JSON.parse(row.version_vector)));
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Update version vector after sync
|
|
187
|
+
*/
|
|
188
|
+
async updateVersionVector(shardId, versionVector) {
|
|
189
|
+
const vvJson = JSON.stringify(Object.fromEntries(versionVector));
|
|
190
|
+
this.db.prepare(`INSERT OR REPLACE INTO shard_metadata (shard_id, version_vector, updated_at)
|
|
191
|
+
VALUES (?, ?, ?)`).run(shardId, vvJson, Date.now());
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
exports.ChangelogReader = ChangelogReader;
|
|
195
|
+
/**
|
|
196
|
+
* Version vector utilities for causal ordering
|
|
197
|
+
*/
|
|
198
|
+
class VersionVector {
|
|
199
|
+
/**
|
|
200
|
+
* Compare two version vectors
|
|
201
|
+
* Returns: -1 (a < b), 0 (concurrent), 1 (a > b)
|
|
202
|
+
*/
|
|
203
|
+
static compare(a, b) {
|
|
204
|
+
let aGreater = false;
|
|
205
|
+
let bGreater = false;
|
|
206
|
+
// Get all node IDs
|
|
207
|
+
const allNodes = new Set([...a.keys(), ...b.keys()]);
|
|
208
|
+
for (const nodeId of allNodes) {
|
|
209
|
+
const aVersion = a.get(nodeId) || 0;
|
|
210
|
+
const bVersion = b.get(nodeId) || 0;
|
|
211
|
+
if (aVersion > bVersion) {
|
|
212
|
+
aGreater = true;
|
|
213
|
+
}
|
|
214
|
+
else if (bVersion > aVersion) {
|
|
215
|
+
bGreater = true;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
if (aGreater && !bGreater)
|
|
219
|
+
return 1;
|
|
220
|
+
if (bGreater && !aGreater)
|
|
221
|
+
return -1;
|
|
222
|
+
return 0; // Concurrent
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Merge two version vectors (element-wise max)
|
|
226
|
+
*/
|
|
227
|
+
static merge(a, b) {
|
|
228
|
+
const merged = new Map(a);
|
|
229
|
+
for (const [nodeId, version] of b) {
|
|
230
|
+
const currentVersion = merged.get(nodeId) || 0;
|
|
231
|
+
merged.set(nodeId, Math.max(currentVersion, version));
|
|
232
|
+
}
|
|
233
|
+
return merged;
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Increment version for node
|
|
237
|
+
*/
|
|
238
|
+
static increment(vv, nodeId) {
|
|
239
|
+
const updated = new Map(vv);
|
|
240
|
+
updated.set(nodeId, (updated.get(nodeId) || 0) + 1);
|
|
241
|
+
return updated;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
exports.VersionVector = VersionVector;
|
|
245
|
+
//# sourceMappingURL=delta.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delta.js","sourceRoot":"","sources":["../../src/sync/delta.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;AAEH,mCAAoC;AACpC,uCAAwC;AAGxC;;GAEG;AACH,MAAa,YAAY;IACvB;;OAEG;IACH,MAAM,CAAC,MAAM,CACX,OAAe,EACf,OAAuB,EACvB,cAA0C,SAAS;QAEnD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEvD,yCAAyC;QACzC,IAAI,gBAAgB,GAAG,OAAO,CAAC;QAC/B,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,gDAAgD;YAChD,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACnC,GAAG,CAAC;gBACJ,0CAA0C;gBAC1C,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC;gBAClD,oDAAoD;gBACpD,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;aACpD,CAAC,CAAQ,CAAC;QACb,CAAC;QAED,qBAAqB;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAEjD,OAAO;YACL,OAAO;YACP,YAAY;YACZ,UAAU;YACV,OAAO,EAAE,gBAAgB;YACzB,QAAQ;YACR,WAAW;SACZ,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,MAAM,CAAC,KAAkB;QAC9B,kBAAkB;QAClB,MAAM,kBAAkB,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjE,IAAI,kBAAkB,KAAK,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACxE,CAAC;QAED,mCAAmC;QACnC,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACpC,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC7B,GAAG,CAAC;gBACJ,6BAA6B;gBAC7B,aAAa,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAE,CAAS,CAAC,aAAa,CAAC,CAAC;gBAChE,qCAAqC;gBACrC,MAAM,EAAG,CAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,YAAY,CAAE,CAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;aAC5E,CAAC,CAAC,CAAC;QACN,CAAC;QAED,OAAO,KAAK,CAAC,OAAO,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,SAAS,CAAC,KAAkB;QACjC,MAAM,MAAM,GAAG,IAAA,eAAI,EAAC,KAAK,CAAC,CAAC;QAC3B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,WAAW,CAAC,KAAiB;QAClC,MAAM,KAAK,GAAG,IAAA,iBAAM,EAAC,KAAK,CAAgB,CAAC;QAE3C,oCAAoC;QACpC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACtC,GAAG,CAAC;YACJ,aAAa,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAE,CAAS,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC;YACtE,MAAM,EAAG,CAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,YAAY,CAAE,CAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;SAC5E,CAAC,CAAC,CAAC;QAEJ,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,iBAAiB,CAAC,OAAuB;QACtD,MAAM,IAAI,GAAG,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC;QAElC,oCAAoC;QACpC,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAExD,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAuB,EAAE,SAAiB;QACrD,MAAM,OAAO,GAAqB,EAAE,CAAC;QAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,QAAQ,CAAC,OAAuB;QACrC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAwB,CAAC;QAEvD,qCAAqC;QACrC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAErD,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,SAAS,GAAG,QAAQ,CAAC,SAAS,EAAE,CAAC;gBACvD,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;CACF;AA3ID,oCA2IC;AAED;;GAEG;AACH,MAAa,eAAe;IAG1B,YAAY,EAAO;QACjB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CACf,OAAe,EACf,YAAoB,EACpB,UAAmB;QAEnB,MAAM,OAAO,GAAmB,EAAE,CAAC;QAEnC,wBAAwB;QACxB,MAAM,KAAK,GAAG,UAAU;YACtB,CAAC,CAAC;;gCAEwB;YAC1B,CAAC,CAAC;;gCAEwB,CAAC;QAE7B,MAAM,MAAM,GAAG,UAAU;YACvB,CAAC,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,UAAU,CAAC;YACrC,CAAC,CAAC,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAE5B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;QAEnD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,GAAG,CAAC,SAAS;gBACjB,SAAS,EAAE,GAAG,CAAC,SAA4B;gBAC3C,OAAO,EAAE,GAAG,CAAC,QAAQ;gBACrB,QAAQ,EAAE,GAAG,CAAC,SAAS;gBACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;gBACnF,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC7D,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,UAAU,EAAE,GAAG,CAAC,WAAW;gBAC3B,aAAa,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC;aAC/D,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,OAAe;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACzB,0EAA0E,CAC3E,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEf,OAAO,GAAG,EAAE,MAAM,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,OAAe;QACpC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACzB,8DAA8D,CAC/D,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEf,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CACvB,OAAe,EACf,aAAkC;QAElC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC;QAEjE,IAAI,CAAC,EAAE,CAAC,OAAO,CACb;wBACkB,CACnB,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACrC,CAAC;CACF;AAzFD,0CAyFC;AAED;;GAEG;AACH,MAAa,aAAa;IACxB;;;OAGG;IACH,MAAM,CAAC,OAAO,CACZ,CAAsB,EACtB,CAAsB;QAEtB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,mBAAmB;QACnB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAErD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACpC,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAEpC,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;gBACxB,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;iBAAM,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;gBAC/B,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,QAAQ;YAAE,OAAO,CAAC,CAAC;QACpC,IAAI,QAAQ,IAAI,CAAC,QAAQ;YAAE,OAAO,CAAC,CAAC,CAAC;QACrC,OAAO,CAAC,CAAC,CAAC,aAAa;IACzB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CACV,CAAsB,EACtB,CAAsB;QAEtB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QAE1B,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,cAAc,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,SAAS,CACd,EAAuB,EACvB,MAAc;QAEd,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AA3DD,sCA2DC"}
|