claude-flow-novice 2.0.3 → 2.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli/commands/guidance.js +487 -668
- package/dist/src/cli/commands/index-validate.js +18 -29
- package/dist/src/cli/commands/mcp-troubleshoot.js +230 -282
- package/dist/src/cli/commands/neural-goal-init.js +92 -125
- package/dist/src/cli/commands/swarm-exec.js +317 -393
- package/dist/src/cli/commands/swarm.js +1 -1
- package/dist/src/cli/commands/validate-framework.js +983 -1100
- package/dist/src/cli/commands/validate.js +144 -223
- package/dist/src/cli/simple-commands/__tests__/agent.test.js +265 -277
- package/dist/src/cli/simple-commands/__tests__/memory.test.js +6 -7
- package/dist/src/cli/simple-commands/__tests__/swarm.test.js +373 -356
- package/dist/src/cli/simple-commands/__tests__/task.test.js +6 -7
- package/dist/src/cli/simple-commands/agent.js +157 -193
- package/dist/src/cli/simple-commands/analysis.js +336 -446
- package/dist/src/cli/simple-commands/automation-executor.js +1095 -1339
- package/dist/src/cli/simple-commands/automation.js +481 -469
- package/dist/src/cli/simple-commands/batch-manager.js +261 -313
- package/dist/src/cli/simple-commands/claude-telemetry.js +241 -267
- package/dist/src/cli/simple-commands/claude-track.js +68 -90
- package/dist/src/cli/simple-commands/concurrent-display.js +266 -320
- package/dist/src/cli/simple-commands/config.js +245 -290
- package/dist/src/cli/simple-commands/coordination.js +182 -234
- package/dist/src/cli/simple-commands/enhanced-ui-views.js +812 -615
- package/dist/src/cli/simple-commands/enhanced-webui-complete.js +922 -981
- package/dist/src/cli/simple-commands/fix-hook-variables.js +274 -294
- package/dist/src/cli/simple-commands/github/gh-coordinator.js +378 -457
- package/dist/src/cli/simple-commands/github/github-api.js +535 -574
- package/dist/src/cli/simple-commands/github/init.js +276 -303
- package/dist/src/cli/simple-commands/github.js +222 -247
- package/dist/src/cli/simple-commands/goal.js +51 -63
- package/dist/src/cli/simple-commands/hive-mind/auto-save-middleware.js +208 -278
- package/dist/src/cli/simple-commands/hive-mind/communication.js +601 -696
- package/dist/src/cli/simple-commands/hive-mind/core.js +907 -979
- package/dist/src/cli/simple-commands/hive-mind/db-optimizer.js +406 -655
- package/dist/src/cli/simple-commands/hive-mind/mcp-wrapper.js +1125 -1245
- package/dist/src/cli/simple-commands/hive-mind/memory.js +854 -1090
- package/dist/src/cli/simple-commands/hive-mind/performance-optimizer.js +459 -574
- package/dist/src/cli/simple-commands/hive-mind/performance-test.js +263 -347
- package/dist/src/cli/simple-commands/hive-mind/queen.js +727 -768
- package/dist/src/cli/simple-commands/hive-mind/session-manager.js +745 -1049
- package/dist/src/cli/simple-commands/hive-mind-optimize.js +227 -283
- package/dist/src/cli/simple-commands/hive-mind-wizard.js +174 -217
- package/dist/src/cli/simple-commands/hive-mind.js +1842 -2283
- package/dist/src/cli/simple-commands/hive.js +90 -79
- package/dist/src/cli/simple-commands/hook-safety.js +431 -521
- package/dist/src/cli/simple-commands/hooks/session-start-soul.js +203 -254
- package/dist/src/cli/simple-commands/hooks.js +1064 -1204
- package/dist/src/cli/simple-commands/init/agent-copier.js +294 -319
- package/dist/src/cli/simple-commands/init/batch-init.js +496 -562
- package/dist/src/cli/simple-commands/init/claude-commands/claude-flow-commands.js +13 -19
- package/dist/src/cli/simple-commands/init/claude-commands/optimized-claude-flow-commands.js +13 -19
- package/dist/src/cli/simple-commands/init/claude-commands/optimized-slash-commands.js +61 -88
- package/dist/src/cli/simple-commands/init/claude-commands/optimized-sparc-commands.js +125 -150
- package/dist/src/cli/simple-commands/init/claude-commands/slash-commands.js +42 -49
- package/dist/src/cli/simple-commands/init/claude-commands/sparc-commands.js +43 -61
- package/dist/src/cli/simple-commands/init/copy-revised-templates.js +141 -147
- package/dist/src/cli/simple-commands/init/executable-wrapper.js +31 -44
- package/dist/src/cli/simple-commands/init/gitignore-updater.js +64 -90
- package/dist/src/cli/simple-commands/init/help.js +104 -107
- package/dist/src/cli/simple-commands/init/hive-mind-init.js +509 -528
- package/dist/src/cli/simple-commands/init/index.js +1510 -1759
- package/dist/src/cli/simple-commands/init/performance-monitor.js +234 -317
- package/dist/src/cli/simple-commands/init/rollback/backup-manager.js +441 -504
- package/dist/src/cli/simple-commands/init/rollback/index.js +289 -364
- package/dist/src/cli/simple-commands/init/rollback/recovery-manager.js +652 -728
- package/dist/src/cli/simple-commands/init/rollback/rollback-executor.js +416 -481
- package/dist/src/cli/simple-commands/init/rollback/state-tracker.js +369 -448
- package/dist/src/cli/simple-commands/init/sparc/roo-readme.js +1 -2
- package/dist/src/cli/simple-commands/init/sparc/roomodes-config.js +122 -99
- package/dist/src/cli/simple-commands/init/sparc/workflows.js +32 -37
- package/dist/src/cli/simple-commands/init/sparc-structure.js +55 -62
- package/dist/src/cli/simple-commands/init/template-copier.js +421 -533
- package/dist/src/cli/simple-commands/init/templates/coordination-md.js +3 -6
- package/dist/src/cli/simple-commands/init/templates/enhanced-templates.js +344 -318
- package/dist/src/cli/simple-commands/init/templates/github-safe-enhanced.js +173 -218
- package/dist/src/cli/simple-commands/init/templates/github-safe.js +65 -75
- package/dist/src/cli/simple-commands/init/templates/memory-bank-md.js +3 -6
- package/dist/src/cli/simple-commands/init/templates/readme-files.js +2 -4
- package/dist/src/cli/simple-commands/init/templates/safe-hook-patterns.js +187 -230
- package/dist/src/cli/simple-commands/init/templates/sparc-modes.js +53 -80
- package/dist/src/cli/simple-commands/init/templates/verification-claude-md.js +101 -85
- package/dist/src/cli/simple-commands/init/validation/config-validator.js +283 -330
- package/dist/src/cli/simple-commands/init/validation/health-checker.js +495 -561
- package/dist/src/cli/simple-commands/init/validation/index.js +302 -358
- package/dist/src/cli/simple-commands/init/validation/mode-validator.js +308 -359
- package/dist/src/cli/simple-commands/init/validation/post-init-validator.js +389 -366
- package/dist/src/cli/simple-commands/init/validation/pre-init-validator.js +270 -268
- package/dist/src/cli/simple-commands/init/validation/test-runner.js +427 -447
- package/dist/src/cli/simple-commands/init.js +1 -2
- package/dist/src/cli/simple-commands/mcp-health.js +131 -158
- package/dist/src/cli/simple-commands/mcp-integration-layer.js +533 -634
- package/dist/src/cli/simple-commands/mcp.js +345 -400
- package/dist/src/cli/simple-commands/memory-consolidation.js +426 -537
- package/dist/src/cli/simple-commands/memory.js +247 -311
- package/dist/src/cli/simple-commands/migrate-hooks.js +39 -46
- package/dist/src/cli/simple-commands/monitor.js +294 -363
- package/dist/src/cli/simple-commands/neural.js +51 -65
- package/dist/src/cli/simple-commands/pair-autofix-only.js +538 -662
- package/dist/src/cli/simple-commands/pair-basic.js +528 -656
- package/dist/src/cli/simple-commands/pair-old.js +430 -543
- package/dist/src/cli/simple-commands/pair-working.js +615 -751
- package/dist/src/cli/simple-commands/pair.js +615 -751
- package/dist/src/cli/simple-commands/performance-hooks.js +83 -111
- package/dist/src/cli/simple-commands/performance-metrics.js +348 -433
- package/dist/src/cli/simple-commands/process-ui-enhanced.js +708 -787
- package/dist/src/cli/simple-commands/process-ui.js +230 -254
- package/dist/src/cli/simple-commands/realtime-update-system.js +525 -611
- package/dist/src/cli/simple-commands/sparc/architecture.js +1704 -1530
- package/dist/src/cli/simple-commands/sparc/commands.js +438 -516
- package/dist/src/cli/simple-commands/sparc/completion.js +1224 -1481
- package/dist/src/cli/simple-commands/sparc/coordinator.js +913 -978
- package/dist/src/cli/simple-commands/sparc/index.js +241 -298
- package/dist/src/cli/simple-commands/sparc/phase-base.js +314 -390
- package/dist/src/cli/simple-commands/sparc/pseudocode.js +965 -869
- package/dist/src/cli/simple-commands/sparc/refinement.js +980 -1273
- package/dist/src/cli/simple-commands/sparc/specification.js +559 -645
- package/dist/src/cli/simple-commands/sparc-modes/architect.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/ask.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/code.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/debug.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/devops.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/docs-writer.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/generic.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/index.js +47 -55
- package/dist/src/cli/simple-commands/sparc-modes/integration.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/mcp.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/monitoring.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/optimization.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/security-review.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/sparc-orchestrator.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/spec-pseudocode.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/supabase-admin.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/swarm.js +101 -87
- package/dist/src/cli/simple-commands/sparc-modes/tdd.js +1 -1
- package/dist/src/cli/simple-commands/sparc-modes/tutorial.js +1 -1
- package/dist/src/cli/simple-commands/sparc.js +465 -493
- package/dist/src/cli/simple-commands/start-ui.js +108 -132
- package/dist/src/cli/simple-commands/start-wrapper.js +240 -268
- package/dist/src/cli/simple-commands/start.js +1 -1
- package/dist/src/cli/simple-commands/status.js +254 -275
- package/dist/src/cli/simple-commands/stream-chain-clean.js +128 -171
- package/dist/src/cli/simple-commands/stream-chain-fixed.js +61 -82
- package/dist/src/cli/simple-commands/stream-chain-real.js +267 -331
- package/dist/src/cli/simple-commands/stream-chain-working.js +211 -263
- package/dist/src/cli/simple-commands/stream-chain.js +260 -318
- package/dist/src/cli/simple-commands/stream-processor.js +290 -315
- package/dist/src/cli/simple-commands/swarm-executor.js +189 -222
- package/dist/src/cli/simple-commands/swarm-metrics-integration.js +208 -300
- package/dist/src/cli/simple-commands/swarm-ui.js +623 -703
- package/dist/src/cli/simple-commands/swarm-webui-integration.js +258 -286
- package/dist/src/cli/simple-commands/swarm.js +887 -1082
- package/dist/src/cli/simple-commands/task.js +161 -206
- package/dist/src/cli/simple-commands/timestamp-fix.js +59 -89
- package/dist/src/cli/simple-commands/token-tracker.js +258 -316
- package/dist/src/cli/simple-commands/tool-execution-framework.js +433 -519
- package/dist/src/cli/simple-commands/train-and-stream.js +275 -331
- package/dist/src/cli/simple-commands/training-pipeline.js +619 -725
- package/dist/src/cli/simple-commands/training.js +170 -227
- package/dist/src/cli/simple-commands/verification-hooks.js +261 -284
- package/dist/src/cli/simple-commands/verification-integration.js +389 -417
- package/dist/src/cli/simple-commands/verification-training-integration.js +486 -606
- package/dist/src/cli/simple-commands/verification.js +493 -513
- package/dist/src/cli/simple-commands/web-server.js +766 -836
- package/dist/src/cli/simple-commands/webui-validator.js +106 -124
- package/dist/src/coordination/event-bus/demo-wasm-integration.js +212 -251
- package/dist/src/coordination/event-bus/qe-event-bus.js +608 -748
- package/dist/src/coordination/event-bus/qe-event-bus.test.js +379 -454
- package/dist/src/coordination/iteration-tracker.js +363 -454
- package/dist/src/enterprise/analytics-manager.js +1135 -0
- package/dist/src/enterprise/audit-manager.js +1115 -0
- package/dist/src/enterprise/cloud-manager.js +891 -0
- package/dist/src/enterprise/deployment-manager.js +966 -0
- package/dist/src/enterprise/index.js +6 -0
- package/dist/src/enterprise/project-manager.js +584 -0
- package/dist/src/enterprise/security-manager.js +991 -0
- package/dist/src/index.js +1 -1
- package/dist/src/mcp/DEPRECATED.js +46 -60
- package/dist/src/mcp/fixes/mcp-error-fixes.js +115 -134
- package/dist/src/mcp/implementations/agent-tracker.js +114 -128
- package/dist/src/mcp/implementations/daa-tools.js +292 -350
- package/dist/src/mcp/implementations/workflow-tools.js +329 -361
- package/dist/src/mcp/mcp-config-manager.js +1183 -1331
- package/dist/src/mcp/mcp-server-novice-simplified.js +11 -17
- package/dist/src/mcp/mcp-server-novice.js +11 -17
- package/dist/src/mcp/mcp-server-sdk.js +11 -17
- package/dist/src/mcp/mcp-server.js +1620 -1484
- package/dist/src/mcp/ruv-swarm-wrapper.js +209 -239
- package/dist/src/memory/advanced-serializer.js +609 -589
- package/dist/src/memory/enhanced-examples.js +220 -305
- package/dist/src/memory/enhanced-memory.js +295 -336
- package/dist/src/memory/enhanced-session-serializer.js +408 -492
- package/dist/src/memory/fallback-memory-system.js +900 -1021
- package/dist/src/memory/fallback-store.js +93 -131
- package/dist/src/memory/high-performance-serialization.js +592 -730
- package/dist/src/memory/in-memory-store.js +161 -213
- package/dist/src/memory/index.js +123 -157
- package/dist/src/memory/lock-free-structures.js +578 -764
- package/dist/src/memory/memory-mapped-persistence.js +585 -766
- package/dist/src/memory/memory-pressure-manager.js +569 -707
- package/dist/src/memory/migration.js +358 -445
- package/dist/src/memory/shared-memory.js +641 -768
- package/dist/src/memory/sqlite-store.js +245 -325
- package/dist/src/memory/sqlite-wrapper.js +122 -151
- package/dist/src/memory/swarm-memory.js +470 -603
- package/dist/src/memory/test-example.js +126 -134
- package/dist/src/memory/ultra-fast-memory-store.js +622 -821
- package/dist/src/memory/unified-memory-manager.js +356 -437
- package/dist/src/migration/index.js +92 -0
- package/dist/src/migration/logger.js +121 -0
- package/dist/src/migration/migration-analyzer.js +268 -0
- package/dist/src/migration/migration-runner.js +522 -0
- package/dist/src/migration/migration-validator.js +285 -0
- package/dist/src/migration/progress-reporter.js +150 -0
- package/dist/src/migration/rollback-manager.js +321 -0
- package/dist/src/migration/tests/migration-system.test.js +7 -0
- package/dist/src/migration/types.js +3 -0
- package/dist/src/swarm/CodeRefactoringSwarm.js +777 -952
- package/dist/src/swarm/__tests__/integration.test.js +227 -0
- package/dist/src/swarm/__tests__/prompt-copier.test.js +344 -0
- package/dist/src/swarm/advanced-orchestrator.js +1095 -0
- package/dist/src/swarm/claude-code-interface.js +961 -0
- package/dist/src/swarm/claude-flow-executor.js +229 -0
- package/dist/src/swarm/consensus-coordinator.js +475 -0
- package/dist/src/swarm/coordinator.js +2993 -0
- package/dist/src/swarm/direct-executor.js +1180 -0
- package/dist/src/swarm/error-recovery/advanced-error-detection.js +691 -0
- package/dist/src/swarm/error-recovery/automated-recovery-workflows.js +998 -0
- package/dist/src/swarm/error-recovery/error-recovery-coordinator.js +1197 -0
- package/dist/src/swarm/error-recovery/recovery-monitoring.js +772 -0
- package/dist/src/swarm/error-recovery/resilience-architecture.js +714 -0
- package/dist/src/swarm/error-recovery/self-healing-mechanisms.js +1319 -0
- package/dist/src/swarm/error-recovery/test-error-recovery-effectiveness.js +808 -0
- package/dist/src/swarm/executor-v2.js +322 -0
- package/dist/src/swarm/executor.js +815 -0
- package/dist/src/swarm/hive-mind-integration.js +703 -0
- package/dist/src/swarm/index.js +41 -0
- package/dist/src/swarm/json-output-aggregator.js +267 -0
- package/dist/src/swarm/large-scale-coordinator.js +542 -0
- package/dist/src/swarm/mcp-integration-wrapper.js +628 -0
- package/dist/src/swarm/memory.js +1117 -0
- package/dist/src/swarm/optimizations/__tests__/optimization.test.js +348 -0
- package/dist/src/swarm/optimizations/async-file-manager.js +285 -0
- package/dist/src/swarm/optimizations/circular-buffer.js +162 -0
- package/dist/src/swarm/optimizations/connection-pool.js +244 -0
- package/dist/src/swarm/optimizations/index.js +28 -0
- package/dist/src/swarm/optimizations/optimized-executor.js +320 -0
- package/dist/src/swarm/optimizations/ttl-map.js +234 -0
- package/dist/src/swarm/prompt-cli.js +200 -0
- package/dist/src/swarm/prompt-copier-enhanced.js +202 -0
- package/dist/src/swarm/prompt-copier.js +381 -0
- package/dist/src/swarm/prompt-manager.js +295 -0
- package/dist/src/swarm/prompt-utils.js +310 -0
- package/dist/src/swarm/result-aggregator.js +718 -0
- package/dist/src/swarm/sparc-executor.js +1568 -0
- package/dist/src/swarm/strategies/auto.js +758 -0
- package/dist/src/swarm/strategies/base.js +128 -0
- package/dist/src/swarm/strategies/research.js +914 -0
- package/dist/src/swarm/strategies/strategy-metrics-patch.js +2 -0
- package/dist/src/swarm/types.js +52 -0
- package/dist/src/swarm/workers/copy-worker.js +56 -0
- package/dist/src/utils/__tests__/github-cli-safety-wrapper.test.js +332 -400
- package/dist/src/utils/github-cli-safe.js +56 -64
- package/dist/src/utils/github-cli-safety-wrapper.js +451 -546
- package/dist/src/utils/npx-isolated-cache.js +104 -119
- package/dist/src/utils/preference-manager.js +622 -652
- package/dist/src/utils/timezone-utils.js +86 -105
- package/dist/src/validators/epic-config-schema.js +214 -0
- package/dist/src/validators/index.js +10 -0
- package/dist/src/validators/swarm-init-validator.js +259 -0
- package/dist/src/validators/todowrite-batching-validator.js +215 -0
- package/dist/src/validators/todowrite-integration.js +187 -0
- package/package.json +2 -2
|
@@ -1,250 +1,188 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Collective Memory System for Hive Mind
|
|
3
3
|
* Shared knowledge base and learning system
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
import
|
|
7
|
-
import
|
|
8
|
-
import path from 'path';
|
|
9
|
-
import { performance } from 'perf_hooks';
|
|
10
|
-
import { Worker } from 'worker_threads';
|
|
11
|
-
|
|
4
|
+
*/ import EventEmitter from "events";
|
|
5
|
+
import Database from "better-sqlite3";
|
|
6
|
+
import path from "path";
|
|
7
|
+
import { performance } from "perf_hooks";
|
|
12
8
|
/**
|
|
13
9
|
* Memory types and their characteristics
|
|
14
|
-
*/
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
10
|
+
*/ const MEMORY_TYPES = {
|
|
11
|
+
knowledge: {
|
|
12
|
+
priority: 1,
|
|
13
|
+
ttl: null,
|
|
14
|
+
compress: false
|
|
15
|
+
},
|
|
16
|
+
context: {
|
|
17
|
+
priority: 2,
|
|
18
|
+
ttl: 3600000,
|
|
19
|
+
compress: false
|
|
20
|
+
},
|
|
21
|
+
task: {
|
|
22
|
+
priority: 3,
|
|
23
|
+
ttl: 1800000,
|
|
24
|
+
compress: true
|
|
25
|
+
},
|
|
26
|
+
result: {
|
|
27
|
+
priority: 2,
|
|
28
|
+
ttl: null,
|
|
29
|
+
compress: true
|
|
30
|
+
},
|
|
31
|
+
error: {
|
|
32
|
+
priority: 1,
|
|
33
|
+
ttl: 86400000,
|
|
34
|
+
compress: false
|
|
35
|
+
},
|
|
36
|
+
metric: {
|
|
37
|
+
priority: 3,
|
|
38
|
+
ttl: 3600000,
|
|
39
|
+
compress: true
|
|
40
|
+
},
|
|
41
|
+
consensus: {
|
|
42
|
+
priority: 1,
|
|
43
|
+
ttl: null,
|
|
44
|
+
compress: false
|
|
45
|
+
},
|
|
46
|
+
system: {
|
|
47
|
+
priority: 1,
|
|
48
|
+
ttl: null,
|
|
49
|
+
compress: false
|
|
50
|
+
}
|
|
24
51
|
};
|
|
25
|
-
|
|
26
52
|
/**
|
|
27
53
|
* Memory Pool for object reuse
|
|
28
|
-
*/
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
this.reused = 0;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
acquire() {
|
|
40
|
-
if (this.pool.length > 0) {
|
|
41
|
-
this.reused++;
|
|
42
|
-
return this.pool.pop();
|
|
54
|
+
*/ let MemoryPool = class MemoryPool {
|
|
55
|
+
acquire() {
|
|
56
|
+
if (this.pool.length > 0) {
|
|
57
|
+
this.reused++;
|
|
58
|
+
return this.pool.pop();
|
|
59
|
+
}
|
|
60
|
+
this.allocated++;
|
|
61
|
+
return this.createFn();
|
|
43
62
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
if (this.pool.length < this.maxSize) {
|
|
50
|
-
this.resetFn(obj);
|
|
51
|
-
this.pool.push(obj);
|
|
63
|
+
release(obj) {
|
|
64
|
+
if (this.pool.length < this.maxSize) {
|
|
65
|
+
this.resetFn(obj);
|
|
66
|
+
this.pool.push(obj);
|
|
67
|
+
}
|
|
52
68
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
69
|
+
getStats() {
|
|
70
|
+
return {
|
|
71
|
+
poolSize: this.pool.length,
|
|
72
|
+
allocated: this.allocated,
|
|
73
|
+
reused: this.reused,
|
|
74
|
+
reuseRate: this.reused / (this.allocated + this.reused) * 100
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
constructor(createFn, resetFn, maxSize = 1000){
|
|
78
|
+
this.createFn = createFn;
|
|
79
|
+
this.resetFn = resetFn;
|
|
80
|
+
this.maxSize = maxSize;
|
|
81
|
+
this.pool = [];
|
|
82
|
+
this.allocated = 0;
|
|
83
|
+
this.reused = 0;
|
|
84
|
+
}
|
|
85
|
+
};
|
|
65
86
|
/**
|
|
66
87
|
* Optimized LRU Cache with memory pressure handling
|
|
67
|
-
*/
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
get(key) {
|
|
80
|
-
if (this.cache.has(key)) {
|
|
81
|
-
const value = this.cache.get(key);
|
|
82
|
-
// Move to end (most recently used)
|
|
83
|
-
this.cache.delete(key);
|
|
84
|
-
this.cache.set(key, value);
|
|
85
|
-
this.hits++;
|
|
86
|
-
return value.data;
|
|
88
|
+
*/ let OptimizedLRUCache = class OptimizedLRUCache {
|
|
89
|
+
get(key) {
|
|
90
|
+
if (this.cache.has(key)) {
|
|
91
|
+
const value = this.cache.get(key);
|
|
92
|
+
// Move to end (most recently used)
|
|
93
|
+
this.cache.delete(key);
|
|
94
|
+
this.cache.set(key, value);
|
|
95
|
+
this.hits++;
|
|
96
|
+
return value.data;
|
|
97
|
+
}
|
|
98
|
+
this.misses++;
|
|
99
|
+
return null;
|
|
87
100
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
101
|
+
set(key, data) {
|
|
102
|
+
const size = this._estimateSize(data);
|
|
103
|
+
// Check memory pressure
|
|
104
|
+
if (this.currentMemory + size > this.maxMemory) {
|
|
105
|
+
this._evictByMemoryPressure(size);
|
|
106
|
+
}
|
|
107
|
+
// Check size limit
|
|
108
|
+
if (this.cache.size >= this.maxSize) {
|
|
109
|
+
this._evictLRU();
|
|
110
|
+
}
|
|
111
|
+
const entry = {
|
|
112
|
+
data,
|
|
113
|
+
size,
|
|
114
|
+
timestamp: Date.now()
|
|
115
|
+
};
|
|
116
|
+
this.cache.set(key, entry);
|
|
117
|
+
this.currentMemory += size;
|
|
98
118
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if (this.cache.size >= this.maxSize) {
|
|
102
|
-
this._evictLRU();
|
|
119
|
+
_estimateSize(obj) {
|
|
120
|
+
return JSON.stringify(obj).length * 2; // Rough estimate
|
|
103
121
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
this.currentMemory += size;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
_estimateSize(obj) {
|
|
116
|
-
return JSON.stringify(obj).length * 2; // Rough estimate
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
_evictLRU() {
|
|
120
|
-
const firstKey = this.cache.keys().next().value;
|
|
121
|
-
if (firstKey) {
|
|
122
|
-
const entry = this.cache.get(firstKey);
|
|
123
|
-
this.cache.delete(firstKey);
|
|
124
|
-
this.currentMemory -= entry.size;
|
|
125
|
-
this.evictions++;
|
|
122
|
+
_evictLRU() {
|
|
123
|
+
const firstKey = this.cache.keys().next().value;
|
|
124
|
+
if (firstKey) {
|
|
125
|
+
const entry = this.cache.get(firstKey);
|
|
126
|
+
this.cache.delete(firstKey);
|
|
127
|
+
this.currentMemory -= entry.size;
|
|
128
|
+
this.evictions++;
|
|
129
|
+
}
|
|
126
130
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
this._evictLRU();
|
|
131
|
+
_evictByMemoryPressure(neededSize) {
|
|
132
|
+
while(this.currentMemory + neededSize > this.maxMemory && this.cache.size > 0){
|
|
133
|
+
this._evictLRU();
|
|
134
|
+
}
|
|
132
135
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
callback(entry, key);
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
delete(key) {
|
|
142
|
-
if (this.cache.has(key)) {
|
|
143
|
-
const entry = this.cache.get(key);
|
|
144
|
-
this.cache.delete(key);
|
|
145
|
-
this.currentMemory -= entry.size;
|
|
146
|
-
return true;
|
|
136
|
+
forEach(callback) {
|
|
137
|
+
this.cache.forEach((entry, key)=>{
|
|
138
|
+
callback(entry, key);
|
|
139
|
+
});
|
|
147
140
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
141
|
+
delete(key) {
|
|
142
|
+
if (this.cache.has(key)) {
|
|
143
|
+
const entry = this.cache.get(key);
|
|
144
|
+
this.cache.delete(key);
|
|
145
|
+
this.currentMemory -= entry.size;
|
|
146
|
+
return true;
|
|
147
|
+
}
|
|
148
|
+
return false;
|
|
149
|
+
}
|
|
150
|
+
getStats() {
|
|
151
|
+
return {
|
|
152
|
+
size: this.cache.size,
|
|
153
|
+
memoryUsage: this.currentMemory,
|
|
154
|
+
hitRate: this.hits / (this.hits + this.misses) * 100,
|
|
155
|
+
evictions: this.evictions
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
constructor(maxSize = 1000, maxMemoryMB = 50){
|
|
159
|
+
this.maxSize = maxSize;
|
|
160
|
+
this.maxMemory = maxMemoryMB * 1024 * 1024;
|
|
161
|
+
this.cache = new Map();
|
|
162
|
+
this.currentMemory = 0;
|
|
163
|
+
this.hits = 0;
|
|
164
|
+
this.misses = 0;
|
|
165
|
+
this.evictions = 0;
|
|
166
|
+
}
|
|
167
|
+
};
|
|
161
168
|
/**
|
|
162
169
|
* Optimized CollectiveMemory class with advanced memory management
|
|
163
|
-
*/
|
|
164
|
-
|
|
165
|
-
constructor(config = {}) {
|
|
166
|
-
super();
|
|
167
|
-
|
|
168
|
-
/** @type {import('better-sqlite3').Database | null} */
|
|
169
|
-
this.db = null;
|
|
170
|
-
|
|
171
|
-
this.config = {
|
|
172
|
-
swarmId: config.swarmId,
|
|
173
|
-
maxSize: config.maxSize || 100, // MB
|
|
174
|
-
dbPath: config.dbPath || path.join(process.cwd(), '.hive-mind', 'hive.db'),
|
|
175
|
-
compressionThreshold: config.compressionThreshold || 1024, // bytes
|
|
176
|
-
gcInterval: config.gcInterval || 300000, // 5 minutes
|
|
177
|
-
cacheSize: config.cacheSize || 1000,
|
|
178
|
-
cacheMemoryMB: config.cacheMemoryMB || 50,
|
|
179
|
-
enablePooling: config.enablePooling !== false,
|
|
180
|
-
enableAsyncOperations: config.enableAsyncOperations !== false,
|
|
181
|
-
...config,
|
|
182
|
-
};
|
|
183
|
-
|
|
184
|
-
this.state = {
|
|
185
|
-
totalSize: 0,
|
|
186
|
-
entryCount: 0,
|
|
187
|
-
compressionRatio: 1,
|
|
188
|
-
lastGC: Date.now(),
|
|
189
|
-
accessPatterns: new Map(),
|
|
190
|
-
performanceMetrics: {
|
|
191
|
-
queryTimes: [],
|
|
192
|
-
avgQueryTime: 0,
|
|
193
|
-
cacheHitRate: 0,
|
|
194
|
-
memoryEfficiency: 0,
|
|
195
|
-
},
|
|
196
|
-
};
|
|
197
|
-
|
|
198
|
-
this.gcTimer = null;
|
|
199
|
-
|
|
200
|
-
// Optimized cache with LRU eviction
|
|
201
|
-
this.cache = new OptimizedLRUCache(this.config.cacheSize, this.config.cacheMemoryMB);
|
|
202
|
-
|
|
203
|
-
// Memory pools for frequently created objects
|
|
204
|
-
this.pools = {
|
|
205
|
-
queryResults: new MemoryPool(
|
|
206
|
-
() => ({ results: [], metadata: {} }),
|
|
207
|
-
(obj) => {
|
|
208
|
-
obj.results.length = 0;
|
|
209
|
-
Object.keys(obj.metadata).forEach((k) => delete obj.metadata[k]);
|
|
210
|
-
},
|
|
211
|
-
),
|
|
212
|
-
memoryEntries: new MemoryPool(
|
|
213
|
-
() => ({ id: '', key: '', value: '', metadata: {} }),
|
|
214
|
-
(obj) => {
|
|
215
|
-
obj.id = obj.key = obj.value = '';
|
|
216
|
-
Object.keys(obj.metadata).forEach((k) => delete obj.metadata[k]);
|
|
217
|
-
},
|
|
218
|
-
),
|
|
219
|
-
};
|
|
220
|
-
|
|
221
|
-
// Prepared statements for better performance
|
|
222
|
-
this.statements = new Map();
|
|
223
|
-
|
|
224
|
-
// Background worker for heavy operations
|
|
225
|
-
this.backgroundWorker = null;
|
|
226
|
-
|
|
227
|
-
this._initialize();
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
/**
|
|
170
|
+
*/ export class CollectiveMemory extends EventEmitter {
|
|
171
|
+
/**
|
|
231
172
|
* Initialize collective memory with optimizations
|
|
232
|
-
*/
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
// Ensure table exists with optimized schema
|
|
247
|
-
this.db.exec(`
|
|
173
|
+
*/ _initialize() {
|
|
174
|
+
try {
|
|
175
|
+
// Open database connection with optimizations
|
|
176
|
+
this.db = new Database(this.config.dbPath);
|
|
177
|
+
// Performance optimizations
|
|
178
|
+
this.db.pragma('journal_mode = WAL');
|
|
179
|
+
this.db.pragma('synchronous = NORMAL');
|
|
180
|
+
this.db.pragma('cache_size = -64000'); // 64MB cache
|
|
181
|
+
this.db.pragma('temp_store = MEMORY');
|
|
182
|
+
this.db.pragma('mmap_size = 268435456'); // 256MB memory mapping
|
|
183
|
+
this.db.pragma('optimize');
|
|
184
|
+
// Ensure table exists with optimized schema
|
|
185
|
+
this.db.exec(`
|
|
248
186
|
CREATE TABLE IF NOT EXISTS collective_memory (
|
|
249
187
|
id TEXT PRIMARY KEY,
|
|
250
188
|
swarm_id TEXT NOT NULL,
|
|
@@ -283,90 +221,61 @@ export class CollectiveMemory extends EventEmitter {
|
|
|
283
221
|
FROM collective_memory
|
|
284
222
|
GROUP BY swarm_id, type;
|
|
285
223
|
`);
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
}
|
|
308
|
-
});
|
|
309
|
-
} catch (error) {
|
|
310
|
-
this.emit('error', error);
|
|
311
|
-
throw error;
|
|
224
|
+
// Prepare optimized statements
|
|
225
|
+
this._prepareStatements();
|
|
226
|
+
// Load initial statistics
|
|
227
|
+
this._updateStatistics();
|
|
228
|
+
// Start background optimization processes
|
|
229
|
+
this._startOptimizationTimers();
|
|
230
|
+
// Initialize background worker for heavy operations
|
|
231
|
+
if (this.config.enableAsyncOperations) {
|
|
232
|
+
this._initializeBackgroundWorker();
|
|
233
|
+
}
|
|
234
|
+
this.emit('memory:initialized', {
|
|
235
|
+
swarmId: this.config.swarmId,
|
|
236
|
+
optimizations: {
|
|
237
|
+
pooling: this.config.enablePooling,
|
|
238
|
+
asyncOps: this.config.enableAsyncOperations,
|
|
239
|
+
cacheSize: this.config.cacheSize
|
|
240
|
+
}
|
|
241
|
+
});
|
|
242
|
+
} catch (error) {
|
|
243
|
+
this.emit('error', error);
|
|
244
|
+
throw error;
|
|
245
|
+
}
|
|
312
246
|
}
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
/**
|
|
247
|
+
/**
|
|
316
248
|
* Prepare optimized SQL statements
|
|
317
|
-
*/
|
|
318
|
-
|
|
319
|
-
this.statements.set(
|
|
320
|
-
'insert',
|
|
321
|
-
this.db.prepare(`
|
|
249
|
+
*/ _prepareStatements() {
|
|
250
|
+
this.statements.set('insert', this.db.prepare(`
|
|
322
251
|
INSERT OR REPLACE INTO collective_memory
|
|
323
252
|
(id, swarm_id, key, value, type, confidence, created_by, compressed, size)
|
|
324
253
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
325
|
-
`)
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
this.statements.set(
|
|
329
|
-
'update',
|
|
330
|
-
this.db.prepare(`
|
|
254
|
+
`));
|
|
255
|
+
this.statements.set('update', this.db.prepare(`
|
|
331
256
|
UPDATE collective_memory
|
|
332
257
|
SET value = ?, accessed_at = strftime('%s','now'), access_count = access_count + 1,
|
|
333
258
|
compressed = ?, size = ?
|
|
334
259
|
WHERE swarm_id = ? AND key = ?
|
|
335
|
-
`)
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
this.statements.set(
|
|
339
|
-
'select',
|
|
340
|
-
this.db.prepare(`
|
|
260
|
+
`));
|
|
261
|
+
this.statements.set('select', this.db.prepare(`
|
|
341
262
|
SELECT value, type, compressed, confidence, access_count
|
|
342
263
|
FROM collective_memory
|
|
343
264
|
WHERE swarm_id = ? AND key = ?
|
|
344
|
-
`)
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
this.statements.set(
|
|
348
|
-
'updateAccess',
|
|
349
|
-
this.db.prepare(`
|
|
265
|
+
`));
|
|
266
|
+
this.statements.set('updateAccess', this.db.prepare(`
|
|
350
267
|
UPDATE collective_memory
|
|
351
268
|
SET accessed_at = strftime('%s','now'), access_count = access_count + 1
|
|
352
269
|
WHERE swarm_id = ? AND key = ?
|
|
353
|
-
`)
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
this.statements.set(
|
|
357
|
-
'searchByPattern',
|
|
358
|
-
this.db.prepare(`
|
|
270
|
+
`));
|
|
271
|
+
this.statements.set('searchByPattern', this.db.prepare(`
|
|
359
272
|
SELECT key, type, confidence, created_at, accessed_at, access_count
|
|
360
273
|
FROM collective_memory
|
|
361
274
|
WHERE swarm_id = ? AND key LIKE ? AND confidence >= ?
|
|
362
275
|
ORDER BY access_count DESC, confidence DESC
|
|
363
276
|
LIMIT ?
|
|
364
|
-
`)
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
this.statements.set(
|
|
368
|
-
'getStats',
|
|
369
|
-
this.db.prepare(`
|
|
277
|
+
`));
|
|
278
|
+
this.statements.set('getStats', this.db.prepare(`
|
|
370
279
|
SELECT
|
|
371
280
|
COUNT(*) as count,
|
|
372
281
|
SUM(size) as totalSize,
|
|
@@ -375,270 +284,191 @@ export class CollectiveMemory extends EventEmitter {
|
|
|
375
284
|
AVG(access_count) as avgAccess
|
|
376
285
|
FROM collective_memory
|
|
377
286
|
WHERE swarm_id = ?
|
|
378
|
-
`)
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
this.statements.set(
|
|
382
|
-
'deleteExpired',
|
|
383
|
-
this.db.prepare(`
|
|
287
|
+
`));
|
|
288
|
+
this.statements.set('deleteExpired', this.db.prepare(`
|
|
384
289
|
DELETE FROM collective_memory
|
|
385
290
|
WHERE swarm_id = ? AND type = ? AND (strftime('%s','now') - accessed_at) > ?
|
|
386
|
-
`)
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
this.statements.set(
|
|
390
|
-
'getLRU',
|
|
391
|
-
this.db.prepare(`
|
|
291
|
+
`));
|
|
292
|
+
this.statements.set('getLRU', this.db.prepare(`
|
|
392
293
|
SELECT id, size FROM collective_memory
|
|
393
294
|
WHERE swarm_id = ? AND type NOT IN ('system', 'consensus')
|
|
394
295
|
ORDER BY accessed_at ASC, access_count ASC
|
|
395
296
|
LIMIT ?
|
|
396
|
-
`)
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
/**
|
|
297
|
+
`));
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
401
300
|
* Start optimization timers
|
|
402
|
-
*/
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
// Performance monitoring
|
|
414
|
-
this.metricsTimer = setInterval(() => this._updatePerformanceMetrics(), 30000); // 30 seconds
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
/**
|
|
301
|
+
*/ _startOptimizationTimers() {
|
|
302
|
+
// Main garbage collection
|
|
303
|
+
this.gcTimer = setInterval(()=>this._garbageCollect(), this.config.gcInterval);
|
|
304
|
+
// Database optimization
|
|
305
|
+
this.optimizeTimer = setInterval(()=>this._optimizeDatabase(), 1800000); // 30 minutes
|
|
306
|
+
// Cache cleanup
|
|
307
|
+
this.cacheTimer = setInterval(()=>this._optimizeCache(), 60000); // 1 minute
|
|
308
|
+
// Performance monitoring
|
|
309
|
+
this.metricsTimer = setInterval(()=>this._updatePerformanceMetrics(), 30000); // 30 seconds
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
418
312
|
* Initialize background worker for heavy operations
|
|
419
|
-
*/
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
/**
|
|
313
|
+
*/ _initializeBackgroundWorker() {
|
|
314
|
+
// Note: In production, this would initialize a proper Worker
|
|
315
|
+
// For now, we'll use async operations
|
|
316
|
+
this.backgroundQueue = [];
|
|
317
|
+
this.backgroundProcessing = false;
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
428
320
|
* Store data in collective memory
|
|
429
|
-
*/
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
const id = `${this.config.swarmId}-${key}-${Date.now()}`;
|
|
447
|
-
|
|
448
|
-
// Check if key already exists
|
|
449
|
-
const existing = this.db
|
|
450
|
-
.prepare(
|
|
451
|
-
`
|
|
321
|
+
*/ async store(key, value, type = 'knowledge', metadata = {}) {
|
|
322
|
+
try {
|
|
323
|
+
const serialized = JSON.stringify(value);
|
|
324
|
+
const size = Buffer.byteLength(serialized);
|
|
325
|
+
const shouldCompress = size > this.config.compressionThreshold && MEMORY_TYPES[type]?.compress;
|
|
326
|
+
let storedValue = serialized;
|
|
327
|
+
let compressed = 0;
|
|
328
|
+
if (shouldCompress) {
|
|
329
|
+
// In production, use proper compression like zlib
|
|
330
|
+
// For now, we'll just mark it as compressed
|
|
331
|
+
compressed = 1;
|
|
332
|
+
}
|
|
333
|
+
const id = `${this.config.swarmId}-${key}-${Date.now()}`;
|
|
334
|
+
// Check if key already exists
|
|
335
|
+
const existing = this.db.prepare(`
|
|
452
336
|
SELECT id FROM collective_memory
|
|
453
337
|
WHERE swarm_id = ? AND key = ?
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
if (existing) {
|
|
459
|
-
// Update existing entry
|
|
460
|
-
this.db
|
|
461
|
-
.prepare(
|
|
462
|
-
`
|
|
338
|
+
`).get(this.config.swarmId, key);
|
|
339
|
+
if (existing) {
|
|
340
|
+
// Update existing entry
|
|
341
|
+
this.db.prepare(`
|
|
463
342
|
UPDATE collective_memory
|
|
464
343
|
SET value = ?, type = ?, confidence = ?,
|
|
465
344
|
accessed_at = CURRENT_TIMESTAMP, access_count = access_count + 1,
|
|
466
345
|
compressed = ?, size = ?
|
|
467
346
|
WHERE swarm_id = ? AND key = ?
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
type,
|
|
473
|
-
metadata.confidence || 1.0,
|
|
474
|
-
compressed,
|
|
475
|
-
size,
|
|
476
|
-
this.config.swarmId,
|
|
477
|
-
key,
|
|
478
|
-
);
|
|
479
|
-
} else {
|
|
480
|
-
// Insert new entry
|
|
481
|
-
this.db
|
|
482
|
-
.prepare(
|
|
483
|
-
`
|
|
347
|
+
`).run(storedValue, type, metadata.confidence || 1.0, compressed, size, this.config.swarmId, key);
|
|
348
|
+
} else {
|
|
349
|
+
// Insert new entry
|
|
350
|
+
this.db.prepare(`
|
|
484
351
|
INSERT INTO collective_memory
|
|
485
352
|
(id, swarm_id, key, value, type, confidence, created_by, compressed, size)
|
|
486
353
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
this._trackAccess(key, 'write');
|
|
515
|
-
|
|
516
|
-
this.emit('memory:stored', { key, type, size });
|
|
517
|
-
|
|
518
|
-
return { success: true, id, size };
|
|
519
|
-
} catch (error) {
|
|
520
|
-
this.emit('error', error);
|
|
521
|
-
throw error;
|
|
354
|
+
`).run(id, this.config.swarmId, key, storedValue, type, metadata.confidence || 1.0, metadata.createdBy || 'system', compressed, size);
|
|
355
|
+
}
|
|
356
|
+
// Update cache
|
|
357
|
+
this.cache.set(key, {
|
|
358
|
+
value,
|
|
359
|
+
type,
|
|
360
|
+
timestamp: Date.now(),
|
|
361
|
+
size
|
|
362
|
+
});
|
|
363
|
+
// Check memory limits
|
|
364
|
+
this._checkMemoryLimits();
|
|
365
|
+
// Track access pattern
|
|
366
|
+
this._trackAccess(key, 'write');
|
|
367
|
+
this.emit('memory:stored', {
|
|
368
|
+
key,
|
|
369
|
+
type,
|
|
370
|
+
size
|
|
371
|
+
});
|
|
372
|
+
return {
|
|
373
|
+
success: true,
|
|
374
|
+
id,
|
|
375
|
+
size
|
|
376
|
+
};
|
|
377
|
+
} catch (error) {
|
|
378
|
+
this.emit('error', error);
|
|
379
|
+
throw error;
|
|
380
|
+
}
|
|
522
381
|
}
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
/**
|
|
382
|
+
/**
|
|
526
383
|
* Retrieve data from collective memory
|
|
527
|
-
*/
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
// Query database
|
|
538
|
-
const result = this.db
|
|
539
|
-
.prepare(
|
|
540
|
-
`
|
|
384
|
+
*/ async retrieve(key) {
|
|
385
|
+
try {
|
|
386
|
+
// Check cache first
|
|
387
|
+
if (this.cache.has(key)) {
|
|
388
|
+
const cached = this.cache.get(key);
|
|
389
|
+
this._trackAccess(key, 'cache_hit');
|
|
390
|
+
return cached.value;
|
|
391
|
+
}
|
|
392
|
+
// Query database
|
|
393
|
+
const result = this.db.prepare(`
|
|
541
394
|
SELECT value, type, compressed, confidence
|
|
542
395
|
FROM collective_memory
|
|
543
396
|
WHERE swarm_id = ? AND key = ?
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
// Update access statistics
|
|
554
|
-
this.db
|
|
555
|
-
.prepare(
|
|
556
|
-
`
|
|
397
|
+
`).get(this.config.swarmId, key);
|
|
398
|
+
if (!result) {
|
|
399
|
+
this._trackAccess(key, 'miss');
|
|
400
|
+
return null;
|
|
401
|
+
}
|
|
402
|
+
// Update access statistics
|
|
403
|
+
this.db.prepare(`
|
|
557
404
|
UPDATE collective_memory
|
|
558
405
|
SET accessed_at = CURRENT_TIMESTAMP,
|
|
559
406
|
access_count = access_count + 1
|
|
560
407
|
WHERE swarm_id = ? AND key = ?
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
this._trackAccess(key, 'read');
|
|
583
|
-
|
|
584
|
-
return parsed;
|
|
585
|
-
} catch (error) {
|
|
586
|
-
this.emit('error', error);
|
|
587
|
-
throw error;
|
|
408
|
+
`).run(this.config.swarmId, key);
|
|
409
|
+
// Decompress if needed
|
|
410
|
+
let value = result.value;
|
|
411
|
+
if (result.compressed) {
|
|
412
|
+
// In production, decompress here
|
|
413
|
+
}
|
|
414
|
+
// Parse JSON
|
|
415
|
+
const parsed = JSON.parse(value);
|
|
416
|
+
// Add to cache
|
|
417
|
+
this.cache.set(key, {
|
|
418
|
+
value: parsed,
|
|
419
|
+
type: result.type,
|
|
420
|
+
timestamp: Date.now(),
|
|
421
|
+
confidence: result.confidence
|
|
422
|
+
});
|
|
423
|
+
this._trackAccess(key, 'read');
|
|
424
|
+
return parsed;
|
|
425
|
+
} catch (error) {
|
|
426
|
+
this.emit('error', error);
|
|
427
|
+
throw error;
|
|
428
|
+
}
|
|
588
429
|
}
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
/**
|
|
430
|
+
/**
|
|
592
431
|
* Search collective memory
|
|
593
|
-
*/
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
let query = `
|
|
432
|
+
*/ async search(pattern, options = {}) {
|
|
433
|
+
try {
|
|
434
|
+
const limit = options.limit || 50;
|
|
435
|
+
const type = options.type || null;
|
|
436
|
+
const minConfidence = options.minConfidence || 0;
|
|
437
|
+
let query = `
|
|
601
438
|
SELECT key, type, confidence, created_at, accessed_at, access_count
|
|
602
439
|
FROM collective_memory
|
|
603
440
|
WHERE swarm_id = ?
|
|
604
441
|
AND key LIKE ?
|
|
605
442
|
AND confidence >= ?
|
|
606
443
|
`;
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
throw error;
|
|
444
|
+
const params = [
|
|
445
|
+
this.config.swarmId,
|
|
446
|
+
`%${pattern}%`,
|
|
447
|
+
minConfidence
|
|
448
|
+
];
|
|
449
|
+
if (type) {
|
|
450
|
+
query += ' AND type = ?';
|
|
451
|
+
params.push(type);
|
|
452
|
+
}
|
|
453
|
+
query += ' ORDER BY access_count DESC, confidence DESC LIMIT ?';
|
|
454
|
+
params.push(limit);
|
|
455
|
+
const results = this.db.prepare(query).all(...params);
|
|
456
|
+
this._trackAccess(`search:${pattern}`, 'search');
|
|
457
|
+
return results;
|
|
458
|
+
} catch (error) {
|
|
459
|
+
this.emit('error', error);
|
|
460
|
+
throw error;
|
|
461
|
+
}
|
|
626
462
|
}
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
/**
|
|
463
|
+
/**
|
|
630
464
|
* Get related memories using association
|
|
631
|
-
*/
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
// Simple association: find memories accessed around the same time
|
|
639
|
-
const result = this.db
|
|
640
|
-
.prepare(
|
|
641
|
-
`
|
|
465
|
+
*/ async getRelated(key, limit = 10) {
|
|
466
|
+
try {
|
|
467
|
+
// Get the original memory
|
|
468
|
+
const original = await this.retrieve(key);
|
|
469
|
+
if (!original) return [];
|
|
470
|
+
// Simple association: find memories accessed around the same time
|
|
471
|
+
const result = this.db.prepare(`
|
|
642
472
|
SELECT m1.key, m1.type, m1.confidence, m1.access_count
|
|
643
473
|
FROM collective_memory m1
|
|
644
474
|
JOIN collective_memory m2 ON m1.swarm_id = m2.swarm_id
|
|
@@ -648,315 +478,250 @@ export class CollectiveMemory extends EventEmitter {
|
|
|
648
478
|
AND ABS(julianday(m1.accessed_at) - julianday(m2.accessed_at)) < 0.01
|
|
649
479
|
ORDER BY m1.confidence DESC, m1.access_count DESC
|
|
650
480
|
LIMIT ?
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
this.emit('error', error);
|
|
658
|
-
throw error;
|
|
481
|
+
`).all(key, key, this.config.swarmId, limit);
|
|
482
|
+
return result;
|
|
483
|
+
} catch (error) {
|
|
484
|
+
this.emit('error', error);
|
|
485
|
+
throw error;
|
|
486
|
+
}
|
|
659
487
|
}
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
/**
|
|
488
|
+
/**
|
|
663
489
|
* Build associations between memories
|
|
664
|
-
*/
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
);
|
|
689
|
-
|
|
690
|
-
this.emit('memory:associated', { key1, key2, strength });
|
|
691
|
-
} catch (error) {
|
|
692
|
-
this.emit('error', error);
|
|
693
|
-
throw error;
|
|
490
|
+
*/ async associate(key1, key2, strength = 1.0) {
|
|
491
|
+
try {
|
|
492
|
+
// Store bidirectional association
|
|
493
|
+
await this.store(`assoc:${key1}:${key2}`, {
|
|
494
|
+
from: key1,
|
|
495
|
+
to: key2,
|
|
496
|
+
strength,
|
|
497
|
+
created: Date.now()
|
|
498
|
+
}, 'system');
|
|
499
|
+
await this.store(`assoc:${key2}:${key1}`, {
|
|
500
|
+
from: key2,
|
|
501
|
+
to: key1,
|
|
502
|
+
strength,
|
|
503
|
+
created: Date.now()
|
|
504
|
+
}, 'system');
|
|
505
|
+
this.emit('memory:associated', {
|
|
506
|
+
key1,
|
|
507
|
+
key2,
|
|
508
|
+
strength
|
|
509
|
+
});
|
|
510
|
+
} catch (error) {
|
|
511
|
+
this.emit('error', error);
|
|
512
|
+
throw error;
|
|
513
|
+
}
|
|
694
514
|
}
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
/**
|
|
515
|
+
/**
|
|
698
516
|
* Consolidate similar memories
|
|
699
|
-
*/
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
const memories = this.db
|
|
704
|
-
.prepare(
|
|
705
|
-
`
|
|
517
|
+
*/ async consolidate() {
|
|
518
|
+
try {
|
|
519
|
+
// Find similar memories
|
|
520
|
+
const memories = this.db.prepare(`
|
|
706
521
|
SELECT key, value, type, confidence, access_count
|
|
707
522
|
FROM collective_memory
|
|
708
523
|
WHERE swarm_id = ?
|
|
709
524
|
AND type IN ('knowledge', 'result')
|
|
710
525
|
ORDER BY created_at DESC
|
|
711
526
|
LIMIT 1000
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
527
|
+
`).all(this.config.swarmId);
|
|
528
|
+
const consolidated = new Map();
|
|
529
|
+
// Group by similarity (simple implementation)
|
|
530
|
+
memories.forEach((memory)=>{
|
|
531
|
+
const value = JSON.parse(memory.value);
|
|
532
|
+
const category = this._categorizeMemory(value);
|
|
533
|
+
if (!consolidated.has(category)) {
|
|
534
|
+
consolidated.set(category, []);
|
|
535
|
+
}
|
|
536
|
+
consolidated.get(category).push({
|
|
537
|
+
...memory,
|
|
538
|
+
value
|
|
539
|
+
});
|
|
540
|
+
});
|
|
541
|
+
// Merge similar memories
|
|
542
|
+
let mergeCount = 0;
|
|
543
|
+
consolidated.forEach((group, category)=>{
|
|
544
|
+
if (group.length > 1) {
|
|
545
|
+
const merged = this._mergeMemories(group);
|
|
546
|
+
// Store merged memory
|
|
547
|
+
this.store(`consolidated:${category}`, merged, 'knowledge', {
|
|
548
|
+
confidence: merged.confidence,
|
|
549
|
+
createdBy: 'consolidation'
|
|
550
|
+
});
|
|
551
|
+
mergeCount++;
|
|
552
|
+
}
|
|
553
|
+
});
|
|
554
|
+
this.emit('memory:consolidated', {
|
|
555
|
+
categories: consolidated.size,
|
|
556
|
+
merged: mergeCount
|
|
557
|
+
});
|
|
558
|
+
return {
|
|
559
|
+
categories: consolidated.size,
|
|
560
|
+
merged: mergeCount
|
|
561
|
+
};
|
|
562
|
+
} catch (error) {
|
|
563
|
+
this.emit('error', error);
|
|
564
|
+
throw error;
|
|
725
565
|
}
|
|
726
|
-
|
|
727
|
-
consolidated.get(category).push({
|
|
728
|
-
...memory,
|
|
729
|
-
value,
|
|
730
|
-
});
|
|
731
|
-
});
|
|
732
|
-
|
|
733
|
-
// Merge similar memories
|
|
734
|
-
let mergeCount = 0;
|
|
735
|
-
consolidated.forEach((group, category) => {
|
|
736
|
-
if (group.length > 1) {
|
|
737
|
-
const merged = this._mergeMemories(group);
|
|
738
|
-
|
|
739
|
-
// Store merged memory
|
|
740
|
-
this.store(`consolidated:${category}`, merged, 'knowledge', {
|
|
741
|
-
confidence: merged.confidence,
|
|
742
|
-
createdBy: 'consolidation',
|
|
743
|
-
});
|
|
744
|
-
|
|
745
|
-
mergeCount++;
|
|
746
|
-
}
|
|
747
|
-
});
|
|
748
|
-
|
|
749
|
-
this.emit('memory:consolidated', { categories: consolidated.size, merged: mergeCount });
|
|
750
|
-
|
|
751
|
-
return { categories: consolidated.size, merged: mergeCount };
|
|
752
|
-
} catch (error) {
|
|
753
|
-
this.emit('error', error);
|
|
754
|
-
throw error;
|
|
755
566
|
}
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
/**
|
|
567
|
+
/**
|
|
759
568
|
* Categorize memory for consolidation
|
|
760
|
-
*/
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
569
|
+
*/ _categorizeMemory(value) {
|
|
570
|
+
// Simple categorization based on content
|
|
571
|
+
if (typeof value === 'string') {
|
|
572
|
+
return 'text';
|
|
573
|
+
}
|
|
574
|
+
if (typeof value === 'object') {
|
|
575
|
+
const keys = Object.keys(value).sort().join(':');
|
|
576
|
+
return `object:${keys.substring(0, 50)}`;
|
|
577
|
+
}
|
|
578
|
+
return 'other';
|
|
770
579
|
}
|
|
771
|
-
|
|
772
|
-
return 'other';
|
|
773
|
-
}
|
|
774
|
-
|
|
775
|
-
/**
|
|
580
|
+
/**
|
|
776
581
|
* Merge similar memories
|
|
777
|
-
*/
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
sourceCount: memories.length,
|
|
799
|
-
};
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
/**
|
|
582
|
+
*/ _mergeMemories(memories) {
|
|
583
|
+
// Calculate weighted average confidence
|
|
584
|
+
let totalWeight = 0;
|
|
585
|
+
let weightedConfidence = 0;
|
|
586
|
+
const mergedValue = {};
|
|
587
|
+
memories.forEach((memory)=>{
|
|
588
|
+
const weight = memory.access_count + 1;
|
|
589
|
+
totalWeight += weight;
|
|
590
|
+
weightedConfidence += memory.confidence * weight;
|
|
591
|
+
// Merge values (simple implementation)
|
|
592
|
+
if (typeof memory.value === 'object') {
|
|
593
|
+
Object.assign(mergedValue, memory.value);
|
|
594
|
+
}
|
|
595
|
+
});
|
|
596
|
+
return {
|
|
597
|
+
value: mergedValue,
|
|
598
|
+
confidence: weightedConfidence / totalWeight,
|
|
599
|
+
sourceCount: memories.length
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
/**
|
|
803
603
|
* Garbage collection
|
|
804
|
-
*/
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
if (config.ttl) {
|
|
813
|
-
const result = this.db
|
|
814
|
-
.prepare(
|
|
815
|
-
`
|
|
604
|
+
*/ _garbageCollect() {
|
|
605
|
+
try {
|
|
606
|
+
const now = Date.now();
|
|
607
|
+
let deletedCount = 0;
|
|
608
|
+
// Delete expired memories based on TTL
|
|
609
|
+
Object.entries(MEMORY_TYPES).forEach(([type, config])=>{
|
|
610
|
+
if (config.ttl) {
|
|
611
|
+
const result = this.db.prepare(`
|
|
816
612
|
DELETE FROM collective_memory
|
|
817
613
|
WHERE swarm_id = ?
|
|
818
614
|
AND type = ?
|
|
819
615
|
AND (julianday('now') - julianday(accessed_at)) * 86400000 > ?
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
616
|
+
`).run(this.config.swarmId, type, config.ttl);
|
|
617
|
+
deletedCount += result.changes;
|
|
618
|
+
}
|
|
619
|
+
});
|
|
620
|
+
// Clear old cache entries
|
|
621
|
+
const cacheTimeout = 300000; // 5 minutes
|
|
622
|
+
this.cache.forEach((value, key)=>{
|
|
623
|
+
if (now - value.timestamp > cacheTimeout) {
|
|
624
|
+
this.cache.delete(key);
|
|
625
|
+
}
|
|
626
|
+
});
|
|
627
|
+
// Update statistics
|
|
628
|
+
this._updateStatistics();
|
|
629
|
+
this.state.lastGC = now;
|
|
630
|
+
if (deletedCount > 0) {
|
|
631
|
+
this.emit('memory:gc', {
|
|
632
|
+
deleted: deletedCount,
|
|
633
|
+
cacheSize: this.cache.size
|
|
634
|
+
});
|
|
635
|
+
}
|
|
636
|
+
} catch (error) {
|
|
637
|
+
this.emit('error', error);
|
|
825
638
|
}
|
|
826
|
-
});
|
|
827
|
-
|
|
828
|
-
// Clear old cache entries
|
|
829
|
-
const cacheTimeout = 300000; // 5 minutes
|
|
830
|
-
this.cache.forEach((value, key) => {
|
|
831
|
-
if (now - value.timestamp > cacheTimeout) {
|
|
832
|
-
this.cache.delete(key);
|
|
833
|
-
}
|
|
834
|
-
});
|
|
835
|
-
|
|
836
|
-
// Update statistics
|
|
837
|
-
this._updateStatistics();
|
|
838
|
-
|
|
839
|
-
this.state.lastGC = now;
|
|
840
|
-
|
|
841
|
-
if (deletedCount > 0) {
|
|
842
|
-
this.emit('memory:gc', { deleted: deletedCount, cacheSize: this.cache.size });
|
|
843
|
-
}
|
|
844
|
-
} catch (error) {
|
|
845
|
-
this.emit('error', error);
|
|
846
639
|
}
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
/**
|
|
640
|
+
/**
|
|
850
641
|
* Check memory limits and evict if necessary
|
|
851
|
-
*/
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
const toEvict = this.db
|
|
856
|
-
.prepare(
|
|
857
|
-
`
|
|
642
|
+
*/ _checkMemoryLimits() {
|
|
643
|
+
if (this.state.totalSize > this.config.maxSize * 1024 * 1024) {
|
|
644
|
+
// Evict least recently used memories
|
|
645
|
+
const toEvict = this.db.prepare(`
|
|
858
646
|
SELECT id, size FROM collective_memory
|
|
859
647
|
WHERE swarm_id = ?
|
|
860
648
|
AND type NOT IN ('system', 'consensus')
|
|
861
649
|
ORDER BY accessed_at ASC, access_count ASC
|
|
862
650
|
LIMIT 100
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
651
|
+
`).all(this.config.swarmId);
|
|
652
|
+
let freedSize = 0;
|
|
653
|
+
toEvict.forEach((memory)=>{
|
|
654
|
+
this.db.prepare('DELETE FROM collective_memory WHERE id = ?').run(memory.id);
|
|
655
|
+
freedSize += memory.size;
|
|
656
|
+
});
|
|
657
|
+
this.emit('memory:evicted', {
|
|
658
|
+
count: toEvict.length,
|
|
659
|
+
freedSize
|
|
660
|
+
});
|
|
661
|
+
}
|
|
874
662
|
}
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
/**
|
|
663
|
+
/**
|
|
878
664
|
* Optimize database performance
|
|
879
|
-
*/
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
} catch (error) {
|
|
892
|
-
this.emit('error', error);
|
|
665
|
+
*/ _optimizeDatabase() {
|
|
666
|
+
try {
|
|
667
|
+
// Run database optimization
|
|
668
|
+
this.db.pragma('optimize');
|
|
669
|
+
this.db.pragma('analysis_limit=1000');
|
|
670
|
+
this.db.exec('ANALYZE');
|
|
671
|
+
// Update database statistics
|
|
672
|
+
this._updateStatistics();
|
|
673
|
+
this.emit('database:optimized');
|
|
674
|
+
} catch (error) {
|
|
675
|
+
this.emit('error', error);
|
|
676
|
+
}
|
|
893
677
|
}
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
/**
|
|
678
|
+
/**
|
|
897
679
|
* Optimize cache performance
|
|
898
|
-
*/
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
} catch (error) {
|
|
917
|
-
this.emit('error', error);
|
|
680
|
+
*/ _optimizeCache() {
|
|
681
|
+
try {
|
|
682
|
+
const now = Date.now();
|
|
683
|
+
const cacheTimeout = 300000; // 5 minutes
|
|
684
|
+
// Clear expired cache entries
|
|
685
|
+
if (this.cache.cache) {
|
|
686
|
+
this.cache.cache.forEach((value, key)=>{
|
|
687
|
+
if (now - value.timestamp > cacheTimeout) {
|
|
688
|
+
this.cache.cache.delete(key);
|
|
689
|
+
}
|
|
690
|
+
});
|
|
691
|
+
}
|
|
692
|
+
this.emit('cache:optimized', {
|
|
693
|
+
size: this.cache.cache ? this.cache.cache.size : 0
|
|
694
|
+
});
|
|
695
|
+
} catch (error) {
|
|
696
|
+
this.emit('error', error);
|
|
697
|
+
}
|
|
918
698
|
}
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
/**
|
|
699
|
+
/**
|
|
922
700
|
* Update performance metrics
|
|
923
|
-
*/
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
if (this.state.performanceMetrics.queryTimes.length > 100) {
|
|
942
|
-
this.state.performanceMetrics.queryTimes =
|
|
943
|
-
this.state.performanceMetrics.queryTimes.slice(-100);
|
|
701
|
+
*/ _updatePerformanceMetrics() {
|
|
702
|
+
try {
|
|
703
|
+
// Calculate cache hit rate
|
|
704
|
+
const cacheStats = this.cache.getStats();
|
|
705
|
+
this.state.performanceMetrics.cacheHitRate = cacheStats.hitRate || 0;
|
|
706
|
+
// Calculate memory efficiency
|
|
707
|
+
this.state.performanceMetrics.memoryEfficiency = this.state.totalSize / (this.config.maxSize * 1024 * 1024) * 100;
|
|
708
|
+
// Update average query time if we have recent measurements
|
|
709
|
+
if (this.state.performanceMetrics.queryTimes.length > 0) {
|
|
710
|
+
this.state.performanceMetrics.avgQueryTime = this.state.performanceMetrics.queryTimes.reduce((sum, time)=>sum + time, 0) / this.state.performanceMetrics.queryTimes.length;
|
|
711
|
+
// Keep only recent query times (last 100)
|
|
712
|
+
if (this.state.performanceMetrics.queryTimes.length > 100) {
|
|
713
|
+
this.state.performanceMetrics.queryTimes = this.state.performanceMetrics.queryTimes.slice(-100);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
this.emit('metrics:updated', this.state.performanceMetrics);
|
|
717
|
+
} catch (error) {
|
|
718
|
+
this.emit('error', error);
|
|
944
719
|
}
|
|
945
|
-
}
|
|
946
|
-
|
|
947
|
-
this.emit('metrics:updated', this.state.performanceMetrics);
|
|
948
|
-
} catch (error) {
|
|
949
|
-
this.emit('error', error);
|
|
950
720
|
}
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
/**
|
|
721
|
+
/**
|
|
954
722
|
* Update memory statistics
|
|
955
|
-
*/
|
|
956
|
-
|
|
957
|
-
const stats = this.db
|
|
958
|
-
.prepare(
|
|
959
|
-
`
|
|
723
|
+
*/ _updateStatistics() {
|
|
724
|
+
const stats = this.db.prepare(`
|
|
960
725
|
SELECT
|
|
961
726
|
COUNT(*) as count,
|
|
962
727
|
SUM(size) as totalSize,
|
|
@@ -964,329 +729,328 @@ export class CollectiveMemory extends EventEmitter {
|
|
|
964
729
|
SUM(compressed) as compressedCount
|
|
965
730
|
FROM collective_memory
|
|
966
731
|
WHERE swarm_id = ?
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
if (stats.compressedCount > 0) {
|
|
976
|
-
// Estimate compression ratio
|
|
977
|
-
this.state.compressionRatio = 0.6; // Assume 40% compression
|
|
732
|
+
`).get(this.config.swarmId);
|
|
733
|
+
this.state.entryCount = stats.count || 0;
|
|
734
|
+
this.state.totalSize = stats.totalSize || 0;
|
|
735
|
+
this.state.avgConfidence = stats.avgConfidence || 1.0;
|
|
736
|
+
if (stats.compressedCount > 0) {
|
|
737
|
+
// Estimate compression ratio
|
|
738
|
+
this.state.compressionRatio = 0.6; // Assume 40% compression
|
|
739
|
+
}
|
|
978
740
|
}
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
/**
|
|
741
|
+
/**
|
|
982
742
|
* Track access patterns
|
|
983
|
-
*/
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
(a, b) => a[1].lastAccess - b[1].lastAccess,
|
|
1020
|
-
);
|
|
1021
|
-
|
|
1022
|
-
sorted.slice(0, 100).forEach(([key]) => {
|
|
1023
|
-
this.state.accessPatterns.delete(key);
|
|
1024
|
-
});
|
|
743
|
+
*/ _trackAccess(key, operation) {
|
|
744
|
+
const pattern = this.state.accessPatterns.get(key) || {
|
|
745
|
+
reads: 0,
|
|
746
|
+
writes: 0,
|
|
747
|
+
searches: 0,
|
|
748
|
+
cacheHits: 0,
|
|
749
|
+
misses: 0,
|
|
750
|
+
lastAccess: Date.now()
|
|
751
|
+
};
|
|
752
|
+
switch(operation){
|
|
753
|
+
case 'read':
|
|
754
|
+
pattern.reads++;
|
|
755
|
+
break;
|
|
756
|
+
case 'write':
|
|
757
|
+
pattern.writes++;
|
|
758
|
+
break;
|
|
759
|
+
case 'search':
|
|
760
|
+
pattern.searches++;
|
|
761
|
+
break;
|
|
762
|
+
case 'cache_hit':
|
|
763
|
+
pattern.cacheHits++;
|
|
764
|
+
break;
|
|
765
|
+
case 'miss':
|
|
766
|
+
pattern.misses++;
|
|
767
|
+
break;
|
|
768
|
+
}
|
|
769
|
+
pattern.lastAccess = Date.now();
|
|
770
|
+
this.state.accessPatterns.set(key, pattern);
|
|
771
|
+
// Keep access patterns size limited
|
|
772
|
+
if (this.state.accessPatterns.size > 1000) {
|
|
773
|
+
// Remove oldest entries
|
|
774
|
+
const sorted = Array.from(this.state.accessPatterns.entries()).sort((a, b)=>a[1].lastAccess - b[1].lastAccess);
|
|
775
|
+
sorted.slice(0, 100).forEach(([key])=>{
|
|
776
|
+
this.state.accessPatterns.delete(key);
|
|
777
|
+
});
|
|
778
|
+
}
|
|
1025
779
|
}
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
/**
|
|
780
|
+
/**
|
|
1029
781
|
* Get enhanced memory statistics
|
|
1030
|
-
*/
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
}
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
/**
|
|
782
|
+
*/ getStatistics() {
|
|
783
|
+
return {
|
|
784
|
+
swarmId: this.config.swarmId,
|
|
785
|
+
entryCount: this.state.entryCount,
|
|
786
|
+
totalSize: this.state.totalSize,
|
|
787
|
+
maxSize: this.config.maxSize * 1024 * 1024,
|
|
788
|
+
utilizationPercent: this.state.totalSize / (this.config.maxSize * 1024 * 1024) * 100,
|
|
789
|
+
avgConfidence: this.state.avgConfidence,
|
|
790
|
+
compressionRatio: this.state.compressionRatio,
|
|
791
|
+
cacheSize: this.cache.cache ? this.cache.cache.size : 0,
|
|
792
|
+
lastGC: new Date(this.state.lastGC).toISOString(),
|
|
793
|
+
accessPatterns: this.state.accessPatterns.size,
|
|
794
|
+
optimization: {
|
|
795
|
+
cacheOptimized: true,
|
|
796
|
+
poolingEnabled: this.config.enablePooling,
|
|
797
|
+
asyncOperations: this.config.enableAsyncOperations,
|
|
798
|
+
compressionRatio: this.state.compressionRatio,
|
|
799
|
+
performanceMetrics: this.state.performanceMetrics
|
|
800
|
+
}
|
|
801
|
+
};
|
|
802
|
+
}
|
|
803
|
+
/**
|
|
1054
804
|
* Export memory snapshot
|
|
1055
|
-
*/
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
const memories = this.db
|
|
1059
|
-
.prepare(
|
|
1060
|
-
`
|
|
805
|
+
*/ async exportSnapshot(filepath) {
|
|
806
|
+
try {
|
|
807
|
+
const memories = this.db.prepare(`
|
|
1061
808
|
SELECT * FROM collective_memory
|
|
1062
809
|
WHERE swarm_id = ?
|
|
1063
810
|
ORDER BY created_at DESC
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
this.emit('error', error);
|
|
1085
|
-
throw error;
|
|
811
|
+
`).all(this.config.swarmId);
|
|
812
|
+
const snapshot = {
|
|
813
|
+
swarmId: this.config.swarmId,
|
|
814
|
+
timestamp: new Date().toISOString(),
|
|
815
|
+
statistics: this.getStatistics(),
|
|
816
|
+
memories: memories.map((m)=>({
|
|
817
|
+
...m,
|
|
818
|
+
value: JSON.parse(m.value)
|
|
819
|
+
}))
|
|
820
|
+
};
|
|
821
|
+
// In production, write to file
|
|
822
|
+
// For now, return the snapshot
|
|
823
|
+
this.emit('memory:exported', {
|
|
824
|
+
count: memories.length
|
|
825
|
+
});
|
|
826
|
+
return snapshot;
|
|
827
|
+
} catch (error) {
|
|
828
|
+
this.emit('error', error);
|
|
829
|
+
throw error;
|
|
830
|
+
}
|
|
1086
831
|
}
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
/**
|
|
832
|
+
/**
|
|
1090
833
|
* Import memory snapshot
|
|
1091
|
-
*/
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
834
|
+
*/ async importSnapshot(snapshot) {
|
|
835
|
+
try {
|
|
836
|
+
let imported = 0;
|
|
837
|
+
for (const memory of snapshot.memories){
|
|
838
|
+
await this.store(memory.key, memory.value, memory.type, {
|
|
839
|
+
confidence: memory.confidence,
|
|
840
|
+
createdBy: memory.created_by
|
|
841
|
+
});
|
|
842
|
+
imported++;
|
|
843
|
+
}
|
|
844
|
+
this.emit('memory:imported', {
|
|
845
|
+
count: imported
|
|
846
|
+
});
|
|
847
|
+
return {
|
|
848
|
+
imported
|
|
849
|
+
};
|
|
850
|
+
} catch (error) {
|
|
851
|
+
this.emit('error', error);
|
|
852
|
+
throw error;
|
|
853
|
+
}
|
|
1110
854
|
}
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
/**
|
|
855
|
+
/**
|
|
1114
856
|
* Enhanced shutdown with cleanup
|
|
1115
|
-
*/
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
857
|
+
*/ close() {
|
|
858
|
+
// Clear all timers
|
|
859
|
+
if (this.gcTimer) clearInterval(this.gcTimer);
|
|
860
|
+
if (this.optimizeTimer) clearInterval(this.optimizeTimer);
|
|
861
|
+
if (this.cacheTimer) clearInterval(this.cacheTimer);
|
|
862
|
+
if (this.metricsTimer) clearInterval(this.metricsTimer);
|
|
863
|
+
// Final optimization before closing
|
|
864
|
+
try {
|
|
865
|
+
this.db.pragma('optimize');
|
|
866
|
+
} catch (error) {
|
|
867
|
+
// Ignore errors during shutdown
|
|
868
|
+
}
|
|
869
|
+
// Close database
|
|
870
|
+
if (this.db) {
|
|
871
|
+
this.db.close();
|
|
872
|
+
}
|
|
873
|
+
// Clear memory pools
|
|
874
|
+
if (this.config.enablePooling) {
|
|
875
|
+
Object.values(this.pools).forEach((pool)=>{
|
|
876
|
+
pool.pool.length = 0;
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
const finalStats = {
|
|
880
|
+
cacheStats: this.cache.getStats ? this.cache.getStats() : {},
|
|
881
|
+
poolStats: this.config.enablePooling ? {
|
|
882
|
+
queryResults: this.pools.queryResults.getStats(),
|
|
883
|
+
memoryEntries: this.pools.memoryEntries.getStats()
|
|
884
|
+
} : null,
|
|
885
|
+
performanceMetrics: this.state.performanceMetrics
|
|
886
|
+
};
|
|
887
|
+
this.emit('memory:closed', finalStats);
|
|
1140
888
|
}
|
|
1141
|
-
|
|
1142
|
-
const finalStats = {
|
|
1143
|
-
cacheStats: this.cache.getStats ? this.cache.getStats() : {},
|
|
1144
|
-
poolStats: this.config.enablePooling
|
|
1145
|
-
? {
|
|
1146
|
-
queryResults: this.pools.queryResults.getStats(),
|
|
1147
|
-
memoryEntries: this.pools.memoryEntries.getStats(),
|
|
1148
|
-
}
|
|
1149
|
-
: null,
|
|
1150
|
-
performanceMetrics: this.state.performanceMetrics,
|
|
1151
|
-
};
|
|
1152
|
-
|
|
1153
|
-
this.emit('memory:closed', finalStats);
|
|
1154
|
-
}
|
|
1155
|
-
|
|
1156
|
-
/**
|
|
889
|
+
/**
|
|
1157
890
|
* Get comprehensive memory analytics
|
|
1158
|
-
*/
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
cacheSize: this.db.pragma('cache_size'),
|
|
1174
|
-
},
|
|
1175
|
-
};
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
|
-
/**
|
|
1179
|
-
* Memory health check
|
|
1180
|
-
*/
|
|
1181
|
-
async healthCheck() {
|
|
1182
|
-
const analytics = this.getAnalytics();
|
|
1183
|
-
const health = {
|
|
1184
|
-
status: 'healthy',
|
|
1185
|
-
issues: [],
|
|
1186
|
-
recommendations: [],
|
|
1187
|
-
};
|
|
1188
|
-
|
|
1189
|
-
// Check cache hit rate
|
|
1190
|
-
if (analytics.cache.hitRate < 50) {
|
|
1191
|
-
health.issues.push('Low cache hit rate');
|
|
1192
|
-
health.recommendations.push('Consider increasing cache size');
|
|
891
|
+
*/ getAnalytics() {
|
|
892
|
+
return {
|
|
893
|
+
basic: this.getStatistics(),
|
|
894
|
+
performance: this.state.performanceMetrics,
|
|
895
|
+
cache: this.cache.getStats ? this.cache.getStats() : {},
|
|
896
|
+
pools: this.config.enablePooling ? {
|
|
897
|
+
queryResults: this.pools.queryResults.getStats(),
|
|
898
|
+
memoryEntries: this.pools.memoryEntries.getStats()
|
|
899
|
+
} : null,
|
|
900
|
+
database: {
|
|
901
|
+
fragmentation: this.db.pragma('freelist_count'),
|
|
902
|
+
pageSize: this.db.pragma('page_size'),
|
|
903
|
+
cacheSize: this.db.pragma('cache_size')
|
|
904
|
+
}
|
|
905
|
+
};
|
|
1193
906
|
}
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
907
|
+
/**
|
|
908
|
+
* Memory health check
|
|
909
|
+
*/ async healthCheck() {
|
|
910
|
+
const analytics = this.getAnalytics();
|
|
911
|
+
const health = {
|
|
912
|
+
status: 'healthy',
|
|
913
|
+
issues: [],
|
|
914
|
+
recommendations: []
|
|
915
|
+
};
|
|
916
|
+
// Check cache hit rate
|
|
917
|
+
if (analytics.cache.hitRate < 50) {
|
|
918
|
+
health.issues.push('Low cache hit rate');
|
|
919
|
+
health.recommendations.push('Consider increasing cache size');
|
|
920
|
+
}
|
|
921
|
+
// Check memory usage
|
|
922
|
+
if (analytics.basic.utilizationPercent > 90) {
|
|
923
|
+
health.status = 'warning';
|
|
924
|
+
health.issues.push('High memory utilization');
|
|
925
|
+
health.recommendations.push('Consider increasing max memory or running garbage collection');
|
|
926
|
+
}
|
|
927
|
+
// Check query performance
|
|
928
|
+
if (analytics.performance.avgQueryTime > 100) {
|
|
929
|
+
health.issues.push('Slow query performance');
|
|
930
|
+
health.recommendations.push('Consider database optimization or indexing');
|
|
931
|
+
}
|
|
932
|
+
return health;
|
|
1200
933
|
}
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
934
|
+
constructor(config = {}){
|
|
935
|
+
super();
|
|
936
|
+
/** @type {import('better-sqlite3').Database | null} */ this.db = null;
|
|
937
|
+
this.config = {
|
|
938
|
+
swarmId: config.swarmId,
|
|
939
|
+
maxSize: config.maxSize || 100,
|
|
940
|
+
dbPath: config.dbPath || path.join(process.cwd(), '.hive-mind', 'hive.db'),
|
|
941
|
+
compressionThreshold: config.compressionThreshold || 1024,
|
|
942
|
+
gcInterval: config.gcInterval || 300000,
|
|
943
|
+
cacheSize: config.cacheSize || 1000,
|
|
944
|
+
cacheMemoryMB: config.cacheMemoryMB || 50,
|
|
945
|
+
enablePooling: config.enablePooling !== false,
|
|
946
|
+
enableAsyncOperations: config.enableAsyncOperations !== false,
|
|
947
|
+
...config
|
|
948
|
+
};
|
|
949
|
+
this.state = {
|
|
950
|
+
totalSize: 0,
|
|
951
|
+
entryCount: 0,
|
|
952
|
+
compressionRatio: 1,
|
|
953
|
+
lastGC: Date.now(),
|
|
954
|
+
accessPatterns: new Map(),
|
|
955
|
+
performanceMetrics: {
|
|
956
|
+
queryTimes: [],
|
|
957
|
+
avgQueryTime: 0,
|
|
958
|
+
cacheHitRate: 0,
|
|
959
|
+
memoryEfficiency: 0
|
|
960
|
+
}
|
|
961
|
+
};
|
|
962
|
+
this.gcTimer = null;
|
|
963
|
+
// Optimized cache with LRU eviction
|
|
964
|
+
this.cache = new OptimizedLRUCache(this.config.cacheSize, this.config.cacheMemoryMB);
|
|
965
|
+
// Memory pools for frequently created objects
|
|
966
|
+
this.pools = {
|
|
967
|
+
queryResults: new MemoryPool(()=>({
|
|
968
|
+
results: [],
|
|
969
|
+
metadata: {}
|
|
970
|
+
}), (obj)=>{
|
|
971
|
+
obj.results.length = 0;
|
|
972
|
+
Object.keys(obj.metadata).forEach((k)=>delete obj.metadata[k]);
|
|
973
|
+
}),
|
|
974
|
+
memoryEntries: new MemoryPool(()=>({
|
|
975
|
+
id: '',
|
|
976
|
+
key: '',
|
|
977
|
+
value: '',
|
|
978
|
+
metadata: {}
|
|
979
|
+
}), (obj)=>{
|
|
980
|
+
obj.id = obj.key = obj.value = '';
|
|
981
|
+
Object.keys(obj.metadata).forEach((k)=>delete obj.metadata[k]);
|
|
982
|
+
})
|
|
983
|
+
};
|
|
984
|
+
// Prepared statements for better performance
|
|
985
|
+
this.statements = new Map();
|
|
986
|
+
// Background worker for heavy operations
|
|
987
|
+
this.backgroundWorker = null;
|
|
988
|
+
this._initialize();
|
|
1206
989
|
}
|
|
1207
|
-
|
|
1208
|
-
return health;
|
|
1209
|
-
}
|
|
1210
990
|
}
|
|
1211
|
-
|
|
1212
991
|
/**
|
|
1213
992
|
* Memory optimization utilities
|
|
1214
|
-
*/
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
duration,
|
|
1228
|
-
analytics: memory.getAnalytics(),
|
|
1229
|
-
health: await memory.healthCheck(),
|
|
1230
|
-
};
|
|
1231
|
-
}
|
|
1232
|
-
|
|
1233
|
-
static calculateOptimalCacheSize(memoryStats, accessPatterns) {
|
|
1234
|
-
const avgEntrySize = memoryStats.totalSize / memoryStats.entryCount;
|
|
1235
|
-
const hotKeys = Array.from(accessPatterns.entries())
|
|
1236
|
-
.sort((a, b) => b[1] - a[1])
|
|
1237
|
-
.slice(0, Math.min(1000, memoryStats.entryCount * 0.2));
|
|
1238
|
-
|
|
1239
|
-
const optimalCacheEntries = hotKeys.length * 1.2; // 20% buffer
|
|
1240
|
-
const optimalCacheMemoryMB = (optimalCacheEntries * avgEntrySize) / (1024 * 1024);
|
|
1241
|
-
|
|
1242
|
-
return {
|
|
1243
|
-
entries: Math.ceil(optimalCacheEntries),
|
|
1244
|
-
memoryMB: Math.ceil(optimalCacheMemoryMB),
|
|
1245
|
-
efficiency: (hotKeys.length / memoryStats.entryCount) * 100,
|
|
1246
|
-
};
|
|
1247
|
-
}
|
|
1248
|
-
|
|
1249
|
-
static generateOptimizationReport(analytics) {
|
|
1250
|
-
const report = {
|
|
1251
|
-
timestamp: new Date().toISOString(),
|
|
1252
|
-
summary: {},
|
|
1253
|
-
recommendations: [],
|
|
1254
|
-
metrics: analytics,
|
|
1255
|
-
};
|
|
1256
|
-
|
|
1257
|
-
// Performance summary
|
|
1258
|
-
report.summary.avgQueryTime = analytics.performance.avgQueryTime;
|
|
1259
|
-
report.summary.cacheHitRate = analytics.cache.hitRate || 0;
|
|
1260
|
-
report.summary.memoryEfficiency = analytics.cache.memoryUsage / (1024 * 1024);
|
|
1261
|
-
|
|
1262
|
-
// Generate recommendations
|
|
1263
|
-
if ((analytics.cache.hitRate || 0) < 70) {
|
|
1264
|
-
report.recommendations.push({
|
|
1265
|
-
type: 'cache',
|
|
1266
|
-
priority: 'high',
|
|
1267
|
-
description: 'Increase cache size to improve hit rate',
|
|
1268
|
-
impact: 'Reduce database queries by up to 30%',
|
|
1269
|
-
});
|
|
993
|
+
*/ export class MemoryOptimizer {
|
|
994
|
+
static async optimizeCollectiveMemory(memory) {
|
|
995
|
+
const startTime = performance.now();
|
|
996
|
+
// Run comprehensive optimization
|
|
997
|
+
await memory._optimizeDatabase();
|
|
998
|
+
memory._optimizeCache();
|
|
999
|
+
memory._garbageCollect();
|
|
1000
|
+
const duration = performance.now() - startTime;
|
|
1001
|
+
return {
|
|
1002
|
+
duration,
|
|
1003
|
+
analytics: memory.getAnalytics(),
|
|
1004
|
+
health: await memory.healthCheck()
|
|
1005
|
+
};
|
|
1270
1006
|
}
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1007
|
+
static calculateOptimalCacheSize(memoryStats, accessPatterns) {
|
|
1008
|
+
const avgEntrySize = memoryStats.totalSize / memoryStats.entryCount;
|
|
1009
|
+
const hotKeys = Array.from(accessPatterns.entries()).sort((a, b)=>b[1] - a[1]).slice(0, Math.min(1000, memoryStats.entryCount * 0.2));
|
|
1010
|
+
const optimalCacheEntries = hotKeys.length * 1.2; // 20% buffer
|
|
1011
|
+
const optimalCacheMemoryMB = optimalCacheEntries * avgEntrySize / (1024 * 1024);
|
|
1012
|
+
return {
|
|
1013
|
+
entries: Math.ceil(optimalCacheEntries),
|
|
1014
|
+
memoryMB: Math.ceil(optimalCacheMemoryMB),
|
|
1015
|
+
efficiency: hotKeys.length / memoryStats.entryCount * 100
|
|
1016
|
+
};
|
|
1279
1017
|
}
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1018
|
+
static generateOptimizationReport(analytics) {
|
|
1019
|
+
const report = {
|
|
1020
|
+
timestamp: new Date().toISOString(),
|
|
1021
|
+
summary: {},
|
|
1022
|
+
recommendations: [],
|
|
1023
|
+
metrics: analytics
|
|
1024
|
+
};
|
|
1025
|
+
// Performance summary
|
|
1026
|
+
report.summary.avgQueryTime = analytics.performance.avgQueryTime;
|
|
1027
|
+
report.summary.cacheHitRate = analytics.cache.hitRate || 0;
|
|
1028
|
+
report.summary.memoryEfficiency = analytics.cache.memoryUsage / (1024 * 1024);
|
|
1029
|
+
// Generate recommendations
|
|
1030
|
+
if ((analytics.cache.hitRate || 0) < 70) {
|
|
1031
|
+
report.recommendations.push({
|
|
1032
|
+
type: 'cache',
|
|
1033
|
+
priority: 'high',
|
|
1034
|
+
description: 'Increase cache size to improve hit rate',
|
|
1035
|
+
impact: 'Reduce database queries by up to 30%'
|
|
1036
|
+
});
|
|
1037
|
+
}
|
|
1038
|
+
if (analytics.performance.avgQueryTime > 50) {
|
|
1039
|
+
report.recommendations.push({
|
|
1040
|
+
type: 'database',
|
|
1041
|
+
priority: 'medium',
|
|
1042
|
+
description: 'Optimize database indexes and run ANALYZE',
|
|
1043
|
+
impact: 'Improve query performance by 20-40%'
|
|
1044
|
+
});
|
|
1045
|
+
}
|
|
1046
|
+
if (analytics.pools?.queryResults?.reuseRate < 50) {
|
|
1047
|
+
report.recommendations.push({
|
|
1048
|
+
type: 'pooling',
|
|
1049
|
+
priority: 'low',
|
|
1050
|
+
description: 'Increase object pool sizes for better reuse',
|
|
1051
|
+
impact: 'Reduce garbage collection pressure'
|
|
1052
|
+
});
|
|
1053
|
+
}
|
|
1054
|
+
return report;
|
|
1288
1055
|
}
|
|
1289
|
-
|
|
1290
|
-
return report;
|
|
1291
|
-
}
|
|
1292
1056
|
}
|