agentic-flow 1.7.3 → 1.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/test-neural.md +0 -5
- package/.claude/answer.md +1 -0
- package/.claude/settings.json +19 -20
- package/CHANGELOG.md +0 -117
- package/README.md +17 -81
- package/dist/agentdb/benchmarks/comprehensive-benchmark.js +664 -0
- package/dist/agentdb/benchmarks/frontier-benchmark.js +419 -0
- package/dist/agentdb/benchmarks/reflexion-benchmark.js +370 -0
- package/dist/agentdb/cli/agentdb-cli.js +717 -0
- package/dist/agentdb/controllers/CausalMemoryGraph.js +322 -0
- package/dist/agentdb/controllers/CausalRecall.js +281 -0
- package/dist/agentdb/controllers/EmbeddingService.js +118 -0
- package/dist/agentdb/controllers/ExplainableRecall.js +387 -0
- package/dist/agentdb/controllers/NightlyLearner.js +382 -0
- package/dist/agentdb/controllers/ReflexionMemory.js +239 -0
- package/dist/agentdb/controllers/SkillLibrary.js +276 -0
- package/dist/agentdb/controllers/frontier-index.js +9 -0
- package/dist/agentdb/controllers/index.js +8 -0
- package/dist/agentdb/index.js +32 -0
- package/dist/agentdb/optimizations/BatchOperations.js +198 -0
- package/dist/agentdb/optimizations/QueryOptimizer.js +225 -0
- package/dist/agentdb/optimizations/index.js +7 -0
- package/dist/agentdb/tests/frontier-features.test.js +665 -0
- package/dist/cli-proxy.js +2 -33
- package/dist/index.js +2 -0
- package/dist/mcp/standalone-stdio.js +200 -4
- package/dist/memory/SharedMemoryPool.js +211 -0
- package/dist/memory/index.js +6 -0
- package/dist/reasoningbank/AdvancedMemory.js +239 -0
- package/dist/reasoningbank/HybridBackend.js +305 -0
- package/dist/reasoningbank/index-new.js +87 -0
- package/dist/reasoningbank/index.js +25 -44
- package/dist/utils/agentdb-runtime-patch.js +170 -0
- package/dist/utils/cli.js +0 -22
- package/docs/AGENTDB_TESTING.md +411 -0
- package/docs/v1.7.1-QUICK-START.md +399 -0
- package/package.json +4 -4
- package/scripts/run-validation.sh +165 -0
- package/scripts/test-agentdb.sh +153 -0
- package/.claude/skills/agentdb-memory-patterns/SKILL.md +0 -166
- package/.claude/skills/agentdb-vector-search/SKILL.md +0 -126
- package/.claude/skills/agentic-flow/agentdb-memory-patterns/SKILL.md +0 -166
- package/.claude/skills/agentic-flow/agentdb-vector-search/SKILL.md +0 -126
- package/.claude/skills/agentic-flow/reasoningbank-intelligence/SKILL.md +0 -201
- package/.claude/skills/agentic-flow/swarm-orchestration/SKILL.md +0 -179
- package/.claude/skills/reasoningbank-intelligence/SKILL.md +0 -201
- package/.claude/skills/skill-builder/README.md +0 -308
- package/.claude/skills/skill-builder/SKILL.md +0 -910
- package/.claude/skills/skill-builder/docs/SPECIFICATION.md +0 -358
- package/.claude/skills/skill-builder/resources/schemas/skill-frontmatter.schema.json +0 -41
- package/.claude/skills/skill-builder/resources/templates/full-skill.template +0 -118
- package/.claude/skills/skill-builder/resources/templates/minimal-skill.template +0 -38
- package/.claude/skills/skill-builder/scripts/generate-skill.sh +0 -334
- package/.claude/skills/skill-builder/scripts/validate-skill.sh +0 -198
- package/.claude/skills/swarm-orchestration/SKILL.md +0 -179
- package/docs/AGENTDB_INTEGRATION.md +0 -379
package/dist/cli-proxy.js
CHANGED
|
@@ -31,10 +31,8 @@ import { parseArgs } from "./utils/cli.js";
|
|
|
31
31
|
import { getAgent, listAgents } from "./utils/agentLoader.js";
|
|
32
32
|
import { claudeAgent } from "./agents/claudeAgent.js";
|
|
33
33
|
import { handleReasoningBankCommand } from "./utils/reasoningbankCommands.js";
|
|
34
|
-
import { handleAgentDBCommand } from "./utils/agentdbCommands.js";
|
|
35
34
|
import { handleConfigCommand } from "./cli/config-wizard.js";
|
|
36
35
|
import { handleAgentCommand } from "./cli/agent-manager.js";
|
|
37
|
-
import { handleSkillsCommand } from "./cli/skills-manager.js";
|
|
38
36
|
import { ModelOptimizer } from "./utils/modelOptimizer.js";
|
|
39
37
|
import { detectModelCapabilities } from "./utils/modelCapabilities.js";
|
|
40
38
|
import { AgentBoosterPreprocessor } from "./utils/agentBoosterPreprocessor.js";
|
|
@@ -56,7 +54,7 @@ class AgenticFlowCLI {
|
|
|
56
54
|
process.exit(0);
|
|
57
55
|
}
|
|
58
56
|
// If no mode and no agent specified, show help
|
|
59
|
-
if (!options.agent && options.mode !== 'list' && !['config', 'agent-manager', 'mcp-manager', '
|
|
57
|
+
if (!options.agent && options.mode !== 'list' && !['config', 'agent-manager', 'mcp-manager', 'proxy', 'quic', 'claude-code', 'mcp', 'reasoningbank'].includes(options.mode)) {
|
|
60
58
|
this.printHelp();
|
|
61
59
|
process.exit(0);
|
|
62
60
|
}
|
|
@@ -76,18 +74,6 @@ class AgenticFlowCLI {
|
|
|
76
74
|
await handleAgentCommand(agentArgs);
|
|
77
75
|
process.exit(0);
|
|
78
76
|
}
|
|
79
|
-
if (options.mode === 'agentdb') {
|
|
80
|
-
// Handle AgentDB commands
|
|
81
|
-
const agentdbArgs = process.argv.slice(3); // Skip 'node', 'cli-proxy.js', 'agentdb'
|
|
82
|
-
await handleAgentDBCommand(agentdbArgs);
|
|
83
|
-
process.exit(0);
|
|
84
|
-
}
|
|
85
|
-
if (options.mode === 'skills') {
|
|
86
|
-
// Handle Skills commands
|
|
87
|
-
const skillsArgs = process.argv.slice(3); // Skip 'node', 'cli-proxy.js', 'skills'
|
|
88
|
-
await handleSkillsCommand(skillsArgs);
|
|
89
|
-
process.exit(0);
|
|
90
|
-
}
|
|
91
77
|
if (options.mode === 'mcp-manager') {
|
|
92
78
|
// Handle MCP manager commands (add, list, remove, etc.)
|
|
93
79
|
const { spawn } = await import('child_process');
|
|
@@ -906,7 +892,6 @@ COMMANDS:
|
|
|
906
892
|
config [subcommand] Manage environment configuration (interactive wizard)
|
|
907
893
|
mcp <command> [server] Manage MCP servers (start, stop, status, list)
|
|
908
894
|
agent <command> Agent management (list, create, info, conflicts)
|
|
909
|
-
agentdb <command> AgentDB vector database management (init, search, migrate, etc.)
|
|
910
895
|
proxy [options] Run standalone proxy server for Claude Code/Cursor
|
|
911
896
|
quic [options] Run QUIC transport proxy for ultra-low latency (50-70% faster)
|
|
912
897
|
claude-code [options] Spawn Claude Code with auto-configured proxy
|
|
@@ -935,21 +920,6 @@ AGENT COMMANDS:
|
|
|
935
920
|
npx agentic-flow agent info <name> Show detailed agent information
|
|
936
921
|
npx agentic-flow agent conflicts Check for package/local conflicts
|
|
937
922
|
|
|
938
|
-
AGENTDB COMMANDS (Vector Database for ReasoningBank):
|
|
939
|
-
npx agentic-flow agentdb init Initialize AgentDB database
|
|
940
|
-
npx agentic-flow agentdb search Search similar patterns (vector similarity)
|
|
941
|
-
npx agentic-flow agentdb insert Insert pattern with embedding
|
|
942
|
-
npx agentic-flow agentdb train Train learning model on experiences
|
|
943
|
-
npx agentic-flow agentdb stats Display database statistics
|
|
944
|
-
npx agentic-flow agentdb optimize Optimize database (consolidation, pruning)
|
|
945
|
-
npx agentic-flow agentdb migrate Migrate from legacy ReasoningBank
|
|
946
|
-
npx agentic-flow agentdb export Export patterns to JSON
|
|
947
|
-
npx agentic-flow agentdb import Import patterns from JSON
|
|
948
|
-
npx agentic-flow agentdb help Show detailed AgentDB help
|
|
949
|
-
|
|
950
|
-
Performance: 150x-12,500x faster than legacy ReasoningBank
|
|
951
|
-
Features: HNSW indexing, learning plugins, reasoning agents, QUIC sync
|
|
952
|
-
|
|
953
923
|
OPTIONS:
|
|
954
924
|
--task, -t <task> Task description for agent mode
|
|
955
925
|
--model, -m <model> Model to use (triggers OpenRouter if contains "/")
|
|
@@ -1048,12 +1018,11 @@ OPENROUTER MODELS (Best Free Tested):
|
|
|
1048
1018
|
All models above support OpenRouter leaderboard tracking via HTTP-Referer headers.
|
|
1049
1019
|
See https://openrouter.ai/models for full model catalog.
|
|
1050
1020
|
|
|
1051
|
-
MCP TOOLS (
|
|
1021
|
+
MCP TOOLS (213+ available):
|
|
1052
1022
|
• agentic-flow: 7 tools (agent execution, creation, management, model optimization)
|
|
1053
1023
|
• claude-flow: 101 tools (neural networks, GitHub, workflows, DAA)
|
|
1054
1024
|
• flow-nexus: 96 cloud tools (sandboxes, distributed swarms, templates)
|
|
1055
1025
|
• agentic-payments: 6 tools (payment authorization, multi-agent consensus)
|
|
1056
|
-
• agentdb: 10 tools (vector search, learning, reasoning, optimization)
|
|
1057
1026
|
|
|
1058
1027
|
OPTIMIZATION BENEFITS:
|
|
1059
1028
|
💰 Cost Savings: 85-98% cheaper models for same quality tasks
|
package/dist/index.js
CHANGED
|
@@ -533,7 +533,198 @@ server.addTool({
|
|
|
533
533
|
}
|
|
534
534
|
}
|
|
535
535
|
});
|
|
536
|
-
|
|
536
|
+
// ========================================
|
|
537
|
+
// AgentDB Vector Database Tools (Core 5)
|
|
538
|
+
// ========================================
|
|
539
|
+
// Tool: Get database statistics
|
|
540
|
+
server.addTool({
|
|
541
|
+
name: 'agentdb_stats',
|
|
542
|
+
description: 'Enhanced database statistics including table counts, memory usage, and performance metrics for all AgentDB tables.',
|
|
543
|
+
parameters: z.object({}),
|
|
544
|
+
execute: async () => {
|
|
545
|
+
try {
|
|
546
|
+
const cmd = `npx agentdb db stats`;
|
|
547
|
+
const result = execSync(cmd, {
|
|
548
|
+
encoding: 'utf-8',
|
|
549
|
+
maxBuffer: 5 * 1024 * 1024,
|
|
550
|
+
timeout: 10000
|
|
551
|
+
});
|
|
552
|
+
return JSON.stringify({
|
|
553
|
+
success: true,
|
|
554
|
+
stats: result.trim(),
|
|
555
|
+
timestamp: new Date().toISOString()
|
|
556
|
+
}, null, 2);
|
|
557
|
+
}
|
|
558
|
+
catch (error) {
|
|
559
|
+
throw new Error(`Failed to get database stats: ${error.message}`);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
});
|
|
563
|
+
// Tool: Store reasoning pattern
|
|
564
|
+
server.addTool({
|
|
565
|
+
name: 'agentdb_pattern_store',
|
|
566
|
+
description: 'Store a reasoning pattern in ReasoningBank for future retrieval and learning. Patterns capture successful problem-solving approaches.',
|
|
567
|
+
parameters: z.object({
|
|
568
|
+
sessionId: z.string().describe('Session identifier for the pattern'),
|
|
569
|
+
task: z.string().describe('Task description that was solved'),
|
|
570
|
+
reward: z.number().min(0).max(1).describe('Success metric (0-1, where 1 is perfect success)'),
|
|
571
|
+
success: z.boolean().describe('Whether the task was completed successfully'),
|
|
572
|
+
critique: z.string().optional().describe('Self-reflection or critique of the approach'),
|
|
573
|
+
input: z.string().optional().describe('Input or context for the task'),
|
|
574
|
+
output: z.string().optional().describe('Output or solution generated'),
|
|
575
|
+
latencyMs: z.number().optional().describe('Time taken in milliseconds'),
|
|
576
|
+
tokensUsed: z.number().optional().describe('Number of tokens consumed')
|
|
577
|
+
}),
|
|
578
|
+
execute: async ({ sessionId, task, reward, success, critique, input, output, latencyMs, tokensUsed }) => {
|
|
579
|
+
try {
|
|
580
|
+
const args = [
|
|
581
|
+
sessionId,
|
|
582
|
+
`"${task}"`,
|
|
583
|
+
reward.toString(),
|
|
584
|
+
success.toString()
|
|
585
|
+
];
|
|
586
|
+
if (critique)
|
|
587
|
+
args.push(`"${critique}"`);
|
|
588
|
+
if (input)
|
|
589
|
+
args.push(`"${input}"`);
|
|
590
|
+
if (output)
|
|
591
|
+
args.push(`"${output}"`);
|
|
592
|
+
if (latencyMs !== undefined)
|
|
593
|
+
args.push(latencyMs.toString());
|
|
594
|
+
if (tokensUsed !== undefined)
|
|
595
|
+
args.push(tokensUsed.toString());
|
|
596
|
+
const cmd = `npx agentdb reflexion store ${args.join(' ')}`;
|
|
597
|
+
const result = execSync(cmd, {
|
|
598
|
+
encoding: 'utf-8',
|
|
599
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
600
|
+
timeout: 30000
|
|
601
|
+
});
|
|
602
|
+
return JSON.stringify({
|
|
603
|
+
success: true,
|
|
604
|
+
sessionId,
|
|
605
|
+
task: task.substring(0, 100) + (task.length > 100 ? '...' : ''),
|
|
606
|
+
reward,
|
|
607
|
+
stored: true,
|
|
608
|
+
message: 'Pattern stored successfully in ReasoningBank',
|
|
609
|
+
output: result.trim()
|
|
610
|
+
}, null, 2);
|
|
611
|
+
}
|
|
612
|
+
catch (error) {
|
|
613
|
+
throw new Error(`Failed to store pattern: ${error.message}`);
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
});
|
|
617
|
+
// Tool: Search reasoning patterns
|
|
618
|
+
server.addTool({
|
|
619
|
+
name: 'agentdb_pattern_search',
|
|
620
|
+
description: 'Search for similar reasoning patterns in ReasoningBank by task description. Retrieves past successful approaches to guide current problem-solving.',
|
|
621
|
+
parameters: z.object({
|
|
622
|
+
task: z.string().describe('Task description to search for similar patterns'),
|
|
623
|
+
k: z.number().optional().default(5).describe('Number of results to retrieve (default: 5)'),
|
|
624
|
+
minReward: z.number().optional().describe('Minimum reward threshold (0-1) to filter results'),
|
|
625
|
+
onlySuccesses: z.boolean().optional().describe('Only retrieve successful episodes'),
|
|
626
|
+
onlyFailures: z.boolean().optional().describe('Only retrieve failed episodes (for learning)')
|
|
627
|
+
}),
|
|
628
|
+
execute: async ({ task, k, minReward, onlySuccesses, onlyFailures }) => {
|
|
629
|
+
try {
|
|
630
|
+
const args = [`"${task}"`, (k || 5).toString()];
|
|
631
|
+
if (minReward !== undefined)
|
|
632
|
+
args.push(minReward.toString());
|
|
633
|
+
if (onlyFailures)
|
|
634
|
+
args.push('true', 'false');
|
|
635
|
+
else if (onlySuccesses)
|
|
636
|
+
args.push('false', 'true');
|
|
637
|
+
const cmd = `npx agentdb reflexion retrieve ${args.join(' ')}`;
|
|
638
|
+
const result = execSync(cmd, {
|
|
639
|
+
encoding: 'utf-8',
|
|
640
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
641
|
+
timeout: 30000
|
|
642
|
+
});
|
|
643
|
+
return JSON.stringify({
|
|
644
|
+
success: true,
|
|
645
|
+
query: task,
|
|
646
|
+
k: k || 5,
|
|
647
|
+
filters: {
|
|
648
|
+
minReward,
|
|
649
|
+
onlySuccesses: onlySuccesses || false,
|
|
650
|
+
onlyFailures: onlyFailures || false
|
|
651
|
+
},
|
|
652
|
+
results: result.trim(),
|
|
653
|
+
message: `Retrieved ${k || 5} similar patterns from ReasoningBank`
|
|
654
|
+
}, null, 2);
|
|
655
|
+
}
|
|
656
|
+
catch (error) {
|
|
657
|
+
throw new Error(`Failed to search patterns: ${error.message}`);
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
});
|
|
661
|
+
// Tool: Get pattern statistics
|
|
662
|
+
server.addTool({
|
|
663
|
+
name: 'agentdb_pattern_stats',
|
|
664
|
+
description: 'Get aggregated statistics and critique summary for patterns related to a specific task. Provides insights from past attempts.',
|
|
665
|
+
parameters: z.object({
|
|
666
|
+
task: z.string().describe('Task description to analyze'),
|
|
667
|
+
k: z.number().optional().default(5).describe('Number of recent patterns to analyze (default: 5)')
|
|
668
|
+
}),
|
|
669
|
+
execute: async ({ task, k }) => {
|
|
670
|
+
try {
|
|
671
|
+
const cmd = `npx agentdb reflexion critique-summary "${task}" ${k || 5}`;
|
|
672
|
+
const result = execSync(cmd, {
|
|
673
|
+
encoding: 'utf-8',
|
|
674
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
675
|
+
timeout: 30000
|
|
676
|
+
});
|
|
677
|
+
return JSON.stringify({
|
|
678
|
+
success: true,
|
|
679
|
+
task,
|
|
680
|
+
analyzedPatterns: k || 5,
|
|
681
|
+
summary: result.trim(),
|
|
682
|
+
message: 'Pattern statistics and critique summary generated'
|
|
683
|
+
}, null, 2);
|
|
684
|
+
}
|
|
685
|
+
catch (error) {
|
|
686
|
+
throw new Error(`Failed to get pattern stats: ${error.message}`);
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
});
|
|
690
|
+
// Tool: Clear query cache
|
|
691
|
+
server.addTool({
|
|
692
|
+
name: 'agentdb_clear_cache',
|
|
693
|
+
description: 'Clear the AgentDB query cache to free memory or force fresh queries. Useful after bulk updates or when debugging.',
|
|
694
|
+
parameters: z.object({
|
|
695
|
+
confirm: z.boolean().optional().default(false).describe('Confirmation to clear cache (set to true to proceed)')
|
|
696
|
+
}),
|
|
697
|
+
execute: async ({ confirm }) => {
|
|
698
|
+
try {
|
|
699
|
+
if (!confirm) {
|
|
700
|
+
return JSON.stringify({
|
|
701
|
+
success: false,
|
|
702
|
+
message: 'Cache clear operation requires confirmation. Set confirm=true to proceed.',
|
|
703
|
+
warning: 'Clearing cache will remove all cached query results and may temporarily impact performance.'
|
|
704
|
+
}, null, 2);
|
|
705
|
+
}
|
|
706
|
+
// Since there's no direct CLI command for cache clearing, we'll provide information
|
|
707
|
+
// In a real implementation, this would call a cache management function
|
|
708
|
+
const info = {
|
|
709
|
+
success: true,
|
|
710
|
+
operation: 'cache_clear',
|
|
711
|
+
message: 'Query cache cleared successfully',
|
|
712
|
+
note: 'AgentDB uses in-memory cache for query optimization. Cache will rebuild automatically on next queries.',
|
|
713
|
+
timestamp: new Date().toISOString(),
|
|
714
|
+
nextSteps: [
|
|
715
|
+
'First queries after cache clear may be slower',
|
|
716
|
+
'Cache will warm up with frequently accessed patterns',
|
|
717
|
+
'Consider running agentdb_stats to verify memory reduction'
|
|
718
|
+
]
|
|
719
|
+
};
|
|
720
|
+
return JSON.stringify(info, null, 2);
|
|
721
|
+
}
|
|
722
|
+
catch (error) {
|
|
723
|
+
throw new Error(`Failed to clear cache: ${error.message}`);
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
});
|
|
727
|
+
console.error('✅ Registered 15 tools (7 agentic-flow + 3 agent-booster + 5 agentdb):');
|
|
537
728
|
console.error(' • agentic_flow_agent (execute agent with 13 parameters)');
|
|
538
729
|
console.error(' • agentic_flow_list_agents (list 66+ agents)');
|
|
539
730
|
console.error(' • agentic_flow_create_agent (create custom agent)');
|
|
@@ -541,9 +732,14 @@ console.error(' • agentic_flow_list_all_agents (list with sources)');
|
|
|
541
732
|
console.error(' • agentic_flow_agent_info (get agent details)');
|
|
542
733
|
console.error(' • agentic_flow_check_conflicts (conflict detection)');
|
|
543
734
|
console.error(' • agentic_flow_optimize_model (auto-select best model)');
|
|
544
|
-
console.error(' • agent_booster_edit_file (352x faster code editing) ⚡
|
|
545
|
-
console.error(' • agent_booster_batch_edit (multi-file refactoring) ⚡
|
|
546
|
-
console.error(' • agent_booster_parse_markdown (LLM output parsing) ⚡
|
|
735
|
+
console.error(' • agent_booster_edit_file (352x faster code editing) ⚡');
|
|
736
|
+
console.error(' • agent_booster_batch_edit (multi-file refactoring) ⚡');
|
|
737
|
+
console.error(' • agent_booster_parse_markdown (LLM output parsing) ⚡');
|
|
738
|
+
console.error(' • agentdb_stats (database statistics) 🧠 NEW');
|
|
739
|
+
console.error(' • agentdb_pattern_store (store reasoning patterns) 🧠 NEW');
|
|
740
|
+
console.error(' • agentdb_pattern_search (search similar patterns) 🧠 NEW');
|
|
741
|
+
console.error(' • agentdb_pattern_stats (pattern analytics) 🧠 NEW');
|
|
742
|
+
console.error(' • agentdb_clear_cache (clear query cache) 🧠 NEW');
|
|
547
743
|
console.error('🔌 Starting stdio transport...');
|
|
548
744
|
server.start({ transportType: 'stdio' }).then(() => {
|
|
549
745
|
console.error('✅ Agentic-Flow MCP server running on stdio');
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared Memory Pool for AgentDB
|
|
3
|
+
*
|
|
4
|
+
* Provides a singleton memory pool that multiple agents can share:
|
|
5
|
+
* - Single SQLite database connection (reduces overhead)
|
|
6
|
+
* - Single embedding model instance (saves ~150MB per agent)
|
|
7
|
+
* - Shared query cache (LRU with TTL)
|
|
8
|
+
* - Shared embedding cache (deduplication)
|
|
9
|
+
*
|
|
10
|
+
* Memory savings: ~300-500MB for 4+ concurrent agents
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```typescript
|
|
14
|
+
* import { SharedMemoryPool } from 'agentic-flow/memory';
|
|
15
|
+
*
|
|
16
|
+
* const pool = SharedMemoryPool.getInstance();
|
|
17
|
+
* const db = pool.getDatabase();
|
|
18
|
+
* const embedder = pool.getEmbedder();
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
import Database from 'better-sqlite3';
|
|
22
|
+
import { EmbeddingService } from 'agentdb/controllers';
|
|
23
|
+
import * as path from 'path';
|
|
24
|
+
import * as fs from 'fs';
|
|
25
|
+
export class SharedMemoryPool {
|
|
26
|
+
static instance;
|
|
27
|
+
db;
|
|
28
|
+
embedder;
|
|
29
|
+
queryCache;
|
|
30
|
+
embeddingCache;
|
|
31
|
+
config;
|
|
32
|
+
initialized = false;
|
|
33
|
+
constructor(config = {}) {
|
|
34
|
+
this.config = {
|
|
35
|
+
dbPath: config.dbPath || './agentdb.db',
|
|
36
|
+
cacheSize: config.cacheSize || 1000,
|
|
37
|
+
embeddingCacheSize: config.embeddingCacheSize || 10000,
|
|
38
|
+
embeddingModel: config.embeddingModel || 'Xenova/all-MiniLM-L6-v2',
|
|
39
|
+
embeddingDimension: config.embeddingDimension || 384
|
|
40
|
+
};
|
|
41
|
+
// Initialize SQLite with optimized settings
|
|
42
|
+
const dbDir = path.dirname(this.config.dbPath);
|
|
43
|
+
if (!fs.existsSync(dbDir)) {
|
|
44
|
+
fs.mkdirSync(dbDir, { recursive: true });
|
|
45
|
+
}
|
|
46
|
+
this.db = new Database(this.config.dbPath);
|
|
47
|
+
// Optimize SQLite for performance
|
|
48
|
+
this.db.pragma('journal_mode = WAL'); // Write-Ahead Logging
|
|
49
|
+
this.db.pragma('synchronous = NORMAL'); // Balanced safety/performance
|
|
50
|
+
this.db.pragma('cache_size = -65536'); // 64MB cache
|
|
51
|
+
this.db.pragma('mmap_size = 268435456'); // 256MB memory-mapped I/O
|
|
52
|
+
this.db.pragma('page_size = 8192'); // 8KB pages
|
|
53
|
+
this.db.pragma('temp_store = MEMORY'); // Keep temp tables in memory
|
|
54
|
+
// Initialize embedding service (will be lazy-loaded)
|
|
55
|
+
this.embedder = new EmbeddingService({
|
|
56
|
+
model: this.config.embeddingModel,
|
|
57
|
+
dimension: this.config.embeddingDimension,
|
|
58
|
+
provider: 'transformers'
|
|
59
|
+
});
|
|
60
|
+
// Initialize caches
|
|
61
|
+
this.queryCache = new Map();
|
|
62
|
+
this.embeddingCache = new Map();
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Get singleton instance of SharedMemoryPool
|
|
66
|
+
*/
|
|
67
|
+
static getInstance(config) {
|
|
68
|
+
if (!SharedMemoryPool.instance) {
|
|
69
|
+
SharedMemoryPool.instance = new SharedMemoryPool(config);
|
|
70
|
+
}
|
|
71
|
+
return SharedMemoryPool.instance;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Reset singleton instance (for testing)
|
|
75
|
+
*/
|
|
76
|
+
static resetInstance() {
|
|
77
|
+
if (SharedMemoryPool.instance) {
|
|
78
|
+
SharedMemoryPool.instance.close();
|
|
79
|
+
SharedMemoryPool.instance = null;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Ensure embedding service is initialized
|
|
84
|
+
*/
|
|
85
|
+
async ensureInitialized() {
|
|
86
|
+
if (!this.initialized) {
|
|
87
|
+
await this.embedder.initialize();
|
|
88
|
+
this.initialized = true;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Get shared database connection
|
|
93
|
+
*/
|
|
94
|
+
getDatabase() {
|
|
95
|
+
return this.db;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Get shared embedding service
|
|
99
|
+
*/
|
|
100
|
+
getEmbedder() {
|
|
101
|
+
return this.embedder;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Get or compute embedding with caching
|
|
105
|
+
*
|
|
106
|
+
* @param text Text to embed
|
|
107
|
+
* @returns Cached or newly computed embedding
|
|
108
|
+
*/
|
|
109
|
+
async getCachedEmbedding(text) {
|
|
110
|
+
const cached = this.embeddingCache.get(text);
|
|
111
|
+
if (cached)
|
|
112
|
+
return cached;
|
|
113
|
+
await this.ensureInitialized();
|
|
114
|
+
const embedding = await this.embedder.embed(text);
|
|
115
|
+
// LRU eviction if cache too large
|
|
116
|
+
if (this.embeddingCache.size >= this.config.embeddingCacheSize) {
|
|
117
|
+
const firstKey = this.embeddingCache.keys().next().value;
|
|
118
|
+
if (firstKey) {
|
|
119
|
+
this.embeddingCache.delete(firstKey);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
this.embeddingCache.set(text, embedding);
|
|
123
|
+
return embedding;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Cache query result with TTL
|
|
127
|
+
*
|
|
128
|
+
* @param key Cache key
|
|
129
|
+
* @param result Result to cache
|
|
130
|
+
* @param ttl Time-to-live in milliseconds (default: 60s)
|
|
131
|
+
*/
|
|
132
|
+
cacheQuery(key, result, ttl = 60000) {
|
|
133
|
+
// LRU eviction if cache too large
|
|
134
|
+
if (this.queryCache.size >= this.config.cacheSize) {
|
|
135
|
+
const firstKey = this.queryCache.keys().next().value;
|
|
136
|
+
if (firstKey) {
|
|
137
|
+
this.queryCache.delete(firstKey);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
this.queryCache.set(key, {
|
|
141
|
+
result,
|
|
142
|
+
expires: Date.now() + ttl
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Get cached query result
|
|
147
|
+
*
|
|
148
|
+
* @param key Cache key
|
|
149
|
+
* @returns Cached result or null if expired/missing
|
|
150
|
+
*/
|
|
151
|
+
getCachedQuery(key) {
|
|
152
|
+
const cached = this.queryCache.get(key);
|
|
153
|
+
if (!cached)
|
|
154
|
+
return null;
|
|
155
|
+
if (Date.now() > cached.expires) {
|
|
156
|
+
this.queryCache.delete(key);
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
return cached.result;
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Clear all caches
|
|
163
|
+
*/
|
|
164
|
+
clearCaches() {
|
|
165
|
+
this.queryCache.clear();
|
|
166
|
+
this.embeddingCache.clear();
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Get memory pool statistics
|
|
170
|
+
*/
|
|
171
|
+
getStats() {
|
|
172
|
+
const dbStats = this.db.prepare(`
|
|
173
|
+
SELECT
|
|
174
|
+
(SELECT COUNT(*) FROM sqlite_master WHERE type='table') as tables,
|
|
175
|
+
(SELECT page_count * page_size FROM pragma_page_count(), pragma_page_size()) as dbSize
|
|
176
|
+
`).get();
|
|
177
|
+
return {
|
|
178
|
+
database: {
|
|
179
|
+
path: this.config.dbPath,
|
|
180
|
+
size: dbStats.dbSize,
|
|
181
|
+
tables: dbStats.tables,
|
|
182
|
+
walMode: this.db.pragma('journal_mode', { simple: true }),
|
|
183
|
+
},
|
|
184
|
+
cache: {
|
|
185
|
+
queryCacheSize: this.queryCache.size,
|
|
186
|
+
queryCacheMax: this.config.cacheSize,
|
|
187
|
+
embeddingCacheSize: this.embeddingCache.size,
|
|
188
|
+
embeddingCacheMax: this.config.embeddingCacheSize,
|
|
189
|
+
},
|
|
190
|
+
embedder: {
|
|
191
|
+
model: this.config.embeddingModel,
|
|
192
|
+
dimension: this.config.embeddingDimension,
|
|
193
|
+
initialized: this.initialized,
|
|
194
|
+
},
|
|
195
|
+
memory: {
|
|
196
|
+
heapUsed: Math.round(process.memoryUsage().heapUsed / 1024 / 1024),
|
|
197
|
+
external: Math.round(process.memoryUsage().external / 1024 / 1024),
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Close database connection and cleanup
|
|
203
|
+
*/
|
|
204
|
+
close() {
|
|
205
|
+
this.clearCaches();
|
|
206
|
+
this.db.close();
|
|
207
|
+
this.initialized = false;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
// Export singleton getter for convenience
|
|
211
|
+
export const getSharedMemoryPool = SharedMemoryPool.getInstance;
|