opencode-fractal-memory 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +493 -0
- package/agent/memory-hints.md +98 -0
- package/agent/memory-researcher.md +56 -0
- package/commands/memory-auto-test.md +10 -0
- package/commands/memory-cache-status.md +13 -0
- package/commands/memory-check-context.md +4 -0
- package/commands/memory-compress.md +13 -0
- package/commands/memory-dashboard.md +23 -0
- package/commands/memory-delete.md +24 -0
- package/commands/memory-detect-topics.md +28 -0
- package/commands/memory-distill.md +35 -0
- package/commands/memory-drilldown-query.md +28 -0
- package/commands/memory-drilldown.md +11 -0
- package/commands/memory-extract-patterns.md +4 -0
- package/commands/memory-generate-embeddings.md +26 -0
- package/commands/memory-get.md +26 -0
- package/commands/memory-help.md +55 -0
- package/commands/memory-injection-feedback.md +26 -0
- package/commands/memory-injection-stats.md +11 -0
- package/commands/memory-list.md +4 -0
- package/commands/memory-llm-compress.md +34 -0
- package/commands/memory-mcp.md +20 -0
- package/commands/memory-prune.md +4 -0
- package/commands/memory-rate.md +48 -0
- package/commands/memory-reflect.md +37 -0
- package/commands/memory-replace.md +26 -0
- package/commands/memory-retrieve.md +34 -0
- package/commands/memory-search.md +28 -0
- package/commands/memory-session-stats.md +4 -0
- package/commands/memory-set.md +31 -0
- package/commands/memory-stats.md +11 -0
- package/commands/memory-summarize.md +29 -0
- package/commands/memory-tool-stats.md +4 -0
- package/commands/memory-total-tokens.md +10 -0
- package/commands/memory-verify.md +4 -0
- package/commands/memory-version.md +9 -0
- package/dist/cache.js +39 -0
- package/dist/config.js +120 -0
- package/dist/embeddings.js +125 -0
- package/dist/ensure-models.js +70 -0
- package/dist/file-summary.js +143 -0
- package/dist/frontmatter.js +28 -0
- package/dist/hnsw-index.js +138 -0
- package/dist/hooks/auto-discover.js +4 -0
- package/dist/hooks/auto-distill.js +120 -0
- package/dist/hooks/auto-retrieve/content.js +47 -0
- package/dist/hooks/auto-retrieve/detection.js +50 -0
- package/dist/hooks/auto-retrieve/formatting.js +19 -0
- package/dist/hooks/auto-retrieve/index.js +163 -0
- package/dist/hooks/auto-retrieve/scoring.js +56 -0
- package/dist/hooks/auto-retrieve.js +1 -0
- package/dist/hooks/index.js +4 -0
- package/dist/hooks/predictive-rating.js +87 -0
- package/dist/journal.js +279 -0
- package/dist/logging.js +147 -0
- package/dist/management/helpers.js +227 -0
- package/dist/management/router.js +48 -0
- package/dist/management/routes.js +197 -0
- package/dist/management-server.js +4 -0
- package/dist/management-standalone.js +31 -0
- package/dist/mcp/logging.js +57 -0
- package/dist/mcp/server.js +251 -0
- package/dist/mcp/transform.js +48 -0
- package/dist/mcp-server.js +18 -0
- package/dist/memory.js +2 -0
- package/dist/ollama.js +74 -0
- package/dist/plugin/hooks.js +168 -0
- package/dist/plugin/index.js +28 -0
- package/dist/plugin/init.js +109 -0
- package/dist/plugin/state.js +75 -0
- package/dist/plugin/tools.js +45 -0
- package/dist/plugin.js +2 -0
- package/dist/procedural/store.js +1 -0
- package/dist/procedural/types.js +1 -0
- package/dist/seed-nodes.js +804 -0
- package/dist/storage/compress-ops.js +129 -0
- package/dist/storage/compression/formatters.js +243 -0
- package/dist/storage/compression/index.js +107 -0
- package/dist/storage/compression/patterns.js +138 -0
- package/dist/storage/expiration.js +66 -0
- package/dist/storage/index.js +1 -0
- package/dist/storage/injection-events.js +82 -0
- package/dist/storage/lifecycle.js +65 -0
- package/dist/storage/maintenance.js +60 -0
- package/dist/storage/migrations/definitions.js +374 -0
- package/dist/storage/migrations/index.js +21 -0
- package/dist/storage/navigation.js +98 -0
- package/dist/storage/queries/base.js +44 -0
- package/dist/storage/queries/links.js +32 -0
- package/dist/storage/queries/nodes.js +189 -0
- package/dist/storage/queries/search-helpers.js +239 -0
- package/dist/storage/scoring.js +36 -0
- package/dist/storage/search.js +233 -0
- package/dist/storage/session-tracking.js +180 -0
- package/dist/storage/sqlite.js +329 -0
- package/dist/storage/tool-usage.js +56 -0
- package/dist/storage/types.js +1 -0
- package/dist/storage/utils.js +94 -0
- package/dist/tools/auto-test.js +24 -0
- package/dist/tools/cache-status.js +36 -0
- package/dist/tools/compress.js +186 -0
- package/dist/tools/core.js +307 -0
- package/dist/tools/dashboard.js +97 -0
- package/dist/tools/help.js +59 -0
- package/dist/tools/index.js +12 -0
- package/dist/tools/inject.js +91 -0
- package/dist/tools/injection-debug.js +48 -0
- package/dist/tools/journal.js +105 -0
- package/dist/tools/llm-compress.js +41 -0
- package/dist/tools/middle-term.js +68 -0
- package/dist/tools/playbook.js +64 -0
- package/dist/tools/reflect.js +291 -0
- package/dist/tools/search.js +188 -0
- package/dist/tools/session.js +189 -0
- package/dist/tools/shared.js +74 -0
- package/dist/tools/skill.js +37 -0
- package/dist/tools/stats.js +256 -0
- package/dist/tools/version.js +13 -0
- package/dist/tools.js +18 -0
- package/dist/utils/hybridScore.js +67 -0
- package/management/public/app.js +1529 -0
- package/management/public/index.html +486 -0
- package/management/public/three.min.js +6 -0
- package/package.json +65 -0
- package/scripts/download-models.ts +16 -0
- package/scripts/postinstall.cjs +30 -0
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { HNSW } from "hnsw";
|
|
2
|
+
const M = 16;
|
|
3
|
+
const EF_CONSTRUCTION = 200;
|
|
4
|
+
const EF_SEARCH = 64;
|
|
5
|
+
export class HNSWIndex {
|
|
6
|
+
globalIndex = null;
|
|
7
|
+
projectIndex = null;
|
|
8
|
+
globalLabelMap;
|
|
9
|
+
projectLabelMap;
|
|
10
|
+
globalIdCounter;
|
|
11
|
+
projectIdCounter;
|
|
12
|
+
dimension;
|
|
13
|
+
initialized = false;
|
|
14
|
+
constructor(dimension = 384) {
|
|
15
|
+
this.dimension = dimension;
|
|
16
|
+
this.globalLabelMap = new Map();
|
|
17
|
+
this.projectLabelMap = new Map();
|
|
18
|
+
this.globalIdCounter = 0;
|
|
19
|
+
this.projectIdCounter = 0;
|
|
20
|
+
}
|
|
21
|
+
ensureInitialized() {
|
|
22
|
+
if (!this.initialized) {
|
|
23
|
+
this.globalIndex = new HNSW(M, EF_CONSTRUCTION, this.dimension, "cosine", EF_SEARCH);
|
|
24
|
+
this.projectIndex = new HNSW(M, EF_CONSTRUCTION, this.dimension, "cosine", EF_SEARCH);
|
|
25
|
+
this.initialized = true;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
getIndex(scope) {
|
|
29
|
+
this.ensureInitialized();
|
|
30
|
+
return scope === "global" ? this.globalIndex : this.projectIndex;
|
|
31
|
+
}
|
|
32
|
+
getMaps(scope) {
|
|
33
|
+
return scope === "global" ? this.globalLabelMap : this.projectLabelMap;
|
|
34
|
+
}
|
|
35
|
+
getCounter(scope) {
|
|
36
|
+
return scope === "global"
|
|
37
|
+
? { value: this.globalIdCounter }
|
|
38
|
+
: { value: this.projectIdCounter };
|
|
39
|
+
}
|
|
40
|
+
setCounter(scope, value) {
|
|
41
|
+
if (scope === "global") {
|
|
42
|
+
this.globalIdCounter = value;
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
this.projectIdCounter = value;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
async addNode(scope, nodeId, embedding) {
|
|
49
|
+
if (embedding.length !== this.dimension) {
|
|
50
|
+
return -1;
|
|
51
|
+
}
|
|
52
|
+
this.ensureInitialized();
|
|
53
|
+
const index = this.getIndex(scope);
|
|
54
|
+
const labelMap = this.getMaps(scope);
|
|
55
|
+
const counter = this.getCounter(scope);
|
|
56
|
+
const hnswId = counter.value;
|
|
57
|
+
labelMap.set(hnswId, nodeId);
|
|
58
|
+
counter.value = hnswId + 1;
|
|
59
|
+
await index.addPoint(hnswId, embedding);
|
|
60
|
+
this.setCounter(scope, counter.value);
|
|
61
|
+
return hnswId;
|
|
62
|
+
}
|
|
63
|
+
async removeNode(scope, nodeId) {
|
|
64
|
+
const labelMap = this.getMaps(scope);
|
|
65
|
+
for (const [hnswId, storedId] of labelMap) {
|
|
66
|
+
if (storedId === nodeId) {
|
|
67
|
+
labelMap.delete(hnswId);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
async search(query, limit, scope, _levelFilter) {
|
|
73
|
+
if (!this.initialized || query.length !== this.dimension) {
|
|
74
|
+
return [];
|
|
75
|
+
}
|
|
76
|
+
let results;
|
|
77
|
+
if (scope === "global") {
|
|
78
|
+
results = this.globalIndex.searchKNN(query, limit);
|
|
79
|
+
}
|
|
80
|
+
else if (scope === "project") {
|
|
81
|
+
results = this.projectIndex.searchKNN(query, limit);
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
const globalResults = this.globalIndex.searchKNN(query, limit);
|
|
85
|
+
const projectResults = this.projectIndex.searchKNN(query, limit);
|
|
86
|
+
const combined = [...globalResults, ...projectResults];
|
|
87
|
+
combined.sort((a, b) => b.score - a.score);
|
|
88
|
+
results = combined.slice(0, limit);
|
|
89
|
+
}
|
|
90
|
+
return results.map(r => {
|
|
91
|
+
let nodeId;
|
|
92
|
+
if (scope === "global") {
|
|
93
|
+
nodeId = this.globalLabelMap.get(r.id) ?? "";
|
|
94
|
+
}
|
|
95
|
+
else if (scope === "project") {
|
|
96
|
+
nodeId = this.projectLabelMap.get(r.id) ?? "";
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
nodeId = this.globalLabelMap.get(r.id) ?? this.projectLabelMap.get(r.id) ?? "";
|
|
100
|
+
}
|
|
101
|
+
return { id: nodeId, score: r.score };
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
async rebuild(nodes) {
|
|
105
|
+
if (nodes.length > 0) {
|
|
106
|
+
this.dimension = nodes[0].embedding.length;
|
|
107
|
+
}
|
|
108
|
+
this.globalIndex = new HNSW(M, EF_CONSTRUCTION, this.dimension, "cosine", EF_SEARCH);
|
|
109
|
+
this.projectIndex = new HNSW(M, EF_CONSTRUCTION, this.dimension, "cosine", EF_SEARCH);
|
|
110
|
+
this.globalLabelMap.clear();
|
|
111
|
+
this.projectLabelMap.clear();
|
|
112
|
+
this.globalIdCounter = 0;
|
|
113
|
+
this.projectIdCounter = 0;
|
|
114
|
+
this.initialized = true;
|
|
115
|
+
for (const node of nodes) {
|
|
116
|
+
if (node.embedding.length === this.dimension) {
|
|
117
|
+
await this.addNode(node.scope, node.id, node.embedding);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
getStats() {
|
|
122
|
+
return {
|
|
123
|
+
globalNodes: this.globalLabelMap.size,
|
|
124
|
+
projectNodes: this.projectLabelMap.size,
|
|
125
|
+
dimension: this.dimension,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
let hnswInstance = null;
|
|
130
|
+
export function getHNSWIndex(dimension = 384) {
|
|
131
|
+
if (!hnswInstance) {
|
|
132
|
+
hnswInstance = new HNSWIndex(dimension);
|
|
133
|
+
}
|
|
134
|
+
return hnswInstance;
|
|
135
|
+
}
|
|
136
|
+
export function resetHNSWIndex() {
|
|
137
|
+
hnswInstance = null;
|
|
138
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { memLog } from "../logging";
|
|
2
|
+
export async function distillRules(store, config, sessionId, client) {
|
|
3
|
+
const lessons = await store.listNodes("global");
|
|
4
|
+
const lessonNodes = lessons.filter(n => n.label?.startsWith("lesson:") && !n.label.includes(":", 7));
|
|
5
|
+
if (lessonNodes.length < config.minLessons) {
|
|
6
|
+
return `Auto-distill: skipped, have ${lessonNodes.length} lessons, need ${config.minLessons}`;
|
|
7
|
+
}
|
|
8
|
+
const recentLessons = lessonNodes
|
|
9
|
+
.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime())
|
|
10
|
+
.slice(0, 10);
|
|
11
|
+
const fixes = [];
|
|
12
|
+
const allLessonNodes = lessons.filter(n => n.label?.startsWith("lesson:"));
|
|
13
|
+
for (const lesson of recentLessons) {
|
|
14
|
+
const fixNode = allLessonNodes.find(n => n.label === `${lesson.label}:fix`);
|
|
15
|
+
if (fixNode?.content) {
|
|
16
|
+
fixes.push(...fixNode.content.split("\n").filter(l => l.trim()));
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
const uniqueFixes = [...new Set(fixes)];
|
|
20
|
+
const distilledRules = [];
|
|
21
|
+
for (const fix of uniqueFixes) {
|
|
22
|
+
const match = fix.match(/^- (.+)$/);
|
|
23
|
+
const action = match?.[1];
|
|
24
|
+
if (action) {
|
|
25
|
+
if (action.includes("memory_drilldown")) {
|
|
26
|
+
distilledRules.push("- Avoid memory_drilldown with vague queries - use memory_search first");
|
|
27
|
+
}
|
|
28
|
+
else if (action.includes("memory_get")) {
|
|
29
|
+
distilledRules.push("- Always verify label exists before memory_get");
|
|
30
|
+
}
|
|
31
|
+
else if (action.includes("memory_replace")) {
|
|
32
|
+
distilledRules.push("- Re-read file before replace to ensure content is current");
|
|
33
|
+
}
|
|
34
|
+
else if (action.includes("read") || action.includes("glob")) {
|
|
35
|
+
distilledRules.push("- Check if file exists before read/glob");
|
|
36
|
+
}
|
|
37
|
+
else if (action.includes("search before get")) {
|
|
38
|
+
// Already in rules, skip
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
distilledRules.push(`- ${action}`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (config.useLlm && client && client?.session?.prompt) {
|
|
46
|
+
try {
|
|
47
|
+
const lessonContent = recentLessons.map(l => l.content).join("\n---\n");
|
|
48
|
+
const prompt = `Based on these lesson summaries, generate specific, actionable rules for the agent.
|
|
49
|
+
|
|
50
|
+
Lessons:
|
|
51
|
+
${lessonContent}
|
|
52
|
+
|
|
53
|
+
Current rules:
|
|
54
|
+
${distilledRules.join("\n")}
|
|
55
|
+
|
|
56
|
+
Generate more specific rules based on the lessons. Focus on:
|
|
57
|
+
1. What specific checks to do before calling tools
|
|
58
|
+
2. What common mistakes to avoid
|
|
59
|
+
3. What order of operations works best
|
|
60
|
+
|
|
61
|
+
Format as bullet points starting with "- ".
|
|
62
|
+
|
|
63
|
+
Your rules:`;
|
|
64
|
+
const result = await client
|
|
65
|
+
.session?.prompt({
|
|
66
|
+
path: { id: sessionId ?? "auto-distill" },
|
|
67
|
+
body: {
|
|
68
|
+
noReply: true,
|
|
69
|
+
parts: [{ type: 'text', text: prompt }],
|
|
70
|
+
},
|
|
71
|
+
});
|
|
72
|
+
if (result) {
|
|
73
|
+
const llmResponse = await (await result.text()).trim();
|
|
74
|
+
if (llmResponse) {
|
|
75
|
+
const llmRules = llmResponse.split("\n")
|
|
76
|
+
.filter((line) => line.trim().startsWith("-"))
|
|
77
|
+
.map((line) => line.trim());
|
|
78
|
+
if (llmRules.length > 0) {
|
|
79
|
+
distilledRules.push(...llmRules);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
memLog("error", "auto-distill", "LLM distillation failed:", { error });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
if (distilledRules.length === 0) {
|
|
89
|
+
return `Auto-distill: no rules extracted from ${lessonNodes.length} lessons`;
|
|
90
|
+
}
|
|
91
|
+
const mandatoryNode = await store.getNodeByLabel("global", "rule:mandatory:memory").catch(() => null);
|
|
92
|
+
if (!mandatoryNode) {
|
|
93
|
+
return "Auto-distill: rule:mandatory:memory node not found";
|
|
94
|
+
}
|
|
95
|
+
const existingRules = mandatoryNode.content;
|
|
96
|
+
const newRules = [];
|
|
97
|
+
for (const rule of distilledRules) {
|
|
98
|
+
const ruleText = rule.replace(/^- /, "");
|
|
99
|
+
if (!existingRules.toLowerCase().includes(ruleText.toLowerCase().slice(0, 30))) {
|
|
100
|
+
newRules.push(rule);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (newRules.length === 0) {
|
|
104
|
+
return `Auto-distill: no new rules (${lessonNodes.length} lessons, all ${distilledRules.length} rules already exist)`;
|
|
105
|
+
}
|
|
106
|
+
const autoLearnedSection = "### Auto-Learned";
|
|
107
|
+
const sectionIdx = existingRules.indexOf(autoLearnedSection);
|
|
108
|
+
let updatedContent;
|
|
109
|
+
if (sectionIdx >= 0) {
|
|
110
|
+
const before = existingRules.slice(0, sectionIdx + autoLearnedSection.length);
|
|
111
|
+
const after = existingRules.slice(sectionIdx + autoLearnedSection.length);
|
|
112
|
+
updatedContent = before + "\n" + newRules.join("\n") + after;
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
updatedContent = existingRules + "\n\n" + autoLearnedSection + "\n" + newRules.join("\n");
|
|
116
|
+
}
|
|
117
|
+
await store.updateNode(mandatoryNode.id, { content: updatedContent });
|
|
118
|
+
memLog("info", "auto-distill", `Added ${newRules.length} rules from ${lessonNodes.length} lessons`);
|
|
119
|
+
return `Auto-distill: added ${newRules.length} new rules from ${lessonNodes.length} lessons`;
|
|
120
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export function truncateContent(content, maxTokens) {
|
|
2
|
+
const maxChars = maxTokens * 4;
|
|
3
|
+
if (content.length <= maxChars)
|
|
4
|
+
return content;
|
|
5
|
+
let truncated = content.slice(0, maxChars);
|
|
6
|
+
const lastSpace = truncated.lastIndexOf(" ");
|
|
7
|
+
if (lastSpace > maxChars * 0.8) {
|
|
8
|
+
truncated = truncated.slice(0, lastSpace);
|
|
9
|
+
}
|
|
10
|
+
const lastNewline = truncated.lastIndexOf("\n");
|
|
11
|
+
if (lastNewline > maxChars * 0.8) {
|
|
12
|
+
truncated = truncated.slice(0, lastNewline);
|
|
13
|
+
}
|
|
14
|
+
return truncated.trim();
|
|
15
|
+
}
|
|
16
|
+
export function cleanContent(content) {
|
|
17
|
+
return content
|
|
18
|
+
.replace(/^#{1,6} .*$/gm, "")
|
|
19
|
+
.replace(/^tag:.*$/gm, "")
|
|
20
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
21
|
+
.trim();
|
|
22
|
+
}
|
|
23
|
+
export function formatNodeForInjection(node, maxTokensPerNode) {
|
|
24
|
+
if (node.label?.startsWith("middle-term:") || node.metadata?.customType === "middle-term") {
|
|
25
|
+
try {
|
|
26
|
+
const parsed = JSON.parse(node.content);
|
|
27
|
+
return {
|
|
28
|
+
label: node.label ?? node.id,
|
|
29
|
+
type: "middle-term",
|
|
30
|
+
sessionId: parsed.sessionId,
|
|
31
|
+
timestamp: parsed.timestamp,
|
|
32
|
+
workingCacheCount: parsed.workingCache?.length || 0,
|
|
33
|
+
recentNodesCount: parsed.recentNodes?.length || 0,
|
|
34
|
+
contextTokens: parsed.contextTokens,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
// Not valid JSON, fall through
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const cleaned = cleanContent(node.content);
|
|
42
|
+
const truncated = truncateContent(cleaned, maxTokensPerNode);
|
|
43
|
+
return {
|
|
44
|
+
label: node.label ?? node.id,
|
|
45
|
+
content: truncated,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { scoreCandidates } from "./scoring";
|
|
2
|
+
export async function detectRelevantSkills(store, query, queryEmbedding, maxSkills = 3, log) {
|
|
3
|
+
try {
|
|
4
|
+
const skillCandidates = await store.searchByEmbedding(queryEmbedding, maxSkills * 3, { bm25Weight: 0.3 });
|
|
5
|
+
const skills = skillCandidates.filter(n => n.type === "skill");
|
|
6
|
+
if (skills.length === 0) {
|
|
7
|
+
log("debug", "No skill nodes found");
|
|
8
|
+
return [];
|
|
9
|
+
}
|
|
10
|
+
log("debug", "Found skill candidates", { count: skills.length });
|
|
11
|
+
const scored = scoreCandidates(skills, query, queryEmbedding);
|
|
12
|
+
const threshold = 0.15;
|
|
13
|
+
const relevant = scored.filter(s => {
|
|
14
|
+
const semantic = s.embedding && queryEmbedding.length > 0
|
|
15
|
+
? s.embedding.reduce((dot, v, i) => dot + v * (queryEmbedding[i] ?? 0), 0) /
|
|
16
|
+
(Math.sqrt(s.embedding.reduce((s, v) => s + v * v, 0)) * Math.sqrt(queryEmbedding.reduce((s, v) => s + v * v, 0)) + 1e-8)
|
|
17
|
+
: 0;
|
|
18
|
+
return semantic > threshold;
|
|
19
|
+
});
|
|
20
|
+
const selected = relevant.slice(0, maxSkills);
|
|
21
|
+
log("info", "Skill detection", { found: skills.length, selected: selected.length });
|
|
22
|
+
return selected;
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
log("warn", "Skill detection failed", { error: String(err) });
|
|
26
|
+
return [];
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export async function detectRelevantPlaybooks(store, queryEmbedding, maxPlaybooks, log) {
|
|
30
|
+
try {
|
|
31
|
+
const candidates = await store.searchByEmbedding(queryEmbedding, maxPlaybooks * 3, { bm25Weight: 0.4 });
|
|
32
|
+
const playbooks = candidates.filter(n => n.type === "playbook");
|
|
33
|
+
if (playbooks.length === 0) {
|
|
34
|
+
log("debug", "No matching playbooks found");
|
|
35
|
+
return [];
|
|
36
|
+
}
|
|
37
|
+
const selected = playbooks.slice(0, maxPlaybooks);
|
|
38
|
+
log("info", "Playbook detection", { found: playbooks.length, selected: selected.length });
|
|
39
|
+
return selected.map(pb => ({
|
|
40
|
+
label: pb.label || pb.id,
|
|
41
|
+
description: pb.content?.slice(0, 200) || "",
|
|
42
|
+
steps: pb.metadata?.steps?.length ?? 0,
|
|
43
|
+
executionCount: pb.metadata?.executionCount ?? 0,
|
|
44
|
+
}));
|
|
45
|
+
}
|
|
46
|
+
catch (err) {
|
|
47
|
+
log("warn", "Playbook detection failed", { error: String(err) });
|
|
48
|
+
return [];
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export function formatPlaybooksAsAvailable(playbooks) {
|
|
2
|
+
if (playbooks.length === 0)
|
|
3
|
+
return "";
|
|
4
|
+
const entries = playbooks.map(pb => ` <playbook>\n <name>${pb.label}</name>\n <description>${pb.description} (${pb.steps} steps, ${pb.executionCount} runs)</description>\n <!-- memory_playbook_execute(playbook_id="${pb.label}") -->\n </playbook>`).join("\n");
|
|
5
|
+
return `<available_playbooks>\n${entries}\n</available_playbooks>`;
|
|
6
|
+
}
|
|
7
|
+
export function formatSkillsAsAvailable(skills) {
|
|
8
|
+
if (skills.length === 0)
|
|
9
|
+
return "";
|
|
10
|
+
const skillEntries = skills.map(skill => {
|
|
11
|
+
const name = skill.label?.replace(/^skill:/, "") ?? skill.id.slice(0, 8);
|
|
12
|
+
const description = skill.summary ?? skill.content.slice(0, 200);
|
|
13
|
+
const triggers = skill.metadata?.triggers
|
|
14
|
+
? ` [triggers: ${Array.isArray(skill.metadata.triggers) ? skill.metadata.triggers.join(", ") : skill.metadata.triggers}]`
|
|
15
|
+
: "";
|
|
16
|
+
return ` <skill>\n <name>${name}</name>\n <description>${description}${triggers}</description>\n <!-- memory_skill_load(name="${name}") -->\n </skill>`;
|
|
17
|
+
}).join("\n");
|
|
18
|
+
return `<available_skills>\n${skillEntries}\n</available_skills>`;
|
|
19
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { generateEmbedding } from "../../embeddings";
|
|
2
|
+
import { rerankDocuments } from "../../ollama";
|
|
3
|
+
import * as fs from "node:fs";
|
|
4
|
+
import * as path from "node:path";
|
|
5
|
+
import * as os from "node:os";
|
|
6
|
+
import { scoreCandidates } from "./scoring";
|
|
7
|
+
import { detectRelevantSkills, detectRelevantPlaybooks } from "./detection";
|
|
8
|
+
import { formatPlaybooksAsAvailable, formatSkillsAsAvailable } from "./formatting";
|
|
9
|
+
import { formatNodeForInjection } from "./content";
|
|
10
|
+
const INJECTION_LOG_FILE = path.join(os.homedir(), ".config", "opencode", "memory-injection.log");
|
|
11
|
+
export function createAutoRetrieveHook(deps) {
|
|
12
|
+
const { store, config, log } = deps;
|
|
13
|
+
const autoConfig = config.autoRetrieve;
|
|
14
|
+
const ollamaConfig = config.ollama;
|
|
15
|
+
const recentToolNames = [];
|
|
16
|
+
log("info", "Auto-retrieve hook created", { enabled: autoConfig?.enabled, ollamaEnabled: ollamaConfig?.enabled });
|
|
17
|
+
return {
|
|
18
|
+
"experimental.chat.messages.transform": async (_input, output) => {
|
|
19
|
+
// Process pending injection queue first
|
|
20
|
+
try {
|
|
21
|
+
const pending = await store.getPendingInjections();
|
|
22
|
+
if (pending.length > 0) {
|
|
23
|
+
log("info", "Processing pending injections", { count: pending.length });
|
|
24
|
+
const pendingNodes = [];
|
|
25
|
+
for (const p of pending) {
|
|
26
|
+
try {
|
|
27
|
+
const node = await store.getNode(p.nodeId);
|
|
28
|
+
pendingNodes.push(node);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
log("warn", "Pending injection node not found", { nodeId: p.nodeId });
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (pendingNodes.length > 0) {
|
|
35
|
+
const MAX_TOKENS_PER_NODE = 300;
|
|
36
|
+
const pendingJson = pendingNodes.map(n => formatNodeForInjection(n, MAX_TOKENS_PER_NODE));
|
|
37
|
+
const pendingBlock = `\n\n### Injected Memory:\nThe following node was selected for injection from the management app.\n\n${JSON.stringify(pendingJson, null, 2)}---`;
|
|
38
|
+
const lastUserIndex = findLastUserMessage(output.messages);
|
|
39
|
+
if (lastUserIndex >= 0) {
|
|
40
|
+
const userMsg = output.messages[lastUserIndex];
|
|
41
|
+
if (userMsg && userMsg.parts) {
|
|
42
|
+
userMsg.parts.unshift({ type: "text", text: pendingBlock + "\n\n" });
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
for (const p of pending) {
|
|
47
|
+
await store.markInjectionProcessed(p.id);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
log("error", "Pending injection processing failed", { error: String(err) });
|
|
53
|
+
}
|
|
54
|
+
if (!autoConfig?.enabled) {
|
|
55
|
+
log("debug", "Auto-retrieve disabled in config");
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
try {
|
|
59
|
+
const lastUserIndex = findLastUserMessage(output.messages);
|
|
60
|
+
if (lastUserIndex === -1) {
|
|
61
|
+
log("debug", "No user message found");
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
const userMsg = output.messages[lastUserIndex];
|
|
65
|
+
if (!userMsg) {
|
|
66
|
+
log("debug", "User message not found at index");
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
const userText = userMsg.parts
|
|
70
|
+
.filter(p => p.type === "text")
|
|
71
|
+
.map(p => p.text || "")
|
|
72
|
+
.join(" ")
|
|
73
|
+
.trim();
|
|
74
|
+
if (!userText || userText.length < 3) {
|
|
75
|
+
log("debug", "User message too short", { length: userText?.length });
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
log("info", "Auto-retrieving for query", { query: userText.slice(0, 50) + "..." });
|
|
79
|
+
const queryEmbedding = await generateEmbedding(userText);
|
|
80
|
+
const maxPlaybooks = autoConfig.maxInjectPlaybooks ?? 3;
|
|
81
|
+
const [candidates, relevantSkills, relevantPlaybooks] = await Promise.all([
|
|
82
|
+
store.searchByEmbedding(queryEmbedding, autoConfig.candidateCount ?? 30, { bm25Weight: 0.4 }),
|
|
83
|
+
detectRelevantSkills(store, userText, queryEmbedding, 3, log),
|
|
84
|
+
detectRelevantPlaybooks(store, queryEmbedding, maxPlaybooks, log),
|
|
85
|
+
]);
|
|
86
|
+
if (candidates.length === 0 && relevantSkills.length === 0 && relevantPlaybooks.length === 0) {
|
|
87
|
+
log("debug", "No candidates, skills, or playbooks found");
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
const filtered = candidates.filter(c => !c.label?.startsWith("rule:") && c.type !== "skill");
|
|
91
|
+
if (filtered.length === 0 && relevantSkills.length === 0 && relevantPlaybooks.length === 0) {
|
|
92
|
+
log("debug", "No candidates after dedup");
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
log("debug", "Candidates after dedup", { count: filtered.length, skills: relevantSkills.length, playbooks: relevantPlaybooks.length });
|
|
96
|
+
let scored = filtered.length > 0 ? scoreCandidates(filtered, userText, queryEmbedding) : [];
|
|
97
|
+
if (ollamaConfig?.enabled && scored.length > 0) {
|
|
98
|
+
log("info", "Ollama reranking", { model: ollamaConfig.model, candidateCount: scored.length });
|
|
99
|
+
const startTime = Date.now();
|
|
100
|
+
const results = await rerankDocuments(userText, scored.map(c => ({ id: c.id, label: c.label ?? c.id, content: c.content })), {
|
|
101
|
+
baseUrl: ollamaConfig.baseUrl,
|
|
102
|
+
model: ollamaConfig.model,
|
|
103
|
+
topK: autoConfig.maxInjectNodes ?? 5,
|
|
104
|
+
});
|
|
105
|
+
const ollamaDuration = Date.now() - startTime;
|
|
106
|
+
const selectedIds = new Set(results.map(r => r.id));
|
|
107
|
+
scored = scored.filter(c => selectedIds.has(c.id));
|
|
108
|
+
log("info", "Ollama reranking done", { selectedCount: scored.length, ollamaDuration });
|
|
109
|
+
}
|
|
110
|
+
const maxNodes = autoConfig.maxInjectNodes ?? 5;
|
|
111
|
+
const maxMemoryNodes = Math.max(0, maxNodes - relevantSkills.length - relevantPlaybooks.length);
|
|
112
|
+
scored = scored.slice(0, maxMemoryNodes);
|
|
113
|
+
const MAX_TOKENS_PER_NODE = 150;
|
|
114
|
+
const memoriesJson = scored.map(n => formatNodeForInjection(n, MAX_TOKENS_PER_NODE));
|
|
115
|
+
let fullBlock = "";
|
|
116
|
+
if (memoriesJson.length > 0) {
|
|
117
|
+
fullBlock += `### Retrieved Context:
|
|
118
|
+
Use the following memories to inform your response. Do not repeat or summarize them.
|
|
119
|
+
|
|
120
|
+
${JSON.stringify(memoriesJson, null, 2)}
|
|
121
|
+
---`;
|
|
122
|
+
}
|
|
123
|
+
if (relevantSkills.length > 0) {
|
|
124
|
+
const skillsBlock = formatSkillsAsAvailable(relevantSkills);
|
|
125
|
+
if (skillsBlock) {
|
|
126
|
+
fullBlock += `\n\n### Available Skills:\nThe following skills are relevant to your task. Load full instructions with:\nmemory_skill_load(name="skill-name")\n\n${skillsBlock}`;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
if (relevantPlaybooks.length > 0) {
|
|
130
|
+
const playbooksBlock = formatPlaybooksAsAvailable(relevantPlaybooks);
|
|
131
|
+
if (playbooksBlock) {
|
|
132
|
+
fullBlock += `\n\n### Available Playbooks:\nThe following playbooks match your current task. Execute one with:\nmemory_playbook_execute playbook_id="<id>"\n\n${playbooksBlock}`;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
if (!fullBlock)
|
|
136
|
+
return;
|
|
137
|
+
const debugLine = `[${new Date().toISOString()}] Query: ${userText.slice(0, 100)}...\n${fullBlock}\n\n`;
|
|
138
|
+
try {
|
|
139
|
+
fs.appendFileSync(INJECTION_LOG_FILE, debugLine);
|
|
140
|
+
}
|
|
141
|
+
catch { /* silent fail */ }
|
|
142
|
+
if (userMsg && userMsg.parts) {
|
|
143
|
+
userMsg.parts.unshift({ type: "text", text: fullBlock + "\n\n" });
|
|
144
|
+
}
|
|
145
|
+
log("info", "Memory injected", { nodeCount: scored.length, skillCount: relevantSkills.length, playbookCount: relevantPlaybooks.length, queryLength: userText.length });
|
|
146
|
+
}
|
|
147
|
+
catch (err) {
|
|
148
|
+
log("error", "Auto-retrieve failed", { error: String(err) });
|
|
149
|
+
}
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
function findLastUserMessage(messages) {
|
|
154
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
155
|
+
const msg = messages[i];
|
|
156
|
+
if (!msg)
|
|
157
|
+
continue;
|
|
158
|
+
if (msg.info?.role === "user" && !msg.info.synthetic) {
|
|
159
|
+
return i;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return -1;
|
|
163
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { tokenize } from "../../storage/sqlite";
|
|
2
|
+
export function scoreCandidates(candidates, query, queryEmbedding) {
|
|
3
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
4
|
+
const now = Date.now();
|
|
5
|
+
const semanticScores = new Map();
|
|
6
|
+
if (queryEmbedding.length > 0) {
|
|
7
|
+
for (const node of candidates) {
|
|
8
|
+
if (node.embedding && node.embedding.length > 0) {
|
|
9
|
+
let dot = 0, normA = 0, normB = 0;
|
|
10
|
+
for (let i = 0; i < queryEmbedding.length && i < node.embedding.length; i++) {
|
|
11
|
+
const qi = queryEmbedding[i] ?? 0;
|
|
12
|
+
const ni = node.embedding[i] ?? 0;
|
|
13
|
+
dot += qi * ni;
|
|
14
|
+
normA += qi * qi;
|
|
15
|
+
normB += ni * ni;
|
|
16
|
+
}
|
|
17
|
+
semanticScores.set(node.id, dot / (Math.sqrt(normA) * Math.sqrt(normB) + 1e-8));
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
const queryTerms = tokenize(query);
|
|
22
|
+
const keywordBoost = new Map();
|
|
23
|
+
if (queryTerms.length > 0) {
|
|
24
|
+
for (const node of candidates) {
|
|
25
|
+
const nodeTokens = tokenize(node.content + " " + (node.label ?? ""));
|
|
26
|
+
let overlap = 0;
|
|
27
|
+
for (const term of queryTerms) {
|
|
28
|
+
if (nodeTokens.includes(term))
|
|
29
|
+
overlap++;
|
|
30
|
+
}
|
|
31
|
+
if (overlap > 0) {
|
|
32
|
+
keywordBoost.set(node.id, Math.min(0.15, (overlap / queryTerms.length) * 0.15));
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
const recencyBoost = new Map();
|
|
37
|
+
for (const node of candidates) {
|
|
38
|
+
if (node.updatedAt && node.updatedAt instanceof Date) {
|
|
39
|
+
const daysSince = (now - node.updatedAt.getTime()) / DAY_MS;
|
|
40
|
+
if (daysSince < 1)
|
|
41
|
+
recencyBoost.set(node.id, 0.1);
|
|
42
|
+
else if (daysSince < 7)
|
|
43
|
+
recencyBoost.set(node.id, 0.05);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
const scored = candidates.map(node => {
|
|
47
|
+
const semantic = semanticScores.get(node.id) ?? 0;
|
|
48
|
+
const base = 0.4 * semantic + 0.3 * (node.importance ?? 0.5) + 0.15 * (node.confidence ?? 0.5) + 0.1 * (node.usefulnessScore ?? 0.5);
|
|
49
|
+
const access = Math.min(0.05, (node.accessCount ?? 0) * 0.005);
|
|
50
|
+
const recency = recencyBoost.get(node.id) ?? 0;
|
|
51
|
+
const keyword = keywordBoost.get(node.id) ?? 0;
|
|
52
|
+
return { node, score: base + access + recency + keyword };
|
|
53
|
+
});
|
|
54
|
+
scored.sort((a, b) => b.score - a.score);
|
|
55
|
+
return scored.map(s => s.node);
|
|
56
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { createAutoRetrieveHook } from "./auto-retrieve/index";
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { createAutoRetrieveHook } from "./auto-retrieve";
|
|
2
|
+
export { distillRules } from "./auto-distill";
|
|
3
|
+
export { predictiveRateToolCall, applyScoreDecay } from "./predictive-rating";
|
|
4
|
+
// Playbooks are now memory nodes (type: "playbook"). Auto-discover is agent-driven.
|