march-cli 0.1.42 → 0.1.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent/code-search/cache.mjs +16 -7
- package/src/agent/code-search/engine.mjs +9 -2
- package/src/agent/code-search/retrieval/resilient-vectorizer.mjs +59 -0
- package/src/agent/code-search/retrieval/safetensors.mjs +16 -10
- package/src/agent/code-search/scanner.mjs +11 -5
- package/src/agent/code-search/tool.mjs +11 -5
- package/src/agent/model-payload-dumper.mjs +1 -1
- package/src/agent/runner/payload/provider-payload-transform.mjs +59 -0
- package/src/agent/runner/recall/mid-turn-recall-bridge.mjs +23 -0
- package/src/agent/runner.mjs +28 -27
- package/src/agent/runtime/remote-ui-client.mjs +1 -1
- package/src/agent/runtime/resource/context-resource-loader.mjs +17 -0
- package/src/agent/runtime/runtime-factory.mjs +5 -1
- package/src/agent/runtime/state/runner-state.mjs +10 -3
- package/src/agent/runtime/ui-event-bridge.mjs +2 -2
- package/src/agent/turn/turn-runner.mjs +35 -24
- package/src/cli/fallback-ui.mjs +2 -2
- package/src/cli/repl-loop.mjs +9 -8
- package/src/cli/startup/app-runtime.mjs +4 -2
- package/src/cli/tui/input/mouse-selection-controller.mjs +19 -0
- package/src/cli/tui/output/selectable-copy.mjs +3 -3
- package/src/cli/tui/output/timeline-block-restore.mjs +1 -1
- package/src/cli/tui/output-buffer.mjs +18 -0
- package/src/cli/tui/recall-rendering.mjs +38 -9
- package/src/cli/tui/selection/ansi-range.mjs +88 -0
- package/src/cli/tui/selection-screen.mjs +31 -99
- package/src/cli/turn/turn-input-preparer.mjs +15 -6
- package/src/cli/ui.mjs +2 -2
- package/src/cli/workspace/tui-timeline-projection.mjs +1 -1
- package/src/context/engine.mjs +15 -5
- package/src/context/system-core/base.md +1 -1
- package/src/memory/markdown/markdown-format.mjs +0 -17
- package/src/memory/markdown/markdown-recall.mjs +11 -19
- package/src/memory/markdown/semantic-preload.mjs +26 -0
- package/src/memory/markdown/semantic-recall.mjs +169 -0
- package/src/memory/markdown/sqlite-index.mjs +1 -13
- package/src/memory/markdown-store.mjs +34 -54
- package/src/web-ui/dist/assets/{index-BQtl1uQs.css → index-BG1Pxf1k.css} +1 -1
- package/src/web-ui/dist/assets/{index-DrlJis_D.js → index-C0xOHlDz.js} +1 -1
- package/src/web-ui/dist/index.html +2 -2
- package/src/web-ui/runtime-host.mjs +18 -3
- package/src/web-ui/src/components/timeline/TimelineBlocks.tsx +27 -0
- package/src/web-ui/src/model.ts +18 -0
- package/src/web-ui/src/runtime/client.ts +2 -1
- package/src/web-ui/src/runtime/runtimeTimeline.ts +5 -0
- package/src/web-ui/src/styles/shell.css +7 -0
- package/src/web-ui/src/timelineAdapter.ts +2 -0
|
@@ -4,19 +4,21 @@ import { formatRecallHints } from "../../memory/markdown-store.mjs";
|
|
|
4
4
|
import { formatMessageAttachmentsForDisplay } from "../../session/attachment-display.mjs";
|
|
5
5
|
import { formatShellHints } from "../../shell/hints.mjs";
|
|
6
6
|
|
|
7
|
-
export function prepareTurnInput({ prompt, runner, memoryStore, currentProject, modeState = null }) {
|
|
7
|
+
export async function prepareTurnInput({ prompt, runner, memoryStore, currentProject, modeState = null }) {
|
|
8
8
|
const engine = runner.engine ?? {};
|
|
9
9
|
const carryoverAlreadyRendered = engine.hasRenderedPendingAssistantRecallHints?.() ?? false;
|
|
10
|
-
const
|
|
11
|
-
const
|
|
10
|
+
const carryoverRecall = normalizePendingAssistantRecall(engine.takePendingAssistantRecallHints?.());
|
|
11
|
+
const carryoverRecallHints = carryoverRecall.hints;
|
|
12
|
+
const userRecallHints = await memoryStore.recallForUser(prompt, {
|
|
12
13
|
currentProject,
|
|
13
14
|
excludedIds: engine.getRecentRecallMemoryIds?.() ?? [],
|
|
14
15
|
});
|
|
16
|
+
const userRecallReport = memoryStore.lastUserRecallReport ?? null;
|
|
15
17
|
const modePrompt = appendModeReminder(prompt, modeState?.get?.());
|
|
16
18
|
const fullPrompt = appendPromptBlocks(
|
|
17
19
|
modePrompt,
|
|
18
|
-
formatRecallHints(
|
|
19
|
-
formatRecallHints(
|
|
20
|
+
formatRecallHints(userRecallHints),
|
|
21
|
+
formatRecallHints(carryoverRecallHints),
|
|
20
22
|
formatShellHints(runner.shellRuntime),
|
|
21
23
|
);
|
|
22
24
|
|
|
@@ -26,8 +28,10 @@ export function prepareTurnInput({ prompt, runner, memoryStore, currentProject,
|
|
|
26
28
|
runOptions: { userRecallHints, currentProject },
|
|
27
29
|
displayMessage: formatUserDisplayMessage(prompt),
|
|
28
30
|
userRecallHints,
|
|
31
|
+
userRecallReport,
|
|
29
32
|
carryoverRecallHints,
|
|
30
|
-
|
|
33
|
+
carryoverRecallReport: carryoverRecall.report,
|
|
34
|
+
shouldRenderCarryoverRecall: (carryoverRecallHints.length > 0 || carryoverRecall.report) && !carryoverAlreadyRendered,
|
|
31
35
|
};
|
|
32
36
|
}
|
|
33
37
|
|
|
@@ -38,3 +42,8 @@ export function formatUserDisplayMessage(prompt) {
|
|
|
38
42
|
function appendPromptBlocks(prompt, ...blocks) {
|
|
39
43
|
return [prompt, ...blocks.filter(Boolean)].join("\n\n");
|
|
40
44
|
}
|
|
45
|
+
|
|
46
|
+
function normalizePendingAssistantRecall(value) {
|
|
47
|
+
if (Array.isArray(value)) return { hints: value, report: null };
|
|
48
|
+
return { hints: value?.hints ?? [], report: value?.report ?? null };
|
|
49
|
+
}
|
package/src/cli/ui.mjs
CHANGED
|
@@ -212,8 +212,8 @@ export function createTuiUI({
|
|
|
212
212
|
status: (text) => {
|
|
213
213
|
ensureStarted(); flushStreamDeltas(); retryStatus.stop(); spinnerStatus.stop(); output.setOverlayStatus([brightBlack(`● ${text}`)]); requestRender();
|
|
214
214
|
},
|
|
215
|
-
recall: ({ hints }) => {
|
|
216
|
-
ensureStarted(); flushStreamDeltas(); retryStatus.stop(); spinnerStatus.stop(); output.ensureNewline(); writeRecall({ output, hints }); requestRender();
|
|
215
|
+
recall: ({ hints, report, variant }) => {
|
|
216
|
+
ensureStarted(); flushStreamDeltas(); retryStatus.stop(); spinnerStatus.stop(); output.ensureNewline(); writeRecall({ output, hints, report, variant }); requestRender();
|
|
217
217
|
},
|
|
218
218
|
|
|
219
219
|
clearOutput: () => {
|
|
@@ -94,7 +94,7 @@ export function createTuiTimelineProjection() {
|
|
|
94
94
|
blocks.push(createBlock("status", event.at, { content: String(first ?? "") }));
|
|
95
95
|
break;
|
|
96
96
|
case "recall":
|
|
97
|
-
blocks.push(createBlock("recall", event.at, { hints: first?.hints ?? [] }));
|
|
97
|
+
blocks.push(createBlock("recall", event.at, { hints: first?.hints ?? [], report: first?.report ?? null }));
|
|
98
98
|
break;
|
|
99
99
|
case "editDiff":
|
|
100
100
|
closeAssistantBlock(event.at);
|
package/src/context/engine.mjs
CHANGED
|
@@ -17,6 +17,7 @@ export class ContextEngine {
|
|
|
17
17
|
this.thinkingLevel = thinkingLevel;
|
|
18
18
|
this.turns = [];
|
|
19
19
|
this.pendingAssistantRecallHints = [];
|
|
20
|
+
this.pendingAssistantRecallReport = null;
|
|
20
21
|
this.pendingAssistantRecallHintsRendered = false;
|
|
21
22
|
this.sessionName = "";
|
|
22
23
|
this.toolDefs = [];
|
|
@@ -96,8 +97,9 @@ export class ContextEngine {
|
|
|
96
97
|
return ids;
|
|
97
98
|
}
|
|
98
99
|
|
|
99
|
-
setPendingAssistantRecallHints(hints = []) {
|
|
100
|
+
setPendingAssistantRecallHints(hints = [], report = null) {
|
|
100
101
|
this.pendingAssistantRecallHints = uniqueHints(hints);
|
|
102
|
+
this.pendingAssistantRecallReport = report;
|
|
101
103
|
this.pendingAssistantRecallHintsRendered = false;
|
|
102
104
|
}
|
|
103
105
|
|
|
@@ -105,19 +107,25 @@ export class ContextEngine {
|
|
|
105
107
|
return this.pendingAssistantRecallHints;
|
|
106
108
|
}
|
|
107
109
|
|
|
110
|
+
peekPendingAssistantRecallReport() {
|
|
111
|
+
return this.pendingAssistantRecallReport;
|
|
112
|
+
}
|
|
113
|
+
|
|
108
114
|
hasRenderedPendingAssistantRecallHints() {
|
|
109
115
|
return this.pendingAssistantRecallHintsRendered;
|
|
110
116
|
}
|
|
111
117
|
|
|
112
118
|
markPendingAssistantRecallHintsRendered() {
|
|
113
|
-
if (this.pendingAssistantRecallHints.length > 0) this.pendingAssistantRecallHintsRendered = true;
|
|
119
|
+
if (this.pendingAssistantRecallHints.length > 0 || this.pendingAssistantRecallReport) this.pendingAssistantRecallHintsRendered = true;
|
|
114
120
|
}
|
|
115
121
|
|
|
116
122
|
takePendingAssistantRecallHints() {
|
|
117
123
|
const hints = this.pendingAssistantRecallHints;
|
|
124
|
+
const report = this.pendingAssistantRecallReport;
|
|
118
125
|
this.pendingAssistantRecallHints = [];
|
|
126
|
+
this.pendingAssistantRecallReport = null;
|
|
119
127
|
this.pendingAssistantRecallHintsRendered = false;
|
|
120
|
-
return hints;
|
|
128
|
+
return { hints, report };
|
|
121
129
|
}
|
|
122
130
|
|
|
123
131
|
resolvePath(raw) {
|
|
@@ -142,12 +150,14 @@ export class ContextEngine {
|
|
|
142
150
|
if (replace) {
|
|
143
151
|
this.turns = [];
|
|
144
152
|
this.pendingAssistantRecallHints = [];
|
|
153
|
+
this.pendingAssistantRecallReport = null;
|
|
145
154
|
this.pendingAssistantRecallHintsRendered = false;
|
|
146
155
|
this.sessionName = "";
|
|
147
156
|
}
|
|
148
157
|
if (data.turns) this.turns = data.turns;
|
|
149
158
|
if (Array.isArray(data.pendingAssistantRecallHints)) {
|
|
150
159
|
this.pendingAssistantRecallHints = uniqueHints(data.pendingAssistantRecallHints);
|
|
160
|
+
this.pendingAssistantRecallReport = data.pendingAssistantRecallReport ?? null;
|
|
151
161
|
this.pendingAssistantRecallHintsRendered = false;
|
|
152
162
|
}
|
|
153
163
|
if (typeof data.sessionName === "string") this.sessionName = data.sessionName;
|
|
@@ -168,14 +178,14 @@ export class ContextEngine {
|
|
|
168
178
|
for (const turn of this.turns) {
|
|
169
179
|
let block = `## Turn ${turn.index}\n` +
|
|
170
180
|
`[user]\n${String(turn.userMessage ?? "")}\n`;
|
|
171
|
-
const userRecall = formatRecallHints(
|
|
181
|
+
const userRecall = formatRecallHints(turn.userRecallHints ?? []);
|
|
172
182
|
if (userRecall) block += `\n${userRecall}\n`;
|
|
173
183
|
block += `\n[assistant]\n`;
|
|
174
184
|
const assistantText = turn.assistantContext || turn.assistantMessage;
|
|
175
185
|
if (assistantText) {
|
|
176
186
|
block += `\n${String(assistantText ?? "")}\n`;
|
|
177
187
|
}
|
|
178
|
-
const assistantRecall = formatRecallHints(
|
|
188
|
+
const assistantRecall = formatRecallHints(turn.assistantRecallHints ?? []);
|
|
179
189
|
if (assistantRecall) block += `\n${assistantRecall}\n`;
|
|
180
190
|
entries.push(block);
|
|
181
191
|
}
|
|
@@ -96,7 +96,7 @@ The user primarily asks for software engineering work: fixing bugs, adding behav
|
|
|
96
96
|
</git_contract>
|
|
97
97
|
|
|
98
98
|
<memory_system>
|
|
99
|
-
- [recall
|
|
99
|
+
- [recall] blocks in recent_chat are lightweight memory hints matched by semantic recall. Treat them as possibly relevant pointers, not as complete facts.
|
|
100
100
|
- A recall hint's description may record key operational constraints, including when the full memory must be opened; factor those constraints into relevance before acting.
|
|
101
101
|
- If a recall hint may help the current task, use memory_open(id) to read the full memory before relying on it. Ignore hints that are clearly unrelated or too low-value for the task.
|
|
102
102
|
- Use memory_search(query) for full-text search across all memories.
|
|
@@ -32,24 +32,7 @@ export function normalizeTags(tags) {
|
|
|
32
32
|
return out;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
export function expandTags(tags) {
|
|
36
|
-
const terms = [];
|
|
37
|
-
for (const tag of tags) {
|
|
38
|
-
terms.push(tag);
|
|
39
|
-
for (const part of tag.split(/[\/_-]+/)) {
|
|
40
|
-
if (part) terms.push(part);
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
return [...new Set(terms.map(normalizeText).filter(Boolean))];
|
|
44
|
-
}
|
|
45
35
|
|
|
46
|
-
export function quoteFtsTerm(term) {
|
|
47
|
-
return `"${String(term).replace(/"/g, '""')}"`;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
export function normalizeText(text) {
|
|
51
|
-
return String(text ?? "").trim().toLowerCase();
|
|
52
|
-
}
|
|
53
36
|
|
|
54
37
|
export function generateMemoryId() {
|
|
55
38
|
return `mem_${randomUUID().replace(/-/g, "").slice(0, 16)}`;
|
|
@@ -1,28 +1,20 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
export function formatRecallHints(source, hints = []) {
|
|
1
|
+
export function formatRecallHints(hints = []) {
|
|
4
2
|
if (!hints.length) return "";
|
|
5
|
-
const lines = [
|
|
3
|
+
const lines = ["[recall]"];
|
|
6
4
|
for (const hint of hints) {
|
|
7
|
-
lines.push(`- ${hint.id} | ${hint.name} | ${hint.description}`);
|
|
5
|
+
lines.push(`- ${hint.id}${formatScoreForPrompt(hint.score)} | ${hint.name} | ${hint.description}`);
|
|
8
6
|
}
|
|
9
7
|
return lines.join("\n");
|
|
10
8
|
}
|
|
11
9
|
|
|
12
|
-
export function
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
}
|
|
19
|
-
if (currentProject) {
|
|
20
|
-
const projectTag = normalizeText(`project/${currentProject}`);
|
|
21
|
-
if (entry.tags.map(normalizeText).includes(projectTag)) score += 2;
|
|
22
|
-
}
|
|
23
|
-
return score;
|
|
10
|
+
export function toHint(entry, metadata = {}) {
|
|
11
|
+
return { id: entry.id, name: entry.name, description: entry.description, ...metadata };
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function formatScoreForPrompt(score) {
|
|
15
|
+
return Number.isFinite(score) ? ` | score=${formatScore(score)}` : "";
|
|
24
16
|
}
|
|
25
17
|
|
|
26
|
-
export function
|
|
27
|
-
return
|
|
18
|
+
export function formatScore(score) {
|
|
19
|
+
return Number.isFinite(score) ? score.toFixed(2) : "--";
|
|
28
20
|
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export async function preloadSemanticMemoryRecall({ memoryStore, ui = null, logger = null } = {}) {
|
|
2
|
+
if (!memoryStore?.semanticRecall?.enabled) return { ok: true, skipped: true };
|
|
3
|
+
try {
|
|
4
|
+
ui?.status?.("Preparing memory recall model...");
|
|
5
|
+
await memoryStore.semanticRecall.preload();
|
|
6
|
+
memoryStore.semanticRecallWarning = memoryStore.semanticRecall.warning;
|
|
7
|
+
if (memoryStore.semanticRecallWarning) ui?.writeln?.(`Memory recall fallback: ${memoryStore.semanticRecallWarning}`);
|
|
8
|
+
logger?.event?.("memory.semantic_model_ready", { modelId: memoryStore.semanticRecall.modelId, status: memoryStore.semanticRecall.status });
|
|
9
|
+
return { ok: true, skipped: false, fallback: memoryStore.semanticRecall.status === "fallback" };
|
|
10
|
+
} catch (err) {
|
|
11
|
+
const message = err?.message ?? String(err);
|
|
12
|
+
memoryStore.semanticRecallWarning = message;
|
|
13
|
+
logger?.error?.("memory.semantic_model_preload_failed", { error: message });
|
|
14
|
+
ui?.writeln?.(`Memory recall model preload failed: ${message}`);
|
|
15
|
+
return { ok: false, error: message };
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function startSemanticMemoryRecallPreload({ memoryStore, ui = null, logger = null, delayMs = 0 } = {}) {
|
|
20
|
+
if (!memoryStore?.semanticRecall?.enabled) return null;
|
|
21
|
+
const start = () => preloadSemanticMemoryRecall({ memoryStore, ui, logger });
|
|
22
|
+
if (delayMs <= 0) return start();
|
|
23
|
+
const timer = setTimeout(start, delayMs);
|
|
24
|
+
timer.unref?.();
|
|
25
|
+
return timer;
|
|
26
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { Model2VecVectorizer } from "../../agent/code-search/retrieval/model2vec.mjs";
|
|
4
|
+
import { ResilientVectorizer } from "../../agent/code-search/retrieval/resilient-vectorizer.mjs";
|
|
5
|
+
import { parseMemoryMarkdown } from "./markdown-format.mjs";
|
|
6
|
+
|
|
7
|
+
export const POTION_RETRIEVAL_MODEL_ID = "minishlab/potion-retrieval-32M";
|
|
8
|
+
|
|
9
|
+
const MAX_CHUNK_CHARS = 1800;
|
|
10
|
+
export const DEFAULT_MEMORY_RECALL_MIN_SCORE = 0.5;
|
|
11
|
+
|
|
12
|
+
export class SemanticMemoryRecallIndex {
|
|
13
|
+
constructor({ stateRoot = null, modelId = POTION_RETRIEVAL_MODEL_ID, modelDir = null, vectorizer = null, minScore = parseMemoryRecallMinScore() } = {}) {
|
|
14
|
+
this.modelId = modelId;
|
|
15
|
+
this.minScore = minScore;
|
|
16
|
+
this.vectorizer = vectorizer ?? createDefaultVectorizer({ stateRoot, modelId, modelDir });
|
|
17
|
+
this.signature = "";
|
|
18
|
+
this.chunks = [];
|
|
19
|
+
this.vectors = [];
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
get enabled() {
|
|
23
|
+
return Boolean(this.vectorizer);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
get warning() {
|
|
27
|
+
return this.vectorizer?.warning ?? null;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
get status() {
|
|
31
|
+
return this.vectorizer?.status ?? "primary";
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
async preload() {
|
|
35
|
+
if (!this.vectorizer) return false;
|
|
36
|
+
if (typeof this.vectorizer.load === "function") await this.vectorizer.load();
|
|
37
|
+
else await this.vectorizer.encode(["memory recall warmup"]);
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async search(query, { entries, excluded = new Set(), limit = 3, candidateLimit = 5 } = {}) {
|
|
42
|
+
const empty = { recalled: [], candidates: [], threshold: this.minScore };
|
|
43
|
+
if (!this.vectorizer || !String(query ?? "").trim()) return empty;
|
|
44
|
+
const activeEntries = [...entries.values()].filter((entry) => entry.status === "active" && entry.description && !excluded.has(entry.id));
|
|
45
|
+
if (activeEntries.length === 0) return empty;
|
|
46
|
+
await this.#ensureIndex(activeEntries);
|
|
47
|
+
const [queryVector] = await this.vectorizer.encode([query]);
|
|
48
|
+
if (!queryVector || queryVector.norm === 0) return empty;
|
|
49
|
+
|
|
50
|
+
const bestByEntry = new Map();
|
|
51
|
+
for (let index = 0; index < this.vectors.length; index += 1) {
|
|
52
|
+
const chunk = this.chunks[index];
|
|
53
|
+
if (excluded.has(chunk.entry.id)) continue;
|
|
54
|
+
const score = cosineSimilarity(queryVector, this.vectors[index]);
|
|
55
|
+
const prev = bestByEntry.get(chunk.entry.id);
|
|
56
|
+
if (!prev || score > prev.score) bestByEntry.set(chunk.entry.id, { entry: chunk.entry, score });
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const ranked = [...bestByEntry.values()]
|
|
60
|
+
.filter(({ score }) => Number.isFinite(score) && score > 0)
|
|
61
|
+
.sort((a, b) => b.score - a.score || a.entry.name.localeCompare(b.entry.name));
|
|
62
|
+
const recalled = ranked.filter(({ score }) => score >= this.minScore).slice(0, limit);
|
|
63
|
+
const recalledIds = new Set(recalled.map(({ entry }) => entry.id));
|
|
64
|
+
const candidates = ranked
|
|
65
|
+
.slice(0, Math.max(limit, candidateLimit))
|
|
66
|
+
.map(({ entry, score }) => ({ entry, score, recalled: recalledIds.has(entry.id) }));
|
|
67
|
+
return {
|
|
68
|
+
recalled,
|
|
69
|
+
candidates,
|
|
70
|
+
threshold: this.minScore,
|
|
71
|
+
vectorizerStatus: this.status,
|
|
72
|
+
warning: this.warning,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async #ensureIndex(entries) {
|
|
77
|
+
const signature = entries.map(entrySignature).join("\n");
|
|
78
|
+
if (signature === this.signature) return;
|
|
79
|
+
this.chunks = entries.flatMap(memoryChunks);
|
|
80
|
+
this.vectors = this.chunks.length > 0
|
|
81
|
+
? await this.vectorizer.encode(this.chunks.map((chunk) => chunk.text))
|
|
82
|
+
: [];
|
|
83
|
+
this.signature = signature;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function parseMemoryRecallMinScore(value = process.env.MARCH_MEMORY_RECALL_MIN_SCORE) {
|
|
88
|
+
if (value == null || value === "") return DEFAULT_MEMORY_RECALL_MIN_SCORE;
|
|
89
|
+
const normalized = String(value).trim().toLowerCase();
|
|
90
|
+
if (["false", "no", "off"].includes(normalized)) return 0;
|
|
91
|
+
const parsed = Number(normalized);
|
|
92
|
+
return Number.isFinite(parsed) && parsed >= 0 ? parsed : DEFAULT_MEMORY_RECALL_MIN_SCORE;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function createDefaultVectorizer({ stateRoot, modelId, modelDir }) {
|
|
96
|
+
const dir = modelDir ?? (stateRoot ? join(stateRoot, "memory", "models", modelId.replaceAll("/", "__")) : null);
|
|
97
|
+
if (!dir) return null;
|
|
98
|
+
return new ResilientVectorizer({
|
|
99
|
+
primary: new Model2VecVectorizer({ modelDir: dir, modelId }),
|
|
100
|
+
label: "memory recall",
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function memoryChunks(entry) {
|
|
105
|
+
const body = readMemoryBody(entry);
|
|
106
|
+
const sections = splitMarkdownBody(body);
|
|
107
|
+
const chunks = sections.length > 0 ? sections : [""];
|
|
108
|
+
return chunks.map((section, index) => ({
|
|
109
|
+
entry,
|
|
110
|
+
index,
|
|
111
|
+
text: [
|
|
112
|
+
entry.name,
|
|
113
|
+
entry.description,
|
|
114
|
+
entry.tags.join(" "),
|
|
115
|
+
section,
|
|
116
|
+
].filter(Boolean).join("\n"),
|
|
117
|
+
}));
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function readMemoryBody(entry) {
|
|
121
|
+
try {
|
|
122
|
+
return parseMemoryMarkdown(readFileSync(entry.path, "utf8")).body.trim();
|
|
123
|
+
} catch {
|
|
124
|
+
return "";
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function splitMarkdownBody(body) {
|
|
129
|
+
const blocks = body
|
|
130
|
+
.split(/\n{2,}/)
|
|
131
|
+
.map((block) => block.trim())
|
|
132
|
+
.filter(Boolean);
|
|
133
|
+
const chunks = [];
|
|
134
|
+
let current = "";
|
|
135
|
+
for (const block of blocks) {
|
|
136
|
+
if (!current) {
|
|
137
|
+
current = block;
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
if (current.length + block.length + 2 <= MAX_CHUNK_CHARS) {
|
|
141
|
+
current = `${current}\n\n${block}`;
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
chunks.push(current);
|
|
145
|
+
current = block;
|
|
146
|
+
}
|
|
147
|
+
if (current) chunks.push(current);
|
|
148
|
+
return chunks.flatMap(splitOversizedChunk);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function splitOversizedChunk(text) {
|
|
152
|
+
if (text.length <= MAX_CHUNK_CHARS) return [text];
|
|
153
|
+
const chunks = [];
|
|
154
|
+
for (let index = 0; index < text.length; index += MAX_CHUNK_CHARS) {
|
|
155
|
+
chunks.push(text.slice(index, index + MAX_CHUNK_CHARS));
|
|
156
|
+
}
|
|
157
|
+
return chunks;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function entrySignature(entry) {
|
|
161
|
+
return `${entry.id}:${entry.path}:${Math.trunc(entry.mtimeMs ?? 0)}:${entry.size ?? 0}`;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function cosineSimilarity(left, right) {
|
|
165
|
+
if (!left?.norm || !right?.norm) return 0;
|
|
166
|
+
let dot = 0;
|
|
167
|
+
for (let index = 0; index < left.values.length; index += 1) dot += left.values[index] * right.values[index];
|
|
168
|
+
return dot / (left.norm * right.norm);
|
|
169
|
+
}
|
|
@@ -15,11 +15,6 @@ CREATE TABLE IF NOT EXISTS memory_index (
|
|
|
15
15
|
mtime_ms REAL NOT NULL,
|
|
16
16
|
size INTEGER NOT NULL
|
|
17
17
|
);
|
|
18
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS memory_tags_fts USING fts5(
|
|
19
|
-
id UNINDEXED,
|
|
20
|
-
tags_text,
|
|
21
|
-
tokenize = 'unicode61'
|
|
22
|
-
);
|
|
23
18
|
`;
|
|
24
19
|
|
|
25
20
|
export function openMarkdownMemoryIndex(path) {
|
|
@@ -33,7 +28,6 @@ export function openMarkdownMemoryIndex(path) {
|
|
|
33
28
|
|
|
34
29
|
export function clearMarkdownMemoryIndex(db) {
|
|
35
30
|
db.exec("DELETE FROM memory_index");
|
|
36
|
-
db.exec("DELETE FROM memory_tags_fts");
|
|
37
31
|
}
|
|
38
32
|
|
|
39
33
|
export function loadMarkdownMemoryIndex(db) {
|
|
@@ -48,17 +42,15 @@ export function loadMarkdownMemoryIndex(db) {
|
|
|
48
42
|
return { entries, pathStats };
|
|
49
43
|
}
|
|
50
44
|
|
|
51
|
-
export function replaceMarkdownMemoryIndex(db, entries
|
|
45
|
+
export function replaceMarkdownMemoryIndex(db, entries) {
|
|
52
46
|
db.exec("BEGIN IMMEDIATE");
|
|
53
47
|
try {
|
|
54
48
|
clearMarkdownMemoryIndex(db);
|
|
55
49
|
const insertMeta = db.prepare(
|
|
56
50
|
"INSERT INTO memory_index (id, path, name, description, tags_json, status, created_at, updated_at, mtime_ms, size) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
57
51
|
);
|
|
58
|
-
const insertFts = db.prepare("INSERT INTO memory_tags_fts (id, tags_text) VALUES (?, ?)");
|
|
59
52
|
for (const entry of entries.values()) {
|
|
60
53
|
insertMeta.run(entry.id, entry.path, entry.name, entry.description, JSON.stringify(entry.tags), entry.status, entry.createdAt, entry.updatedAt, entry.mtimeMs, entry.size);
|
|
61
|
-
insertFts.run(entry.id, expandTags(entry.tags).join(" "));
|
|
62
54
|
}
|
|
63
55
|
db.exec("COMMIT");
|
|
64
56
|
} catch (err) {
|
|
@@ -67,10 +59,6 @@ export function replaceMarkdownMemoryIndex(db, entries, expandTags) {
|
|
|
67
59
|
}
|
|
68
60
|
}
|
|
69
61
|
|
|
70
|
-
export function queryMarkdownMemoryIndex(db, query) {
|
|
71
|
-
return db.prepare("SELECT id FROM memory_tags_fts WHERE tags_text MATCH ? LIMIT 50").all(query);
|
|
72
|
-
}
|
|
73
|
-
|
|
74
62
|
function rowToEntry(row) {
|
|
75
63
|
return {
|
|
76
64
|
id: String(row.id),
|
|
@@ -1,17 +1,15 @@
|
|
|
1
1
|
import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { basename, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
3
3
|
import {
|
|
4
|
-
expandTags,
|
|
5
4
|
formatMemoryMarkdown,
|
|
6
5
|
generateMemoryId,
|
|
7
6
|
normalizeTags,
|
|
8
|
-
normalizeText,
|
|
9
7
|
parseMemoryMarkdown,
|
|
10
|
-
quoteFtsTerm,
|
|
11
8
|
walkMarkdownFiles,
|
|
12
9
|
} from "./markdown/markdown-format.mjs";
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
10
|
+
import { toHint } from "./markdown/markdown-recall.mjs";
|
|
11
|
+
import { SemanticMemoryRecallIndex } from "./markdown/semantic-recall.mjs";
|
|
12
|
+
import { clearMarkdownMemoryIndex, loadMarkdownMemoryIndex, openMarkdownMemoryIndex, replaceMarkdownMemoryIndex } from "./markdown/sqlite-index.mjs";
|
|
15
13
|
import { softDeleteMemoryFile } from "./markdown/markdown-delete.mjs";
|
|
16
14
|
import { isMemoryIdLike, isSingleEditAway } from "./markdown/memory-id.mjs";
|
|
17
15
|
import { openMarkdownRoot, searchMarkdownRoot } from "./search.mjs";
|
|
@@ -22,15 +20,17 @@ export { normalizeTags } from "./markdown/markdown-format.mjs";
|
|
|
22
20
|
const DEFAULT_SCAN_INTERVAL_MS = 5000;
|
|
23
21
|
|
|
24
22
|
export class MarkdownMemoryStore {
|
|
25
|
-
constructor({ root, now = () => new Date(), indexPath = null } = {}) {
|
|
23
|
+
constructor({ root, now = () => new Date(), indexPath = null, stateRoot = null, semanticRecall = true, semanticVectorizer = null, semanticModelId = undefined, semanticModelDir = null, semanticMinScore = undefined } = {}) {
|
|
26
24
|
if (!root) throw new Error("MarkdownMemoryStore requires a root path");
|
|
27
25
|
this.root = resolve(root);
|
|
28
26
|
this.now = now;
|
|
27
|
+
this.semanticRecall = semanticRecall ? new SemanticMemoryRecallIndex({ stateRoot, modelId: semanticModelId, modelDir: semanticModelDir, vectorizer: semanticVectorizer, minScore: semanticMinScore }) : null;
|
|
28
|
+
this.semanticRecallWarning = null;
|
|
29
|
+
this.lastUserRecallReport = null;
|
|
29
30
|
this.indexPath = indexPath ? resolve(indexPath) : join(this.root, ".march-memory-index.sqlite");
|
|
30
31
|
this.db = openMarkdownMemoryIndex(this.indexPath);
|
|
31
32
|
this.entries = new Map();
|
|
32
33
|
this.pathStats = new Map();
|
|
33
|
-
this.tagDictionary = new Set();
|
|
34
34
|
this.diagnostics = [];
|
|
35
35
|
this.lastScanAt = 0;
|
|
36
36
|
this.scanIntervalMs = DEFAULT_SCAN_INTERVAL_MS;
|
|
@@ -99,8 +99,7 @@ export class MarkdownMemoryStore {
|
|
|
99
99
|
this.entries = nextEntries;
|
|
100
100
|
this.pathStats = nextStats;
|
|
101
101
|
this.diagnostics = diagnostics;
|
|
102
|
-
this
|
|
103
|
-
replaceMarkdownMemoryIndex(this.db, this.entries, expandTags);
|
|
102
|
+
replaceMarkdownMemoryIndex(this.db, this.entries);
|
|
104
103
|
this.lastScanAt = Date.now();
|
|
105
104
|
return { entries: this.entries.size, diagnostics };
|
|
106
105
|
}
|
|
@@ -121,20 +120,18 @@ export class MarkdownMemoryStore {
|
|
|
121
120
|
this.db.close?.();
|
|
122
121
|
}
|
|
123
122
|
|
|
124
|
-
recallForUser(text, { limit = 3,
|
|
123
|
+
async recallForUser(text, { limit = 3, excludedIds = [] } = {}) {
|
|
125
124
|
const excluded = new Set([...excludedIds, ...this.turnSeenMemoryIds]);
|
|
126
|
-
const hints = this.#
|
|
127
|
-
for (const hint of hints)
|
|
128
|
-
this.turnSeenMemoryIds.add(hint.id);
|
|
129
|
-
}
|
|
125
|
+
const hints = await this.#recallSemantic(text, { limit, excluded });
|
|
126
|
+
for (const hint of hints) this.turnSeenMemoryIds.add(hint.id);
|
|
130
127
|
return hints;
|
|
131
128
|
}
|
|
132
129
|
|
|
133
|
-
recallForAssistant(text, { limit = 2,
|
|
130
|
+
async recallForAssistant(text, { limit = 2, excludedIds = [], candidateLimit = 3 } = {}) {
|
|
134
131
|
const excluded = new Set([...excludedIds, ...this.turnSeenMemoryIds]);
|
|
135
|
-
const hints = this.#
|
|
132
|
+
const { hints, report } = await this.#recallSemantic(text, { limit, excluded, candidateLimit, recordReport: false, returnReport: true });
|
|
136
133
|
for (const hint of hints) this.turnSeenMemoryIds.add(hint.id);
|
|
137
|
-
return hints;
|
|
134
|
+
return { hints, report };
|
|
138
135
|
}
|
|
139
136
|
|
|
140
137
|
searchRipgrep(query, { limit = 20, context = 2, syntax = "regex", case: caseMode = "smart", caseMode: explicitCaseMode = null, glob = [] } = {}) {
|
|
@@ -205,45 +202,28 @@ export class MarkdownMemoryStore {
|
|
|
205
202
|
return result;
|
|
206
203
|
}
|
|
207
204
|
|
|
208
|
-
#
|
|
205
|
+
async #recallSemantic(text, { limit, excluded, candidateLimit, recordReport = true, returnReport = false }) {
|
|
209
206
|
this.ensureFresh();
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
let rows = [];
|
|
207
|
+
if (recordReport) this.lastUserRecallReport = null;
|
|
208
|
+
const empty = { hints: [], report: null };
|
|
209
|
+
if (!this.semanticRecall?.enabled) return returnReport ? empty : [];
|
|
214
210
|
try {
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
#extractKnownTagTerms(text) {
|
|
233
|
-
const normalized = normalizeText(text);
|
|
234
|
-
if (!normalized) return [];
|
|
235
|
-
const terms = [];
|
|
236
|
-
for (const term of this.tagDictionary) {
|
|
237
|
-
if (term.length < 2) continue;
|
|
238
|
-
if (normalized.includes(term)) terms.push(term);
|
|
239
|
-
}
|
|
240
|
-
return [...new Set(terms)].sort((a, b) => b.length - a.length).slice(0, 16);
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
#rebuildTagDictionary() {
|
|
244
|
-
this.tagDictionary = new Set();
|
|
245
|
-
for (const entry of this.entries.values()) {
|
|
246
|
-
for (const term of expandTags(entry.tags)) this.tagDictionary.add(normalizeText(term));
|
|
211
|
+
const result = await this.semanticRecall.search(text, { entries: this.entries, excluded, limit, candidateLimit });
|
|
212
|
+
const hints = result.recalled.map(({ entry, score }) => toHint(entry, { score }));
|
|
213
|
+
const report = {
|
|
214
|
+
threshold: result.threshold,
|
|
215
|
+
vectorizerStatus: result.vectorizerStatus,
|
|
216
|
+
warning: result.warning,
|
|
217
|
+
hints,
|
|
218
|
+
candidates: result.candidates.map(({ entry, score, recalled }) => ({ ...toHint(entry, { score }), recalled })),
|
|
219
|
+
};
|
|
220
|
+
if (recordReport) this.lastUserRecallReport = report;
|
|
221
|
+
return returnReport ? { hints, report } : hints;
|
|
222
|
+
} catch (err) {
|
|
223
|
+
this.semanticRecallWarning = err?.message ?? String(err);
|
|
224
|
+
const report = { threshold: this.semanticRecall.minScore, vectorizerStatus: this.semanticRecall.status, warning: this.semanticRecallWarning, hints: [], candidates: [] };
|
|
225
|
+
if (recordReport) this.lastUserRecallReport = report;
|
|
226
|
+
return returnReport ? { hints: [], report } : [];
|
|
247
227
|
}
|
|
248
228
|
}
|
|
249
229
|
|