march-cli 0.1.42 → 0.1.46

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/package.json +1 -1
  2. package/src/agent/code-search/cache.mjs +16 -7
  3. package/src/agent/code-search/engine.mjs +9 -2
  4. package/src/agent/code-search/retrieval/resilient-vectorizer.mjs +59 -0
  5. package/src/agent/code-search/retrieval/safetensors.mjs +16 -10
  6. package/src/agent/code-search/scanner.mjs +11 -5
  7. package/src/agent/code-search/tool.mjs +11 -5
  8. package/src/agent/model-payload-dumper.mjs +1 -1
  9. package/src/agent/runner/payload/provider-payload-transform.mjs +59 -0
  10. package/src/agent/runner/recall/mid-turn-recall-bridge.mjs +23 -0
  11. package/src/agent/runner.mjs +28 -27
  12. package/src/agent/runtime/remote-ui-client.mjs +1 -1
  13. package/src/agent/runtime/resource/context-resource-loader.mjs +17 -0
  14. package/src/agent/runtime/runtime-factory.mjs +5 -1
  15. package/src/agent/runtime/state/runner-state.mjs +10 -3
  16. package/src/agent/runtime/ui-event-bridge.mjs +2 -2
  17. package/src/agent/turn/turn-runner.mjs +35 -24
  18. package/src/cli/fallback-ui.mjs +2 -2
  19. package/src/cli/repl-loop.mjs +9 -8
  20. package/src/cli/startup/app-runtime.mjs +4 -2
  21. package/src/cli/tui/input/mouse-selection-controller.mjs +19 -0
  22. package/src/cli/tui/output/selectable-copy.mjs +3 -3
  23. package/src/cli/tui/output/timeline-block-restore.mjs +1 -1
  24. package/src/cli/tui/output-buffer.mjs +18 -0
  25. package/src/cli/tui/recall-rendering.mjs +38 -9
  26. package/src/cli/tui/selection/ansi-range.mjs +88 -0
  27. package/src/cli/tui/selection-screen.mjs +31 -99
  28. package/src/cli/turn/turn-input-preparer.mjs +15 -6
  29. package/src/cli/ui.mjs +2 -2
  30. package/src/cli/workspace/tui-timeline-projection.mjs +1 -1
  31. package/src/context/engine.mjs +15 -5
  32. package/src/context/system-core/base.md +1 -1
  33. package/src/memory/markdown/markdown-format.mjs +0 -17
  34. package/src/memory/markdown/markdown-recall.mjs +11 -19
  35. package/src/memory/markdown/semantic-preload.mjs +26 -0
  36. package/src/memory/markdown/semantic-recall.mjs +169 -0
  37. package/src/memory/markdown/sqlite-index.mjs +1 -13
  38. package/src/memory/markdown-store.mjs +34 -54
  39. package/src/web-ui/dist/assets/{index-BQtl1uQs.css → index-BG1Pxf1k.css} +1 -1
  40. package/src/web-ui/dist/assets/{index-DrlJis_D.js → index-C0xOHlDz.js} +1 -1
  41. package/src/web-ui/dist/index.html +2 -2
  42. package/src/web-ui/runtime-host.mjs +18 -3
  43. package/src/web-ui/src/components/timeline/TimelineBlocks.tsx +27 -0
  44. package/src/web-ui/src/model.ts +18 -0
  45. package/src/web-ui/src/runtime/client.ts +2 -1
  46. package/src/web-ui/src/runtime/runtimeTimeline.ts +5 -0
  47. package/src/web-ui/src/styles/shell.css +7 -0
  48. package/src/web-ui/src/timelineAdapter.ts +2 -0
@@ -4,19 +4,21 @@ import { formatRecallHints } from "../../memory/markdown-store.mjs";
4
4
  import { formatMessageAttachmentsForDisplay } from "../../session/attachment-display.mjs";
5
5
  import { formatShellHints } from "../../shell/hints.mjs";
6
6
 
7
- export function prepareTurnInput({ prompt, runner, memoryStore, currentProject, modeState = null }) {
7
+ export async function prepareTurnInput({ prompt, runner, memoryStore, currentProject, modeState = null }) {
8
8
  const engine = runner.engine ?? {};
9
9
  const carryoverAlreadyRendered = engine.hasRenderedPendingAssistantRecallHints?.() ?? false;
10
- const carryoverRecallHints = engine.takePendingAssistantRecallHints?.() ?? [];
11
- const userRecallHints = memoryStore.recallForUser(prompt, {
10
+ const carryoverRecall = normalizePendingAssistantRecall(engine.takePendingAssistantRecallHints?.());
11
+ const carryoverRecallHints = carryoverRecall.hints;
12
+ const userRecallHints = await memoryStore.recallForUser(prompt, {
12
13
  currentProject,
13
14
  excludedIds: engine.getRecentRecallMemoryIds?.() ?? [],
14
15
  });
16
+ const userRecallReport = memoryStore.lastUserRecallReport ?? null;
15
17
  const modePrompt = appendModeReminder(prompt, modeState?.get?.());
16
18
  const fullPrompt = appendPromptBlocks(
17
19
  modePrompt,
18
- formatRecallHints("user", userRecallHints),
19
- formatRecallHints("assistant", carryoverRecallHints),
20
+ formatRecallHints(userRecallHints),
21
+ formatRecallHints(carryoverRecallHints),
20
22
  formatShellHints(runner.shellRuntime),
21
23
  );
22
24
 
@@ -26,8 +28,10 @@ export function prepareTurnInput({ prompt, runner, memoryStore, currentProject,
26
28
  runOptions: { userRecallHints, currentProject },
27
29
  displayMessage: formatUserDisplayMessage(prompt),
28
30
  userRecallHints,
31
+ userRecallReport,
29
32
  carryoverRecallHints,
30
- shouldRenderCarryoverRecall: carryoverRecallHints.length > 0 && !carryoverAlreadyRendered,
33
+ carryoverRecallReport: carryoverRecall.report,
34
+ shouldRenderCarryoverRecall: (carryoverRecallHints.length > 0 || carryoverRecall.report) && !carryoverAlreadyRendered,
31
35
  };
32
36
  }
33
37
 
@@ -38,3 +42,8 @@ export function formatUserDisplayMessage(prompt) {
38
42
  function appendPromptBlocks(prompt, ...blocks) {
39
43
  return [prompt, ...blocks.filter(Boolean)].join("\n\n");
40
44
  }
45
+
46
+ function normalizePendingAssistantRecall(value) {
47
+ if (Array.isArray(value)) return { hints: value, report: null };
48
+ return { hints: value?.hints ?? [], report: value?.report ?? null };
49
+ }
package/src/cli/ui.mjs CHANGED
@@ -212,8 +212,8 @@ export function createTuiUI({
212
212
  status: (text) => {
213
213
  ensureStarted(); flushStreamDeltas(); retryStatus.stop(); spinnerStatus.stop(); output.setOverlayStatus([brightBlack(`● ${text}`)]); requestRender();
214
214
  },
215
- recall: ({ hints }) => {
216
- ensureStarted(); flushStreamDeltas(); retryStatus.stop(); spinnerStatus.stop(); output.ensureNewline(); writeRecall({ output, hints }); requestRender();
215
+ recall: ({ hints, report, variant }) => {
216
+ ensureStarted(); flushStreamDeltas(); retryStatus.stop(); spinnerStatus.stop(); output.ensureNewline(); writeRecall({ output, hints, report, variant }); requestRender();
217
217
  },
218
218
 
219
219
  clearOutput: () => {
@@ -94,7 +94,7 @@ export function createTuiTimelineProjection() {
94
94
  blocks.push(createBlock("status", event.at, { content: String(first ?? "") }));
95
95
  break;
96
96
  case "recall":
97
- blocks.push(createBlock("recall", event.at, { hints: first?.hints ?? [] }));
97
+ blocks.push(createBlock("recall", event.at, { hints: first?.hints ?? [], report: first?.report ?? null }));
98
98
  break;
99
99
  case "editDiff":
100
100
  closeAssistantBlock(event.at);
@@ -17,6 +17,7 @@ export class ContextEngine {
17
17
  this.thinkingLevel = thinkingLevel;
18
18
  this.turns = [];
19
19
  this.pendingAssistantRecallHints = [];
20
+ this.pendingAssistantRecallReport = null;
20
21
  this.pendingAssistantRecallHintsRendered = false;
21
22
  this.sessionName = "";
22
23
  this.toolDefs = [];
@@ -96,8 +97,9 @@ export class ContextEngine {
96
97
  return ids;
97
98
  }
98
99
 
99
- setPendingAssistantRecallHints(hints = []) {
100
+ setPendingAssistantRecallHints(hints = [], report = null) {
100
101
  this.pendingAssistantRecallHints = uniqueHints(hints);
102
+ this.pendingAssistantRecallReport = report;
101
103
  this.pendingAssistantRecallHintsRendered = false;
102
104
  }
103
105
 
@@ -105,19 +107,25 @@ export class ContextEngine {
105
107
  return this.pendingAssistantRecallHints;
106
108
  }
107
109
 
110
+ peekPendingAssistantRecallReport() {
111
+ return this.pendingAssistantRecallReport;
112
+ }
113
+
108
114
  hasRenderedPendingAssistantRecallHints() {
109
115
  return this.pendingAssistantRecallHintsRendered;
110
116
  }
111
117
 
112
118
  markPendingAssistantRecallHintsRendered() {
113
- if (this.pendingAssistantRecallHints.length > 0) this.pendingAssistantRecallHintsRendered = true;
119
+ if (this.pendingAssistantRecallHints.length > 0 || this.pendingAssistantRecallReport) this.pendingAssistantRecallHintsRendered = true;
114
120
  }
115
121
 
116
122
  takePendingAssistantRecallHints() {
117
123
  const hints = this.pendingAssistantRecallHints;
124
+ const report = this.pendingAssistantRecallReport;
118
125
  this.pendingAssistantRecallHints = [];
126
+ this.pendingAssistantRecallReport = null;
119
127
  this.pendingAssistantRecallHintsRendered = false;
120
- return hints;
128
+ return { hints, report };
121
129
  }
122
130
 
123
131
  resolvePath(raw) {
@@ -142,12 +150,14 @@ export class ContextEngine {
142
150
  if (replace) {
143
151
  this.turns = [];
144
152
  this.pendingAssistantRecallHints = [];
153
+ this.pendingAssistantRecallReport = null;
145
154
  this.pendingAssistantRecallHintsRendered = false;
146
155
  this.sessionName = "";
147
156
  }
148
157
  if (data.turns) this.turns = data.turns;
149
158
  if (Array.isArray(data.pendingAssistantRecallHints)) {
150
159
  this.pendingAssistantRecallHints = uniqueHints(data.pendingAssistantRecallHints);
160
+ this.pendingAssistantRecallReport = data.pendingAssistantRecallReport ?? null;
151
161
  this.pendingAssistantRecallHintsRendered = false;
152
162
  }
153
163
  if (typeof data.sessionName === "string") this.sessionName = data.sessionName;
@@ -168,14 +178,14 @@ export class ContextEngine {
168
178
  for (const turn of this.turns) {
169
179
  let block = `## Turn ${turn.index}\n` +
170
180
  `[user]\n${String(turn.userMessage ?? "")}\n`;
171
- const userRecall = formatRecallHints("user", turn.userRecallHints ?? []);
181
+ const userRecall = formatRecallHints(turn.userRecallHints ?? []);
172
182
  if (userRecall) block += `\n${userRecall}\n`;
173
183
  block += `\n[assistant]\n`;
174
184
  const assistantText = turn.assistantContext || turn.assistantMessage;
175
185
  if (assistantText) {
176
186
  block += `\n${String(assistantText ?? "")}\n`;
177
187
  }
178
- const assistantRecall = formatRecallHints("assistant", turn.assistantRecallHints ?? []);
188
+ const assistantRecall = formatRecallHints(turn.assistantRecallHints ?? []);
179
189
  if (assistantRecall) block += `\n${assistantRecall}\n`;
180
190
  entries.push(block);
181
191
  }
@@ -96,7 +96,7 @@ The user primarily asks for software engineering work: fixing bugs, adding behav
96
96
  </git_contract>
97
97
 
98
98
  <memory_system>
99
- - [recall source="..."] blocks in recent_chat are lightweight recall hints matched from prior thinking output. Treat them as possibly relevant pointers, not as complete facts.
99
+ - [recall] blocks in recent_chat are lightweight memory hints matched by semantic recall. Treat them as possibly relevant pointers, not as complete facts.
100
100
  - A recall hint's description may record key operational constraints, including when the full memory must be opened; factor those constraints into relevance before acting.
101
101
  - If a recall hint may help the current task, use memory_open(id) to read the full memory before relying on it. Ignore hints that are clearly unrelated or too low-value for the task.
102
102
  - Use memory_search(query) for full-text search across all memories.
@@ -32,24 +32,7 @@ export function normalizeTags(tags) {
32
32
  return out;
33
33
  }
34
34
 
35
- export function expandTags(tags) {
36
- const terms = [];
37
- for (const tag of tags) {
38
- terms.push(tag);
39
- for (const part of tag.split(/[\/_-]+/)) {
40
- if (part) terms.push(part);
41
- }
42
- }
43
- return [...new Set(terms.map(normalizeText).filter(Boolean))];
44
- }
45
35
 
46
- export function quoteFtsTerm(term) {
47
- return `"${String(term).replace(/"/g, '""')}"`;
48
- }
49
-
50
- export function normalizeText(text) {
51
- return String(text ?? "").trim().toLowerCase();
52
- }
53
36
 
54
37
  export function generateMemoryId() {
55
38
  return `mem_${randomUUID().replace(/-/g, "").slice(0, 16)}`;
@@ -1,28 +1,20 @@
1
- import { expandTags, normalizeText } from "./markdown-format.mjs";
2
-
3
- export function formatRecallHints(source, hints = []) {
1
+ export function formatRecallHints(hints = []) {
4
2
  if (!hints.length) return "";
5
- const lines = [`[recall source="${source}"]`];
3
+ const lines = ["[recall]"];
6
4
  for (const hint of hints) {
7
- lines.push(`- ${hint.id} | ${hint.name} | ${hint.description}`);
5
+ lines.push(`- ${hint.id}${formatScoreForPrompt(hint.score)} | ${hint.name} | ${hint.description}`);
8
6
  }
9
7
  return lines.join("\n");
10
8
  }
11
9
 
12
- export function scoreEntry(entry, terms, currentProject) {
13
- const expanded = expandTags(entry.tags);
14
- let score = 0;
15
- for (const term of terms) {
16
- if (entry.tags.map(normalizeText).includes(term)) score += 10;
17
- else if (expanded.includes(term)) score += 5;
18
- }
19
- if (currentProject) {
20
- const projectTag = normalizeText(`project/${currentProject}`);
21
- if (entry.tags.map(normalizeText).includes(projectTag)) score += 2;
22
- }
23
- return score;
10
+ export function toHint(entry, metadata = {}) {
11
+ return { id: entry.id, name: entry.name, description: entry.description, ...metadata };
12
+ }
13
+
14
+ function formatScoreForPrompt(score) {
15
+ return Number.isFinite(score) ? ` | score=${formatScore(score)}` : "";
24
16
  }
25
17
 
26
- export function toHint(entry) {
27
- return { id: entry.id, name: entry.name, description: entry.description };
18
+ export function formatScore(score) {
19
+ return Number.isFinite(score) ? score.toFixed(2) : "--";
28
20
  }
@@ -0,0 +1,26 @@
1
+ export async function preloadSemanticMemoryRecall({ memoryStore, ui = null, logger = null } = {}) {
2
+ if (!memoryStore?.semanticRecall?.enabled) return { ok: true, skipped: true };
3
+ try {
4
+ ui?.status?.("Preparing memory recall model...");
5
+ await memoryStore.semanticRecall.preload();
6
+ memoryStore.semanticRecallWarning = memoryStore.semanticRecall.warning;
7
+ if (memoryStore.semanticRecallWarning) ui?.writeln?.(`Memory recall fallback: ${memoryStore.semanticRecallWarning}`);
8
+ logger?.event?.("memory.semantic_model_ready", { modelId: memoryStore.semanticRecall.modelId, status: memoryStore.semanticRecall.status });
9
+ return { ok: true, skipped: false, fallback: memoryStore.semanticRecall.status === "fallback" };
10
+ } catch (err) {
11
+ const message = err?.message ?? String(err);
12
+ memoryStore.semanticRecallWarning = message;
13
+ logger?.error?.("memory.semantic_model_preload_failed", { error: message });
14
+ ui?.writeln?.(`Memory recall model preload failed: ${message}`);
15
+ return { ok: false, error: message };
16
+ }
17
+ }
18
+
19
+ export function startSemanticMemoryRecallPreload({ memoryStore, ui = null, logger = null, delayMs = 0 } = {}) {
20
+ if (!memoryStore?.semanticRecall?.enabled) return null;
21
+ const start = () => preloadSemanticMemoryRecall({ memoryStore, ui, logger });
22
+ if (delayMs <= 0) return start();
23
+ const timer = setTimeout(start, delayMs);
24
+ timer.unref?.();
25
+ return timer;
26
+ }
@@ -0,0 +1,169 @@
1
+ import { readFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { Model2VecVectorizer } from "../../agent/code-search/retrieval/model2vec.mjs";
4
+ import { ResilientVectorizer } from "../../agent/code-search/retrieval/resilient-vectorizer.mjs";
5
+ import { parseMemoryMarkdown } from "./markdown-format.mjs";
6
+
7
+ export const POTION_RETRIEVAL_MODEL_ID = "minishlab/potion-retrieval-32M";
8
+
9
+ const MAX_CHUNK_CHARS = 1800;
10
+ export const DEFAULT_MEMORY_RECALL_MIN_SCORE = 0.5;
11
+
12
+ export class SemanticMemoryRecallIndex {
13
+ constructor({ stateRoot = null, modelId = POTION_RETRIEVAL_MODEL_ID, modelDir = null, vectorizer = null, minScore = parseMemoryRecallMinScore() } = {}) {
14
+ this.modelId = modelId;
15
+ this.minScore = minScore;
16
+ this.vectorizer = vectorizer ?? createDefaultVectorizer({ stateRoot, modelId, modelDir });
17
+ this.signature = "";
18
+ this.chunks = [];
19
+ this.vectors = [];
20
+ }
21
+
22
+ get enabled() {
23
+ return Boolean(this.vectorizer);
24
+ }
25
+
26
+ get warning() {
27
+ return this.vectorizer?.warning ?? null;
28
+ }
29
+
30
+ get status() {
31
+ return this.vectorizer?.status ?? "primary";
32
+ }
33
+
34
+ async preload() {
35
+ if (!this.vectorizer) return false;
36
+ if (typeof this.vectorizer.load === "function") await this.vectorizer.load();
37
+ else await this.vectorizer.encode(["memory recall warmup"]);
38
+ return true;
39
+ }
40
+
41
+ async search(query, { entries, excluded = new Set(), limit = 3, candidateLimit = 5 } = {}) {
42
+ const empty = { recalled: [], candidates: [], threshold: this.minScore };
43
+ if (!this.vectorizer || !String(query ?? "").trim()) return empty;
44
+ const activeEntries = [...entries.values()].filter((entry) => entry.status === "active" && entry.description && !excluded.has(entry.id));
45
+ if (activeEntries.length === 0) return empty;
46
+ await this.#ensureIndex(activeEntries);
47
+ const [queryVector] = await this.vectorizer.encode([query]);
48
+ if (!queryVector || queryVector.norm === 0) return empty;
49
+
50
+ const bestByEntry = new Map();
51
+ for (let index = 0; index < this.vectors.length; index += 1) {
52
+ const chunk = this.chunks[index];
53
+ if (excluded.has(chunk.entry.id)) continue;
54
+ const score = cosineSimilarity(queryVector, this.vectors[index]);
55
+ const prev = bestByEntry.get(chunk.entry.id);
56
+ if (!prev || score > prev.score) bestByEntry.set(chunk.entry.id, { entry: chunk.entry, score });
57
+ }
58
+
59
+ const ranked = [...bestByEntry.values()]
60
+ .filter(({ score }) => Number.isFinite(score) && score > 0)
61
+ .sort((a, b) => b.score - a.score || a.entry.name.localeCompare(b.entry.name));
62
+ const recalled = ranked.filter(({ score }) => score >= this.minScore).slice(0, limit);
63
+ const recalledIds = new Set(recalled.map(({ entry }) => entry.id));
64
+ const candidates = ranked
65
+ .slice(0, Math.max(limit, candidateLimit))
66
+ .map(({ entry, score }) => ({ entry, score, recalled: recalledIds.has(entry.id) }));
67
+ return {
68
+ recalled,
69
+ candidates,
70
+ threshold: this.minScore,
71
+ vectorizerStatus: this.status,
72
+ warning: this.warning,
73
+ };
74
+ }
75
+
76
+ async #ensureIndex(entries) {
77
+ const signature = entries.map(entrySignature).join("\n");
78
+ if (signature === this.signature) return;
79
+ this.chunks = entries.flatMap(memoryChunks);
80
+ this.vectors = this.chunks.length > 0
81
+ ? await this.vectorizer.encode(this.chunks.map((chunk) => chunk.text))
82
+ : [];
83
+ this.signature = signature;
84
+ }
85
+ }
86
+
87
+ export function parseMemoryRecallMinScore(value = process.env.MARCH_MEMORY_RECALL_MIN_SCORE) {
88
+ if (value == null || value === "") return DEFAULT_MEMORY_RECALL_MIN_SCORE;
89
+ const normalized = String(value).trim().toLowerCase();
90
+ if (["false", "no", "off"].includes(normalized)) return 0;
91
+ const parsed = Number(normalized);
92
+ return Number.isFinite(parsed) && parsed >= 0 ? parsed : DEFAULT_MEMORY_RECALL_MIN_SCORE;
93
+ }
94
+
95
+ function createDefaultVectorizer({ stateRoot, modelId, modelDir }) {
96
+ const dir = modelDir ?? (stateRoot ? join(stateRoot, "memory", "models", modelId.replaceAll("/", "__")) : null);
97
+ if (!dir) return null;
98
+ return new ResilientVectorizer({
99
+ primary: new Model2VecVectorizer({ modelDir: dir, modelId }),
100
+ label: "memory recall",
101
+ });
102
+ }
103
+
104
+ function memoryChunks(entry) {
105
+ const body = readMemoryBody(entry);
106
+ const sections = splitMarkdownBody(body);
107
+ const chunks = sections.length > 0 ? sections : [""];
108
+ return chunks.map((section, index) => ({
109
+ entry,
110
+ index,
111
+ text: [
112
+ entry.name,
113
+ entry.description,
114
+ entry.tags.join(" "),
115
+ section,
116
+ ].filter(Boolean).join("\n"),
117
+ }));
118
+ }
119
+
120
+ function readMemoryBody(entry) {
121
+ try {
122
+ return parseMemoryMarkdown(readFileSync(entry.path, "utf8")).body.trim();
123
+ } catch {
124
+ return "";
125
+ }
126
+ }
127
+
128
+ function splitMarkdownBody(body) {
129
+ const blocks = body
130
+ .split(/\n{2,}/)
131
+ .map((block) => block.trim())
132
+ .filter(Boolean);
133
+ const chunks = [];
134
+ let current = "";
135
+ for (const block of blocks) {
136
+ if (!current) {
137
+ current = block;
138
+ continue;
139
+ }
140
+ if (current.length + block.length + 2 <= MAX_CHUNK_CHARS) {
141
+ current = `${current}\n\n${block}`;
142
+ continue;
143
+ }
144
+ chunks.push(current);
145
+ current = block;
146
+ }
147
+ if (current) chunks.push(current);
148
+ return chunks.flatMap(splitOversizedChunk);
149
+ }
150
+
151
+ function splitOversizedChunk(text) {
152
+ if (text.length <= MAX_CHUNK_CHARS) return [text];
153
+ const chunks = [];
154
+ for (let index = 0; index < text.length; index += MAX_CHUNK_CHARS) {
155
+ chunks.push(text.slice(index, index + MAX_CHUNK_CHARS));
156
+ }
157
+ return chunks;
158
+ }
159
+
160
+ function entrySignature(entry) {
161
+ return `${entry.id}:${entry.path}:${Math.trunc(entry.mtimeMs ?? 0)}:${entry.size ?? 0}`;
162
+ }
163
+
164
+ function cosineSimilarity(left, right) {
165
+ if (!left?.norm || !right?.norm) return 0;
166
+ let dot = 0;
167
+ for (let index = 0; index < left.values.length; index += 1) dot += left.values[index] * right.values[index];
168
+ return dot / (left.norm * right.norm);
169
+ }
@@ -15,11 +15,6 @@ CREATE TABLE IF NOT EXISTS memory_index (
15
15
  mtime_ms REAL NOT NULL,
16
16
  size INTEGER NOT NULL
17
17
  );
18
- CREATE VIRTUAL TABLE IF NOT EXISTS memory_tags_fts USING fts5(
19
- id UNINDEXED,
20
- tags_text,
21
- tokenize = 'unicode61'
22
- );
23
18
  `;
24
19
 
25
20
  export function openMarkdownMemoryIndex(path) {
@@ -33,7 +28,6 @@ export function openMarkdownMemoryIndex(path) {
33
28
 
34
29
  export function clearMarkdownMemoryIndex(db) {
35
30
  db.exec("DELETE FROM memory_index");
36
- db.exec("DELETE FROM memory_tags_fts");
37
31
  }
38
32
 
39
33
  export function loadMarkdownMemoryIndex(db) {
@@ -48,17 +42,15 @@ export function loadMarkdownMemoryIndex(db) {
48
42
  return { entries, pathStats };
49
43
  }
50
44
 
51
- export function replaceMarkdownMemoryIndex(db, entries, expandTags) {
45
+ export function replaceMarkdownMemoryIndex(db, entries) {
52
46
  db.exec("BEGIN IMMEDIATE");
53
47
  try {
54
48
  clearMarkdownMemoryIndex(db);
55
49
  const insertMeta = db.prepare(
56
50
  "INSERT INTO memory_index (id, path, name, description, tags_json, status, created_at, updated_at, mtime_ms, size) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
57
51
  );
58
- const insertFts = db.prepare("INSERT INTO memory_tags_fts (id, tags_text) VALUES (?, ?)");
59
52
  for (const entry of entries.values()) {
60
53
  insertMeta.run(entry.id, entry.path, entry.name, entry.description, JSON.stringify(entry.tags), entry.status, entry.createdAt, entry.updatedAt, entry.mtimeMs, entry.size);
61
- insertFts.run(entry.id, expandTags(entry.tags).join(" "));
62
54
  }
63
55
  db.exec("COMMIT");
64
56
  } catch (err) {
@@ -67,10 +59,6 @@ export function replaceMarkdownMemoryIndex(db, entries, expandTags) {
67
59
  }
68
60
  }
69
61
 
70
- export function queryMarkdownMemoryIndex(db, query) {
71
- return db.prepare("SELECT id FROM memory_tags_fts WHERE tags_text MATCH ? LIMIT 50").all(query);
72
- }
73
-
74
62
  function rowToEntry(row) {
75
63
  return {
76
64
  id: String(row.id),
@@ -1,17 +1,15 @@
1
1
  import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
2
2
  import { basename, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
3
3
  import {
4
- expandTags,
5
4
  formatMemoryMarkdown,
6
5
  generateMemoryId,
7
6
  normalizeTags,
8
- normalizeText,
9
7
  parseMemoryMarkdown,
10
- quoteFtsTerm,
11
8
  walkMarkdownFiles,
12
9
  } from "./markdown/markdown-format.mjs";
13
- import { scoreEntry, toHint } from "./markdown/markdown-recall.mjs";
14
- import { clearMarkdownMemoryIndex, loadMarkdownMemoryIndex, openMarkdownMemoryIndex, queryMarkdownMemoryIndex, replaceMarkdownMemoryIndex } from "./markdown/sqlite-index.mjs";
10
+ import { toHint } from "./markdown/markdown-recall.mjs";
11
+ import { SemanticMemoryRecallIndex } from "./markdown/semantic-recall.mjs";
12
+ import { clearMarkdownMemoryIndex, loadMarkdownMemoryIndex, openMarkdownMemoryIndex, replaceMarkdownMemoryIndex } from "./markdown/sqlite-index.mjs";
15
13
  import { softDeleteMemoryFile } from "./markdown/markdown-delete.mjs";
16
14
  import { isMemoryIdLike, isSingleEditAway } from "./markdown/memory-id.mjs";
17
15
  import { openMarkdownRoot, searchMarkdownRoot } from "./search.mjs";
@@ -22,15 +20,17 @@ export { normalizeTags } from "./markdown/markdown-format.mjs";
22
20
  const DEFAULT_SCAN_INTERVAL_MS = 5000;
23
21
 
24
22
  export class MarkdownMemoryStore {
25
- constructor({ root, now = () => new Date(), indexPath = null } = {}) {
23
+ constructor({ root, now = () => new Date(), indexPath = null, stateRoot = null, semanticRecall = true, semanticVectorizer = null, semanticModelId = undefined, semanticModelDir = null, semanticMinScore = undefined } = {}) {
26
24
  if (!root) throw new Error("MarkdownMemoryStore requires a root path");
27
25
  this.root = resolve(root);
28
26
  this.now = now;
27
+ this.semanticRecall = semanticRecall ? new SemanticMemoryRecallIndex({ stateRoot, modelId: semanticModelId, modelDir: semanticModelDir, vectorizer: semanticVectorizer, minScore: semanticMinScore }) : null;
28
+ this.semanticRecallWarning = null;
29
+ this.lastUserRecallReport = null;
29
30
  this.indexPath = indexPath ? resolve(indexPath) : join(this.root, ".march-memory-index.sqlite");
30
31
  this.db = openMarkdownMemoryIndex(this.indexPath);
31
32
  this.entries = new Map();
32
33
  this.pathStats = new Map();
33
- this.tagDictionary = new Set();
34
34
  this.diagnostics = [];
35
35
  this.lastScanAt = 0;
36
36
  this.scanIntervalMs = DEFAULT_SCAN_INTERVAL_MS;
@@ -99,8 +99,7 @@ export class MarkdownMemoryStore {
99
99
  this.entries = nextEntries;
100
100
  this.pathStats = nextStats;
101
101
  this.diagnostics = diagnostics;
102
- this.#rebuildTagDictionary();
103
- replaceMarkdownMemoryIndex(this.db, this.entries, expandTags);
102
+ replaceMarkdownMemoryIndex(this.db, this.entries);
104
103
  this.lastScanAt = Date.now();
105
104
  return { entries: this.entries.size, diagnostics };
106
105
  }
@@ -121,20 +120,18 @@ export class MarkdownMemoryStore {
121
120
  this.db.close?.();
122
121
  }
123
122
 
124
- recallForUser(text, { limit = 3, currentProject = "", excludedIds = [] } = {}) {
123
+ async recallForUser(text, { limit = 3, excludedIds = [] } = {}) {
125
124
  const excluded = new Set([...excludedIds, ...this.turnSeenMemoryIds]);
126
- const hints = this.#recall(text, { limit, excluded, currentProject });
127
- for (const hint of hints) {
128
- this.turnSeenMemoryIds.add(hint.id);
129
- }
125
+ const hints = await this.#recallSemantic(text, { limit, excluded });
126
+ for (const hint of hints) this.turnSeenMemoryIds.add(hint.id);
130
127
  return hints;
131
128
  }
132
129
 
133
- recallForAssistant(text, { limit = 2, currentProject = "", excludedIds = [] } = {}) {
130
+ async recallForAssistant(text, { limit = 2, excludedIds = [], candidateLimit = 3 } = {}) {
134
131
  const excluded = new Set([...excludedIds, ...this.turnSeenMemoryIds]);
135
- const hints = this.#recall(text, { limit, excluded, currentProject });
132
+ const { hints, report } = await this.#recallSemantic(text, { limit, excluded, candidateLimit, recordReport: false, returnReport: true });
136
133
  for (const hint of hints) this.turnSeenMemoryIds.add(hint.id);
137
- return hints;
134
+ return { hints, report };
138
135
  }
139
136
 
140
137
  searchRipgrep(query, { limit = 20, context = 2, syntax = "regex", case: caseMode = "smart", caseMode: explicitCaseMode = null, glob = [] } = {}) {
@@ -205,45 +202,28 @@ export class MarkdownMemoryStore {
205
202
  return result;
206
203
  }
207
204
 
208
- #recall(text, { limit, excluded, currentProject }) {
205
+ async #recallSemantic(text, { limit, excluded, candidateLimit, recordReport = true, returnReport = false }) {
209
206
  this.ensureFresh();
210
- const queryTerms = this.#extractKnownTagTerms(text);
211
- if (queryTerms.length === 0) return [];
212
- const query = queryTerms.map(quoteFtsTerm).join(" OR ");
213
- let rows = [];
207
+ if (recordReport) this.lastUserRecallReport = null;
208
+ const empty = { hints: [], report: null };
209
+ if (!this.semanticRecall?.enabled) return returnReport ? empty : [];
214
210
  try {
215
- rows = queryMarkdownMemoryIndex(this.db, query);
216
- } catch {
217
- return [];
218
- }
219
- const scored = [];
220
- for (const row of rows) {
221
- if (excluded.has(row.id)) continue;
222
- const entry = this.entries.get(row.id);
223
- if (!entry || entry.status !== "active" || !entry.description) continue;
224
- const score = scoreEntry(entry, queryTerms, currentProject);
225
- if (score <= 0) continue;
226
- scored.push({ score, entry });
227
- }
228
- scored.sort((a, b) => b.score - a.score || a.entry.name.localeCompare(b.entry.name));
229
- return scored.slice(0, limit).map(({ entry }) => toHint(entry));
230
- }
231
-
232
- #extractKnownTagTerms(text) {
233
- const normalized = normalizeText(text);
234
- if (!normalized) return [];
235
- const terms = [];
236
- for (const term of this.tagDictionary) {
237
- if (term.length < 2) continue;
238
- if (normalized.includes(term)) terms.push(term);
239
- }
240
- return [...new Set(terms)].sort((a, b) => b.length - a.length).slice(0, 16);
241
- }
242
-
243
- #rebuildTagDictionary() {
244
- this.tagDictionary = new Set();
245
- for (const entry of this.entries.values()) {
246
- for (const term of expandTags(entry.tags)) this.tagDictionary.add(normalizeText(term));
211
+ const result = await this.semanticRecall.search(text, { entries: this.entries, excluded, limit, candidateLimit });
212
+ const hints = result.recalled.map(({ entry, score }) => toHint(entry, { score }));
213
+ const report = {
214
+ threshold: result.threshold,
215
+ vectorizerStatus: result.vectorizerStatus,
216
+ warning: result.warning,
217
+ hints,
218
+ candidates: result.candidates.map(({ entry, score, recalled }) => ({ ...toHint(entry, { score }), recalled })),
219
+ };
220
+ if (recordReport) this.lastUserRecallReport = report;
221
+ return returnReport ? { hints, report } : hints;
222
+ } catch (err) {
223
+ this.semanticRecallWarning = err?.message ?? String(err);
224
+ const report = { threshold: this.semanticRecall.minScore, vectorizerStatus: this.semanticRecall.status, warning: this.semanticRecallWarning, hints: [], candidates: [] };
225
+ if (recordReport) this.lastUserRecallReport = report;
226
+ return returnReport ? { hints: [], report } : [];
247
227
  }
248
228
  }
249
229