mind-palace-graph 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INSTALL.md +387 -0
- package/README.md +602 -0
- package/dist/api.d.ts +682 -0
- package/dist/api.js +660 -0
- package/dist/api.js.map +1 -0
- package/dist/cli.d.ts +95 -0
- package/dist/cli.js +856 -0
- package/dist/cli.js.map +1 -0
- package/dist/format.d.ts +16 -0
- package/dist/format.js +199 -0
- package/dist/format.js.map +1 -0
- package/dist/fuzzy.d.ts +45 -0
- package/dist/fuzzy.js +150 -0
- package/dist/fuzzy.js.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +528 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp-server.d.ts +24 -0
- package/dist/mcp-server.js +187 -0
- package/dist/mcp-server.js.map +1 -0
- package/dist/mind-palace.d.ts +148 -0
- package/dist/mind-palace.js +780 -0
- package/dist/mind-palace.js.map +1 -0
- package/dist/nodes.d.ts +57 -0
- package/dist/nodes.js +220 -0
- package/dist/nodes.js.map +1 -0
- package/dist/pagination.d.ts +41 -0
- package/dist/pagination.js +63 -0
- package/dist/pagination.js.map +1 -0
- package/dist/palace-format.d.ts +30 -0
- package/dist/palace-format.js +146 -0
- package/dist/palace-format.js.map +1 -0
- package/dist/rg.d.ts +34 -0
- package/dist/rg.js +288 -0
- package/dist/rg.js.map +1 -0
- package/dist/sources.d.ts +87 -0
- package/dist/sources.js +457 -0
- package/dist/sources.js.map +1 -0
- package/dist/tokens.d.ts +35 -0
- package/dist/tokens.js +95 -0
- package/dist/tokens.js.map +1 -0
- package/dist/types.d.ts +236 -0
- package/dist/types.js +8 -0
- package/dist/types.js.map +1 -0
- package/package.json +67 -0
- package/skills/mpg-context/SKILL.md +556 -0
- package/skills/mpg-context/references/anti-patterns.md +133 -0
- package/skills/mpg-context/references/integration.md +123 -0
- package/skills/mpg-context/references/mind-palace.md +217 -0
- package/skills/mpg-context/references/multi-agent.md +147 -0
- package/skills/mpg-context/references/sources.md +120 -0
package/dist/api.js
ADDED
|
@@ -0,0 +1,660 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Programmatic API for mpg.
|
|
3
|
+
*
|
|
4
|
+
* This is the surface an LLM harness should embed against instead of
|
|
5
|
+
* shelling out to the CLI. It exposes the same operations the CLI
|
|
6
|
+
* does, but as async functions returning structured data.
|
|
7
|
+
*
|
|
8
|
+
* The CLI is a thin wrapper over this module. Conversely, this module
|
|
9
|
+
* is what powers `mpg search`, `mpg stash`, etc. for in-process use.
|
|
10
|
+
*/
|
|
11
|
+
import { closeSync, openSync, readSync, statSync } from "node:fs";
|
|
12
|
+
import { applyTotalBudget, applyWindowCurve, buildNode, loadSourceContent } from "./nodes.js";
|
|
13
|
+
import { buildFuzzyRegex, verifyFuzzy } from "./fuzzy.js";
|
|
14
|
+
import { RgError, runRg } from "./rg.js";
|
|
15
|
+
import { captureCommand, captureStdin, captureUrl, classifyPathSpecs, } from "./sources.js";
|
|
16
|
+
import { addStash as addStashToPalace, composeToSources, defaultPalacePath, dropStash as dropStashFromPalace, getStash as getStashFromPalace, listStashes as listStashesFromPalace, loadPalace, savePalace, } from "./mind-palace.js";
|
|
17
|
+
// ─── Search ─────────────────────────────────────────────────────────
|
|
18
|
+
const EFFORT_DEFAULTS = {
|
|
19
|
+
// "scan" — index mode. Many nodes with tiny disambiguating windows
|
|
20
|
+
// (~20 tokens on each side of the match). Purpose: see all hits at
|
|
21
|
+
// once, then pick which file/page to dig into with quick/normal/deep.
|
|
22
|
+
// This is the "index -> detail" pattern: scan first to find what's
|
|
23
|
+
// relevant; targeted small queries follow up on the chosen file(s).
|
|
24
|
+
// Tokens scale O(n) with hit count (~60 tok/match on average);
|
|
25
|
+
// maxNodes is intentionally high so scan matches rg's recall AND
|
|
26
|
+
// precision regardless of hit count. Use --max-tokens if you want
|
|
27
|
+
// to cap by budget instead.
|
|
28
|
+
scan: { before: 20, after: 20, maxNodes: 100000 },
|
|
29
|
+
quick: { before: 200, after: 200, maxNodes: 10 },
|
|
30
|
+
normal: { before: 500, after: 500, maxNodes: 30 },
|
|
31
|
+
deep: { before: 2000, after: 2000, maxNodes: 100 },
|
|
32
|
+
auto: { before: 500, after: 500, maxNodes: 30 },
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* Threshold above which auto-tune treats the corpus as "wide-record"
|
|
36
|
+
* (JSONL events, log lines with embedded JSON, etc) and drops
|
|
37
|
+
* before/after padding. Chosen so that typical source code (lines
|
|
38
|
+
* usually under 200 chars) stays in the line-based regime, while
|
|
39
|
+
* single-line serialized events trip the switch.
|
|
40
|
+
*/
|
|
41
|
+
export const WIDE_RECORD_MEDIAN_THRESHOLD = 500;
|
|
42
|
+
/** Per-file sample budget — read only the head, never the whole file. */
|
|
43
|
+
const SAMPLE_BYTES_PER_FILE = 64 * 1024;
|
|
44
|
+
/** Across all sampled files, never exceed this much I/O on auto-tune. */
|
|
45
|
+
const SAMPLE_BYTES_TOTAL = 256 * 1024;
|
|
46
|
+
/**
|
|
47
|
+
* Estimate median line length across the first chunk of up to 3 files.
|
|
48
|
+
* Used by the wide-record auto-tune.
|
|
49
|
+
*
|
|
50
|
+
* Bounded I/O: we read at most 64KB per file and at most 256KB total,
|
|
51
|
+
* regardless of file size. A file whose head contains any NUL byte is
|
|
52
|
+
* treated as binary and skipped — otherwise a stray .png in a dir spec
|
|
53
|
+
* would feed garbage into the median.
|
|
54
|
+
*/
|
|
55
|
+
export function sampleMedianLineLength(files) {
|
|
56
|
+
if (files.length === 0)
|
|
57
|
+
return 0;
|
|
58
|
+
const lengths = [];
|
|
59
|
+
let totalRead = 0;
|
|
60
|
+
for (const f of files.slice(0, 3)) {
|
|
61
|
+
if (totalRead >= SAMPLE_BYTES_TOTAL)
|
|
62
|
+
break;
|
|
63
|
+
let fd = -1;
|
|
64
|
+
try {
|
|
65
|
+
const stat = statSync(f);
|
|
66
|
+
if (!stat.isFile())
|
|
67
|
+
continue;
|
|
68
|
+
const want = Math.min(SAMPLE_BYTES_PER_FILE, SAMPLE_BYTES_TOTAL - totalRead, Number(stat.size));
|
|
69
|
+
if (want <= 0)
|
|
70
|
+
continue;
|
|
71
|
+
fd = openSync(f, "r");
|
|
72
|
+
const buf = Buffer.alloc(want);
|
|
73
|
+
const n = readSync(fd, buf, 0, want, 0);
|
|
74
|
+
totalRead += n;
|
|
75
|
+
// Binary heuristic: a NUL byte in the head means rg won't search
|
|
76
|
+
// this as text anyway, so it shouldn't influence auto-tune.
|
|
77
|
+
let binary = false;
|
|
78
|
+
for (let i = 0; i < Math.min(n, 4096); i++) {
|
|
79
|
+
if (buf[i] === 0) {
|
|
80
|
+
binary = true;
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (binary)
|
|
85
|
+
continue;
|
|
86
|
+
const text = buf.toString("utf8", 0, n);
|
|
87
|
+
const lines = text.split(/\r?\n/).slice(0, 100);
|
|
88
|
+
for (const ln of lines) {
|
|
89
|
+
if (ln.length > 0)
|
|
90
|
+
lengths.push(ln.length);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
catch { /* skip unreadable files */ }
|
|
94
|
+
finally {
|
|
95
|
+
if (fd >= 0) {
|
|
96
|
+
try {
|
|
97
|
+
closeSync(fd);
|
|
98
|
+
}
|
|
99
|
+
catch { /* ignore */ }
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (lengths.length === 0)
|
|
104
|
+
return 0;
|
|
105
|
+
lengths.sort((a, b) => a - b);
|
|
106
|
+
return lengths[Math.floor(lengths.length / 2)];
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Run a search and return structured result.
|
|
110
|
+
*
|
|
111
|
+
* @example
|
|
112
|
+
* const r = await search({ pattern: "TODO", in: ["src/"], effort: "quick" });
|
|
113
|
+
* console.log(r.total_nodes, r.nodes[0].match_text);
|
|
114
|
+
*/
|
|
115
|
+
export async function search(opts) {
|
|
116
|
+
// Resolve effort defaults. Default is "quick" — cheap by design,
|
|
117
|
+
// following a "scan first, dig deeper on demand" philosophy.
|
|
118
|
+
// Agents bump to normal/deep when one shot returned ambiguous nodes.
|
|
119
|
+
const effort = opts.effort ?? "quick";
|
|
120
|
+
const preset = EFFORT_DEFAULTS[effort];
|
|
121
|
+
const userSetBefore = opts.before !== undefined;
|
|
122
|
+
const userSetAfter = opts.after !== undefined;
|
|
123
|
+
let before = opts.before ?? preset.before;
|
|
124
|
+
let after = opts.after ?? preset.after;
|
|
125
|
+
const maxNodes = opts.maxNodes ?? preset.maxNodes;
|
|
126
|
+
const strategy = opts.strategy ?? "fill";
|
|
127
|
+
// Build source list. Path inputs go through resolvePathSpecs which
|
|
128
|
+
// handles globs, dirs, @- and @file. Other sources are captured
|
|
129
|
+
// inline.
|
|
130
|
+
const pathInputs = [...(opts.in ?? [])];
|
|
131
|
+
let palace = null;
|
|
132
|
+
const palacePath = opts.palacePath ?? defaultPalacePath();
|
|
133
|
+
if (opts.from || (opts.compose && opts.compose.length > 0)) {
|
|
134
|
+
palace = loadPalace(palacePath);
|
|
135
|
+
const names = opts.from ? [opts.from] : opts.compose;
|
|
136
|
+
for (const s of composeToSources(palace, names)) {
|
|
137
|
+
pathInputs.unshift(s.id);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
// Split into literal files (cheap per-file fan-out, hot content
|
|
141
|
+
// cache) vs bulk specs (dirs / globs / @file expansions of dirs)
|
|
142
|
+
// that we hand to rg as-is so it can do its own parallel walk.
|
|
143
|
+
// The old code expanded everything to per-file specs up front, which
|
|
144
|
+
// turned a `--in .` scan into hundreds of rg spawns — measured 30s+
|
|
145
|
+
// on the perf bench. Letting rg walk one spec is ~100ms.
|
|
146
|
+
const { files, bulk } = pathInputs.length > 0
|
|
147
|
+
? await classifyPathSpecs(pathInputs)
|
|
148
|
+
: { files: [], bulk: [] };
|
|
149
|
+
// Wide-record auto-tune. If the user didn't pass explicit
|
|
150
|
+
// before/after and the corpus has very long lines (e.g. JSONL
|
|
151
|
+
// events), drop padding to 0 so we don't pull in entire neighboring
|
|
152
|
+
// records around each match. This is the headline product fix for
|
|
153
|
+
// the conversational-corpus benchmark.
|
|
154
|
+
let autoTuneApplied = false;
|
|
155
|
+
if (!opts.noAutoTune && !userSetBefore && !userSetAfter && files.length > 0) {
|
|
156
|
+
const median = sampleMedianLineLength(files);
|
|
157
|
+
if (median > WIDE_RECORD_MEDIAN_THRESHOLD) {
|
|
158
|
+
before = 0;
|
|
159
|
+
after = 0;
|
|
160
|
+
autoTuneApplied = true;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
// Explicit files first, then bulk specs. The ordering matters when
|
|
164
|
+
// `max_nodes` caps the result: a user who lists specific files
|
|
165
|
+
// alongside a dir means "I care about these specifically — show me
|
|
166
|
+
// their hits before you start mining the dir."
|
|
167
|
+
const resolved = files.map((f) => ({
|
|
168
|
+
source: { id: f, type: "file" },
|
|
169
|
+
content: null,
|
|
170
|
+
}));
|
|
171
|
+
for (const b of bulk) {
|
|
172
|
+
resolved.push({ source: { id: b, type: "bulk" }, content: null });
|
|
173
|
+
}
|
|
174
|
+
if (opts.cmd) {
|
|
175
|
+
const content = await captureCommand(opts.cmd);
|
|
176
|
+
resolved.push({
|
|
177
|
+
source: { id: `cmd:${opts.cmd}`, type: "command", label: `$ ${opts.cmd}` },
|
|
178
|
+
content,
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
if (opts.url) {
|
|
182
|
+
const content = await captureUrl(opts.url);
|
|
183
|
+
resolved.push({ source: { id: opts.url, type: "url" }, content });
|
|
184
|
+
}
|
|
185
|
+
if (opts.stdin) {
|
|
186
|
+
const content = await captureStdin();
|
|
187
|
+
resolved.push({ source: { id: "stdin", type: "stdin" }, content });
|
|
188
|
+
}
|
|
189
|
+
// Run rg + build nodes.
|
|
190
|
+
const t0 = Date.now();
|
|
191
|
+
const errors = [];
|
|
192
|
+
// Fuzzy: trigram-union regex driver + Levenshtein post-filter (./fuzzy.ts).
|
|
193
|
+
const effectivePattern = opts.fuzzy ? buildFuzzyRegex(opts.pattern) : opts.pattern;
|
|
194
|
+
const rgOpts = {
|
|
195
|
+
case_insensitive: opts.rg?.caseInsensitive,
|
|
196
|
+
word_match: opts.rg?.word,
|
|
197
|
+
fixed_strings: opts.rg?.fixedStrings,
|
|
198
|
+
multiline: opts.rg?.multiline,
|
|
199
|
+
hidden: opts.rg?.hidden,
|
|
200
|
+
no_ignore: opts.rg?.noIgnore,
|
|
201
|
+
include_globs: opts.rg?.include,
|
|
202
|
+
exclude_globs: opts.rg?.exclude,
|
|
203
|
+
type: opts.rg?.type,
|
|
204
|
+
};
|
|
205
|
+
// Per-source scan worker. Returns the nodes for that source plus an
|
|
206
|
+
// optional error.
|
|
207
|
+
//
|
|
208
|
+
// Content caching: each match comes back tagged with its actual file
|
|
209
|
+
// path (rg.ts overrides match.source for matches from file/bulk
|
|
210
|
+
// searches). We cache per-file so a 1000-match file reads once, AND
|
|
211
|
+
// a bulk search across many files doesn't accidentally reuse one
|
|
212
|
+
// file's content for another file's matches.
|
|
213
|
+
async function scanSource(rs, nodeBudget) {
|
|
214
|
+
const out = [];
|
|
215
|
+
// For inline-content sources (cmd/url/stdin) the source content is
|
|
216
|
+
// always `rs.content`; for file/bulk sources we cache per file path.
|
|
217
|
+
const inlineContent = rs.content;
|
|
218
|
+
const fileCache = new Map();
|
|
219
|
+
// rg emits one match record per submatch. Two TODOs on one line
|
|
220
|
+
// (e.g. `// TODO: x; // TODO: y`) become two `Match` objects with
|
|
221
|
+
// identical `(source.id, line)`. We always dedup by line so each
|
|
222
|
+
// line contributes one node, regardless of auto-tune state.
|
|
223
|
+
const seenLines = new Set();
|
|
224
|
+
try {
|
|
225
|
+
for await (const match of runRg(effectivePattern, rs.source, rs.content, rgOpts)) {
|
|
226
|
+
if (out.length >= nodeBudget)
|
|
227
|
+
break;
|
|
228
|
+
if (opts.fuzzy) {
|
|
229
|
+
if (!verifyFuzzy(match.text, match.match_start, opts.pattern, 2))
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
const lineKey = `${match.source.id}:${match.line}`;
|
|
233
|
+
if (seenLines.has(lineKey))
|
|
234
|
+
continue;
|
|
235
|
+
seenLines.add(lineKey);
|
|
236
|
+
let content;
|
|
237
|
+
if (inlineContent !== null) {
|
|
238
|
+
content = inlineContent;
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
241
|
+
const cached = fileCache.get(match.source.id);
|
|
242
|
+
if (cached !== undefined) {
|
|
243
|
+
content = cached;
|
|
244
|
+
}
|
|
245
|
+
else {
|
|
246
|
+
content = loadSourceContent(match.source, null);
|
|
247
|
+
fileCache.set(match.source.id, content);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
const node = buildNode(match, content, {
|
|
251
|
+
beforeTokens: before,
|
|
252
|
+
afterTokens: after,
|
|
253
|
+
clipChars: opts.clipChars,
|
|
254
|
+
});
|
|
255
|
+
out.push(node);
|
|
256
|
+
if (out.length >= nodeBudget)
|
|
257
|
+
break;
|
|
258
|
+
}
|
|
259
|
+
return { nodes: out, error: null };
|
|
260
|
+
}
|
|
261
|
+
catch (err) {
|
|
262
|
+
const msg = err instanceof RgError
|
|
263
|
+
? err.message
|
|
264
|
+
: `${err.message}`;
|
|
265
|
+
return { nodes: out, error: msg };
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
// Bounded-parallel scan across resolved sources. Each worker still
|
|
269
|
+
// honors the overall `maxNodes` cap conservatively by partitioning
|
|
270
|
+
// the budget; we re-truncate after merging to enforce the hard cap.
|
|
271
|
+
// Parallelism = min(resolved.length, RG_CONCURRENCY).
|
|
272
|
+
const RG_CONCURRENCY = Math.max(1, parseInt(process.env.MPG_RG_CONCURRENCY ?? "4", 10) || 4);
|
|
273
|
+
const allNodes = [];
|
|
274
|
+
// Track which file paths we've already attributed a node to so a
|
|
275
|
+
// bulk source can't blot out the explicit files alongside it under
|
|
276
|
+
// a tight maxNodes cap.
|
|
277
|
+
const dedupKey = new Set();
|
|
278
|
+
for (let i = 0; i < resolved.length; i += RG_CONCURRENCY) {
|
|
279
|
+
if (allNodes.length >= maxNodes)
|
|
280
|
+
break;
|
|
281
|
+
const batch = resolved.slice(i, i + RG_CONCURRENCY);
|
|
282
|
+
// Per-worker budget. For file/cmd/url/stdin sources we cap at
|
|
283
|
+
// maxNodes (one source can't legally produce more than the global
|
|
284
|
+
// cap). For bulk sources we let the worker over-collect — a bulk
|
|
285
|
+
// walk can produce matches from many files, and we want the
|
|
286
|
+
// orchestrator's round-robin interleave to do the fair-share
|
|
287
|
+
// distribution rather than rg's walk order silently picking
|
|
288
|
+
// winners. The 4x spread is enough to cover ~4 files at the cap
|
|
289
|
+
// before we have to worry about real memory pressure.
|
|
290
|
+
const results = await Promise.all(batch.map((rs) => {
|
|
291
|
+
const budget = rs.source.type === "bulk" ? maxNodes * 4 : maxNodes;
|
|
292
|
+
return scanSource(rs, budget);
|
|
293
|
+
}));
|
|
294
|
+
// Merge in input order, deduping by (file, match_line) so that a
|
|
295
|
+
// bulk source that re-walks a file already covered by an explicit
|
|
296
|
+
// file spec doesn't double-count.
|
|
297
|
+
for (let j = 0; j < batch.length; j++) {
|
|
298
|
+
const { nodes: workerNodes, error } = results[j];
|
|
299
|
+
if (error) {
|
|
300
|
+
errors.push({ source: batch[j].source.id, message: error });
|
|
301
|
+
}
|
|
302
|
+
for (const n of workerNodes) {
|
|
303
|
+
if (allNodes.length >= maxNodes)
|
|
304
|
+
break;
|
|
305
|
+
const k = `${n.source.id}:${n.match_line}`;
|
|
306
|
+
if (dedupKey.has(k))
|
|
307
|
+
continue;
|
|
308
|
+
dedupKey.add(k);
|
|
309
|
+
allNodes.push(n);
|
|
310
|
+
}
|
|
311
|
+
if (allNodes.length >= maxNodes)
|
|
312
|
+
break;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
// Interleave round-robin so when a tight maxNodes cap is in play,
|
|
316
|
+
// every distinct source contributes before any one source repeats.
|
|
317
|
+
// This preserves the test-23-style semantic where listing a dir +
|
|
318
|
+
// an explicit file should surface both in sources_count even at
|
|
319
|
+
// very tight caps. We compute the round-robin AFTER the in-order
|
|
320
|
+
// merge so the natural file-order is preserved when budget allows.
|
|
321
|
+
if (allNodes.length > 0 && allNodes.length < maxNodes * 2) {
|
|
322
|
+
const bySource = new Map();
|
|
323
|
+
for (const n of allNodes) {
|
|
324
|
+
const id = n.source.id;
|
|
325
|
+
let arr = bySource.get(id);
|
|
326
|
+
if (!arr) {
|
|
327
|
+
arr = [];
|
|
328
|
+
bySource.set(id, arr);
|
|
329
|
+
}
|
|
330
|
+
arr.push(n);
|
|
331
|
+
}
|
|
332
|
+
if (bySource.size > 1) {
|
|
333
|
+
const interleaved = [];
|
|
334
|
+
const buckets = [...bySource.values()];
|
|
335
|
+
let still = true;
|
|
336
|
+
let idx = 0;
|
|
337
|
+
while (still && interleaved.length < allNodes.length) {
|
|
338
|
+
still = false;
|
|
339
|
+
for (const b of buckets) {
|
|
340
|
+
if (idx < b.length) {
|
|
341
|
+
interleaved.push(b[idx]);
|
|
342
|
+
still = true;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
idx++;
|
|
346
|
+
}
|
|
347
|
+
allNodes.length = 0;
|
|
348
|
+
for (const n of interleaved)
|
|
349
|
+
allNodes.push(n);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
// Optional ordering by source file mtime.
|
|
353
|
+
if (opts.sort === "recent" || opts.sort === "oldest") {
|
|
354
|
+
const mtimes = new Map();
|
|
355
|
+
for (const n of allNodes) {
|
|
356
|
+
const id = n.source.id;
|
|
357
|
+
if (mtimes.has(id))
|
|
358
|
+
continue;
|
|
359
|
+
if (n.source.type === "file") {
|
|
360
|
+
try {
|
|
361
|
+
mtimes.set(id, statSync(id).mtimeMs);
|
|
362
|
+
}
|
|
363
|
+
catch {
|
|
364
|
+
mtimes.set(id, 0);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
else {
|
|
368
|
+
// Non-file sources have no mtime; push to one end.
|
|
369
|
+
mtimes.set(id, opts.sort === "recent" ? -Infinity : Infinity);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
const dir = opts.sort === "recent" ? -1 : 1;
|
|
373
|
+
allNodes.sort((a, b) => {
|
|
374
|
+
const ma = mtimes.get(a.source.id) ?? 0;
|
|
375
|
+
const mb = mtimes.get(b.source.id) ?? 0;
|
|
376
|
+
if (ma !== mb)
|
|
377
|
+
return dir * (ma - mb);
|
|
378
|
+
// Stable within a file: preserve match-line order.
|
|
379
|
+
return (a.match_line ?? 0) - (b.match_line ?? 0);
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
// Apply window-decay curve before total-budget enforcement so the
|
|
383
|
+
// smaller windows count against the cap accurately.
|
|
384
|
+
const windowCurve = opts.windowCurve ?? "flat";
|
|
385
|
+
if (windowCurve !== "flat") {
|
|
386
|
+
applyWindowCurve(allNodes, windowCurve, before, after);
|
|
387
|
+
}
|
|
388
|
+
const { nodes: budgeted, truncated } = applyTotalBudget(allNodes, opts.maxTokens, strategy);
|
|
389
|
+
// Apply pagination if requested.
|
|
390
|
+
const { paginate } = await import("./pagination.js");
|
|
391
|
+
const { items: paged, pagination } = paginate(budgeted, {
|
|
392
|
+
page: opts.page,
|
|
393
|
+
pageSize: opts.pageSize,
|
|
394
|
+
all: opts.all,
|
|
395
|
+
});
|
|
396
|
+
paged.forEach((n, i) => { n.id = i + 1; });
|
|
397
|
+
const sources = new Set(budgeted.map((n) => n.source.id));
|
|
398
|
+
const totalTokens = budgeted.reduce((s, n) => s + n.tokens, 0);
|
|
399
|
+
const pageTokens = paged.reduce((s, n) => s + n.tokens, 0);
|
|
400
|
+
// Status: if any source errored, classify by whether anything came
|
|
401
|
+
// back. Agents branching on `status` will see "partial" / "error"
|
|
402
|
+
// instead of a misleading "no_matches".
|
|
403
|
+
let status;
|
|
404
|
+
if (errors.length > 0 && budgeted.length === 0) {
|
|
405
|
+
status = "error";
|
|
406
|
+
}
|
|
407
|
+
else if (errors.length > 0) {
|
|
408
|
+
status = "partial";
|
|
409
|
+
}
|
|
410
|
+
else if (budgeted.length === 0) {
|
|
411
|
+
status = "no_matches";
|
|
412
|
+
}
|
|
413
|
+
else if (truncated) {
|
|
414
|
+
status = "truncated";
|
|
415
|
+
}
|
|
416
|
+
else {
|
|
417
|
+
status = "ok";
|
|
418
|
+
}
|
|
419
|
+
return {
|
|
420
|
+
pattern: opts.pattern,
|
|
421
|
+
effort,
|
|
422
|
+
strategy,
|
|
423
|
+
status,
|
|
424
|
+
total_nodes: budgeted.length,
|
|
425
|
+
total_tokens: totalTokens,
|
|
426
|
+
page_tokens: pageTokens,
|
|
427
|
+
sources_count: sources.size,
|
|
428
|
+
truncated,
|
|
429
|
+
nodes: paged,
|
|
430
|
+
errors,
|
|
431
|
+
duration_ms: Date.now() - t0,
|
|
432
|
+
before_tokens: before,
|
|
433
|
+
after_tokens: after,
|
|
434
|
+
max_nodes: maxNodes,
|
|
435
|
+
max_tokens: opts.maxTokens,
|
|
436
|
+
auto_tune_applied: autoTuneApplied || undefined,
|
|
437
|
+
pagination,
|
|
438
|
+
};
|
|
439
|
+
}
|
|
440
|
+
// ─── Mind palace operations ─────────────────────────────────────────
|
|
441
|
+
/**
|
|
442
|
+
* Stash a search result in the mind palace.
|
|
443
|
+
*
|
|
444
|
+
* The `result` can be a SearchResult, or just an array of Nodes.
|
|
445
|
+
* Returns the action taken (created/merged/replaced) and the full stash.
|
|
446
|
+
*/
|
|
447
|
+
export async function stash(result, opts) {
|
|
448
|
+
const palacePath = opts.palacePath ?? defaultPalacePath();
|
|
449
|
+
const palace = loadPalace(palacePath);
|
|
450
|
+
const nodes = Array.isArray(result) ? result : result.nodes;
|
|
451
|
+
const sources = Array.isArray(result)
|
|
452
|
+
? [...new Set(result.map((n) => n.source.id))]
|
|
453
|
+
: [...new Set(result.nodes.map((n) => n.source.id))];
|
|
454
|
+
const pattern = Array.isArray(result) ? "" : result.pattern;
|
|
455
|
+
const effort = Array.isArray(result) ? "normal" : result.effort;
|
|
456
|
+
const { action, stash: newStash } = addStashToPalace(palace, opts.name, opts.note ?? "", nodes, { pattern, effort, sources_count: sources.length }, sources, opts.tags ?? [], {
|
|
457
|
+
replace: opts.replace ?? false,
|
|
458
|
+
ttl: opts.ttl,
|
|
459
|
+
locations: opts.locations,
|
|
460
|
+
});
|
|
461
|
+
savePalace(palacePath, palace);
|
|
462
|
+
return { action, stash: newStash, palace_path: palacePath };
|
|
463
|
+
}
|
|
464
|
+
export function listStashes(palacePath, tagFilter) {
|
|
465
|
+
const path = palacePath ?? defaultPalacePath();
|
|
466
|
+
const palace = loadPalace(path);
|
|
467
|
+
return listStashesFromPalace(palace, tagFilter);
|
|
468
|
+
}
|
|
469
|
+
export function getStash(name, palacePath) {
|
|
470
|
+
const path = palacePath ?? defaultPalacePath();
|
|
471
|
+
const palace = loadPalace(path);
|
|
472
|
+
return getStashFromPalace(palace, name);
|
|
473
|
+
}
|
|
474
|
+
export function dropStash(name, palacePath) {
|
|
475
|
+
const path = palacePath ?? defaultPalacePath();
|
|
476
|
+
const palace = loadPalace(path);
|
|
477
|
+
const ok = dropStashFromPalace(palace, name);
|
|
478
|
+
if (ok)
|
|
479
|
+
savePalace(path, palace);
|
|
480
|
+
return ok;
|
|
481
|
+
}
|
|
482
|
+
/** Resolve a stash (or composition of stashes) to its source paths. */
|
|
483
|
+
export function stashToSources(names, palacePath) {
|
|
484
|
+
const path = palacePath ?? defaultPalacePath();
|
|
485
|
+
const palace = loadPalace(path);
|
|
486
|
+
const arr = Array.isArray(names) ? names : [names];
|
|
487
|
+
return composeToSources(palace, arr).map((s) => s.id);
|
|
488
|
+
}
|
|
489
|
+
// ─── Tool calling schemas (Claude, Gemini, OpenAI) ──────────────────
|
|
490
|
+
// Claude and Gemini work better with SEPARATE tools per operation
|
|
491
|
+
// because each tool_use result is atomic. Five tools = five decisions.
|
|
492
|
+
const SEARCH_PARAMS = {
|
|
493
|
+
type: "object",
|
|
494
|
+
properties: {
|
|
495
|
+
pattern: { type: "string", description: "Regex pattern to search for (ripgrep syntax)." },
|
|
496
|
+
in: { type: "array", items: { type: "string" },
|
|
497
|
+
description: "Paths to search: files, directories (recurses), globs, @file, @-." },
|
|
498
|
+
cmd: { type: "string", description: "Search the stdout of a shell command." },
|
|
499
|
+
url: { type: "string", description: "Fetch and search a URL." },
|
|
500
|
+
before: { type: "number", description: "Tokens of context before each match. Default 500." },
|
|
501
|
+
after: { type: "number", description: "Tokens of context after each match. Default 500." },
|
|
502
|
+
max_nodes: { type: "number", description: "Cap on number of nodes. Default 30." },
|
|
503
|
+
max_tokens: { type: "number", description: "Total token budget across all nodes." },
|
|
504
|
+
effort: { type: "string", enum: ["quick", "normal", "deep", "auto"],
|
|
505
|
+
description: "Preset. quick=200t/10n, normal=500t/30n, deep=2000t/100n." },
|
|
506
|
+
strategy: { type: "string", enum: ["fill", "deep"],
|
|
507
|
+
description: "How to use max_tokens. fill prefers more nodes, deep prefers deeper per node." },
|
|
508
|
+
from: { type: "string", description: "Scope search to files from a stashed mind-palace slot." },
|
|
509
|
+
compose: { type: "array", items: { type: "string" },
|
|
510
|
+
description: "Scope search to the union of multiple stashed slots' file lists." },
|
|
511
|
+
page: { type: "number", description: "1-indexed page number. Set to 1 to enable pagination." },
|
|
512
|
+
page_size: { type: "number", description: "Nodes per page. Default 10." },
|
|
513
|
+
},
|
|
514
|
+
required: ["pattern"],
|
|
515
|
+
};
|
|
516
|
+
const STASH_PARAMS = {
|
|
517
|
+
type: "object",
|
|
518
|
+
properties: {
|
|
519
|
+
name: { type: "string", description: "Name for this memory slot. Use kebab-case." },
|
|
520
|
+
note: { type: "string", description: "Free-form note describing what this stash contains." },
|
|
521
|
+
tags: { type: "array", items: { type: "string" },
|
|
522
|
+
description: "Tags for filtering: e.g. ['auth', 'p0', 'security']." },
|
|
523
|
+
pattern: { type: "string", description: "The regex pattern used in the search (for provenance)." },
|
|
524
|
+
in: { type: "array", items: { type: "string" }, description: "Paths searched (for provenance)." },
|
|
525
|
+
effort: { type: "string", enum: ["quick", "normal", "deep", "auto"] },
|
|
526
|
+
replace: { type: "boolean", description: "Overwrite an existing slot. Default: merge (dedup by file:line)." },
|
|
527
|
+
palace_path: { type: "string", description: "Override mind-palace file location." },
|
|
528
|
+
},
|
|
529
|
+
required: ["name", "note"],
|
|
530
|
+
};
|
|
531
|
+
/** Claude-compatible tool definitions. Drop into Claude's API. */
|
|
532
|
+
export const claudeTools = [
|
|
533
|
+
{
|
|
534
|
+
type: "function",
|
|
535
|
+
function: {
|
|
536
|
+
name: "mpg_search",
|
|
537
|
+
description: "Search code, markdown, command output, and URLs for a regex " +
|
|
538
|
+
"pattern. Returns token-budgeted context nodes with file:line " +
|
|
539
|
+
"attribution. Each node is sized in tokens (not lines). Supports " +
|
|
540
|
+
"effort presets (quick/normal/deep), pagination, and scoped " +
|
|
541
|
+
"re-search from mind-palace stashes via the 'from'/'compose' fields.",
|
|
542
|
+
parameters: SEARCH_PARAMS,
|
|
543
|
+
},
|
|
544
|
+
},
|
|
545
|
+
{
|
|
546
|
+
type: "function",
|
|
547
|
+
function: {
|
|
548
|
+
name: "mpg_stash",
|
|
549
|
+
description: "Save the result of a search into a named mind-palace slot " +
|
|
550
|
+
"(the LLM's instantiable short-term memory). Stashed slots can " +
|
|
551
|
+
"be used as search targets via mpg_search(from:name) or " +
|
|
552
|
+
"mpg_search(compose:[a,b]). Merges by default (dedup by file:line); " +
|
|
553
|
+
"pass replace:true to overwrite.",
|
|
554
|
+
parameters: STASH_PARAMS,
|
|
555
|
+
},
|
|
556
|
+
},
|
|
557
|
+
{
|
|
558
|
+
type: "function",
|
|
559
|
+
function: {
|
|
560
|
+
name: "mpg_list_stashes",
|
|
561
|
+
description: "List all named memory slots in the mind palace. Optionally " +
|
|
562
|
+
"filter by tag. Use this to see what you've stashed before deciding " +
|
|
563
|
+
"to compose or re-search. Supports pagination.",
|
|
564
|
+
parameters: {
|
|
565
|
+
type: "object",
|
|
566
|
+
properties: {
|
|
567
|
+
tag_filter: { type: "array", items: { type: "string" },
|
|
568
|
+
description: "Only show stashes with all of these tags." },
|
|
569
|
+
page: { type: "number", description: "1-indexed page number." },
|
|
570
|
+
page_size: { type: "number", description: "Stashes per page. Default 20." },
|
|
571
|
+
palace_path: { type: "string", description: "Override mind-palace file." },
|
|
572
|
+
},
|
|
573
|
+
required: [],
|
|
574
|
+
},
|
|
575
|
+
},
|
|
576
|
+
},
|
|
577
|
+
{
|
|
578
|
+
type: "function",
|
|
579
|
+
function: {
|
|
580
|
+
name: "mpg_get_stash",
|
|
581
|
+
description: "Show a single mind-palace slot. Returns the CARD VIEW by default: " +
|
|
582
|
+
"the synthesized intel an agent almost always wants — note, tags, " +
|
|
583
|
+
"search provenance, source paths (passable as mpg_search 'from'), " +
|
|
584
|
+
"relations, and node/source counts. The captured node bodies are " +
|
|
585
|
+
"omitted (5–6× cheaper). Pass with_nodes:true to include the full " +
|
|
586
|
+
"stashed nodes block; pagination only applies in that mode.",
|
|
587
|
+
parameters: {
|
|
588
|
+
type: "object",
|
|
589
|
+
properties: {
|
|
590
|
+
name: { type: "string", description: "Name of the stash to retrieve." },
|
|
591
|
+
with_nodes: {
|
|
592
|
+
type: "boolean",
|
|
593
|
+
description: "If true, include the captured node bodies. Default false " +
|
|
594
|
+
"returns the card view (metadata + relations + sources only).",
|
|
595
|
+
},
|
|
596
|
+
page: { type: "number", description: "1-indexed page number (only when with_nodes:true)." },
|
|
597
|
+
page_size: { type: "number", description: "Nodes per page. Default 10 (only when with_nodes:true)." },
|
|
598
|
+
palace_path: { type: "string", description: "Override mind-palace file." },
|
|
599
|
+
},
|
|
600
|
+
required: ["name"],
|
|
601
|
+
},
|
|
602
|
+
},
|
|
603
|
+
},
|
|
604
|
+
{
|
|
605
|
+
type: "function",
|
|
606
|
+
function: {
|
|
607
|
+
name: "mpg_drop_stash",
|
|
608
|
+
description: "Remove a slot from the mind palace. Use this to free memory when " +
|
|
609
|
+
"a line of investigation is complete. Dropped stashes are gone " +
|
|
610
|
+
"permanently (the JSON file is rewritten).",
|
|
611
|
+
parameters: {
|
|
612
|
+
type: "object",
|
|
613
|
+
properties: {
|
|
614
|
+
name: { type: "string", description: "Name of the stash to drop." },
|
|
615
|
+
palace_path: { type: "string", description: "Override mind-palace file." },
|
|
616
|
+
},
|
|
617
|
+
required: ["name"],
|
|
618
|
+
},
|
|
619
|
+
},
|
|
620
|
+
},
|
|
621
|
+
];
|
|
622
|
+
/** Gemini-compatible tool definitions (function_declarations array). */
|
|
623
|
+
export const geminiTools = claudeTools.map((t) => ({
|
|
624
|
+
name: t.function.name,
|
|
625
|
+
description: t.function.description,
|
|
626
|
+
parameters: t.function.parameters,
|
|
627
|
+
}));
|
|
628
|
+
/** Legacy: single-tool definition for OpenAI-compatible APIs. */
|
|
629
|
+
export const toolDefinition = {
|
|
630
|
+
name: "mpg",
|
|
631
|
+
description: "Search code, markdown, command output, and URLs. " +
|
|
632
|
+
"Use mpg_list_stashes first to see available memory slots, " +
|
|
633
|
+
"then mpg_search to find content, and mpg_stash to save results.",
|
|
634
|
+
parameters: {
|
|
635
|
+
type: "object",
|
|
636
|
+
properties: {
|
|
637
|
+
action: {
|
|
638
|
+
type: "string",
|
|
639
|
+
enum: ["search", "stash", "list", "get", "drop"],
|
|
640
|
+
description: "Which operation to perform.",
|
|
641
|
+
},
|
|
642
|
+
pattern: { type: "string", description: "Regex pattern (search)." },
|
|
643
|
+
in: { type: "array", items: { type: "string" }, description: "Paths to search." },
|
|
644
|
+
name: { type: "string", description: "Stash name (stash/get/drop)." },
|
|
645
|
+
note: { type: "string", description: "Stash note (stash)." },
|
|
646
|
+
tags: { type: "array", items: { type: "string" }, description: "Tags (stash/filter)." },
|
|
647
|
+
before: { type: "number" },
|
|
648
|
+
after: { type: "number" },
|
|
649
|
+
max_nodes: { type: "number" },
|
|
650
|
+
max_tokens: { type: "number" },
|
|
651
|
+
effort: { type: "string", enum: ["quick", "normal", "deep", "auto"] },
|
|
652
|
+
from: { type: "string", description: "Stash name as search target." },
|
|
653
|
+
compose: { type: "array", items: { type: "string" }, description: "Stash names as union target." },
|
|
654
|
+
page: { type: "number", description: "1-indexed page number." },
|
|
655
|
+
page_size: { type: "number", description: "Items per page." },
|
|
656
|
+
},
|
|
657
|
+
required: ["action"],
|
|
658
|
+
},
|
|
659
|
+
};
|
|
660
|
+
//# sourceMappingURL=api.js.map
|