@plebeiantech/lat.md 0.12.3-fork.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +97 -0
- package/dist/src/cli/check-frontmatter.d.ts +12 -0
- package/dist/src/cli/check-frontmatter.js +44 -0
- package/dist/src/cli/check-mode.d.ts +6 -0
- package/dist/src/cli/check-mode.js +303 -0
- package/dist/src/cli/check-status.d.ts +74 -0
- package/dist/src/cli/check-status.js +217 -0
- package/dist/src/cli/check.d.ts +45 -0
- package/dist/src/cli/check.js +777 -0
- package/dist/src/cli/checklist-menu.d.ts +12 -0
- package/dist/src/cli/checklist-menu.js +111 -0
- package/dist/src/cli/comment-guard.d.ts +16 -0
- package/dist/src/cli/comment-guard.js +111 -0
- package/dist/src/cli/comment-reminder.d.ts +62 -0
- package/dist/src/cli/comment-reminder.js +210 -0
- package/dist/src/cli/context.d.ts +11 -0
- package/dist/src/cli/context.js +58 -0
- package/dist/src/cli/expand.d.ts +8 -0
- package/dist/src/cli/expand.js +94 -0
- package/dist/src/cli/gen-index.d.ts +45 -0
- package/dist/src/cli/gen-index.js +235 -0
- package/dist/src/cli/gen.d.ts +6 -0
- package/dist/src/cli/gen.js +42 -0
- package/dist/src/cli/graph.d.ts +7 -0
- package/dist/src/cli/graph.js +47 -0
- package/dist/src/cli/hook.d.ts +1 -0
- package/dist/src/cli/hook.js +432 -0
- package/dist/src/cli/index.d.ts +2 -0
- package/dist/src/cli/index.js +307 -0
- package/dist/src/cli/init.d.ts +9 -0
- package/dist/src/cli/init.js +1031 -0
- package/dist/src/cli/link-scheme.d.ts +54 -0
- package/dist/src/cli/link-scheme.js +101 -0
- package/dist/src/cli/locate.d.ts +2 -0
- package/dist/src/cli/locate.js +17 -0
- package/dist/src/cli/refs.d.ts +22 -0
- package/dist/src/cli/refs.js +229 -0
- package/dist/src/cli/reindex.d.ts +13 -0
- package/dist/src/cli/reindex.js +136 -0
- package/dist/src/cli/search.d.ts +36 -0
- package/dist/src/cli/search.js +217 -0
- package/dist/src/cli/section.d.ts +43 -0
- package/dist/src/cli/section.js +252 -0
- package/dist/src/cli/select-menu.d.ts +13 -0
- package/dist/src/cli/select-menu.js +100 -0
- package/dist/src/cli/templates.d.ts +1 -0
- package/dist/src/cli/templates.js +15 -0
- package/dist/src/cli/ui.d.ts +11 -0
- package/dist/src/cli/ui.js +34 -0
- package/dist/src/code-refs.d.ts +17 -0
- package/dist/src/code-refs.js +277 -0
- package/dist/src/config.d.ts +29 -0
- package/dist/src/config.js +83 -0
- package/dist/src/context.d.ts +23 -0
- package/dist/src/context.js +11 -0
- package/dist/src/extensions/wiki-link/from-markdown.d.ts +5 -0
- package/dist/src/extensions/wiki-link/from-markdown.js +34 -0
- package/dist/src/extensions/wiki-link/index.d.ts +4 -0
- package/dist/src/extensions/wiki-link/index.js +3 -0
- package/dist/src/extensions/wiki-link/syntax.d.ts +14 -0
- package/dist/src/extensions/wiki-link/syntax.js +121 -0
- package/dist/src/extensions/wiki-link/to-markdown.d.ts +5 -0
- package/dist/src/extensions/wiki-link/to-markdown.js +28 -0
- package/dist/src/extensions/wiki-link/types.d.ts +34 -0
- package/dist/src/extensions/wiki-link/types.js +1 -0
- package/dist/src/format.d.ts +8 -0
- package/dist/src/format.js +43 -0
- package/dist/src/graph-export.d.ts +60 -0
- package/dist/src/graph-export.js +309 -0
- package/dist/src/init-version.d.ts +10 -0
- package/dist/src/init-version.js +49 -0
- package/dist/src/knowledge/bd.d.ts +2 -0
- package/dist/src/knowledge/bd.js +107 -0
- package/dist/src/knowledge/claude-memory.d.ts +13 -0
- package/dist/src/knowledge/claude-memory.js +204 -0
- package/dist/src/knowledge/cq.d.ts +2 -0
- package/dist/src/knowledge/cq.js +74 -0
- package/dist/src/knowledge/index.d.ts +49 -0
- package/dist/src/knowledge/index.js +271 -0
- package/dist/src/knowledge/ranking.d.ts +17 -0
- package/dist/src/knowledge/ranking.js +62 -0
- package/dist/src/knowledge/session.d.ts +46 -0
- package/dist/src/knowledge/session.js +203 -0
- package/dist/src/knowledge/types.d.ts +47 -0
- package/dist/src/knowledge/types.js +10 -0
- package/dist/src/lattice.d.ts +108 -0
- package/dist/src/lattice.js +816 -0
- package/dist/src/mcp/server.d.ts +1 -0
- package/dist/src/mcp/server.js +58 -0
- package/dist/src/parser.d.ts +3 -0
- package/dist/src/parser.js +18 -0
- package/dist/src/search/db.d.ts +16 -0
- package/dist/src/search/db.js +54 -0
- package/dist/src/search/embedder.d.ts +25 -0
- package/dist/src/search/embedder.js +58 -0
- package/dist/src/search/index.d.ts +9 -0
- package/dist/src/search/index.js +66 -0
- package/dist/src/search/search.d.ts +9 -0
- package/dist/src/search/search.js +16 -0
- package/dist/src/source-parser.d.ts +27 -0
- package/dist/src/source-parser.js +916 -0
- package/dist/src/untrusted.d.ts +24 -0
- package/dist/src/untrusted.js +97 -0
- package/dist/src/version.d.ts +7 -0
- package/dist/src/version.js +39 -0
- package/dist/src/view/client/assets/index-B0n0u03y.css +1 -0
- package/dist/src/view/client/assets/index-xm2_px4q.js +23 -0
- package/dist/src/view/client/index.html +14 -0
- package/dist/src/view/diagnostics.d.ts +5 -0
- package/dist/src/view/diagnostics.js +171 -0
- package/dist/src/view/git-diff.d.ts +3 -0
- package/dist/src/view/git-diff.js +255 -0
- package/dist/src/view/git.d.ts +22 -0
- package/dist/src/view/git.js +193 -0
- package/dist/src/view/highlight.d.ts +2 -0
- package/dist/src/view/highlight.js +3767 -0
- package/dist/src/view/markdown.d.ts +22 -0
- package/dist/src/view/markdown.js +322 -0
- package/dist/src/view/protocol.d.ts +89 -0
- package/dist/src/view/protocol.js +1 -0
- package/dist/src/view/references.d.ts +62 -0
- package/dist/src/view/references.js +345 -0
- package/dist/src/view/repository.d.ts +12 -0
- package/dist/src/view/repository.js +167 -0
- package/dist/src/view/search.d.ts +14 -0
- package/dist/src/view/search.js +64 -0
- package/dist/src/view/server.d.ts +20 -0
- package/dist/src/view/server.js +233 -0
- package/dist/src/view/store.d.ts +57 -0
- package/dist/src/view/store.js +404 -0
- package/dist/src/walk.d.ts +18 -0
- package/dist/src/walk.js +27 -0
- package/package.json +85 -0
- package/templates/AGENTS.md +118 -0
- package/templates/README +1 -0
- package/templates/cursor-rules.md +77 -0
- package/templates/init/lat.md +1 -0
- package/templates/logo-dark.svg +182 -0
- package/templates/logo-light.svg +182 -0
- package/templates/logo.txt +7 -0
- package/templates/opencode-plugin.ts +180 -0
- package/templates/pi-extension.ts +364 -0
- package/templates/skill/SKILL.md +189 -0
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import { execFile, } from 'node:child_process';
|
|
2
|
+
import { existsSync, lstatSync, readFileSync, readdirSync } from 'node:fs';
|
|
3
|
+
import { homedir } from 'node:os';
|
|
4
|
+
import { dirname, join, resolve } from 'node:path';
|
|
5
|
+
const CALL_TIMEOUT_MS = 2000;
|
|
6
|
+
const MAX_FILE_BYTES = 64 * 1024;
|
|
7
|
+
/** Escape regex metacharacters so a raw term can be interpolated safely. */
|
|
8
|
+
function escapeRegExp(s) {
|
|
9
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Resolve the checkout whose memory store applies to `projectRoot`. Claude
|
|
13
|
+
* Code shares one memory store per repo, keyed on the *main* checkout's
|
|
14
|
+
* absolute path — a worktree has no store of its own. Without resolving to
|
|
15
|
+
* the main checkout, every lookup made from inside a worktree session
|
|
16
|
+
* silently finds nothing, which is invisible unless you know to check: it
|
|
17
|
+
* never errors, it just never matches. `git rev-parse --git-common-dir`
|
|
18
|
+
* inside a worktree returns the main checkout's `.git` dir; its parent is
|
|
19
|
+
* the main checkout. Outside a git repo (or on any failure) we fall back to
|
|
20
|
+
* `projectRoot` itself.
|
|
21
|
+
*/
|
|
22
|
+
function resolveMainCheckout(projectRoot) {
|
|
23
|
+
return new Promise((resolvePromise) => {
|
|
24
|
+
// Async execFile, not execFileSync: this runs on the UserPromptSubmit
|
|
25
|
+
// critical path alongside the bd and cq stores under
|
|
26
|
+
// `Promise.allSettled`, and a synchronous subprocess call here would
|
|
27
|
+
// block the event loop and prevent the other stores' I/O from
|
|
28
|
+
// overlapping with it.
|
|
29
|
+
// `stdio` isn't in the string-encoding overload's declared option type
|
|
30
|
+
// even though Node accepts it at runtime, hence the explicit cast.
|
|
31
|
+
const options = {
|
|
32
|
+
cwd: projectRoot,
|
|
33
|
+
stdio: ['ignore', 'pipe', 'ignore'],
|
|
34
|
+
timeout: CALL_TIMEOUT_MS,
|
|
35
|
+
encoding: 'utf-8',
|
|
36
|
+
};
|
|
37
|
+
execFile('git', ['rev-parse', '--git-common-dir'], options, (error, stdout) => {
|
|
38
|
+
if (error || !stdout) {
|
|
39
|
+
resolvePromise(projectRoot);
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
const out = stdout.trim();
|
|
43
|
+
if (!out) {
|
|
44
|
+
resolvePromise(projectRoot);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const gitDir = resolve(projectRoot, out);
|
|
48
|
+
resolvePromise(dirname(gitDir));
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Match Claude Code's own project-directory slug: every character that
|
|
54
|
+
* isn't a letter or digit becomes `-`, including `.` — a project root like
|
|
55
|
+
* `/Users/dave/projects/claude-code/lat.md` slugifies to
|
|
56
|
+
* `-Users-dave-projects-claude-code-lat-md`, verified against a real
|
|
57
|
+
* `~/.claude/projects/` listing. Replacing only `/` (the previous
|
|
58
|
+
* implementation) leaves dots in the slug and never matches the real
|
|
59
|
+
* directory, so the store silently finds nothing for any project root
|
|
60
|
+
* containing a dot.
|
|
61
|
+
*/
|
|
62
|
+
export function slugify(absPath) {
|
|
63
|
+
return absPath.replace(/[^a-zA-Z0-9]/g, '-');
|
|
64
|
+
}
|
|
65
|
+
// `resolveMainCheckout` spawns `git rev-parse` and `memoryDirFor` is called
|
|
66
|
+
// once per matching tag within a single hook invocation, where the project
|
|
67
|
+
// root and its main checkout never change mid-process. Memoize on
|
|
68
|
+
// `projectRoot` so repeated `rank()` calls in one process spawn git at most
|
|
69
|
+
// once per distinct root.
|
|
70
|
+
const memoryDirCache = new Map();
|
|
71
|
+
async function memoryDirFor(projectRoot) {
|
|
72
|
+
const cached = memoryDirCache.get(projectRoot);
|
|
73
|
+
if (cached !== undefined)
|
|
74
|
+
return cached;
|
|
75
|
+
const mainCheckout = await resolveMainCheckout(projectRoot);
|
|
76
|
+
const dir = join(homedir(), '.claude', 'projects', slugify(mainCheckout), 'memory');
|
|
77
|
+
memoryDirCache.set(projectRoot, dir);
|
|
78
|
+
return dir;
|
|
79
|
+
}
|
|
80
|
+
/** Strip a leading/trailing pair of double quotes, if both are present. */
|
|
81
|
+
function unquote(value) {
|
|
82
|
+
if (value.length >= 2 && value.startsWith('"') && value.endsWith('"')) {
|
|
83
|
+
return value.slice(1, -1);
|
|
84
|
+
}
|
|
85
|
+
return value;
|
|
86
|
+
}
|
|
87
|
+
/** Pull a `key: value` line out of YAML-ish frontmatter, quotes stripped. */
|
|
88
|
+
function frontmatterField(text, key) {
|
|
89
|
+
const re = new RegExp(`^${key}:\\s*(.*)$`, 'm');
|
|
90
|
+
const match = re.exec(text);
|
|
91
|
+
return match ? unquote(match[1].trim()) : undefined;
|
|
92
|
+
}
|
|
93
|
+
function readFileCapped(path) {
|
|
94
|
+
try {
|
|
95
|
+
const buf = readFileSync(path);
|
|
96
|
+
return buf.subarray(0, MAX_FILE_BYTES).toString('utf-8');
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
async function query(q) {
|
|
103
|
+
try {
|
|
104
|
+
return await rank(q);
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
// Unconditional, for the same reason as the bd store: every individual
|
|
108
|
+
// filesystem call below is already guarded, and the contract is worth
|
|
109
|
+
// more than the assumption that the list of them stays complete.
|
|
110
|
+
return [];
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
// memoryDir -> ordered list of { key, text } for its non-symlinked .md
|
|
114
|
+
// files. `federateTags` calls `rank()` once per matching document within a
|
|
115
|
+
// single hook invocation, where the memory store on disk cannot change
|
|
116
|
+
// between calls — so the directory listing and file contents are read once
|
|
117
|
+
// per process and reused, instead of re-stat'ing and re-reading every file
|
|
118
|
+
// on every call.
|
|
119
|
+
const fileListCache = new Map();
|
|
120
|
+
function loadFiles(memoryDir) {
|
|
121
|
+
const cached = fileListCache.get(memoryDir);
|
|
122
|
+
if (cached !== undefined)
|
|
123
|
+
return cached;
|
|
124
|
+
let result = [];
|
|
125
|
+
if (existsSync(memoryDir)) {
|
|
126
|
+
let entries = [];
|
|
127
|
+
try {
|
|
128
|
+
entries = readdirSync(memoryDir);
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
entries = [];
|
|
132
|
+
}
|
|
133
|
+
const files = [];
|
|
134
|
+
for (const entry of entries) {
|
|
135
|
+
if (!entry.endsWith('.md'))
|
|
136
|
+
continue;
|
|
137
|
+
const full = join(memoryDir, entry);
|
|
138
|
+
try {
|
|
139
|
+
// The directory name is derived from a filesystem path; a symlink here
|
|
140
|
+
// could point outside the memory store and leak an unrelated file's
|
|
141
|
+
// contents into a prompt, so symlinks are never followed.
|
|
142
|
+
if (lstatSync(full).isSymbolicLink())
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
files.push(full);
|
|
149
|
+
}
|
|
150
|
+
for (const file of files) {
|
|
151
|
+
const text = readFileCapped(file);
|
|
152
|
+
if (text === undefined)
|
|
153
|
+
continue;
|
|
154
|
+
result.push({ key: file, text });
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
fileListCache.set(memoryDir, result);
|
|
158
|
+
return result;
|
|
159
|
+
}
|
|
160
|
+
async function rank(q) {
|
|
161
|
+
const memoryDir = await memoryDirFor(q.projectRoot);
|
|
162
|
+
const files = loadFiles(memoryDir);
|
|
163
|
+
if (files.length === 0)
|
|
164
|
+
return [];
|
|
165
|
+
// key -> { text, count }, built in first-seen (readdir) order so ties keep
|
|
166
|
+
// that order after the stable sort below, matching the bd store's rule.
|
|
167
|
+
const seen = new Map();
|
|
168
|
+
for (const { key, text } of files) {
|
|
169
|
+
seen.set(key, { text, count: 0 });
|
|
170
|
+
}
|
|
171
|
+
for (const term of q.terms) {
|
|
172
|
+
if (!term)
|
|
173
|
+
continue;
|
|
174
|
+
// `\b` is ASCII-only in JS regex, so it fails to bound a term like
|
|
175
|
+
// "café" (`\bcafé\b` never matches "a café here"). Lookarounds on
|
|
176
|
+
// "word character" via a Unicode-aware class, with the `u` flag, bound
|
|
177
|
+
// the match without excluding non-ASCII letters.
|
|
178
|
+
const wordRe = new RegExp(`(?<![\\p{L}\\p{N}_])${escapeRegExp(term)}(?![\\p{L}\\p{N}_])`, 'iu');
|
|
179
|
+
for (const v of seen.values()) {
|
|
180
|
+
if (wordRe.test(v.text))
|
|
181
|
+
v.count += 1;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
const hits = [...seen.entries()]
|
|
185
|
+
.filter(([, v]) => v.count > 0)
|
|
186
|
+
.map(([key, v]) => {
|
|
187
|
+
const name = frontmatterField(v.text, 'name');
|
|
188
|
+
const description = frontmatterField(v.text, 'description');
|
|
189
|
+
const basename = key.slice(memoryDir.length + 1).replace(/\.md$/, '');
|
|
190
|
+
return {
|
|
191
|
+
store: 'claude-memory',
|
|
192
|
+
key,
|
|
193
|
+
title: name ?? basename,
|
|
194
|
+
detail: description ?? '',
|
|
195
|
+
score: v.count,
|
|
196
|
+
};
|
|
197
|
+
});
|
|
198
|
+
hits.sort((a, b) => b.score - a.score);
|
|
199
|
+
return hits.slice(0, q.limit);
|
|
200
|
+
}
|
|
201
|
+
export const claudeMemoryStore = {
|
|
202
|
+
name: 'claude-memory',
|
|
203
|
+
query,
|
|
204
|
+
};
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { createClient } from '@libsql/client';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import xdg from '@folder/xdg';
|
|
5
|
+
/** Only letters (any script), digits, and internal hyphens survive. Terms
|
|
6
|
+
* originate in repository frontmatter, which is attacker-controlled the
|
|
7
|
+
* moment an agent runs `lat` in a repo nobody here owns — a term containing
|
|
8
|
+
* a double quote would break out of the quoted FTS5 MATCH string, so
|
|
9
|
+
* anything that doesn't match this shape is dropped before it ever reaches
|
|
10
|
+
* SQL. Unicode letters/digits are allowed (not just ASCII) so a tag written
|
|
11
|
+
* in a non-Latin script still reaches the store instead of being silently
|
|
12
|
+
* dropped — sqlite's FTS5 tokenizer handles them fine, and they carry no
|
|
13
|
+
* special meaning to the MATCH syntax the way quotes or parens would. */
|
|
14
|
+
const SAFE_TERM = /^[\p{L}\p{N}][\p{L}\p{N}-]*$/u;
|
|
15
|
+
function dbPath() {
|
|
16
|
+
return process.env.CQ_LOCAL_DB_PATH ?? join(xdg().data, 'cq', 'local.db');
|
|
17
|
+
}
|
|
18
|
+
export const cqStore = {
|
|
19
|
+
name: 'cq',
|
|
20
|
+
async query(q) {
|
|
21
|
+
try {
|
|
22
|
+
const path = dbPath();
|
|
23
|
+
if (!existsSync(path))
|
|
24
|
+
return [];
|
|
25
|
+
const terms = q.terms.filter((t) => SAFE_TERM.test(t));
|
|
26
|
+
if (terms.length === 0)
|
|
27
|
+
return [];
|
|
28
|
+
const match = terms.map((t) => `"${t}"`).join(' OR ');
|
|
29
|
+
const client = createClient({ url: `file:${path}` });
|
|
30
|
+
try {
|
|
31
|
+
// A client returning structured rows (rather than shelling out to the
|
|
32
|
+
// `sqlite3` CLI, whose one-row-per-line list output needs newlines
|
|
33
|
+
// flattened and a control character as column separator to survive a
|
|
34
|
+
// summary containing '\n' or '|') removes a whole class of parsing
|
|
35
|
+
// bugs by construction.
|
|
36
|
+
const result = await client.execute({
|
|
37
|
+
sql: `SELECT rowid, summary, substr(action, 1, 400) AS action
|
|
38
|
+
FROM knowledge_units_fts
|
|
39
|
+
WHERE knowledge_units_fts MATCH ?
|
|
40
|
+
ORDER BY rank
|
|
41
|
+
LIMIT ?`,
|
|
42
|
+
args: [match, q.limit],
|
|
43
|
+
});
|
|
44
|
+
const hits = [];
|
|
45
|
+
for (const row of result.rows) {
|
|
46
|
+
const summary = row.summary;
|
|
47
|
+
if (typeof summary !== 'string' || summary.length === 0)
|
|
48
|
+
continue;
|
|
49
|
+
const action = row.action;
|
|
50
|
+
hits.push({
|
|
51
|
+
store: 'cq',
|
|
52
|
+
key: `cq:${row.rowid}`,
|
|
53
|
+
title: summary,
|
|
54
|
+
detail: typeof action === 'string' ? action : '',
|
|
55
|
+
// cq ranks internally via bm25 and cannot attribute a match to
|
|
56
|
+
// individual terms, so it reports the full term count and relies
|
|
57
|
+
// on the caller's stable sort to preserve this query's order.
|
|
58
|
+
score: terms.length,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
return hits;
|
|
62
|
+
}
|
|
63
|
+
finally {
|
|
64
|
+
client.close();
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
// Missing table, malformed FTS syntax, corrupt database, unexpected
|
|
69
|
+
// schema — every failure mode here is an absent-optional-store result,
|
|
70
|
+
// never a thrown error that could fail the user's prompt.
|
|
71
|
+
return [];
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
};
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { Store, TaggedDoc } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Loads and returns the three built-in stores. A function, not a static
|
|
4
|
+
* array: cq.ts, bd.ts, and claude-memory.ts are owned by other agents and may
|
|
5
|
+
* not exist yet, so the import must happen dynamically at call time (inside
|
|
6
|
+
* federateTags, via this function) rather than at module load — a top-level
|
|
7
|
+
* static import would fail this module's typecheck while they're mid-write.
|
|
8
|
+
* Each import is wrapped individually so one missing/broken store module
|
|
9
|
+
* leaves the other two available, per the "every store is optional" contract
|
|
10
|
+
* in types.ts.
|
|
11
|
+
*/
|
|
12
|
+
export declare function DEFAULT_STORES(): Promise<Store[]>;
|
|
13
|
+
export type FederateOptions = {
|
|
14
|
+
projectRoot: string;
|
|
15
|
+
/** Stores to query. Defaults to all three (loaded dynamically). */
|
|
16
|
+
stores?: Store[];
|
|
17
|
+
/** Max hits one store may return for one document. Default 3. */
|
|
18
|
+
perStoreLimit?: number;
|
|
19
|
+
/** Max terms taken from one tag list. Default 2. */
|
|
20
|
+
maxTerms?: number;
|
|
21
|
+
/** Max documents that actually EMIT a block. Default 3. */
|
|
22
|
+
maxEmitted?: number;
|
|
23
|
+
/** Max documents a lookup is attempted for. Default 20. */
|
|
24
|
+
maxAttempts?: number;
|
|
25
|
+
/** Document ids already surfaced. Mutated as documents are emitted. */
|
|
26
|
+
seen?: Set<string>;
|
|
27
|
+
/**
|
|
28
|
+
* Document ids that were looked up on a previous call and came back
|
|
29
|
+
* empty. Distinct from `seen`: an entry here is never emitted, so it must
|
|
30
|
+
* stay retryable (a store may later have something to say), but it should
|
|
31
|
+
* not re-spend an attempt on every call — otherwise a batch of zero-yield
|
|
32
|
+
* documents ahead of an answerable one starves that document forever,
|
|
33
|
+
* since nothing about the situation changes between calls. Skipping an
|
|
34
|
+
* already-attempted-empty document for free lets the attempt budget
|
|
35
|
+
* advance past it on the next call, rotating the scan window forward.
|
|
36
|
+
* Mutated as documents come back empty; not persisted by this function.
|
|
37
|
+
*/
|
|
38
|
+
attemptedEmpty?: Set<string>;
|
|
39
|
+
/** Max documents queried concurrently. Default 4. */
|
|
40
|
+
concurrency?: number;
|
|
41
|
+
/** Overall wall-clock budget for the whole call, in ms. Default 5000. */
|
|
42
|
+
deadlineMs?: number;
|
|
43
|
+
};
|
|
44
|
+
export declare function federateTags(docs: TaggedDoc[], opts: FederateOptions): Promise<string | null>;
|
|
45
|
+
/**
|
|
46
|
+
* Tagged documents for the given project-relative markdown paths, in order,
|
|
47
|
+
* deduped by path.
|
|
48
|
+
*/
|
|
49
|
+
export declare function taggedDocsForFiles(projectRoot: string, filePaths: string[]): Promise<TaggedDoc[]>;
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { parseFrontmatter } from '../lattice.js';
|
|
4
|
+
import { UNTRUSTED_NOTICE, quoteUntrusted, cleanUntrustedId, } from '../untrusted.js';
|
|
5
|
+
import { tagsToTerms, capHits } from './ranking.js';
|
|
6
|
+
const STORE_GROUP_LABELS = {
|
|
7
|
+
cq: 'cq:',
|
|
8
|
+
bd: 'bd memories:',
|
|
9
|
+
'claude-memory': 'Claude Code memory:',
|
|
10
|
+
};
|
|
11
|
+
function storeExportName(name) {
|
|
12
|
+
switch (name) {
|
|
13
|
+
case 'cq':
|
|
14
|
+
return 'cqStore';
|
|
15
|
+
case 'bd':
|
|
16
|
+
return 'bdStore';
|
|
17
|
+
case 'claude-memory':
|
|
18
|
+
return 'claudeMemoryStore';
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Loads and returns the three built-in stores. A function, not a static
|
|
23
|
+
* array: cq.ts, bd.ts, and claude-memory.ts are owned by other agents and may
|
|
24
|
+
* not exist yet, so the import must happen dynamically at call time (inside
|
|
25
|
+
* federateTags, via this function) rather than at module load — a top-level
|
|
26
|
+
* static import would fail this module's typecheck while they're mid-write.
|
|
27
|
+
* Each import is wrapped individually so one missing/broken store module
|
|
28
|
+
* leaves the other two available, per the "every store is optional" contract
|
|
29
|
+
* in types.ts.
|
|
30
|
+
*/
|
|
31
|
+
export async function DEFAULT_STORES() {
|
|
32
|
+
const stores = [];
|
|
33
|
+
const specs = [
|
|
34
|
+
{ path: './cq.js', name: 'cq' },
|
|
35
|
+
{ path: './bd.js', name: 'bd' },
|
|
36
|
+
{ path: './claude-memory.js', name: 'claude-memory' },
|
|
37
|
+
];
|
|
38
|
+
for (const spec of specs) {
|
|
39
|
+
try {
|
|
40
|
+
const mod = await import(spec.path);
|
|
41
|
+
const store = mod[storeExportName(spec.name)];
|
|
42
|
+
if (store && typeof store.query === 'function') {
|
|
43
|
+
stores.push(store);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
// Absent module, missing binary, etc. — treated as "store not present".
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return stores;
|
|
51
|
+
}
|
|
52
|
+
async function queryStore(store, terms, projectRoot, limit) {
|
|
53
|
+
try {
|
|
54
|
+
return await store.query({ terms, projectRoot, limit });
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
// A store must never throw per its contract, but a caller-injected fake
|
|
58
|
+
// in tests might — treat a rejection the same as an empty result so one
|
|
59
|
+
// misbehaving store can't take down federation for every document.
|
|
60
|
+
return [];
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Resolve `promise`, or `null` if `ms` elapses first.
|
|
65
|
+
*
|
|
66
|
+
* The abandoned promise keeps running — an in-flight store query cannot be
|
|
67
|
+
* cancelled — but nothing awaits it, so it cannot hold the prompt open. The
|
|
68
|
+
* timer is unref'd so a pending race never keeps the process alive.
|
|
69
|
+
*/
|
|
70
|
+
function raceDeadline(promise, ms) {
|
|
71
|
+
if (ms <= 0)
|
|
72
|
+
return Promise.resolve(null);
|
|
73
|
+
return new Promise((resolve) => {
|
|
74
|
+
const timer = setTimeout(() => resolve(null), ms);
|
|
75
|
+
timer.unref?.();
|
|
76
|
+
const settle = (value) => {
|
|
77
|
+
clearTimeout(timer);
|
|
78
|
+
resolve(value);
|
|
79
|
+
};
|
|
80
|
+
promise.then(settle, () => settle(null));
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
export async function federateTags(docs, opts) {
|
|
84
|
+
const stores = opts.stores ?? (await DEFAULT_STORES());
|
|
85
|
+
const perStoreLimit = opts.perStoreLimit ?? 3;
|
|
86
|
+
const maxTerms = opts.maxTerms ?? 2;
|
|
87
|
+
const maxEmitted = opts.maxEmitted ?? 3;
|
|
88
|
+
const maxAttempts = opts.maxAttempts ?? 20;
|
|
89
|
+
const seen = opts.seen ?? new Set();
|
|
90
|
+
const attemptedEmpty = opts.attemptedEmpty ?? new Set();
|
|
91
|
+
const concurrency = Math.max(1, opts.concurrency ?? 4);
|
|
92
|
+
const deadlineAt = Date.now() + (opts.deadlineMs ?? 5000);
|
|
93
|
+
// A document with a hit is buffered here by its original index rather
|
|
94
|
+
// than accepted immediately: under concurrency, several documents can
|
|
95
|
+
// resolve with hits before any of them has been checked against
|
|
96
|
+
// maxEmitted, so acceptance is decided once, in document order, after all
|
|
97
|
+
// workers finish. That is what keeps the final `seen`/emitted count
|
|
98
|
+
// identical to the fully-sequential version — see the acceptance step
|
|
99
|
+
// below — even though the queries themselves now overlap.
|
|
100
|
+
const candidates = new Map();
|
|
101
|
+
let attempts = 0;
|
|
102
|
+
let cursor = 0;
|
|
103
|
+
// Each worker pulls the next document off a shared cursor and races the
|
|
104
|
+
// others via Promise.all below. Everything up to and including the
|
|
105
|
+
// `attempts++` happens synchronously (no `await` in between), so — since
|
|
106
|
+
// JS runs one microtask at a time — the cursor, the skip checks, and the
|
|
107
|
+
// attempt count advance in exactly the same document order and with
|
|
108
|
+
// exactly the same skip/spend decisions as the single-worker (fully
|
|
109
|
+
// sequential) version. Only the store queries themselves overlap.
|
|
110
|
+
//
|
|
111
|
+
// One divergence, deliberate and bounded: `candidates.size` is a soft
|
|
112
|
+
// stopping signal standing in for the true accepted count, which isn't
|
|
113
|
+
// known until the acceptance step below runs. Workers claim their next
|
|
114
|
+
// document before any in-flight query has resolved, so up to
|
|
115
|
+
// `concurrency - 1` documents beyond the maxEmitted cutoff can still be
|
|
116
|
+
// attempted. That never yields FEWER accepted blocks (accepted is always
|
|
117
|
+
// <= candidates.size), and anything it records is true — a document marked
|
|
118
|
+
// attemptedEmpty really was queried and really came back empty. It costs
|
|
119
|
+
// at most a few extra queries per call and rotates through the list
|
|
120
|
+
// slightly faster than the sequential version would.
|
|
121
|
+
async function worker() {
|
|
122
|
+
for (;;) {
|
|
123
|
+
if (candidates.size >= maxEmitted || attempts >= maxAttempts)
|
|
124
|
+
return;
|
|
125
|
+
if (Date.now() >= deadlineAt)
|
|
126
|
+
return;
|
|
127
|
+
const index = cursor;
|
|
128
|
+
if (index >= docs.length)
|
|
129
|
+
return;
|
|
130
|
+
cursor++;
|
|
131
|
+
const doc = docs[index];
|
|
132
|
+
// A skip does NOT spend an attempt: other documents later in this same
|
|
133
|
+
// batch may be new, and letting them slide into the attempt window as
|
|
134
|
+
// earlier ones are skipped is what keeps a repeated batch from
|
|
135
|
+
// starving documents that have never been looked up.
|
|
136
|
+
if (seen.has(doc.id))
|
|
137
|
+
continue;
|
|
138
|
+
// A previous call already spent an attempt on this document and found
|
|
139
|
+
// nothing. Skipping it for free (no attempt spent) lets the attempt
|
|
140
|
+
// budget reach further into the list on this call than it did last
|
|
141
|
+
// time, instead of re-spending the same budget on the same dead end.
|
|
142
|
+
if (attemptedEmpty.has(doc.id))
|
|
143
|
+
continue;
|
|
144
|
+
// Tags are only chosen; empty means nothing to search on. Also free —
|
|
145
|
+
// don't spend an attempt on a document with nothing to query.
|
|
146
|
+
const terms = tagsToTerms(doc.tags, maxTerms);
|
|
147
|
+
if (terms.length === 0)
|
|
148
|
+
continue;
|
|
149
|
+
attempts++;
|
|
150
|
+
// Never pool tags from two documents: each document is queried with
|
|
151
|
+
// only its own terms, because two unrelated subjects intersect at
|
|
152
|
+
// nothing and pooling would just produce noise attributed to the
|
|
153
|
+
// wrong document.
|
|
154
|
+
// The deadline has to bound total wall clock, not just dispatch. bd and
|
|
155
|
+
// claude-memory cap their own subprocess calls at CALL_TIMEOUT_MS, but
|
|
156
|
+
// cq's libsql `execute` carries no timeout of its own, so a locked
|
|
157
|
+
// database file would otherwise hold this worker — and the prompt —
|
|
158
|
+
// open with no bound at all. Racing the queries against the remaining
|
|
159
|
+
// budget is what makes the deadline hard rather than advisory: an
|
|
160
|
+
// overrunning store is abandoned, never awaited.
|
|
161
|
+
const results = await raceDeadline(Promise.allSettled(stores.map((store) => queryStore(store, terms, opts.projectRoot, perStoreLimit))), deadlineAt - Date.now());
|
|
162
|
+
// Deadline expired mid-query. Return what has been gathered. The
|
|
163
|
+
// document is deliberately NOT marked attemptedEmpty: it was never
|
|
164
|
+
// answered, so a later call must be free to try it again.
|
|
165
|
+
if (results === null)
|
|
166
|
+
return;
|
|
167
|
+
const hitsByStore = new Map();
|
|
168
|
+
for (let i = 0; i < stores.length; i++) {
|
|
169
|
+
const result = results[i];
|
|
170
|
+
const hits = result.status === 'fulfilled' ? result.value : [];
|
|
171
|
+
if (hits.length > 0) {
|
|
172
|
+
hitsByStore.set(stores[i].name, hits);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
if (hitsByStore.size === 0) {
|
|
176
|
+
// Nothing found: don't mark seen, don't count as emitted. A document
|
|
177
|
+
// that produced nothing stays unmarked so a later run — once the
|
|
178
|
+
// stores have something to say — can still surface it.
|
|
179
|
+
attemptedEmpty.add(doc.id);
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
candidates.set(index, { doc, hitsByStore });
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, docs.length) }, () => worker()));
|
|
186
|
+
// Accept candidates in document order, exactly as the sequential version
|
|
187
|
+
// would have, and only up to maxEmitted — this is what turns the soft,
|
|
188
|
+
// racy `candidates.size` stop above into an exact budget. A candidate
|
|
189
|
+
// beyond the cutoff is dropped: not marked seen, not counted, left
|
|
190
|
+
// exactly as untouched as a document the sequential version never reached.
|
|
191
|
+
const acceptedIndices = [...candidates.keys()]
|
|
192
|
+
.sort((a, b) => a - b)
|
|
193
|
+
.slice(0, maxEmitted);
|
|
194
|
+
const blocks = acceptedIndices.map((index) => {
|
|
195
|
+
const { doc, hitsByStore } = candidates.get(index);
|
|
196
|
+
seen.add(doc.id);
|
|
197
|
+
return formatDocBlock(doc, hitsByStore, perStoreLimit);
|
|
198
|
+
});
|
|
199
|
+
if (blocks.length === 0)
|
|
200
|
+
return null;
|
|
201
|
+
// UNTRUSTED_NOTICE appears exactly once, at the top of the whole result —
|
|
202
|
+
// never once per document.
|
|
203
|
+
return [
|
|
204
|
+
UNTRUSTED_NOTICE,
|
|
205
|
+
'Section tags matched stored knowledge (auto-searched, verify before trusting):',
|
|
206
|
+
'',
|
|
207
|
+
blocks.join('\n\n'),
|
|
208
|
+
].join('\n');
|
|
209
|
+
}
|
|
210
|
+
function formatDocBlock(doc, hitsByStore, perStoreLimit) {
|
|
211
|
+
// The heading lists the document's WHOLE authored tag list, even though
|
|
212
|
+
// only the first two (maxTerms) drove the search — that keeps the heading
|
|
213
|
+
// honest about what the document is tagged with. doc.id and doc.tags come
|
|
214
|
+
// from repository frontmatter and are as untrusted as any hit text, so
|
|
215
|
+
// both go through cleanUntrustedId before joining.
|
|
216
|
+
const cleanId = cleanUntrustedId(doc.id);
|
|
217
|
+
const cleanTags = doc.tags.map((tag) => cleanUntrustedId(tag));
|
|
218
|
+
const lines = [`${cleanId} (${cleanTags.join(', ')}):`];
|
|
219
|
+
for (const [storeName, hits] of hitsByStore) {
|
|
220
|
+
const capped = capHits(hits, perStoreLimit);
|
|
221
|
+
const hitLines = [];
|
|
222
|
+
for (const hit of capped) {
|
|
223
|
+
const title = quoteUntrusted(hit.title, 200);
|
|
224
|
+
if (!hit.detail) {
|
|
225
|
+
// Omit a hit whose detail is empty rather than printing empty
|
|
226
|
+
// quotes; keep just the title.
|
|
227
|
+
hitLines.push(`- ${title}`);
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
const detail = quoteUntrusted(hit.detail, 220);
|
|
231
|
+
hitLines.push(`- ${title} -> ${detail}`);
|
|
232
|
+
}
|
|
233
|
+
if (hitLines.length === 0)
|
|
234
|
+
continue;
|
|
235
|
+
lines.push(STORE_GROUP_LABELS[storeName]);
|
|
236
|
+
lines.push(...hitLines);
|
|
237
|
+
}
|
|
238
|
+
return lines.join('\n');
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Tagged documents for the given project-relative markdown paths, in order,
|
|
242
|
+
* deduped by path.
|
|
243
|
+
*/
|
|
244
|
+
export async function taggedDocsForFiles(projectRoot, filePaths) {
|
|
245
|
+
const docs = [];
|
|
246
|
+
const seenPaths = new Set();
|
|
247
|
+
for (const filePath of filePaths) {
|
|
248
|
+
const id = filePath.split('\\').join('/');
|
|
249
|
+
if (seenPaths.has(id))
|
|
250
|
+
continue;
|
|
251
|
+
seenPaths.add(id);
|
|
252
|
+
let content;
|
|
253
|
+
try {
|
|
254
|
+
content = await readFile(join(projectRoot, filePath), 'utf-8');
|
|
255
|
+
}
|
|
256
|
+
catch {
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
const fm = parseFrontmatter(content);
|
|
260
|
+
const rawTags = fm.raw['tags'];
|
|
261
|
+
const tags = Array.isArray(rawTags)
|
|
262
|
+
? rawTags.filter((t) => typeof t === 'string')
|
|
263
|
+
: typeof rawTags === 'string'
|
|
264
|
+
? [rawTags]
|
|
265
|
+
: [];
|
|
266
|
+
if (tags.length === 0)
|
|
267
|
+
continue;
|
|
268
|
+
docs.push({ id, tags });
|
|
269
|
+
}
|
|
270
|
+
return docs;
|
|
271
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { KnowledgeHit } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Turn a frontmatter `tags:` value into search terms.
|
|
4
|
+
*
|
|
5
|
+
* Two terms by default: each store call costs a subprocess or a query, and
|
|
6
|
+
* two terms is the least that still lets matched-term-count ranking mean
|
|
7
|
+
* anything (a hit that matched both terms outranks one that matched one).
|
|
8
|
+
*/
|
|
9
|
+
export declare function tagsToTerms(value: unknown, maxTerms?: number): string[];
|
|
10
|
+
/**
|
|
11
|
+
* Stable sort by score, descending, then cap. Scores are only ever compared
|
|
12
|
+
* within one store's hits — a store that cannot attribute a match to
|
|
13
|
+
* individual terms and reports one flat score should never be capped
|
|
14
|
+
* alongside a store that scores per-term, so callers must call this once per
|
|
15
|
+
* store's hit list, not once over a merged list.
|
|
16
|
+
*/
|
|
17
|
+
export declare function capHits(hits: KnowledgeHit[], limit: number): KnowledgeHit[];
|