@phren/cli 0.3.49 → 0.3.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bridge/project-memory.d.ts +2 -0
- package/dist/bridge/project-memory.js +27 -13
- package/dist/bridge-hook.mjs +1947 -1672
- package/dist/cli/hooks-output.js +21 -3
- package/dist/content/citation-freshness.d.ts +29 -0
- package/dist/content/citation-freshness.js +311 -0
- package/dist/shared/retrieval.d.ts +2 -1
- package/dist/shared/retrieval.js +40 -8
- package/dist/tools/search.js +34 -4
- package/package.json +1 -1
package/dist/cli/hooks-output.js
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import * as path from "node:path";
|
|
2
|
+
import { cachedCitationFreshness, citationFreshnessAnnotation } from "../content/citation-freshness.js";
|
|
3
|
+
import { parseCitationComment } from "../content/citation.js";
|
|
4
|
+
import { getProjectRemote, readProjectConfig } from "../project-config.js";
|
|
1
5
|
import { recordInjection, recordRetrieval, } from "../shared/governance.js";
|
|
2
6
|
import { getDocSourceKey, } from "../shared/index.js";
|
|
3
7
|
import { logImpact, extractFindingIdsFromSnippet, } from "../finding/impact.js";
|
|
@@ -9,6 +13,7 @@ import { logger } from "../logger.js";
|
|
|
9
13
|
// ── Clanker mode: compact index ──────────────────────────────────────────────
|
|
10
14
|
function buildCompactIndex(selected, phrenPathLocal, terms) {
|
|
11
15
|
const lines = [];
|
|
16
|
+
const options = new Map();
|
|
12
17
|
for (const { doc, snippet, key } of selected) {
|
|
13
18
|
// A snippet that is one finding or task gets that entry's own id, so
|
|
14
19
|
// expanding it returns the entry rather than the whole file.
|
|
@@ -19,15 +24,28 @@ function buildCompactIndex(selected, phrenPathLocal, terms) {
|
|
|
19
24
|
const candidates = textLines.filter((l) => !/^\s*(#|\|?[\s:|-]+\|?\s*$)/.test(l));
|
|
20
25
|
const firstLine = candidates.reduce((best, l) => (best === undefined || termScore(l, terms) > termScore(best, terms) ? l : best), undefined) ?? textLines[0] ?? "";
|
|
21
26
|
const bullets = snippet.split("\n").filter((l) => /^-\s/.test(l));
|
|
22
|
-
const prefix = bullets.length === 1 ? bullets[0].replace(/…$/, "").slice(0, 120) : "";
|
|
27
|
+
const prefix = bullets.length === 1 ? bullets[0].replace(/ \[may be out of date:[\s\S]*$/, "").replace(/…$/, "").slice(0, 120) : "";
|
|
23
28
|
const entry = prefix ? docEntries(doc.content).find((e) => e.line.startsWith(prefix)) : undefined;
|
|
24
29
|
const id = entry ? entry.id : `mem:${getDocSourceKey(doc, phrenPathLocal)}`;
|
|
25
|
-
const text = entry ? entry.line : firstLine;
|
|
30
|
+
const text = (entry ? entry.line : firstLine).replace(/ \[may be out of date:[\s\S]*$/, "");
|
|
31
|
+
const citation = entry ? entry.text.split("\n").map(parseCitationComment).find(cite => cite !== null)
|
|
32
|
+
: parseCitationComment(snippet.split("\n")[snippet.split("\n").indexOf(firstLine) + 1] ?? "");
|
|
33
|
+
let annotation = "";
|
|
34
|
+
if (citation) {
|
|
35
|
+
const root = doc.path ? path.dirname(path.dirname(doc.path)) : phrenPathLocal;
|
|
36
|
+
const optionsKey = JSON.stringify([root, doc.project]);
|
|
37
|
+
let opts = options.get(optionsKey);
|
|
38
|
+
if (!opts) {
|
|
39
|
+
opts = { phrenPath: root, projectRemote: getProjectRemote(root, doc.project, readProjectConfig(root, doc.project)) };
|
|
40
|
+
options.set(optionsKey, opts);
|
|
41
|
+
}
|
|
42
|
+
annotation = citationFreshnessAnnotation(citation, cachedCitationFreshness(citation, opts));
|
|
43
|
+
}
|
|
26
44
|
const keywords = rowKeywords(text, terms);
|
|
27
45
|
// The fb: key rides here too. memory_feedback's description promises it
|
|
28
46
|
// appears in injected headers without qualification, and this path renders
|
|
29
47
|
// instead of the full one whenever clanker mode is on.
|
|
30
|
-
lines.push(`${id} fb:${key} ${rowTitle(text, HOOK_TITLE_CHARS)}${keywords.length ? ` [${keywords.join(", ")}]` : ""}`);
|
|
48
|
+
lines.push(`${id} fb:${key} ${rowTitle(text, HOOK_TITLE_CHARS)}${annotation}${keywords.length ? ` [${keywords.join(", ")}]` : ""}`);
|
|
31
49
|
}
|
|
32
50
|
return lines;
|
|
33
51
|
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { FindingCitation } from "./citation.js";
|
|
2
|
+
export interface CitationFreshness {
|
|
3
|
+
state: "current" | "changed" | "unknown";
|
|
4
|
+
checkedAt: string;
|
|
5
|
+
head?: string;
|
|
6
|
+
lastCommit?: {
|
|
7
|
+
sha: string;
|
|
8
|
+
summary: string;
|
|
9
|
+
authoredAt: string;
|
|
10
|
+
} | null;
|
|
11
|
+
}
|
|
12
|
+
export interface CitationFreshnessOptions {
|
|
13
|
+
phrenPath?: string;
|
|
14
|
+
projectRemote?: string | null;
|
|
15
|
+
}
|
|
16
|
+
export type FreshnessCitation = FindingCitation & {
|
|
17
|
+
projectRemote?: string | null;
|
|
18
|
+
};
|
|
19
|
+
/** Reads persisted resolutions and checks only; never runs git or accesses the network. Changed evidence survives refresh failures for seven days. */
|
|
20
|
+
export declare function cachedCitationFreshness(citation: FreshnessCitation, opts?: CitationFreshnessOptions): CitationFreshness | undefined;
|
|
21
|
+
interface RefreshOptions extends CitationFreshnessOptions {
|
|
22
|
+
budgetMs: number;
|
|
23
|
+
signal?: AbortSignal;
|
|
24
|
+
fetchImpl?: typeof fetch;
|
|
25
|
+
}
|
|
26
|
+
/** Best effort, bounded, and deduplicated per cache key, including background Hook polls. */
|
|
27
|
+
export declare function refreshCitationFreshness(citations: readonly FreshnessCitation[], opts: RefreshOptions): Promise<Map<FreshnessCitation, CitationFreshness>>;
|
|
28
|
+
export declare function citationFreshnessAnnotation(citation: FindingCitation, value?: CitationFreshness): string;
|
|
29
|
+
export {};
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { readFileSync, realpathSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { findPhrenPath, runtimeFile } from "../phren-paths.js";
|
|
6
|
+
import { kindFromDomain, parseRemoteUrl } from "../bridge/git-hosts.js";
|
|
7
|
+
import { resolveHostToken } from "../bridge/git-host-auth.js";
|
|
8
|
+
const MINUTE = 60_000;
|
|
9
|
+
// Refresh path only. Process spawns on Windows runners often exceed 200 ms.
|
|
10
|
+
const GIT_TIMEOUT_MS = 1000;
|
|
11
|
+
const RETENTION = 7 * 24 * 60 * MINUTE;
|
|
12
|
+
const RESOLUTION_TTL = 24 * 60 * MINUTE;
|
|
13
|
+
const cacheMemo = new Map();
|
|
14
|
+
const isChanged = (entry) => !!entry && "state" in entry && entry.state === "changed";
|
|
15
|
+
const resolutionKey = (citation, opts) => `repo:${citation.repo ?? ""}\0${citation.projectRemote ?? opts.projectRemote ?? ""}`;
|
|
16
|
+
const pending = new Map();
|
|
17
|
+
const off = () => process.env.PHREN_CITATION_FRESHNESS?.trim().toLowerCase() === "off";
|
|
18
|
+
function git(root, timeoutMs, ...args) {
|
|
19
|
+
if (timeoutMs <= 0)
|
|
20
|
+
return "";
|
|
21
|
+
try {
|
|
22
|
+
return execFileSync("git", [...(root ? ["-C", root] : []), ...args], {
|
|
23
|
+
encoding: "utf8", timeout: Math.max(1, Math.ceil(timeoutMs)), stdio: ["ignore", "pipe", "ignore"],
|
|
24
|
+
}).trim();
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return "";
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
function checkoutRoot(repo, deadline) {
|
|
31
|
+
if (!repo || !path.isAbsolute(repo))
|
|
32
|
+
return;
|
|
33
|
+
try {
|
|
34
|
+
const root = realpathSync(repo);
|
|
35
|
+
const toplevel = git(root, Math.min(GIT_TIMEOUT_MS, deadline - Date.now()), "rev-parse", "--show-toplevel");
|
|
36
|
+
if (!toplevel || !path.isAbsolute(toplevel))
|
|
37
|
+
return;
|
|
38
|
+
// Git prints forward slashes and long names on Windows; compare canonical forms.
|
|
39
|
+
const canonical = (value) => {
|
|
40
|
+
// The JS realpath keeps 8.3 short names (RUNNER~1); the native one expands them.
|
|
41
|
+
const resolved = path.resolve((realpathSync.native ?? realpathSync)(value));
|
|
42
|
+
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
43
|
+
};
|
|
44
|
+
return canonical(toplevel) === canonical(root) ? root : undefined;
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
function validDomain(domain) {
|
|
51
|
+
const match = /^([a-z0-9-]+(?:\.[a-z0-9-]+)*)(?::([0-9]+))?$/.exec(domain);
|
|
52
|
+
return !!match && (!match[2] || (Number(match[2]) >= 1 && Number(match[2]) <= 65535));
|
|
53
|
+
}
|
|
54
|
+
function repository(citation, opts, entries, deadline) {
|
|
55
|
+
const fallback = citation.projectRemote ?? opts.projectRemote;
|
|
56
|
+
const key = resolutionKey(citation, opts);
|
|
57
|
+
const cached = entries[key];
|
|
58
|
+
if (cached && "value" in cached && fresh(cached))
|
|
59
|
+
return cached.value;
|
|
60
|
+
const local = checkoutRoot(citation.repo, deadline);
|
|
61
|
+
const read = (...args) => git(local, Math.min(GIT_TIMEOUT_MS, deadline - Date.now()), ...args);
|
|
62
|
+
const remote = local ? read("remote", "get-url", "origin") : "";
|
|
63
|
+
const parsed = parseRemoteUrl(remote || fallback || "");
|
|
64
|
+
let value = null;
|
|
65
|
+
if (parsed && validDomain(parsed.domain) && /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(parsed.path)
|
|
66
|
+
&& !parsed.path.split("/").some(part => part === "." || part === "..")) {
|
|
67
|
+
const override = (remote ? read("config", "--get", "remote.origin.phrenHost") : "")
|
|
68
|
+
|| read("config", ...(local ? [] : ["--global"]), "--get", `phren.${parsed.domain}.host`);
|
|
69
|
+
const named = override.trim().toLowerCase();
|
|
70
|
+
const kind = ["github", "gitlab", "gitboy"].includes(named) ? named : kindFromDomain(parsed.domain);
|
|
71
|
+
if (kind === "gitboy")
|
|
72
|
+
value = { ...parsed, root: remote ? local : undefined };
|
|
73
|
+
}
|
|
74
|
+
// A budget-limited git lookup must not become a day-long negative resolution.
|
|
75
|
+
if (Date.now() < deadline)
|
|
76
|
+
entries[key] = { checkedAt: new Date().toISOString(), value };
|
|
77
|
+
return value;
|
|
78
|
+
}
|
|
79
|
+
function target(citation, repo) {
|
|
80
|
+
if (!citation.file || !citation.commit || !/^[a-f0-9]{7,64}$/i.test(citation.commit))
|
|
81
|
+
return;
|
|
82
|
+
let file = citation.file.split(path.sep).join("/");
|
|
83
|
+
if (file.includes("\\") || file.includes("\0") || file.split("/").includes(".."))
|
|
84
|
+
return;
|
|
85
|
+
if (!repo || !validDomain(repo.domain) || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo.path)
|
|
86
|
+
|| repo.path.split("/").some(part => part === "." || part === ".."))
|
|
87
|
+
return;
|
|
88
|
+
if (path.isAbsolute(file)) {
|
|
89
|
+
if (!repo.root || !path.isAbsolute(repo.root))
|
|
90
|
+
return;
|
|
91
|
+
// git canonicalizes macOS /var and other checkout aliases; citations may retain the alias.
|
|
92
|
+
const absolute = citation.repo && path.isAbsolute(citation.repo)
|
|
93
|
+
? path.resolve(repo.root, path.relative(citation.repo, file)) : file;
|
|
94
|
+
file = path.relative(repo.root, absolute).split(path.sep).join("/");
|
|
95
|
+
if (file === ".." || file.startsWith("../") || path.isAbsolute(file))
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
file = path.posix.normalize(file);
|
|
99
|
+
if (!file || file === "." || file.startsWith("../"))
|
|
100
|
+
return;
|
|
101
|
+
const result = { domain: repo.domain, repo: repo.path, commit: citation.commit, file };
|
|
102
|
+
return { ...result, key: JSON.stringify([result.domain, result.repo, result.commit, file]) };
|
|
103
|
+
}
|
|
104
|
+
function cacheFile(opts) {
|
|
105
|
+
const root = opts.phrenPath ?? findPhrenPath();
|
|
106
|
+
return root ? runtimeFile(root, "citation-freshness.json") : undefined;
|
|
107
|
+
}
|
|
108
|
+
function readCache(file) {
|
|
109
|
+
if (!file)
|
|
110
|
+
return {};
|
|
111
|
+
try {
|
|
112
|
+
const info = statSync(file);
|
|
113
|
+
const memo = cacheMemo.get(file);
|
|
114
|
+
if (memo && memo.mtimeMs === info.mtimeMs && memo.size === info.size)
|
|
115
|
+
return memo.entries;
|
|
116
|
+
const raw = JSON.parse(readFileSync(file, "utf8"));
|
|
117
|
+
const entries = Object.fromEntries(Object.entries(raw).filter(([key, value]) => value
|
|
118
|
+
&& Number.isFinite(Date.parse(value.checkedAt))
|
|
119
|
+
&& (key.startsWith("repo:") ? "value" in value : "state" in value && ["current", "changed", "unknown"].includes(value.state))));
|
|
120
|
+
rememberCache(file, entries, info);
|
|
121
|
+
return entries;
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
cacheMemo.delete(file);
|
|
125
|
+
return {};
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
function rememberCache(file, entries, info = statSync(file)) {
|
|
129
|
+
const { mtimeMs, size } = info;
|
|
130
|
+
if (cacheMemo.size >= 64)
|
|
131
|
+
cacheMemo.delete(cacheMemo.keys().next().value);
|
|
132
|
+
cacheMemo.set(file, { mtimeMs, size, entries });
|
|
133
|
+
}
|
|
134
|
+
function ttl(value) {
|
|
135
|
+
return "value" in value ? RESOLUTION_TTL : (value.state === "changed" ? 60 : value.state === "current" ? 15 : 5) * MINUTE;
|
|
136
|
+
}
|
|
137
|
+
function fresh(value) {
|
|
138
|
+
return !!value && Date.now() - Date.parse(value.checkedAt) < ttl(value);
|
|
139
|
+
}
|
|
140
|
+
function save(file, entries) {
|
|
141
|
+
if (!file)
|
|
142
|
+
return;
|
|
143
|
+
const temp = `${file}.${randomUUID()}.tmp`;
|
|
144
|
+
try {
|
|
145
|
+
const previous = readCache(file);
|
|
146
|
+
const merged = { ...previous, ...entries };
|
|
147
|
+
for (const [key, value] of Object.entries(previous)) {
|
|
148
|
+
const incoming = entries[key];
|
|
149
|
+
if (Date.now() - Date.parse(value.checkedAt) > ("value" in value ? RESOLUTION_TTL : RETENTION))
|
|
150
|
+
continue;
|
|
151
|
+
if (!incoming || (isChanged(value) && !isChanged(incoming))
|
|
152
|
+
|| (isChanged(value) === isChanged(incoming) && Date.parse(value.checkedAt) > Date.parse(incoming.checkedAt)))
|
|
153
|
+
merged[key] = value;
|
|
154
|
+
}
|
|
155
|
+
const sorted = Object.entries(merged).filter(([, value]) => Date.now() - Date.parse(value.checkedAt) <= ("value" in value ? RESOLUTION_TTL : RETENTION)).sort((a, b) => Date.parse(b[1].checkedAt) - Date.parse(a[1].checkedAt)).slice(0, 5000);
|
|
156
|
+
const saved = Object.fromEntries(sorted);
|
|
157
|
+
writeFileSync(temp, JSON.stringify(saved), { mode: 0o600 });
|
|
158
|
+
renameSync(temp, file);
|
|
159
|
+
rememberCache(file, saved);
|
|
160
|
+
}
|
|
161
|
+
catch { /* Cache failures never fail retrieval. */ }
|
|
162
|
+
finally {
|
|
163
|
+
try {
|
|
164
|
+
unlinkSync(temp);
|
|
165
|
+
}
|
|
166
|
+
catch { /* Already renamed. */ }
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
/** Reads persisted resolutions and checks only; never runs git or accesses the network. Changed evidence survives refresh failures for seven days. */
|
|
170
|
+
export function cachedCitationFreshness(citation, opts = {}) {
|
|
171
|
+
if (off())
|
|
172
|
+
return;
|
|
173
|
+
try {
|
|
174
|
+
const entries = readCache(cacheFile(opts));
|
|
175
|
+
const resolution = entries[resolutionKey(citation, opts)];
|
|
176
|
+
if (!resolution || !("value" in resolution) || !fresh(resolution))
|
|
177
|
+
return;
|
|
178
|
+
const resolved = target(citation, resolution.value);
|
|
179
|
+
const value = resolved ? entries[resolved.key] : undefined;
|
|
180
|
+
return value && "state" in value && Date.now() - Date.parse(value.checkedAt) <= RETENTION
|
|
181
|
+
&& (isChanged(value) || fresh(value)) ? value : undefined;
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
return undefined;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
function answer(raw, head, checkedAt) {
|
|
188
|
+
const row = raw;
|
|
189
|
+
if (typeof row?.changed !== "boolean")
|
|
190
|
+
return { state: "unknown", checkedAt };
|
|
191
|
+
const last = row.last_commit;
|
|
192
|
+
return { state: row.changed ? "changed" : "current", checkedAt,
|
|
193
|
+
...(typeof (row.head ?? head) === "string" ? { head: (row.head ?? head) } : {}),
|
|
194
|
+
...(row.changed ? { lastCommit: last && typeof last.sha === "string" && typeof last.summary === "string" && typeof last.authored_at === "string"
|
|
195
|
+
? { sha: last.sha, summary: last.summary, authoredAt: last.authored_at } : null } : {}) };
|
|
196
|
+
}
|
|
197
|
+
async function request(batch, opts, remaining, entries) {
|
|
198
|
+
const controller = new AbortController();
|
|
199
|
+
const abort = () => controller.abort();
|
|
200
|
+
opts.signal?.addEventListener("abort", abort, { once: true });
|
|
201
|
+
if (opts.signal?.aborted)
|
|
202
|
+
abort();
|
|
203
|
+
let timer;
|
|
204
|
+
const work = async () => {
|
|
205
|
+
const first = batch[0];
|
|
206
|
+
if (!validDomain(first.domain))
|
|
207
|
+
throw new Error("Invalid git host");
|
|
208
|
+
const url = new URL("https://" + first.domain);
|
|
209
|
+
url.pathname = `/api/v1/repos/${first.repo.split("/").map(encodeURIComponent).join("/")}/changed`;
|
|
210
|
+
const token = await resolveHostToken("gitboy", first.domain);
|
|
211
|
+
if (controller.signal.aborted)
|
|
212
|
+
throw new Error("Aborted");
|
|
213
|
+
url.searchParams.set("since", first.commit);
|
|
214
|
+
for (const item of batch)
|
|
215
|
+
url.searchParams.append("path", item.file);
|
|
216
|
+
const response = await (opts.fetchImpl ?? fetch)(url, { redirect: "manual", signal: controller.signal,
|
|
217
|
+
headers: { accept: "application/json", ...(token ? { authorization: `Bearer ${token.token}` } : {}) } });
|
|
218
|
+
if (!response.ok)
|
|
219
|
+
throw new Error("Host could not answer");
|
|
220
|
+
return await response.json();
|
|
221
|
+
};
|
|
222
|
+
try {
|
|
223
|
+
const expired = new Promise((_, reject) => {
|
|
224
|
+
timer = setTimeout(() => { abort(); reject(new Error("Deadline")); }, Math.max(1, Math.min(remaining, 5000)));
|
|
225
|
+
controller.signal.addEventListener("abort", () => reject(new Error("Aborted")), { once: true });
|
|
226
|
+
});
|
|
227
|
+
const body = await Promise.race([work(), expired]);
|
|
228
|
+
if (controller.signal.aborted)
|
|
229
|
+
throw new Error("Aborted");
|
|
230
|
+
const checkedAt = new Date().toISOString();
|
|
231
|
+
for (const item of batch) {
|
|
232
|
+
const value = answer(Array.isArray(body.paths) ? body.paths.find(row => row.path === item.file) : undefined, body.head, checkedAt);
|
|
233
|
+
if (!isChanged(entries[item.key]) || value.state === "changed")
|
|
234
|
+
entries[item.key] = value;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
for (const item of batch)
|
|
239
|
+
if (!isChanged(entries[item.key]))
|
|
240
|
+
entries[item.key] = { state: "unknown", checkedAt: new Date().toISOString() };
|
|
241
|
+
}
|
|
242
|
+
finally {
|
|
243
|
+
clearTimeout(timer);
|
|
244
|
+
opts.signal?.removeEventListener("abort", abort);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
/** Best effort, bounded, and deduplicated per cache key, including background Hook polls. */
|
|
248
|
+
export async function refreshCitationFreshness(citations, opts) {
|
|
249
|
+
const result = new Map();
|
|
250
|
+
if (off())
|
|
251
|
+
return result;
|
|
252
|
+
try {
|
|
253
|
+
const deadline = Date.now() + Math.max(0, opts.budgetMs);
|
|
254
|
+
const file = cacheFile(opts), entries = { ...readCache(file) };
|
|
255
|
+
const resolved = new Map();
|
|
256
|
+
const groups = new Map();
|
|
257
|
+
for (const citation of citations) {
|
|
258
|
+
const item = target(citation, repository(citation, opts, entries, deadline));
|
|
259
|
+
if (!item)
|
|
260
|
+
continue;
|
|
261
|
+
resolved.set(citation, item);
|
|
262
|
+
if (fresh(entries[item.key]))
|
|
263
|
+
continue;
|
|
264
|
+
const groupKey = JSON.stringify([item.domain, item.repo, item.commit]);
|
|
265
|
+
if (!groups.has(groupKey))
|
|
266
|
+
groups.set(groupKey, new Map());
|
|
267
|
+
groups.get(groupKey).set(item.key, item);
|
|
268
|
+
}
|
|
269
|
+
for (const group of groups.values()) {
|
|
270
|
+
const items = [...group.values()];
|
|
271
|
+
for (let i = 0; i < items.length; i += 100) {
|
|
272
|
+
const batch = items.slice(i, i + 100).filter(item => !pending.has(`${file}:${item.key}`));
|
|
273
|
+
const remaining = deadline - Date.now();
|
|
274
|
+
if (remaining <= 0 || opts.signal?.aborted) {
|
|
275
|
+
for (const item of items.slice(i))
|
|
276
|
+
if (!isChanged(entries[item.key]))
|
|
277
|
+
entries[item.key] = { state: "unknown", checkedAt: new Date().toISOString() };
|
|
278
|
+
break;
|
|
279
|
+
}
|
|
280
|
+
if (!batch.length)
|
|
281
|
+
continue;
|
|
282
|
+
const work = request(batch, opts, remaining, entries).then(() => save(file, entries));
|
|
283
|
+
for (const item of batch)
|
|
284
|
+
pending.set(`${file}:${item.key}`, work);
|
|
285
|
+
try {
|
|
286
|
+
await work;
|
|
287
|
+
}
|
|
288
|
+
finally {
|
|
289
|
+
for (const item of batch)
|
|
290
|
+
pending.delete(`${file}:${item.key}`);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
save(file, entries);
|
|
295
|
+
const latest = { ...entries, ...readCache(file) };
|
|
296
|
+
for (const [citation, item] of resolved) {
|
|
297
|
+
const value = latest[item.key];
|
|
298
|
+
if (value && "state" in value)
|
|
299
|
+
result.set(citation, value);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
catch { /* Offline, invalid citations and disk failures must never fail retrieval. */ }
|
|
303
|
+
return result;
|
|
304
|
+
}
|
|
305
|
+
export function citationFreshnessAnnotation(citation, value) {
|
|
306
|
+
if (value?.state !== "changed")
|
|
307
|
+
return "";
|
|
308
|
+
const clean = (text) => text.replace(/[\r\n\x00-\x1f]/g, " ").slice(0, 300);
|
|
309
|
+
const detail = value.lastCommit ? `changed in ${clean(value.lastCommit.sha).slice(0, 7)} "${clean(value.lastCommit.summary)}"` : "changed since cited";
|
|
310
|
+
return ` [may be out of date: ${clean(citation.file ?? "file")} ${detail}]`;
|
|
311
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type RetentionPolicy } from "./governance.js";
|
|
2
2
|
import { type DocRow, type SqlJsDatabase } from "./index.js";
|
|
3
|
+
import { type CitationFreshnessOptions } from "../content/citation-freshness.js";
|
|
3
4
|
import type { GitContext } from "../cli/hooks-session.js";
|
|
4
5
|
export type { GitContext } from "../cli/hooks-session.js";
|
|
5
6
|
/**
|
|
@@ -136,7 +137,7 @@ export interface SelectedSnippet {
|
|
|
136
137
|
}
|
|
137
138
|
/** Mark snippet lines with stale citations (cited file missing or line content changed).
|
|
138
139
|
* @internal Exported for tests. */
|
|
139
|
-
export declare function markStaleCitations(snippet: string): string;
|
|
140
|
+
export declare function markStaleCitations(snippet: string, opts?: CitationFreshnessOptions): string;
|
|
140
141
|
export { DEFAULT_MIN_QUERY_RELEVANCE };
|
|
141
142
|
/**
|
|
142
143
|
* Relevance floor: drop ranked docs that aren't actually relevant to *this*
|
package/dist/shared/retrieval.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
// shared-retrieval.ts — shared retrieval core used by hooks and MCP search.
|
|
2
|
+
import { getProjectRemote, readProjectConfig } from "../project-config.js";
|
|
2
3
|
import { getQualityMultiplier, entryScoreKey, } from "./governance.js";
|
|
3
4
|
import { queryDocRows, queryRows, cosineFallback, extractSnippet, getDocSourceKey, getFragmentBoostDocs, decodeFiniteNumber, rowToDocWithRowid, buildIndex, } from "./index.js";
|
|
4
5
|
import { filterTrustedFindingsDetailed, } from "./content.js";
|
|
6
|
+
import { cachedCitationFreshness, citationFreshnessAnnotation } from "../content/citation-freshness.js";
|
|
5
7
|
import { parseCitationComment } from "../content/citation.js";
|
|
6
8
|
import { getHighImpactFindings } from "../finding/impact.js";
|
|
7
9
|
import { buildFtsQueryVariants, buildRelaxedFtsQuery, isFeatureEnabled, STOP_WORDS, errorMessage } from "../utils.js";
|
|
@@ -689,6 +691,26 @@ export const NON_INJECTABLE_TYPES = new Set(["notes", "review-queue"]);
|
|
|
689
691
|
export function isInjectableDocType(type) {
|
|
690
692
|
return !NON_INJECTABLE_TYPES.has(type);
|
|
691
693
|
}
|
|
694
|
+
/** Restore only citation metadata for retained bullets; the FTS index omits comments. */
|
|
695
|
+
function restoreCitationComments(content, sourceFile) {
|
|
696
|
+
try {
|
|
697
|
+
const key = (line) => /<!--\s*fid:([a-z0-9]{8})/i.exec(line)?.[1]
|
|
698
|
+
?? line.replace(/<!--[\s\S]*?-->/g, "").trim();
|
|
699
|
+
const source = fs.readFileSync(sourceFile, "utf8").split("\n");
|
|
700
|
+
const citations = new Map();
|
|
701
|
+
for (let i = 0; i < source.length - 1; i++) {
|
|
702
|
+
if (source[i].startsWith("- ") && parseCitationComment(source[i + 1]))
|
|
703
|
+
citations.set(key(source[i]), source[i + 1]);
|
|
704
|
+
}
|
|
705
|
+
return content.split("\n").map((line, index, lines) => {
|
|
706
|
+
const citation = line.startsWith("- ") ? citations.get(key(line)) : undefined;
|
|
707
|
+
return citation && !parseCitationComment(lines[index + 1] ?? "") ? `${line}\n${citation}` : line;
|
|
708
|
+
}).join("\n");
|
|
709
|
+
}
|
|
710
|
+
catch {
|
|
711
|
+
return content;
|
|
712
|
+
}
|
|
713
|
+
}
|
|
692
714
|
/** Apply trust filter to rows. Returns filtered rows plus any queue/audit items to be written
|
|
693
715
|
* by the caller — retrieval itself should remain side-effect-free.
|
|
694
716
|
*
|
|
@@ -719,7 +741,7 @@ export function applyTrustFilter(rows, ttlDays, minConfidence, decay, phrenPath)
|
|
|
719
741
|
queueItems.push({ project: doc.project, section: "Conflicts", items: conflicts });
|
|
720
742
|
auditEntries.push(`project=${doc.project} type=${doc.type} stale=${stale.length} invalid_citation=${conflicts.length}`);
|
|
721
743
|
}
|
|
722
|
-
return { ...doc, content: trust.content };
|
|
744
|
+
return { ...doc, content: restoreCitationComments(trust.content, doc.path) };
|
|
723
745
|
})
|
|
724
746
|
.filter((doc) => {
|
|
725
747
|
return !TRUST_FILTERED_TYPES.has(doc.type) || Boolean(doc.content.trim());
|
|
@@ -907,16 +929,17 @@ export function rankResults(rows, intent, gitCtx, detectedProject, phrenPathLoca
|
|
|
907
929
|
}
|
|
908
930
|
/** Mark snippet lines with stale citations (cited file missing or line content changed).
|
|
909
931
|
* @internal Exported for tests. */
|
|
910
|
-
export function markStaleCitations(snippet) {
|
|
932
|
+
export function markStaleCitations(snippet, opts = {}) {
|
|
911
933
|
const lines = snippet.split("\n");
|
|
912
934
|
const result = [];
|
|
913
935
|
for (let i = 0; i < lines.length; i++) {
|
|
914
|
-
|
|
936
|
+
let line = lines[i];
|
|
915
937
|
// Check if the next line is a citation comment
|
|
916
938
|
const nextLine = lines[i + 1];
|
|
917
939
|
if (nextLine) {
|
|
918
940
|
const citation = parseCitationComment(nextLine);
|
|
919
941
|
if (citation && citation.file) {
|
|
942
|
+
line += citationFreshnessAnnotation(citation, cachedCitationFreshness(citation, opts));
|
|
920
943
|
const resolvedFile = citation.repo
|
|
921
944
|
? path.resolve(citation.repo, citation.file)
|
|
922
945
|
: (path.isAbsolute(citation.file) ? citation.file : null);
|
|
@@ -999,6 +1022,7 @@ export function selectSnippets(rows, keywords, tokenBudget, lineBudget, charBudg
|
|
|
999
1022
|
let usedTokens = 36;
|
|
1000
1023
|
const queryTokens = tokenizeForOverlap(keywords);
|
|
1001
1024
|
const seenBullets = new Set();
|
|
1025
|
+
const citationOptions = new Map();
|
|
1002
1026
|
// For each snippet being added, hash its bullet lines and skip duplicates
|
|
1003
1027
|
function dedupSnippetBullets(snippet) {
|
|
1004
1028
|
return snippet.split('\n').filter(line => {
|
|
@@ -1020,13 +1044,21 @@ export function selectSnippets(rows, keywords, tokenBudget, lineBudget, charBudg
|
|
|
1020
1044
|
// The bullet that matched, not the doc's opening lines.
|
|
1021
1045
|
const match = bestChunkMatch(queryTokens, doc);
|
|
1022
1046
|
const source = match && match.distinct >= MIN_CHUNK_MATCHES ? match.chunk : extractSnippet(doc.content, keywords, 8);
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
continue;
|
|
1026
|
-
// Mark findings with stale citations before injection
|
|
1047
|
+
// Read cached freshness before compaction can cut a citation comment. No refresh here.
|
|
1048
|
+
let annotated = source;
|
|
1027
1049
|
if (TRUST_FILTERED_TYPES.has(doc.type)) {
|
|
1028
|
-
|
|
1050
|
+
const root = doc.path ? path.dirname(path.dirname(doc.path)) : undefined;
|
|
1051
|
+
const optionsKey = JSON.stringify([root, doc.project]);
|
|
1052
|
+
let opts = citationOptions.get(optionsKey);
|
|
1053
|
+
if (!opts) {
|
|
1054
|
+
opts = root ? { phrenPath: root, projectRemote: getProjectRemote(root, doc.project, readProjectConfig(root, doc.project)) } : {};
|
|
1055
|
+
citationOptions.set(optionsKey, opts);
|
|
1056
|
+
}
|
|
1057
|
+
annotated = markStaleCitations(source, opts);
|
|
1029
1058
|
}
|
|
1059
|
+
let snippet = compactSnippet(annotated, lineBudget, charBudget);
|
|
1060
|
+
if (!snippet.trim())
|
|
1061
|
+
continue;
|
|
1030
1062
|
snippet = dedupSnippetBullets(snippet);
|
|
1031
1063
|
if (!snippet.trim())
|
|
1032
1064
|
continue;
|
package/dist/tools/search.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { cachedCitationFreshness, refreshCitationFreshness, citationFreshnessAnnotation } from "../content/citation-freshness.js";
|
|
2
|
+
import { getProjectRemote, readProjectConfig } from "../project-config.js";
|
|
1
3
|
import { nonInteractiveGitEnv } from "../utils-helpers.js";
|
|
2
4
|
import { projectMemoryCounts } from "../content/summarize.js";
|
|
3
5
|
import { moduleEnabled } from "../modules/runtime.js";
|
|
@@ -17,8 +19,8 @@ import { runCustomHooks } from "../hooks.js";
|
|
|
17
19
|
import { entryScoreKey, getQualityMultiplier, getRetentionPolicy, recordLookupEvents } from "../shared/governance.js";
|
|
18
20
|
import { bestFindingNodeId } from "../finding-graph-id.js";
|
|
19
21
|
import { callLlm } from "../content/dedup.js";
|
|
20
|
-
import { rankResults, searchKnowledgeRows, applyTrustFilter, searchFederatedStores } from "../shared/retrieval.js";
|
|
21
|
-
import { formatActorAttribution, parseScopeComment, parseSourceComment, collectSymbolCitations } from "../content/citation.js";
|
|
22
|
+
import { markStaleCitations, rankResults, searchKnowledgeRows, applyTrustFilter, searchFederatedStores } from "../shared/retrieval.js";
|
|
23
|
+
import { parseCitationComment, formatActorAttribution, parseScopeComment, parseSourceComment, collectSymbolCitations } from "../content/citation.js";
|
|
22
24
|
import { resolveActiveSessionScope } from "./session.js";
|
|
23
25
|
import { logger } from "../logger.js";
|
|
24
26
|
import { CLANKER_FETCH_HINT, clankerEnabled, findEntry, formatRow, queryTerms, rowKeywords, rowTitle, rowsForDoc } from "../clanker.js";
|
|
@@ -421,11 +423,31 @@ async function handleSearchKnowledge(ctx, { query, limit, project, type, tag, si
|
|
|
421
423
|
if (rows.length > maxResults) {
|
|
422
424
|
rows = rows.slice(0, maxResults);
|
|
423
425
|
}
|
|
426
|
+
const citationOptionsByProject = new Map();
|
|
427
|
+
const citationOptions = (row) => {
|
|
428
|
+
// Indexed paths also identify findings from registered secondary stores.
|
|
429
|
+
const root = row.path ? path.dirname(path.dirname(row.path)) : phrenPath;
|
|
430
|
+
const key = JSON.stringify([root, row.project]);
|
|
431
|
+
let opts = citationOptionsByProject.get(key);
|
|
432
|
+
if (!opts) {
|
|
433
|
+
opts = { phrenPath, projectRemote: getProjectRemote(root, row.project, readProjectConfig(root, row.project)) };
|
|
434
|
+
citationOptionsByProject.set(key, opts);
|
|
435
|
+
}
|
|
436
|
+
return opts;
|
|
437
|
+
};
|
|
438
|
+
const citationsByRow = rows.map(row => row.type === "findings" ? row.content.split("\n")
|
|
439
|
+
.map(parseCitationComment).filter(cite => cite !== null)
|
|
440
|
+
.map(cite => ({ ...cite, projectRemote: citationOptions(row).projectRemote })) : []);
|
|
441
|
+
await refreshCitationFreshness(citationsByRow.flat(), { phrenPath, budgetMs: 1500 });
|
|
424
442
|
// Snippets share one character budget so a response stays well under the
|
|
425
443
|
// client's tool-result limit however long the matched lines are.
|
|
426
444
|
const perResultChars = snippetBudget(rows.length);
|
|
427
445
|
const results = rows.map((row) => {
|
|
428
|
-
const
|
|
446
|
+
const excerpt = extractSnippet(row.content, query, 5, perResultChars);
|
|
447
|
+
const snippet = row.type === "findings" ? markStaleCitations(excerpt, citationOptions(row)) : excerpt;
|
|
448
|
+
const citations = row.type === "findings" ? snippet.split("\n").map(parseCitationComment)
|
|
449
|
+
.filter(citation => citation !== null).slice(0, 20).map(citation => ({ file: citation.file, commit: citation.commit,
|
|
450
|
+
citationFreshness: cachedCitationFreshness(citation, citationOptions(row)) })) : [];
|
|
429
451
|
const lifecycle = row.type === "findings" ? lifecycleByRowKey.get(findingRowKey(row)) : undefined;
|
|
430
452
|
const federationSource = "federationSource" in row ? row.federationSource : undefined;
|
|
431
453
|
const symbols = row.type === "findings" ? collectSymbolCitations(row.content) : [];
|
|
@@ -436,6 +458,7 @@ async function handleSearchKnowledge(ctx, { query, limit, project, type, tag, si
|
|
|
436
458
|
filename: row.filename,
|
|
437
459
|
type: row.type,
|
|
438
460
|
snippet,
|
|
461
|
+
...(citations.length ? { citations } : {}),
|
|
439
462
|
truncated: snippet.length < row.content.length,
|
|
440
463
|
path: row.path,
|
|
441
464
|
status: lifecycle?.primaryStatus,
|
|
@@ -500,7 +523,14 @@ async function handleSearchKnowledge(ctx, { query, limit, project, type, tag, si
|
|
|
500
523
|
const federationSource = "federationSource" in row ? row.federationSource : undefined;
|
|
501
524
|
const docId = federationSource ? `${federationSource}:${row.project}/${row.filename}` : `mem:${getDocSourceKey(row, phrenPath)}`;
|
|
502
525
|
const bestLine = extractSnippet(row.content, query, 1, 400);
|
|
503
|
-
|
|
526
|
+
const selected = rowsForDoc(row, docId, terms, bestLine, Math.min(3, rowCap - clankerRows.length));
|
|
527
|
+
for (const hit of selected) {
|
|
528
|
+
const entry = findEntry(row.content, hit.id);
|
|
529
|
+
const citation = entry?.text.split("\n").map(parseCitationComment).find(cite => cite !== null);
|
|
530
|
+
if (citation)
|
|
531
|
+
hit.title += citationFreshnessAnnotation(citation, cachedCitationFreshness(citation, citationOptions(row)));
|
|
532
|
+
}
|
|
533
|
+
clankerRows.push(...selected);
|
|
504
534
|
}
|
|
505
535
|
runCustomHooks(phrenPath, "post-search", { PHREN_QUERY: query, PHREN_RESULT_COUNT: String(clankerRows.length) });
|
|
506
536
|
return mcpResponse({
|