nlm-memory 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli/classify-parity.js +1 -1
- package/dist/cli/classify-parity.js.map +1 -1
- package/dist/cli/nlm.js +314 -55
- package/dist/cli/nlm.js.map +1 -1
- package/dist/core/classifier/prompt.d.ts +70 -0
- package/dist/core/classifier/prompt.js +71 -2
- package/dist/core/classifier/prompt.js.map +1 -1
- package/dist/core/exemplars/extract-exemplar.d.ts +36 -0
- package/dist/core/exemplars/extract-exemplar.js +177 -0
- package/dist/core/exemplars/extract-exemplar.js.map +1 -0
- package/dist/core/exemplars/ingest-exemplar.d.ts +30 -0
- package/dist/core/exemplars/ingest-exemplar.js +71 -0
- package/dist/core/exemplars/ingest-exemplar.js.map +1 -0
- package/dist/core/exemplars/recall-code.d.ts +25 -0
- package/dist/core/exemplars/recall-code.js +49 -0
- package/dist/core/exemplars/recall-code.js.map +1 -0
- package/dist/core/facts/backfill-facts.d.ts +4 -2
- package/dist/core/facts/backfill-facts.js +18 -27
- package/dist/core/facts/backfill-facts.js.map +1 -1
- package/dist/core/hook/strip-injected-context.d.ts +24 -0
- package/dist/core/hook/strip-injected-context.js +73 -0
- package/dist/core/hook/strip-injected-context.js.map +1 -0
- package/dist/core/ingest/ingest-session.d.ts +4 -2
- package/dist/core/ingest/ingest-session.js +12 -4
- package/dist/core/ingest/ingest-session.js.map +1 -1
- package/dist/core/providers/provider-registry.d.ts +31 -11
- package/dist/core/providers/provider-registry.js +27 -13
- package/dist/core/providers/provider-registry.js.map +1 -1
- package/dist/core/recall-facts/fact-recall-service.js +75 -28
- package/dist/core/recall-facts/fact-recall-service.js.map +1 -1
- package/dist/core/scheduler/scheduler.d.ts +4 -2
- package/dist/core/scheduler/scheduler.js +26 -15
- package/dist/core/scheduler/scheduler.js.map +1 -1
- package/dist/core/sources/source-registry.d.ts +36 -12
- package/dist/core/sources/source-registry.js +59 -44
- package/dist/core/sources/source-registry.js.map +1 -1
- package/dist/core/storage/backup-rotation.d.ts +38 -0
- package/dist/core/storage/backup-rotation.js +82 -0
- package/dist/core/storage/backup-rotation.js.map +1 -0
- package/dist/core/storage/null-code-exemplar-store.d.ts +21 -0
- package/dist/core/storage/null-code-exemplar-store.js +19 -0
- package/dist/core/storage/null-code-exemplar-store.js.map +1 -0
- package/dist/core/storage/pg-fact-store.d.ts +2 -0
- package/dist/core/storage/pg-fact-store.js +34 -3
- package/dist/core/storage/pg-fact-store.js.map +1 -1
- package/dist/core/storage/pg-session-store.d.ts +14 -4
- package/dist/core/storage/pg-session-store.js +123 -2
- package/dist/core/storage/pg-session-store.js.map +1 -1
- package/dist/core/storage/pg-storage.d.ts +11 -2
- package/dist/core/storage/pg-storage.js +14 -2
- package/dist/core/storage/pg-storage.js.map +1 -1
- package/dist/core/storage/pg-tx-context.d.ts +2 -0
- package/dist/core/storage/pg-tx-context.js +12 -0
- package/dist/core/storage/pg-tx-context.js.map +1 -1
- package/dist/core/storage/sqlite-code-exemplar-store.d.ts +40 -0
- package/dist/core/storage/sqlite-code-exemplar-store.js +247 -0
- package/dist/core/storage/sqlite-code-exemplar-store.js.map +1 -0
- package/dist/core/storage/sqlite-fact-store.d.ts +2 -0
- package/dist/core/storage/sqlite-fact-store.js +30 -2
- package/dist/core/storage/sqlite-fact-store.js.map +1 -1
- package/dist/core/storage/sqlite-session-store.d.ts +21 -0
- package/dist/core/storage/sqlite-session-store.js +23 -0
- package/dist/core/storage/sqlite-session-store.js.map +1 -1
- package/dist/core/storage/sqlite-storage.d.ts +6 -0
- package/dist/core/storage/sqlite-storage.js +14 -2
- package/dist/core/storage/sqlite-storage.js.map +1 -1
- package/dist/core/storage/supersedence-log.d.ts +8 -0
- package/dist/core/storage/supersedence-log.js +19 -0
- package/dist/core/storage/supersedence-log.js.map +1 -1
- package/dist/hook/prompt-recall-hook.d.ts +5 -4
- package/dist/hook/prompt-recall-hook.js +12 -7
- package/dist/hook/prompt-recall-hook.js.map +1 -1
- package/dist/http/app.d.ts +7 -1
- package/dist/http/app.js +226 -83
- package/dist/http/app.js.map +1 -1
- package/dist/install/codex.d.ts +29 -1
- package/dist/install/codex.js +117 -15
- package/dist/install/codex.js.map +1 -1
- package/dist/install/cursor.d.ts +4 -4
- package/dist/install/cursor.js +7 -7
- package/dist/install/cursor.js.map +1 -1
- package/dist/install/health.d.ts +56 -0
- package/dist/install/health.js +133 -0
- package/dist/install/health.js.map +1 -0
- package/dist/install/setup.js +1 -1
- package/dist/install/setup.js.map +1 -1
- package/dist/install/windsurf.d.ts +4 -4
- package/dist/install/windsurf.js +7 -7
- package/dist/install/windsurf.js.map +1 -1
- package/dist/llm/classifier-box.d.ts +4 -0
- package/dist/llm/classifier-box.js +9 -2
- package/dist/llm/classifier-box.js.map +1 -1
- package/dist/llm/ollama-client.d.ts +19 -1
- package/dist/llm/ollama-client.js +14 -6
- package/dist/llm/ollama-client.js.map +1 -1
- package/dist/llm/ollama-code-embedder.d.ts +28 -0
- package/dist/llm/ollama-code-embedder.js +73 -0
- package/dist/llm/ollama-code-embedder.js.map +1 -0
- package/dist/mcp/server.d.ts +10 -0
- package/dist/mcp/server.js +132 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/ports/code-embedder.d.ts +27 -0
- package/dist/ports/code-embedder.js +15 -0
- package/dist/ports/code-embedder.js.map +1 -0
- package/dist/ports/code-exemplar-store.d.ts +45 -0
- package/dist/ports/code-exemplar-store.js +11 -0
- package/dist/ports/code-exemplar-store.js.map +1 -0
- package/dist/ports/fact-store.d.ts +0 -0
- package/dist/ports/fact-store.js.map +1 -1
- package/dist/ports/storage.d.ts +8 -0
- package/dist/shared/types.d.ts +35 -0
- package/dist/ui/assets/{index-Fr5AMEPj.js → index-CVo7KKw2.js} +1 -1
- package/dist/ui/assets/{index-C74N8APQ.css → index-DCuQrzJH.css} +1 -1
- package/dist/ui/index.html +2 -2
- package/migrations/021_code_exemplars.sql +51 -0
- package/migrations/022_facts_retired_at.sql +17 -0
- package/migrations/pg/021_code_exemplars.sql +45 -0
- package/migrations/pg/022_facts_retired_at.sql +12 -0
- package/nlm/index.js +6 -1
- package/package.json +3 -2
- package/plugin/.codex-plugin/plugin.json +2 -2
- package/plugin/hooks/hooks.json +13 -2
- package/plugin/scripts/prompt-recall-hook.mjs +7 -1
- package/plugin/scripts/session-start-hook.mjs +275 -0
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Exemplar extraction from a Signal.
|
|
3
|
+
*
|
|
4
|
+
* Two sourcing paths (plan section 2):
|
|
5
|
+
* (a) git diff at the linked commit — language-agnostic, preferred for
|
|
6
|
+
* gate/test signals tied to a commit; uses git's hunk headers for
|
|
7
|
+
* free funcname capture.
|
|
8
|
+
* (b) producer-supplied code in signal.detail.code — for pre-commit gates
|
|
9
|
+
* and eval harnesses where no commit exists yet.
|
|
10
|
+
*
|
|
11
|
+
* Returns null when neither path produces a valid chunk, so the caller can
|
|
12
|
+
* skip without error.
|
|
13
|
+
*/
|
|
14
|
+
import { execFileSync } from "node:child_process";
|
|
15
|
+
import { codeHash, normalizeExemplar } from "./ingest-exemplar.js";
|
|
16
|
+
/** Extract a git hunk header funcname: `@@ ... @@ funcname` */
|
|
17
|
+
function parseFuncname(hunkHeader) {
|
|
18
|
+
const m = hunkHeader.match(/@@ [^@]+ @@ (.+)/);
|
|
19
|
+
return m?.[1]?.trim() ?? null;
|
|
20
|
+
}
|
|
21
|
+
/** Detect likely language from file extension. */
|
|
22
|
+
function detectLang(filePath) {
|
|
23
|
+
const ext = filePath.split(".").pop()?.toLowerCase() ?? "";
|
|
24
|
+
const map = {
|
|
25
|
+
ts: "ts", tsx: "ts", js: "js", jsx: "js", mjs: "js",
|
|
26
|
+
py: "py", go: "go", rb: "rb", rs: "rs", java: "java",
|
|
27
|
+
c: "c", cpp: "cpp", cs: "cs", swift: "swift", kt: "kt",
|
|
28
|
+
sh: "sh", bash: "sh", zsh: "sh",
|
|
29
|
+
};
|
|
30
|
+
return map[ext] ?? null;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Parse a unified diff into individual hunks. Returns (file, hunkHeader, body) triples.
|
|
34
|
+
*/
|
|
35
|
+
function parseHunks(diff) {
|
|
36
|
+
const result = [];
|
|
37
|
+
let currentFile = "";
|
|
38
|
+
let currentHeader = "";
|
|
39
|
+
let bodyLines = [];
|
|
40
|
+
for (const line of diff.split("\n")) {
|
|
41
|
+
if (line.startsWith("+++ b/")) {
|
|
42
|
+
currentFile = line.slice(6);
|
|
43
|
+
}
|
|
44
|
+
else if (line.startsWith("@@ ")) {
|
|
45
|
+
if (currentHeader && bodyLines.length > 0) {
|
|
46
|
+
result.push({ file: currentFile, hunkHeader: currentHeader, body: bodyLines.join("\n") });
|
|
47
|
+
}
|
|
48
|
+
currentHeader = line;
|
|
49
|
+
bodyLines = [];
|
|
50
|
+
}
|
|
51
|
+
else if (currentHeader && (line.startsWith("+") || line.startsWith("-") || line.startsWith(" "))) {
|
|
52
|
+
bodyLines.push(line);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (currentHeader && bodyLines.length > 0) {
|
|
56
|
+
result.push({ file: currentFile, hunkHeader: currentHeader, body: bodyLines.join("\n") });
|
|
57
|
+
}
|
|
58
|
+
return result;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Extract the added/changed lines from a hunk body (strip context and removed lines).
|
|
62
|
+
*/
|
|
63
|
+
function extractAddedLines(hunkBody) {
|
|
64
|
+
return hunkBody
|
|
65
|
+
.split("\n")
|
|
66
|
+
.filter((l) => l.startsWith("+"))
|
|
67
|
+
.map((l) => l.slice(1))
|
|
68
|
+
.join("\n");
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Path (a): extract exemplar from a git commit's diff.
|
|
72
|
+
* Returns null if the git command fails or no valid hunks are found.
|
|
73
|
+
*/
|
|
74
|
+
export function extractFromGitSha(signal, gitSha, opts) {
|
|
75
|
+
const repoPath = opts.repoPath ?? signal.repo;
|
|
76
|
+
let diff;
|
|
77
|
+
try {
|
|
78
|
+
diff = execFileSync("git", ["show", "--unified=3", "--diff-filter=AM", gitSha], { cwd: repoPath, encoding: "utf8", timeout: 10_000 });
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
const hunks = parseHunks(diff);
|
|
84
|
+
if (hunks.length === 0)
|
|
85
|
+
return null;
|
|
86
|
+
// Pick the largest hunk by added-line count (most representative).
|
|
87
|
+
let best;
|
|
88
|
+
let bestCount = 0;
|
|
89
|
+
for (const h of hunks) {
|
|
90
|
+
const added = extractAddedLines(h.body);
|
|
91
|
+
const count = added.split("\n").filter((l) => l.trim()).length;
|
|
92
|
+
if (count > bestCount) {
|
|
93
|
+
bestCount = count;
|
|
94
|
+
best = h;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (!best)
|
|
98
|
+
return null;
|
|
99
|
+
const code = extractAddedLines(best.body);
|
|
100
|
+
const funcname = parseFuncname(best.hunkHeader);
|
|
101
|
+
const taskContext = funcname
|
|
102
|
+
? `${funcname} (${best.file})`
|
|
103
|
+
: `changes in ${best.file}`;
|
|
104
|
+
try {
|
|
105
|
+
return normalizeExemplar({
|
|
106
|
+
installScope: opts.installScope,
|
|
107
|
+
signalId: signal.id,
|
|
108
|
+
sessionId: signal.sessionId,
|
|
109
|
+
repo: signal.repo,
|
|
110
|
+
model: signal.model,
|
|
111
|
+
lang: detectLang(best.file),
|
|
112
|
+
taskContext,
|
|
113
|
+
code,
|
|
114
|
+
outcome: signal.outcome,
|
|
115
|
+
gitSha,
|
|
116
|
+
survived: null,
|
|
117
|
+
ts: signal.ts,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Path (b): extract exemplar from signal.detail.code (producer-supplied).
|
|
126
|
+
* Returns null if detail.code is absent or doesn't pass size validation.
|
|
127
|
+
*/
|
|
128
|
+
export function extractFromDetail(signal, opts) {
|
|
129
|
+
const detail = signal.detail;
|
|
130
|
+
if (!detail || typeof detail["code"] !== "string")
|
|
131
|
+
return null;
|
|
132
|
+
const code = detail["code"];
|
|
133
|
+
const taskContext = typeof detail["task"] === "string"
|
|
134
|
+
? detail["task"]
|
|
135
|
+
: typeof signal.step === "string"
|
|
136
|
+
? signal.step
|
|
137
|
+
: "agent-supplied code";
|
|
138
|
+
const lang = typeof detail["lang"] === "string" ? detail["lang"] : null;
|
|
139
|
+
try {
|
|
140
|
+
return normalizeExemplar({
|
|
141
|
+
installScope: opts.installScope,
|
|
142
|
+
signalId: signal.id,
|
|
143
|
+
sessionId: signal.sessionId,
|
|
144
|
+
repo: signal.repo,
|
|
145
|
+
model: signal.model,
|
|
146
|
+
lang,
|
|
147
|
+
taskContext,
|
|
148
|
+
code,
|
|
149
|
+
outcome: signal.outcome,
|
|
150
|
+
gitSha: typeof detail["git_sha"] === "string" ? detail["git_sha"] : null,
|
|
151
|
+
survived: null,
|
|
152
|
+
ts: signal.ts,
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Top-level: try path (a) first, fall back to path (b).
|
|
161
|
+
* Inline with signal ingest; called after a signal is stored.
|
|
162
|
+
*/
|
|
163
|
+
export function extractExemplar(signal, opts) {
|
|
164
|
+
// Path (a): git sha from detail, if present.
|
|
165
|
+
const gitSha = signal.detail && typeof signal.detail["git_sha"] === "string"
|
|
166
|
+
? signal.detail["git_sha"]
|
|
167
|
+
: null;
|
|
168
|
+
if (gitSha) {
|
|
169
|
+
const repoPath = opts.repoBasePath ? `${opts.repoBasePath}/${signal.repo}` : signal.repo;
|
|
170
|
+
const fromGit = extractFromGitSha(signal, gitSha, { installScope: opts.installScope, repoPath });
|
|
171
|
+
if (fromGit)
|
|
172
|
+
return fromGit;
|
|
173
|
+
}
|
|
174
|
+
// Path (b): producer-supplied code in detail.
|
|
175
|
+
return extractFromDetail(signal, opts);
|
|
176
|
+
}
|
|
177
|
+
//# sourceMappingURL=extract-exemplar.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract-exemplar.js","sourceRoot":"","sources":["../../../src/core/exemplars/extract-exemplar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AASnE,+DAA+D;AAC/D,SAAS,aAAa,CAAC,UAAkB;IACvC,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAC/C,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;AAChC,CAAC;AAED,kDAAkD;AAClD,SAAS,UAAU,CAAC,QAAgB;IAClC,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC3D,MAAM,GAAG,GAA2B;QAClC,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI;QACnD,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM;QACpD,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI;QACtD,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI;KAChC,CAAC;IACF,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,MAAM,GAA8D,EAAE,CAAC;IAC7E,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,SAAS,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,IAAI,aAAa,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC5F,CAAC;YACD,aAAa,GAAG,IAAI,CAAC;YACrB,SAAS,GAAG,EAAE,CAAC;QACjB,CAAC;aAAM,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACnG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IACD,IAAI,aAAa,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5F,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAgB;IACzC,OAAO,QAAQ;SACZ,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;SAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACtB,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAC/B,MAAc,EACd,MAAc,EACd,IAAoB;IAEpB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC;IAC9C,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,YAAY,CACjB,KAAK,EACL,CAAC,MAAM,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,CAAC,EACnD,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CACrD,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,mEAAmE;IACnE,IAAI,IAAoE,CAAC;IACzE,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAC/D,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YAAC,SAAS,GAAG,KAAK,CAAC;YAAC,IAAI,GAAG,CAAC,CAAC;QAAC,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,QAAQ;QAC1B,CAAC,CAAC,GAAG,QAAQ,KAAK,IAAI,CAAC,IAAI,GAAG;QAC9B,CAAC,CAAC,cAAc,IAAI,CAAC,IAAI,EAAE,CAAC;IAE9B,IAAI,CAAC;QACH,OAAO,iBAAiB,CAAC;YACvB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,QAAQ,EAAE,MAAM,CAAC,EAAE;YACnB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3B,WAAW;YACX,IAAI;YACJ,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM;YACN,QAAQ,EAAE,IAAI;YACd,EAAE,EAAE,MAAM,CAAC,EAAE;SACd,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc,EAAE,IAAoB;IACpE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAW,CAAC;IAEtC,MAAM,WAAW,GACf,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,QAAQ;QAChC,CAAC,CAAE,MAAM,CAAC,MAAM,CAAY;QAC5B,CAAC,CAAC,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ;YACjC,CAAC,CAAC,MAAM,CAAC,IAAI;YACb,CAAC,CAAC,qBAAqB,CAAC;IAE5B,MAAM,IAAI,GACR,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,MAAM,CAAC,MAAM,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAEzE,IAAI,CAAC;QACH,OAAO,iBAAiB,CAAC;YACvB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,QAAQ,EAAE,MAAM,CAAC,EAAE;YACnB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,IAAI;YACJ,WAAW;YACX,IAAI;YACJ,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,MAAM,CAAC,SAAS,CAAY,CAAC,CAAC,CAAC,IAAI;YACpF,QAAQ,EAAE,IAAI;YACd,EAAE,EAAE,MAAM,CAAC,EAAE;SACd,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAc,EACd,IAAgD;IAEhD,6CAA6C;IAC7C,MAAM,MAAM,GACV,MAAM,CAAC,MAAM,IAAI,OAAO,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,QAAQ;QAC3D,CAAC,CAAE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAY;QACtC,CAAC,CAAC,IAAI,CAAC;IACX,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;QACzF,MAAM,OAAO,GAAG,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,CAAC,CAAC;QACjG,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;IAC9B,CAAC;IAED,8CAA8C;IAC9C,OAAO,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Boundary validation + normalization for inbound code exemplars.
|
|
3
|
+
*
|
|
4
|
+
* Parallel to ingest-signal.ts: validates at the boundary (fail loud),
|
|
5
|
+
* soft-defaults optional fields. The id is deterministic so re-ingesting
|
|
6
|
+
* the same chunk is a no-op at the store layer (INSERT OR IGNORE).
|
|
7
|
+
*
|
|
8
|
+
* Gated by NLM_CODE_EXEMPLARS_ENABLED. Callers must check the flag.
|
|
9
|
+
*/
|
|
10
|
+
import type { CodeExemplarInput } from "../../shared/types.js";
|
|
11
|
+
/**
|
|
12
|
+
* sha256 of the code after whitespace normalization.
|
|
13
|
+
* Normalization: trim trailing whitespace per line + collapse blank runs.
|
|
14
|
+
*/
|
|
15
|
+
export declare function codeHash(code: string): string;
|
|
16
|
+
export interface RawExemplarPayload {
|
|
17
|
+
readonly installScope: string;
|
|
18
|
+
readonly signalId?: string | null;
|
|
19
|
+
readonly sessionId?: string | null;
|
|
20
|
+
readonly repo: string;
|
|
21
|
+
readonly model: string;
|
|
22
|
+
readonly lang?: string | null;
|
|
23
|
+
readonly taskContext: string;
|
|
24
|
+
readonly code: string;
|
|
25
|
+
readonly outcome: string;
|
|
26
|
+
readonly gitSha?: string | null;
|
|
27
|
+
readonly survived?: 0 | 1 | null;
|
|
28
|
+
readonly ts?: string;
|
|
29
|
+
}
|
|
30
|
+
export declare function normalizeExemplar(raw: RawExemplarPayload, now?: () => string): CodeExemplarInput;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Boundary validation + normalization for inbound code exemplars.
|
|
3
|
+
*
|
|
4
|
+
* Parallel to ingest-signal.ts: validates at the boundary (fail loud),
|
|
5
|
+
* soft-defaults optional fields. The id is deterministic so re-ingesting
|
|
6
|
+
* the same chunk is a no-op at the store layer (INSERT OR IGNORE).
|
|
7
|
+
*
|
|
8
|
+
* Gated by NLM_CODE_EXEMPLARS_ENABLED. Callers must check the flag.
|
|
9
|
+
*/
|
|
10
|
+
import { createHash } from "node:crypto";
|
|
11
|
+
const OUTCOMES = new Set(["pass", "fail", "fix", "exhausted"]);
|
|
12
|
+
const MIN_MEANINGFUL_LINES = 2;
|
|
13
|
+
const MAX_LINES = 200;
|
|
14
|
+
/**
|
|
15
|
+
* sha256 of the code after whitespace normalization.
|
|
16
|
+
* Normalization: trim trailing whitespace per line + collapse blank runs.
|
|
17
|
+
*/
|
|
18
|
+
export function codeHash(code) {
|
|
19
|
+
const normalized = code
|
|
20
|
+
.split("\n")
|
|
21
|
+
.map((l) => l.trimEnd())
|
|
22
|
+
.filter((l, i, arr) => l !== "" || arr[i - 1] !== "")
|
|
23
|
+
.join("\n")
|
|
24
|
+
.trim();
|
|
25
|
+
return createHash("sha256").update(normalized).digest("hex");
|
|
26
|
+
}
|
|
27
|
+
/** Count non-blank, non-brace-only lines. */
|
|
28
|
+
function meaningfulLineCount(code) {
|
|
29
|
+
return code.split("\n").filter((l) => {
|
|
30
|
+
const t = l.trim();
|
|
31
|
+
return t.length > 0 && t !== "{" && t !== "}" && t !== "}" + ";" && t !== "};";
|
|
32
|
+
}).length;
|
|
33
|
+
}
|
|
34
|
+
export function normalizeExemplar(raw, now = () => new Date().toISOString()) {
|
|
35
|
+
const outcome = raw.outcome;
|
|
36
|
+
if (!OUTCOMES.has(outcome)) {
|
|
37
|
+
throw new Error(`exemplar.outcome must be one of pass|fail|fix|exhausted (got ${String(outcome)})`);
|
|
38
|
+
}
|
|
39
|
+
if (!raw.repo || typeof raw.repo !== "string") {
|
|
40
|
+
throw new Error("exemplar.repo is required");
|
|
41
|
+
}
|
|
42
|
+
if (!raw.taskContext || typeof raw.taskContext !== "string") {
|
|
43
|
+
throw new Error("exemplar.taskContext is required");
|
|
44
|
+
}
|
|
45
|
+
if (!raw.code || typeof raw.code !== "string") {
|
|
46
|
+
throw new Error("exemplar.code is required");
|
|
47
|
+
}
|
|
48
|
+
const meaningful = meaningfulLineCount(raw.code);
|
|
49
|
+
if (meaningful < MIN_MEANINGFUL_LINES) {
|
|
50
|
+
throw new Error(`exemplar.code too small (${meaningful} meaningful lines, minimum ${MIN_MEANINGFUL_LINES})`);
|
|
51
|
+
}
|
|
52
|
+
if (meaningful > MAX_LINES) {
|
|
53
|
+
throw new Error(`exemplar.code too large (${meaningful} meaningful lines, maximum ${MAX_LINES}); split into smaller hunks`);
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
installScope: raw.installScope,
|
|
57
|
+
signalId: raw.signalId ?? null,
|
|
58
|
+
sessionId: raw.sessionId ?? null,
|
|
59
|
+
repo: raw.repo,
|
|
60
|
+
model: raw.model || "unknown",
|
|
61
|
+
lang: raw.lang ?? null,
|
|
62
|
+
taskContext: raw.taskContext,
|
|
63
|
+
code: raw.code,
|
|
64
|
+
codeHash: codeHash(raw.code),
|
|
65
|
+
outcome: outcome,
|
|
66
|
+
gitSha: raw.gitSha ?? null,
|
|
67
|
+
survived: raw.survived ?? null,
|
|
68
|
+
ts: raw.ts ?? now(),
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
//# sourceMappingURL=ingest-exemplar.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingest-exemplar.js","sourceRoot":"","sources":["../../../src/core/exemplars/ingest-exemplar.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,MAAM,QAAQ,GAAwB,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC;AAEpF,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAC/B,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,MAAM,UAAU,GAAG,IAAI;SACpB,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;SACvB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;SACpD,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,EAAE,CAAC;IACV,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC/D,CAAC;AAED,6CAA6C;AAC7C,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACnC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACnB,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC;IACjF,CAAC,CAAC,CAAC,MAAM,CAAC;AACZ,CAAC;AAiBD,MAAM,UAAU,iBAAiB,CAC/B,GAAuB,EACvB,MAAoB,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;IAElD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,gEAAgE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACtG,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,WAAW,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,UAAU,GAAG,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,UAAU,GAAG,oBAAoB,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,4BAA4B,UAAU,8BAA8B,oBAAoB,GAAG,CAC5F,CAAC;IACJ,CAAC;IACD,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,4BAA4B,UAAU,8BAA8B,SAAS,6BAA6B,CAC3G,CAAC;IACJ,CAAC;IAED,OAAO;QACL,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI;QAC9B,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,IAAI;QAChC,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,SAAS;QAC7B,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,IAAI;QACtB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;QAC5B,OAAO,EAAE,OAA8B;QACvC,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI;QAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI;QAC9B,EAAE,EAAE,GAAG,CAAC,EAAE,IAAI,GAAG,EAAE;KACpB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recall-code — semantic search over the code-exemplar lane.
|
|
3
|
+
*
|
|
4
|
+
* Embeds the query with the code embedder (or falls back to the prose embedder),
|
|
5
|
+
* delegates to CodeExemplarStore.searchByVector, and formats results.
|
|
6
|
+
* Negatives are returned clearly labeled — "code that failed the gate" is
|
|
7
|
+
* as useful to the caller as the positive.
|
|
8
|
+
*/
|
|
9
|
+
import type { CodeEmbedder } from "../../ports/code-embedder.js";
|
|
10
|
+
import type { CodeExemplarSearchFilter, CodeExemplarStore } from "../../ports/code-exemplar-store.js";
|
|
11
|
+
import type { LLMClient } from "../../ports/llm-client.js";
|
|
12
|
+
import type { CodeExemplarHit } from "../../shared/types.js";
|
|
13
|
+
export interface RecallCodeOptions extends CodeExemplarSearchFilter {
|
|
14
|
+
/** Natural-language description of the task. Used to build the query vector. */
|
|
15
|
+
readonly query: string;
|
|
16
|
+
}
|
|
17
|
+
export interface RecallCodeResult {
|
|
18
|
+
readonly positives: ReadonlyArray<CodeExemplarHit>;
|
|
19
|
+
readonly negatives: ReadonlyArray<CodeExemplarHit>;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Recall code exemplars. Returns positives (pass/fix) and negatives (fail/exhausted)
|
|
23
|
+
* as separate lists so the caller can surface them with distinct framing.
|
|
24
|
+
*/
|
|
25
|
+
export declare function recallCode(opts: RecallCodeOptions, store: CodeExemplarStore, codeEmbedder: CodeEmbedder | null, proseFallback: LLMClient | null): Promise<RecallCodeResult>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recall-code — semantic search over the code-exemplar lane.
|
|
3
|
+
*
|
|
4
|
+
* Embeds the query with the code embedder (or falls back to the prose embedder),
|
|
5
|
+
* delegates to CodeExemplarStore.searchByVector, and formats results.
|
|
6
|
+
* Negatives are returned clearly labeled — "code that failed the gate" is
|
|
7
|
+
* as useful to the caller as the positive.
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Recall code exemplars. Returns positives (pass/fix) and negatives (fail/exhausted)
|
|
11
|
+
* as separate lists so the caller can surface them with distinct framing.
|
|
12
|
+
*/
|
|
13
|
+
export async function recallCode(opts, store, codeEmbedder, proseFallback) {
|
|
14
|
+
let queryVector = null;
|
|
15
|
+
if (codeEmbedder) {
|
|
16
|
+
try {
|
|
17
|
+
const result = await codeEmbedder.embed(opts.query, "query");
|
|
18
|
+
queryVector = result.vector;
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
// Degraded: fall through to prose fallback
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
if (!queryVector && proseFallback) {
|
|
25
|
+
try {
|
|
26
|
+
const result = await proseFallback.embed(opts.query, "query");
|
|
27
|
+
queryVector = result.vector;
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
// No embedder available
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
if (!queryVector) {
|
|
34
|
+
return { positives: [], negatives: [] };
|
|
35
|
+
}
|
|
36
|
+
const filter = {
|
|
37
|
+
installScope: opts.installScope,
|
|
38
|
+
...(opts.repo !== undefined && { repo: opts.repo }),
|
|
39
|
+
...(opts.lang !== undefined && { lang: opts.lang }),
|
|
40
|
+
...(opts.model !== undefined && { model: opts.model }),
|
|
41
|
+
includeNegatives: opts.includeNegatives ?? true,
|
|
42
|
+
k: opts.k ?? 5,
|
|
43
|
+
};
|
|
44
|
+
const hits = await store.searchByVector(queryVector, filter);
|
|
45
|
+
const positives = hits.filter((h) => h.outcome === "pass" || h.outcome === "fix");
|
|
46
|
+
const negatives = hits.filter((h) => h.outcome === "fail" || h.outcome === "exhausted");
|
|
47
|
+
return { positives, negatives };
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=recall-code.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recall-code.js","sourceRoot":"","sources":["../../../src/core/exemplars/recall-code.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAiBH;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAuB,EACvB,KAAwB,EACxB,YAAiC,EACjC,aAA+B;IAE/B,IAAI,WAAW,GAAwB,IAAI,CAAC;IAE5C,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC7D,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,2CAA2C;QAC7C,CAAC;IACH,CAAC;IAED,IAAI,CAAC,WAAW,IAAI,aAAa,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC9D,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,wBAAwB;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC1C,CAAC;IAED,MAAM,MAAM,GAA6B;QACvC,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QACnD,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QACnD,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QACtD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,IAAI,IAAI;QAC/C,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;KACf,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAE7D,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,IAAI,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC;IAClF,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,IAAI,CAAC,CAAC,OAAO,KAAK,WAAW,CAAC,CAAC;IAExF,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClC,CAAC"}
|
|
@@ -17,10 +17,12 @@
|
|
|
17
17
|
*/
|
|
18
18
|
import type { SqliteFactStore } from "../storage/sqlite-fact-store.js";
|
|
19
19
|
import type { SqliteSessionStore } from "../storage/sqlite-session-store.js";
|
|
20
|
+
import { PgSessionStore } from "../storage/pg-session-store.js";
|
|
21
|
+
import type { PgFactStore } from "../storage/pg-fact-store.js";
|
|
20
22
|
import type { LLMClient } from "../../ports/llm-client.js";
|
|
21
23
|
export interface BackfillFactsOptions {
|
|
22
|
-
readonly store: SqliteSessionStore;
|
|
23
|
-
readonly factStore: SqliteFactStore;
|
|
24
|
+
readonly store: SqliteSessionStore | PgSessionStore;
|
|
25
|
+
readonly factStore: SqliteFactStore | PgFactStore;
|
|
24
26
|
readonly classifier: LLMClient;
|
|
25
27
|
/** Optional embedder. When omitted, facts are written without semantic vectors. */
|
|
26
28
|
readonly embedder?: LLMClient | null;
|
|
@@ -19,6 +19,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
|
19
19
|
import { dirname, join } from "node:path";
|
|
20
20
|
import { homedir } from "node:os";
|
|
21
21
|
import { extractFacts } from "../facts/extract-facts.js";
|
|
22
|
+
import { PgSessionStore } from "../storage/pg-session-store.js";
|
|
22
23
|
import { LLMUnreachableError } from "../../ports/llm-client.js";
|
|
23
24
|
const DEFAULT_STATE_PATH = join(homedir(), ".nlm", "backfill_facts.state");
|
|
24
25
|
const SAVE_EVERY = 25;
|
|
@@ -43,34 +44,16 @@ export async function backfillFacts(opts) {
|
|
|
43
44
|
const startedAtCutoff = new Date().toISOString();
|
|
44
45
|
const statePath = opts.statePath ?? DEFAULT_STATE_PATH;
|
|
45
46
|
const done = opts.dryRun ? new Set() : loadState(statePath);
|
|
46
|
-
const db = opts.store.rawDb();
|
|
47
47
|
// Eligible sessions: started strictly before this run's cutoff (don't
|
|
48
48
|
// race with live ingest), with a non-empty body (the classifier needs
|
|
49
49
|
// transcript text). When reprocess=false, exclude sessions that already
|
|
50
|
-
// have facts attributed to them.
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
${opts.from ? "AND id > ?" : ""}
|
|
58
|
-
ORDER BY started_at ASC, id ASC
|
|
59
|
-
`
|
|
60
|
-
: `
|
|
61
|
-
SELECT s.id, s.started_at, s.body
|
|
62
|
-
FROM sessions s
|
|
63
|
-
WHERE s.started_at < ?
|
|
64
|
-
AND s.body IS NOT NULL AND length(s.body) > 0
|
|
65
|
-
AND NOT EXISTS (
|
|
66
|
-
SELECT 1 FROM facts f WHERE f.source_session_id = s.id
|
|
67
|
-
)
|
|
68
|
-
${opts.from ? "AND s.id > ?" : ""}
|
|
69
|
-
ORDER BY s.started_at ASC, s.id ASC
|
|
70
|
-
`;
|
|
71
|
-
const rows = opts.from
|
|
72
|
-
? db.prepare(sql).all(startedAtCutoff, opts.from)
|
|
73
|
-
: db.prepare(sql).all(startedAtCutoff);
|
|
50
|
+
// have facts attributed to them. SQL lives in each adapter's
|
|
51
|
+
// listBackfillCandidates so backfill is backend-agnostic.
|
|
52
|
+
const rows = await opts.store.listBackfillCandidates({
|
|
53
|
+
cutoff: startedAtCutoff,
|
|
54
|
+
...(opts.from ? { from: opts.from } : {}),
|
|
55
|
+
reprocess: opts.reprocess ?? false,
|
|
56
|
+
});
|
|
74
57
|
// Filter state-file-known done ids BEFORE applying limit. Without this,
|
|
75
58
|
// a dense cluster of previously-skipped (low-confidence) sessions would
|
|
76
59
|
// burn the batch's --limit on no-op skips. With it, --limit N means
|
|
@@ -119,7 +102,7 @@ export async function backfillFacts(opts) {
|
|
|
119
102
|
break;
|
|
120
103
|
continue;
|
|
121
104
|
}
|
|
122
|
-
const facts = extractFacts(classification, sid, row.
|
|
105
|
+
const facts = extractFacts(classification, sid, row.startedAt);
|
|
123
106
|
if (facts.length === 0) {
|
|
124
107
|
skippedLowConfidence += 1;
|
|
125
108
|
opts.onProgress?.(i + 1, total, sid, "skipped_low_confidence", `confidence=${classification.confidence}`);
|
|
@@ -137,7 +120,15 @@ export async function backfillFacts(opts) {
|
|
|
137
120
|
continue;
|
|
138
121
|
}
|
|
139
122
|
try {
|
|
140
|
-
|
|
123
|
+
// store + factStore share a backend (composed together in buildStorage),
|
|
124
|
+
// so the factStore cast is sound — TS just can't correlate the two union
|
|
125
|
+
// members at one call site.
|
|
126
|
+
if (opts.store instanceof PgSessionStore) {
|
|
127
|
+
await opts.store.insertFactsForSession(sid, opts.factStore, facts, opts.embedder ?? null);
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
await opts.store.insertFactsForSession(sid, opts.factStore, facts, opts.embedder ?? null);
|
|
131
|
+
}
|
|
141
132
|
}
|
|
142
133
|
catch (err) {
|
|
143
134
|
storageFailures += 1;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backfill-facts.js","sourceRoot":"","sources":["../../../src/core/facts/backfill-facts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"backfill-facts.js","sourceRoot":"","sources":["../../../src/core/facts/backfill-facts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAG5D,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AAGnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,sBAAsB,CAAC,CAAC;AAC3E,MAAM,UAAU,GAAG,EAAE,CAAC;AAuDtB,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IACxC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAwB,CAAC;QAC3E,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,IAAiB;IAChD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAA0B;IAE1B,MAAM,eAAe,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;IACvD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,EAAU,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAEpE,sEAAsE;IACtE,sEAAsE;IACtE,wEAAwE;IACxE,6DAA6D;IAC7D,0DAA0D;IAC1D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;QACnD,MAAM,EAAE,eAAe;QACvB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,KAAK;KACnC,CAAC,CAAC;IAEH,wEAAwE;IACxE,wEAAwE;IACxE,oEAAoE;IACpE,uEAAuE;IACvE,qEAAqE;IACrE,yEAAyE;IACzE,+BAA+B;IAC/B,MAAM,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IACrE,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,UAAU,CAAC,MAAM,CAAC;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;IAE1B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,kBAAkB,GAAG,kBAAkB,CAAC;IAC5C,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACrB,MAAM,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC;QAEnB,oEAAoE;QACpE,gCAAgC;QAEhC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,aAAa,IAAI,CAAC,CAAC;YACnB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,iBAAiB,CAAC,CAAC;YACxD,SAAS;QACX,CAAC;QAED,IAAI,cAAc,CAAC;QACnB,IAAI,CAAC;YACH,cAAc,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC5D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,gBAAgB,IAAI,CAAC,CAAC;YACtB,MAAM,MAAM,GACV,GAAG,YAAY,mBAAmB;gBAChC,CAAC,CAAC,mCAAmC;gBACrC,CAAC,CAAC,GAAG,YAAY,KAAK;oBACpB,CAAC,CAAC,GAAG,CAAC,OAAO;oBACb,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACpB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;YAChE,kEAAkE;YAClE,2CAA2C;YAC3C,IAAI,GAAG,YAAY,mBAAmB;gBAAE,MAAM;YAC9C,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,YAAY,CAAC,cAAc,EAAE,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,oBAAoB,IAAI,CAAC,CAAC;YAC1B,IAAI,CAAC,UAAU,EAAE,CACf,CAAC,GAAG,CAAC,EACL,KAAK,EACL,GAAG,EACH,wBAAwB,EACxB,cAAc,cAAc,CAAC,UAAU,EAAE,CAC1C,CAAC;YACF,mEAAmE;YACnE,kDAAkD;YAClD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,SAAS,GAAG,UAAU,KAAK,CAAC;gBAAE,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YAC7E,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,YAAY,IAAI,KAAK,CAAC,MAAM,CAAC;YAC7B,SAAS,IAAI,CAAC,CAAC;YACf,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,eAAe,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1E,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,yEAAyE;YACzE,yEAAyE;YACzE,4BAA4B;YAC5B,IAAI,IAAI,CAAC,KAAK,YAAY,cAAc,EAAE,CAAC;gBACzC,MAAM,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,GAAG,EAAE,IAAI,CAAC,SAAwB,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC;YAC3G,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,GAAG,EAAE,IAAI,CAAC,SAA4B,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC;YAC/G,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,eAAe,IAAI,CAAC,CAAC;YACrB,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;YAC/D,SAAS;QACX,CAAC;QAED,YAAY,IAAI,KAAK,CAAC,MAAM,CAAC;QAC7B,SAAS,IAAI,CAAC,CAAC;QACf,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACpE,IAAI,SAAS,GAAG,UAAU,KAAK,CAAC;YAAE,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE7C,OAAO;QACL,KAAK;QACL,SAAS;QACT,YAAY;QACZ,kBAAkB;QAClB,oBAAoB;QACpB,aAAa;QACb,oBAAoB;QACpB,gBAAgB;QAChB,eAAe;KAChB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Removes NLM's own injected recall pointer block from a transcript before it
|
|
3
|
+
* is classified. The recall hook prepends a "Possibly-relevant prior sessions"
|
|
4
|
+
* / "Known facts about top entities" block (see pointer-block.ts) to the user
|
|
5
|
+
* prompt every turn; that block lands in the stored transcript and the
|
|
6
|
+
* classifier then extracted "facts" FROM it — a feedback loop that produced
|
|
7
|
+
* garbage facts like `acme-app owner: user` sourced from the recall block
|
|
8
|
+
* itself (NLM #325).
|
|
9
|
+
*
|
|
10
|
+
* The block is bounded: it starts at one of the two known headers and always
|
|
11
|
+
* ends with the `NLM tools:` footer (formatPointerBlock emits it
|
|
12
|
+
* unconditionally). We strip header→footer inclusive. A header with no
|
|
13
|
+
* following footer is left in place — that signals truncated/real content, and
|
|
14
|
+
* dropping to end-of-text could eat the user's actual message.
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* The recall pointer block's marker strings — the single source of truth.
|
|
18
|
+
* The shipped pi recall hook (nlm/index.js) carries its own copy of these in
|
|
19
|
+
* its self-contained formatPointerBlock; recall-marker-contract.test.ts asserts
|
|
20
|
+
* the hook still emits these so marker drift can't silently reopen the loop.
|
|
21
|
+
*/
|
|
22
|
+
export declare const BLOCK_HEADERS: string[];
|
|
23
|
+
export declare const FOOTER_PREFIX = "NLM tools:";
|
|
24
|
+
export declare function stripInjectedContext(text: string): string;
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Removes NLM's own injected recall pointer block from a transcript before it
|
|
3
|
+
* is classified. The recall hook prepends a "Possibly-relevant prior sessions"
|
|
4
|
+
* / "Known facts about top entities" block (see pointer-block.ts) to the user
|
|
5
|
+
* prompt every turn; that block lands in the stored transcript and the
|
|
6
|
+
* classifier then extracted "facts" FROM it — a feedback loop that produced
|
|
7
|
+
* garbage facts like `acme-app owner: user` sourced from the recall block
|
|
8
|
+
* itself (NLM #325).
|
|
9
|
+
*
|
|
10
|
+
* The block is bounded: it starts at one of the two known headers and always
|
|
11
|
+
* ends with the `NLM tools:` footer (formatPointerBlock emits it
|
|
12
|
+
* unconditionally). We strip header→footer inclusive. A header with no
|
|
13
|
+
* following footer is left in place — that signals truncated/real content, and
|
|
14
|
+
* dropping to end-of-text could eat the user's actual message.
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* The recall pointer block's marker strings — the single source of truth.
|
|
18
|
+
* The shipped pi recall hook (nlm/index.js) carries its own copy of these in
|
|
19
|
+
* its self-contained formatPointerBlock; recall-marker-contract.test.ts asserts
|
|
20
|
+
* the hook still emits these so marker drift can't silently reopen the loop.
|
|
21
|
+
*/
|
|
22
|
+
export const BLOCK_HEADERS = [
|
|
23
|
+
"## Possibly-relevant prior sessions (nlm-memory)",
|
|
24
|
+
"## Known facts about top entities",
|
|
25
|
+
];
|
|
26
|
+
export const FOOTER_PREFIX = "NLM tools:";
|
|
27
|
+
export function stripInjectedContext(text) {
|
|
28
|
+
if (!text.includes(FOOTER_PREFIX))
|
|
29
|
+
return text;
|
|
30
|
+
const lines = text.split("\n");
|
|
31
|
+
const kept = [];
|
|
32
|
+
for (let i = 0; i < lines.length; i++) {
|
|
33
|
+
const line = lines[i] ?? "";
|
|
34
|
+
if (BLOCK_HEADERS.includes(line.trim())) {
|
|
35
|
+
const footerIdx = findFooter(lines, i + 1);
|
|
36
|
+
if (footerIdx !== -1) {
|
|
37
|
+
i = footerIdx; // skip header..footer inclusive
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
kept.push(line);
|
|
42
|
+
}
|
|
43
|
+
return collapseBlankRuns(kept).join("\n");
|
|
44
|
+
}
|
|
45
|
+
function findFooter(lines, from) {
|
|
46
|
+
for (let i = from; i < lines.length; i++) {
|
|
47
|
+
if ((lines[i] ?? "").startsWith(FOOTER_PREFIX))
|
|
48
|
+
return i;
|
|
49
|
+
}
|
|
50
|
+
return -1;
|
|
51
|
+
}
|
|
52
|
+
/** Collapse 3+ consecutive blank lines (left by removal) down to one. */
|
|
53
|
+
function collapseBlankRuns(lines) {
|
|
54
|
+
const out = [];
|
|
55
|
+
let blanks = 0;
|
|
56
|
+
for (const line of lines) {
|
|
57
|
+
if (line.trim() === "") {
|
|
58
|
+
blanks += 1;
|
|
59
|
+
if (blanks > 1)
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
blanks = 0;
|
|
64
|
+
}
|
|
65
|
+
out.push(line);
|
|
66
|
+
}
|
|
67
|
+
while (out.length > 0 && (out[0] ?? "").trim() === "")
|
|
68
|
+
out.shift();
|
|
69
|
+
while (out.length > 0 && (out[out.length - 1] ?? "").trim() === "")
|
|
70
|
+
out.pop();
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=strip-injected-context.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"strip-injected-context.js","sourceRoot":"","sources":["../../../src/core/hook/strip-injected-context.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,kDAAkD;IAClD,mCAAmC;CACpC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,YAAY,CAAC;AAE1C,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;QAAE,OAAO,IAAI,CAAC;IAE/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YACxC,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3C,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;gBACrB,CAAC,GAAG,SAAS,CAAC,CAAC,gCAAgC;gBAC/C,SAAS;YACX,CAAC;QACH,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,UAAU,CAAC,KAA4B,EAAE,IAAY;IAC5D,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC;YAAE,OAAO,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED,yEAAyE;AACzE,SAAS,iBAAiB,CAAC,KAA4B;IACrD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,CAAC;YACZ,IAAI,MAAM,GAAG,CAAC;gBAAE,SAAS;QAC3B,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,CAAC;QACb,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,GAAG,CAAC,KAAK,EAAE,CAAC;IACnE,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,GAAG,CAAC,GAAG,EAAE,CAAC;IAC9E,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import type { SqliteFactStore } from "../storage/sqlite-fact-store.js";
|
|
10
10
|
import type { SqliteSessionStore } from "../storage/sqlite-session-store.js";
|
|
11
|
+
import { PgSessionStore } from "../storage/pg-session-store.js";
|
|
12
|
+
import type { PgFactStore } from "../storage/pg-fact-store.js";
|
|
11
13
|
import type { LLMClient } from "../../ports/llm-client.js";
|
|
12
14
|
export interface IngestInput {
|
|
13
15
|
/** Optional — if omitted, derived from a hash of (runtime + startedAt + text). */
|
|
@@ -24,8 +26,8 @@ export interface IngestInput {
|
|
|
24
26
|
export interface IngestDeps {
|
|
25
27
|
readonly classifier: LLMClient;
|
|
26
28
|
readonly embedder: LLMClient;
|
|
27
|
-
readonly store: SqliteSessionStore;
|
|
28
|
-
readonly factStore?: SqliteFactStore;
|
|
29
|
+
readonly store: SqliteSessionStore | PgSessionStore;
|
|
30
|
+
readonly factStore?: SqliteFactStore | PgFactStore;
|
|
29
31
|
/** Optional logger — defaults to console.error. */
|
|
30
32
|
readonly log?: (msg: string) => void;
|
|
31
33
|
}
|