@unblocklabs/unblock-memory 0.7.1 → 0.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/src/analysis.js +36 -25
  2. package/dist/src/config.js +2 -17
  3. package/dist/src/contracts.d.ts +1 -1
  4. package/dist/src/extraction-chunk.d.ts +1 -0
  5. package/dist/src/extraction-chunk.js +1 -0
  6. package/dist/src/extraction-model.js +5 -4
  7. package/dist/src/extraction-runtime.js +2 -7
  8. package/dist/src/extraction-source.js +6 -2
  9. package/dist/src/extraction-store.d.ts +3 -0
  10. package/dist/src/extraction-store.js +14 -10
  11. package/dist/src/extraction-worker.js +31 -15
  12. package/dist/src/manager.d.ts +1 -1
  13. package/dist/src/manager.js +66 -85
  14. package/dist/src/memory-whisperer.js +6 -10
  15. package/dist/src/people-primer-config.js +1 -4
  16. package/dist/src/people-primer.js +6 -8
  17. package/dist/src/people-store.js +9 -1
  18. package/dist/src/people-tools.js +2 -6
  19. package/dist/src/plugin.js +6 -14
  20. package/dist/src/quality-audit.js +11 -2
  21. package/dist/src/query-contract.js +18 -10
  22. package/dist/src/response-audit.js +3 -3
  23. package/dist/src/response-judge.js +1 -2
  24. package/dist/src/response-reviews.d.ts +1 -1
  25. package/dist/src/response-reviews.js +4 -2
  26. package/dist/src/response-store.js +3 -2
  27. package/dist/src/retrieval-telemetry.d.ts +1 -1
  28. package/dist/src/review-tools.d.ts +24 -0
  29. package/dist/src/review-tools.js +22 -26
  30. package/dist/src/runtime.js +2 -1
  31. package/dist/src/session-projector.js +8 -4
  32. package/dist/src/session-sync.js +2 -1
  33. package/dist/src/skill-whisperer.js +51 -24
  34. package/dist/src/training-candidates.d.ts +12 -13
  35. package/dist/src/training-candidates.js +11 -81
  36. package/dist/src/training-gate.js +2 -2
  37. package/dist/src/training-queries.js +5 -2
  38. package/dist/src/training-retrieval.d.ts +15 -2
  39. package/dist/src/training-retrieval.js +63 -39
  40. package/dist/src/training-store.d.ts +2 -8
  41. package/dist/src/training-store.js +15 -20
  42. package/dist/src/typesafe-client.d.ts +5 -0
  43. package/dist/src/typesafe-client.js +4 -0
  44. package/dist/src/typesafe-review.js +4 -6
  45. package/dist/src/typesafe.d.ts +1 -9
  46. package/dist/src/typesafe.js +4 -10
  47. package/dist/src/whisperer-context.d.ts +5 -1
  48. package/dist/src/whisperer-context.js +12 -7
  49. package/dist/src/workspace-path-classifier.js +39 -32
  50. package/docs/configuration.md +3 -2
  51. package/docs/memory-training.md +8 -7
  52. package/docs/retrieval.md +15 -1
  53. package/openclaw.plugin.json +2 -2
  54. package/package.json +6 -4
@@ -13,6 +13,18 @@ const claimParameters = Type.Object({
13
13
  }, { additionalProperties: false });
14
14
  const clusterParameters = Type.Object({ clusterId: Type.String({ pattern: "^[0-9a-f]{10}$" }) }, { additionalProperties: false });
15
15
  const noParameters = Type.Object({}, { additionalProperties: false });
16
+ /** Bound initialization as well as inference, without cancelling shared managers. */
17
+ export async function reviewRequestContext(runtime, active, typesafe, signal) {
18
+ const deadline = AbortSignal.timeout(30_000);
19
+ const combined = signal ? AbortSignal.any([signal, deadline]) : deadline;
20
+ const apiKey = await abortable(resolveTypeSafeApiKey(typesafe), combined);
21
+ if (!apiKey)
22
+ return { status: "unavailable", reason: "TypeSafe API key not configured" };
23
+ const { manager } = await abortable(runtime.getMemorySearchManager(active), combined);
24
+ if (!manager)
25
+ return { status: "unavailable", reason: "Memory manager unavailable" };
26
+ return { status: "ready", manager, apiKey, signal: combined, timeoutMs: typesafe.timeoutMs };
27
+ }
16
28
  export function registerReviewTools(api, runtime, config, diagnostics) {
17
29
  api.registerTool(ctx => {
18
30
  const active = getContext(ctx);
@@ -59,20 +71,12 @@ export function registerReviewTools(api, runtime, config, diagnostics) {
59
71
  const parsed = Value.Parse(claimParameters, params);
60
72
  if (!config.evidenceReview.enabled || !config.typesafe.enabled)
61
73
  return jsonResult({ status: "disabled" });
62
- const deadline = AbortSignal.timeout(30_000);
63
- const combined = signal ? AbortSignal.any([signal, deadline]) : deadline;
64
74
  try {
65
- combined.throwIfAborted();
66
- const apiKey = await abortable(resolveTypeSafeApiKey(config.typesafe), combined);
67
- combined.throwIfAborted();
68
- if (!apiKey)
69
- return jsonResult({ status: "unavailable", reason: "TypeSafe API key not configured" });
70
- const { manager } = await abortable(runtime.getMemorySearchManager(active), combined);
71
- combined.throwIfAborted();
72
- if (!manager)
73
- return jsonResult({ status: "unavailable" });
74
- return jsonResult(await manager.reviewClaim({ ...parsed, corpora: config.evidenceReview.corpora,
75
- apiKey, timeoutMs: config.typesafe.timeoutMs, signal: combined }));
75
+ const context = await reviewRequestContext(runtime, active, config.typesafe, signal);
76
+ if (context.status !== "ready")
77
+ return jsonResult(context);
78
+ return jsonResult(await context.manager.reviewClaim({ ...parsed, corpora: config.evidenceReview.corpora,
79
+ apiKey: context.apiKey, timeoutMs: context.timeoutMs, signal: context.signal }));
76
80
  }
77
81
  catch {
78
82
  return jsonResult({ status: "unavailable", needsReview: true, reason: "Claim review failed or was cancelled; no claim verified" });
@@ -92,20 +96,12 @@ export function registerReviewTools(api, runtime, config, diagnostics) {
92
96
  const parsed = Value.Parse(clusterParameters, params);
93
97
  if (!config.qualityAudit.enabled || !config.typesafe.enabled)
94
98
  return jsonResult({ status: "disabled" });
95
- const deadline = AbortSignal.timeout(30_000);
96
- const combined = signal ? AbortSignal.any([signal, deadline]) : deadline;
97
99
  try {
98
- combined.throwIfAborted();
99
- const apiKey = await abortable(resolveTypeSafeApiKey(config.typesafe), combined);
100
- combined.throwIfAborted();
101
- if (!apiKey)
102
- return jsonResult({ status: "unavailable", reason: "TypeSafe API key not configured" });
103
- const { manager } = await abortable(runtime.getMemorySearchManager(active), combined);
104
- combined.throwIfAborted();
105
- if (!manager)
106
- return jsonResult({ status: "unavailable" });
107
- return jsonResult(await manager.reviewCluster({ ...parsed, corpora: config.qualityAudit.corpora,
108
- apiKey, timeoutMs: config.typesafe.timeoutMs, signal: combined }));
100
+ const context = await reviewRequestContext(runtime, active, config.typesafe, signal);
101
+ if (context.status !== "ready")
102
+ return jsonResult(context);
103
+ return jsonResult(await context.manager.reviewCluster({ ...parsed, corpora: config.qualityAudit.corpora,
104
+ apiKey: context.apiKey, timeoutMs: context.timeoutMs, signal: context.signal }));
109
105
  }
110
106
  catch {
111
107
  return jsonResult({ status: "unavailable", reason: "Cluster review failed or was cancelled; no cluster judgment made" });
@@ -106,7 +106,8 @@ export class QmdMemoryRuntime {
106
106
  return { manager: await pending };
107
107
  }
108
108
  catch (error) {
109
- this.#managers.delete(params.agentId);
109
+ if (this.#managers.get(params.agentId) === pending)
110
+ this.#managers.delete(params.agentId);
110
111
  return { manager: null, error: error instanceof Error ? error.message : String(error) };
111
112
  }
112
113
  }
@@ -109,8 +109,8 @@ export function projectMessage(row, input) {
109
109
  timestamp: timestamp(eventRecord.timestamp) ?? row.createdAt ?? timestamp(message.timestamp) ?? input.startedAt,
110
110
  };
111
111
  }
112
- function formatTimestamp(value, timezone) {
113
- const parts = new Intl.DateTimeFormat("en-CA", {
112
+ function timestampFormatter(timezone) {
113
+ return new Intl.DateTimeFormat("en-CA", {
114
114
  timeZone: timezone,
115
115
  year: "numeric",
116
116
  month: "2-digit",
@@ -120,7 +120,10 @@ function formatTimestamp(value, timezone) {
120
120
  second: "2-digit",
121
121
  hourCycle: "h23",
122
122
  timeZoneName: "short",
123
- }).formatToParts(value);
123
+ });
124
+ }
125
+ function formatTimestamp(value, formatter) {
126
+ const parts = formatter.formatToParts(value);
124
127
  const part = (type) => parts.find((entry) => entry.type === type)?.value ?? "";
125
128
  return `${part("year")}-${part("month")}-${part("day")} ` +
126
129
  `${part("hour")}:${part("minute")}:${part("second")} ${part("timeZoneName")}`.trim();
@@ -169,11 +172,12 @@ export function projectSessionDocument(input) {
169
172
  }
170
173
  let content = "# Transcript\n\n";
171
174
  const spans = [];
175
+ const formatter = timestampFormatter(input.timezone);
172
176
  for (const message of messages.filter(message => !hidden.has(message))) {
173
177
  if (spans.length)
174
178
  content += "\n\n";
175
179
  const start = content.length;
176
- const timestamp = formatTimestamp(message.timestamp, input.timezone);
180
+ const timestamp = formatTimestamp(message.timestamp, formatter);
177
181
  content += `## ${message.role === "user" ? "User" : "Assistant"} — ${message.speaker} — ${timestamp}\n\n`;
178
182
  const bodyStart = content.length;
179
183
  content += message.text;
@@ -332,8 +332,9 @@ export async function syncSessionProjections(params) {
332
332
  else
333
333
  counts.unchanged += 1;
334
334
  }
335
+ const liveIds = new Set(snapshot.windows.map(window => window.sessionId));
335
336
  for (const [sessionId, session] of Object.entries(previousManifest.sessions)) {
336
- if (sessions[sessionId] || snapshot.windows.some((window) => window.sessionId === sessionId))
337
+ if (sessions[sessionId] || liveIds.has(sessionId))
337
338
  continue;
338
339
  await remove(projectionPath(params.outputDir, session.documentPath));
339
340
  counts.removed += 1;
@@ -1,25 +1,24 @@
1
1
  import { basename } from "node:path";
2
2
  import { selectTypeSafeSkill } from "./typesafe.js";
3
3
  import { resolveTypeSafeApiKey, TypeSafeRequestError } from "./typesafe-client.js";
4
- import { buildSkillWhispererQuery, messageText } from "./whisperer-context.js";
4
+ import { buildSkillWhispererQuery, recentSkillMessages } from "./whisperer-context.js";
5
+ import { abortable } from "./abortable.js";
5
6
  const CANDIDATE_LIMIT = 10;
6
7
  const MAX_QUERY_CHARS = 12_000;
7
8
  const TYPESAFE_CANDIDATE_LIMIT = 3;
9
+ const TOTAL_TIMEOUT_MS = 3000;
8
10
  function isRecord(value) {
9
11
  return value !== null && typeof value === "object" && !Array.isArray(value);
10
12
  }
11
- function typeSafeConversation(prompt, messages, historyMessages) {
13
+ function typeSafeConversation(prompt, messages) {
12
14
  const currentRequest = prompt.trim().slice(-MAX_QUERY_CHARS);
13
15
  let remaining = MAX_QUERY_CHARS - currentRequest.length;
14
- const available = messages.flatMap(message => {
15
- const parsed = messageText(message);
16
- return parsed ? [{ role: parsed.role, content: parsed.text }] : [];
17
- });
18
16
  const history = [];
19
- for (const message of (historyMessages ? available.slice(-historyMessages) : []).reverse()) {
17
+ for (let i = messages.length - 1; i >= 0; i--) {
20
18
  if (remaining <= 0)
21
19
  break;
22
- const content = message.content.slice(-remaining);
20
+ const message = messages[i];
21
+ const content = message.text.slice(-remaining);
23
22
  history.unshift({ role: message.role, content });
24
23
  remaining -= content.length;
25
24
  }
@@ -56,16 +55,24 @@ export function registerSkillWhisperer(api, runtime, config, typesafe, diagnosti
56
55
  const state = stateFor(scope);
57
56
  if (state.lastRunId === runId)
58
57
  return state.result;
58
+ state.controller?.abort();
59
+ const controller = new AbortController();
60
+ state.controller = controller;
61
+ const { signal } = controller;
62
+ const timer = setTimeout(() => controller.abort(new DOMException("Skill deadline", "TimeoutError")), TOTAL_TIMEOUT_MS);
59
63
  state.lastRunId = runId;
60
64
  state.turn += 1;
61
65
  // Reuse even an in-flight result when the host rebuilds this run's prompt.
62
66
  const run = async () => {
63
67
  try {
64
68
  const runtimeParams = active(agentId);
65
- const apiKey = await resolveTypeSafeApiKey(typesafe);
69
+ const apiKey = await abortable(resolveTypeSafeApiKey(typesafe), signal);
70
+ signal.throwIfAborted();
66
71
  if (!apiKey)
67
72
  diagnostics?.record(agentId, "skill", typesafe.enabled ? "missing_key" : "typesafe_disabled");
68
- const candidates = await runtime.searchSkills(runtimeParams, buildSkillWhispererQuery(event.prompt, event.messages, config.historyMessages), apiKey ? -1 : config.minScore, CANDIDATE_LIMIT);
73
+ const recentMessages = recentSkillMessages(event.messages, config.historyMessages);
74
+ const candidates = await abortable(runtime.searchSkills(runtimeParams, buildSkillWhispererQuery(event.prompt, recentMessages), apiKey ? -1 : config.minScore, CANDIDATE_LIMIT), signal);
75
+ signal.throwIfAborted();
69
76
  const resolvedCandidates = candidates.flatMap((candidate) => {
70
77
  const canonicalPath = runtime.resolveSkillPath(runtimeParams, candidate.path);
71
78
  return canonicalPath ? [{ candidate, canonicalPath }] : [];
@@ -75,19 +82,29 @@ export function registerSkillWhisperer(api, runtime, config, typesafe, diagnosti
75
82
  diagnostics?.record(agentId, "skill", "no_candidates");
76
83
  return;
77
84
  }
85
+ const cooling = (path) => {
86
+ const previous = state.skills.get(path);
87
+ return state.turn - Math.max(previous?.suggested ?? -Infinity, previous?.opened ?? -Infinity) <= config.cooldownTurns;
88
+ };
78
89
  if (apiKey) {
79
90
  const shortlist = resolvedCandidates.slice(0, TYPESAFE_CANDIDATE_LIMIT);
80
- const selectedIndex = await selectTypeSafeSkill({
81
- apiKey, timeoutMs: typesafe.timeoutMs,
82
- ...typeSafeConversation(event.prompt, event.messages, config.historyMessages),
91
+ if (shortlist.every(({ canonicalPath }) => cooling(canonicalPath))) {
92
+ diagnostics?.record(agentId, "skill", "cooldown");
93
+ return;
94
+ }
95
+ const selectedIndex = await abortable(selectTypeSafeSkill({
96
+ apiKey, timeoutMs: typesafe.timeoutMs, signal,
97
+ ...typeSafeConversation(event.prompt, recentMessages),
83
98
  candidates: shortlist.map(({ candidate }) => candidate),
84
99
  onCandidateFailure: (candidateIndex, error) => {
100
+ if (signal.aborted)
101
+ return;
85
102
  diagnostics?.record(agentId, "skill", "judge_candidate_failed");
86
103
  api.logger.warn("unblock-memory skill_whisperer " + JSON.stringify({ event: "candidate_failed",
87
104
  agentId, runId, candidateIndex, errorCode: error instanceof TypeSafeRequestError ? error.code : "unexpected",
88
105
  httpStatus: error instanceof TypeSafeRequestError ? error.status : undefined }));
89
106
  },
90
- });
107
+ }), signal);
91
108
  if (selectedIndex === undefined) {
92
109
  diagnostics?.record(agentId, "skill", "rejected");
93
110
  return;
@@ -108,9 +125,7 @@ export function registerSkillWhisperer(api, runtime, config, typesafe, diagnosti
108
125
  const { candidate: selected, canonicalPath } = resolved;
109
126
  if (apiKey && runtime.resolveSkillPath(runtimeParams, selected.path) !== canonicalPath)
110
127
  return;
111
- const previous = state.skills.get(canonicalPath);
112
- const lastSeen = Math.max(previous?.suggested ?? -Infinity, previous?.opened ?? -Infinity);
113
- if (state.turn - lastSeen <= config.cooldownTurns) {
128
+ if (cooling(canonicalPath)) {
114
129
  diagnostics?.record(agentId, "skill", "cooldown");
115
130
  return;
116
131
  }
@@ -123,10 +138,16 @@ export function registerSkillWhisperer(api, runtime, config, typesafe, diagnosti
123
138
  };
124
139
  }
125
140
  catch (error) {
126
- diagnostics?.record(agentId, "skill", error instanceof TypeSafeRequestError && error.code === "timeout" ? "timed_out" : "failed");
127
- api.logger.warn("unblock-memory skill whisperer failed; no hint emitted");
141
+ const timedOut = (error instanceof TypeSafeRequestError && error.code === "timeout") ||
142
+ (signal.aborted && signal.reason instanceof Error && signal.reason.name === "TimeoutError");
143
+ diagnostics?.record(agentId, "skill", timedOut ? "timed_out" : signal.aborted ? "cancelled" : "failed");
144
+ if (!signal.aborted)
145
+ api.logger.warn("unblock-memory skill whisperer failed; no hint emitted");
128
146
  return;
129
147
  }
148
+ finally {
149
+ clearTimeout(timer);
150
+ }
130
151
  };
131
152
  state.result = run();
132
153
  return state.result;
@@ -153,11 +174,17 @@ export function registerSkillWhisperer(api, runtime, config, typesafe, diagnosti
153
174
  }
154
175
  }, { matcher: ["read"] });
155
176
  api.on("session_end", (event, context) => {
156
- sessions.delete(event.sessionId);
157
- if (event.sessionKey)
158
- sessions.delete(event.sessionKey);
159
- if (context.sessionKey)
160
- sessions.delete(context.sessionKey);
177
+ for (const scope of [event.sessionId, event.sessionKey, context.sessionKey]) {
178
+ if (!scope)
179
+ continue;
180
+ sessions.get(scope)?.controller?.abort();
181
+ sessions.delete(scope);
182
+ }
183
+ });
184
+ api.on("gateway_stop", () => {
185
+ for (const state of sessions.values())
186
+ state.controller?.abort();
187
+ sessions.clear();
161
188
  });
162
189
  return beforePrompt;
163
190
  }
@@ -1,22 +1,21 @@
1
1
  import type { QMDStore } from "@unblocklabs/qmd";
2
2
  import type { QueryLane } from "./query-contract.js";
3
- type Candidate = {
4
- file: string;
5
- body: string;
6
- bestChunk: string;
7
- bestChunkPos: number;
8
- score: number;
9
- explain: {
10
- methods: string[];
11
- };
3
+ /** Runtime and training share the exact ten-candidate recipe, not hybrid ranking. */
4
+ export declare function trainingCandidates(qmd: QMDStore, query: string, collection: string | string[], lane: QueryLane, signal?: AbortSignal): Promise<{
12
5
  vector?: {
13
6
  score: number;
14
7
  rank: number;
15
- };
8
+ } | undefined;
16
9
  bm25?: {
17
10
  score: number;
18
11
  rank: number;
12
+ } | undefined;
13
+ file: string;
14
+ body: string;
15
+ bestChunk: string;
16
+ bestChunkPos: number;
17
+ score: number;
18
+ explain: {
19
+ methods: ("vector" | "bm25")[];
19
20
  };
20
- };
21
- export declare function trainingCandidates(qmd: QMDStore, query: string, collection: string | string[], lane: QueryLane, signal?: AbortSignal): Promise<Candidate[]>;
22
- export {};
21
+ }[]>;
@@ -1,83 +1,13 @@
1
- import { randomUUID } from "node:crypto";
2
- const stopWords = new Set("a an and are as at be by can did do does for from how i in is it of on or that the their this to was were what when where which who why will with you".split(" "));
3
- function queryTerms(query) {
4
- const words = [...new Set(query.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [])];
5
- const meaningful = words.filter(word => !stopWords.has(word));
6
- return (meaningful.length ? meaningful : words).slice(0, 64);
7
- }
8
- function lexicalChunk(chunks, body, highlighted, marker, intent) {
9
- const compactLength = (text) => text.replace(/\s/gu, "").length;
10
- const ranges = [];
11
- let offset = 0;
12
- for (const [i, part] of highlighted.split(marker).entries()) {
13
- const end = offset + compactLength(part);
14
- if (i % 2 === 1)
15
- ranges.push({ start: offset, end });
16
- offset = end;
17
- }
18
- const intentTerms = queryTerms(intent);
19
- let sourcePos = 0, compactPos = 0;
20
- return chunks.map(chunk => {
21
- compactPos += compactLength(body.slice(sourcePos, chunk.pos));
22
- sourcePos = chunk.pos;
23
- const end = compactPos + compactLength(chunk.text);
24
- const matches = ranges.reduce((sum, range) => sum + Math.max(0, Math.min(end, range.end) - Math.max(compactPos, range.start)) / Math.max(1, range.end - range.start), 0);
25
- const lower = chunk.text.toLowerCase();
26
- return { chunk, matches, intentMatches: intentTerms.filter(term => lower.includes(term)).length };
27
- }).sort((a, b) => b.matches - a.matches || b.intentMatches - a.intentMatches || a.chunk.pos - b.chunk.pos)[0]?.chunk;
28
- }
1
+ /** Runtime and training share the exact ten-candidate recipe, not hybrid ranking. */
29
2
  export async function trainingCandidates(qmd, query, collection, lane, signal) {
30
- signal?.throwIfAborted();
31
- if (!query.trim() || query.length > 12_000)
32
- throw new Error("Invalid training query");
33
- // QMD exposes its store but not these chunk helpers at the package root.
34
- // Resolve relative to its installed SDK, never a global QMD or modified copy.
35
- const chunksApi = await import(new URL("./store.js", import.meta.resolve("@unblocklabs/qmd")).href);
36
- const candidates = new Map();
37
- const add = (hit, method, rank, rawScore) => {
38
- if (!hit.bestChunk.trim() || hit.bestChunk.length > 12_000)
39
- return;
40
- const key = JSON.stringify([hit.file, hit.bestChunk.trim()]), existing = candidates.get(key);
41
- if (existing) {
42
- if (!existing.explain.methods.includes(method))
43
- existing.explain.methods.push(method);
44
- existing.score = Math.max(existing.score, 1 / (rank + 1));
45
- existing[method] ??= { score: rawScore, rank: rank + 1 };
46
- }
47
- else
48
- candidates.set(key, { ...hit, score: 1 / (rank + 1), explain: { methods: [method] },
49
- [method]: { score: rawScore, rank: rank + 1 } });
50
- };
51
- if (lane === "vec") {
52
- const vectors = await qmd.searchVector(query, { limit: 10, collection });
53
- signal?.throwIfAborted();
54
- for (const [rank, hit] of vectors.entries()) {
55
- const pos = hit.chunkPos, len = hit.chunkLen, body = hit.body ?? "";
56
- if (pos === undefined || len === undefined || pos < 0 || len <= 0 || pos + len > body.length)
57
- continue;
58
- add({ file: hit.filepath, body, bestChunk: body.slice(pos, pos + len), bestChunkPos: pos }, "vector", rank, hit.score);
59
- }
60
- return [...candidates.values()];
61
- }
62
- const expression = queryTerms(query).map(term => `"${chunksApi.normalizeCjkForFTS(term).trim()}"`).join(" OR ");
63
- if (expression) {
64
- const marker = `qmd-match-${randomUUID()}`;
65
- const rows = qmd.internal.db.prepare(`SELECT d.collection,d.path,d.hash,c.doc,
66
- bm25(documents_fts,1.5,4.0,1.0) AS rank, highlight(documents_fts,2,?,?) AS highlighted
67
- FROM documents_fts JOIN documents d ON d.id=documents_fts.rowid JOIN content c ON c.hash=d.hash
68
- WHERE documents_fts MATCH ? AND d.active=1 AND d.collection IN (SELECT value FROM json_each(?))
69
- ORDER BY rank,d.collection,d.path LIMIT 10`).all(marker, marker, expression, JSON.stringify(typeof collection === "string" ? [collection] : collection));
70
- for (const [rank, row] of rows.entries()) {
71
- signal?.throwIfAborted();
72
- const file = `qmd://${row.collection}/${row.path}`;
73
- const stored = chunksApi.getStoredChunkSpans(qmd.internal.db, row.hash)
74
- .filter(span => span.pos >= 0 && span.chunk_len > 0 && span.pos + span.chunk_len <= row.doc.length)
75
- .map(span => ({ pos: span.pos, text: row.doc.slice(span.pos, span.pos + span.chunk_len) }));
76
- const chunks = stored.length ? stored : await chunksApi.chunkDocumentAsync(row.doc, undefined, undefined, undefined, file);
77
- const selected = lexicalChunk(chunks, row.doc, row.highlighted, marker, query);
78
- if (selected)
79
- add({ file, body: row.doc, bestChunk: selected.text, bestChunkPos: selected.pos }, "bm25", rank, row.rank);
80
- }
81
- }
82
- return [...candidates.values()].sort((a, b) => b.score - a.score);
3
+ const hits = await qmd.discoverCandidates({ query, lane, collection, limit: 10, signal,
4
+ // Preserve keyword chunk tie-breaking against this lane's own query.
5
+ intent: query });
6
+ return hits.map(({ method, rank, score, file, body, bestChunk, bestChunkPos }) => {
7
+ const retrieval = {
8
+ [method]: { score, rank: rank + 1 },
9
+ };
10
+ return { file, body, bestChunk, bestChunkPos, score: 1 / (rank + 1),
11
+ explain: { methods: [method] }, ...retrieval };
12
+ });
83
13
  }
@@ -1,6 +1,6 @@
1
1
  import { Type } from "typebox";
2
2
  import { Value } from "typebox/value";
3
- import { requestTypeSafe, TYPESAFE_MODEL, TypeSafeRequestError } from "./typesafe-client.js";
3
+ import { requestTypeSafe, TYPESAFE_MODEL, TYPESAFE_NOUL_SCHEMA, TypeSafeRequestError } from "./typesafe-client.js";
4
4
  export const TRAINING_GATE_VERSION = "historical-recall-v2";
5
5
  export const TRAINING_GATE_MODEL = TYPESAFE_MODEL;
6
6
  export const TRAINING_GATE_THRESHOLD = 0.7;
@@ -27,7 +27,7 @@ export const TRAINING_GATE_QUESTIONS = { recall_needed: {
27
27
  } };
28
28
  const resultSchema = Type.Object({
29
29
  model: Type.Literal(TRAINING_GATE_MODEL),
30
- answers: Type.Object({ recall_needed: Type.Object({ type: Type.Literal("noul"), noul: Type.Number({ minimum: 0, maximum: 1 }) }) }),
30
+ answers: Type.Object({ recall_needed: TYPESAFE_NOUL_SCHEMA }),
31
31
  usage: Type.Object({ input_tokens: Type.Integer({ minimum: 0 }), output_tokens: Type.Integer({ minimum: 0 }) }),
32
32
  });
33
33
  export async function judgeTrainingInput(input, apiKey, signal) {
@@ -3,7 +3,7 @@ import { trainingHash } from "./training-input.js";
3
3
  import { TRAINING_GATE_THRESHOLD } from "./training-gate.js";
4
4
  import { resolveTypeSafeApiKey, TypeSafeRequestError } from "./typesafe-client.js";
5
5
  import { trainingTeacher, trainingTeacherMessage, trainingTeacherPrompt, TRAINING_TEACHER_MODEL, TRAINING_TEACHER_VERSION } from "./training-models.js";
6
- import { historicalTrainingSearch, HistoricalCorpusUnavailableError, TRAINING_RETRIEVAL_VERSION, TRAINING_SEARCH_OPTIONS } from "./training-retrieval.js";
6
+ import { historicalTrainingSearch, historicalTrainingSource, HistoricalCorpusUnavailableError, TRAINING_RETRIEVAL_VERSION, TRAINING_SEARCH_OPTIONS } from "./training-retrieval.js";
7
7
  import { contextJudgeRequest, judgeTrainingPassage, CONTEXT_JUDGE_VERSION } from "./training-judge.js";
8
8
  const LANES = ["lex", "vec"];
9
9
  const SELECTION_VERSION = "independent-lanes-top3-mean-v2";
@@ -162,6 +162,7 @@ export async function evaluateTrainingQueries(source, store, config, runtime, op
162
162
  const refreshed = collectTraining(source, store, { existingOnly: true });
163
163
  const result = { ...summary(), retrievals: 0, evaluated: 0, awaitingTeacher: 0 };
164
164
  const concurrency = options.concurrency ?? TRAINING_EVALUATION_CONCURRENCY;
165
+ let historicalSource;
165
166
  let key, teacher;
166
167
  const apiKey = () => key ??= resolveTypeSafeApiKey(config.typesafe).then(value => {
167
168
  if (!value)
@@ -214,7 +215,9 @@ export async function evaluateTrainingQueries(source, store, config, runtime, op
214
215
  store.renew();
215
216
  let snapshot;
216
217
  try {
217
- snapshot = await createSearch(source.stateDir, corpus.chatTypes, example.timestamp);
218
+ snapshot = createSearch === historicalTrainingSearch
219
+ ? await (await (historicalSource ??= historicalTrainingSource(source.stateDir, corpus.chatTypes)))(example.timestamp)
220
+ : await createSearch(source.stateDir, corpus.chatTypes, example.timestamp);
218
221
  }
219
222
  catch (error) {
220
223
  if (!(error instanceof HistoricalCorpusUnavailableError))
@@ -27,8 +27,21 @@ export declare function historicalPrefix(body: string, spans: readonly SessionMe
27
27
  body: string;
28
28
  spans: SessionMessageSpan[];
29
29
  } | undefined;
30
- /** Fingerprint a read-only source snapshot; build its search index only on demand.
31
- * No filesystem projection, live-index mutation, model re-embedding or dependency patch. */
30
+ /** Capture source bytes once per run; each cutoff still owns its index and native model context. */
31
+ export declare function historicalTrainingSource(stateDir: string, chatTypes: readonly ChatType[]): Promise<(cutoff: number, openStore?: typeof import("@unblocklabs/qmd")["createStore"]) => Promise<{
32
+ corpusHash: string;
33
+ report: {
34
+ sessions: number;
35
+ chunks: number;
36
+ excluded: number;
37
+ truncated: number;
38
+ excludedChunks: number;
39
+ };
40
+ maxDate: string;
41
+ search: (query: string, lane: QueryLane) => Promise<TrainingHit[]>;
42
+ close: () => Promise<void>;
43
+ }>>;
44
+ /** Standalone calls get a fresh source capture; evaluation runs reuse their own capture. */
32
45
  export declare function historicalTrainingSearch(stateDir: string, chatTypes: readonly ChatType[], cutoff: number, openStore?: typeof import("@unblocklabs/qmd")["createStore"]): Promise<{
33
46
  corpusHash: string;
34
47
  report: {