@loreai/core 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/bun/agents-file.d.ts +13 -1
- package/dist/bun/agents-file.d.ts.map +1 -1
- package/dist/bun/config.d.ts +20 -1
- package/dist/bun/config.d.ts.map +1 -1
- package/dist/bun/data.d.ts +174 -0
- package/dist/bun/data.d.ts.map +1 -0
- package/dist/bun/db.d.ts +65 -0
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +49 -6
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/embedding-vendor.d.ts +66 -0
- package/dist/bun/embedding-vendor.d.ts.map +1 -0
- package/dist/bun/embedding-worker-types.d.ts +66 -0
- package/dist/bun/embedding-worker-types.d.ts.map +1 -0
- package/dist/bun/embedding-worker.d.ts +16 -0
- package/dist/bun/embedding-worker.d.ts.map +1 -0
- package/dist/bun/embedding-worker.js +100 -0
- package/dist/bun/embedding-worker.js.map +7 -0
- package/dist/bun/embedding.d.ts +91 -8
- package/dist/bun/embedding.d.ts.map +1 -1
- package/dist/bun/git.d.ts +47 -0
- package/dist/bun/git.d.ts.map +1 -0
- package/dist/bun/gradient.d.ts +19 -1
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/index.d.ts +9 -6
- package/dist/bun/index.d.ts.map +1 -1
- package/dist/bun/index.js +13029 -10885
- package/dist/bun/index.js.map +4 -4
- package/dist/bun/lat-reader.d.ts +1 -1
- package/dist/bun/lat-reader.d.ts.map +1 -1
- package/dist/bun/ltm.d.ts.map +1 -1
- package/dist/bun/markdown.d.ts +11 -0
- package/dist/bun/markdown.d.ts.map +1 -1
- package/dist/bun/prompt.d.ts +1 -1
- package/dist/bun/prompt.d.ts.map +1 -1
- package/dist/bun/recall.d.ts +53 -0
- package/dist/bun/recall.d.ts.map +1 -1
- package/dist/bun/search.d.ts +29 -0
- package/dist/bun/search.d.ts.map +1 -1
- package/dist/bun/temporal.d.ts +2 -0
- package/dist/bun/temporal.d.ts.map +1 -1
- package/dist/bun/types.d.ts +15 -0
- package/dist/bun/types.d.ts.map +1 -1
- package/dist/bun/worker-model.d.ts +12 -9
- package/dist/bun/worker-model.d.ts.map +1 -1
- package/dist/node/agents-file.d.ts +13 -1
- package/dist/node/agents-file.d.ts.map +1 -1
- package/dist/node/config.d.ts +20 -1
- package/dist/node/config.d.ts.map +1 -1
- package/dist/node/data.d.ts +174 -0
- package/dist/node/data.d.ts.map +1 -0
- package/dist/node/db.d.ts +65 -0
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +49 -6
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/embedding-vendor.d.ts +66 -0
- package/dist/node/embedding-vendor.d.ts.map +1 -0
- package/dist/node/embedding-worker-types.d.ts +66 -0
- package/dist/node/embedding-worker-types.d.ts.map +1 -0
- package/dist/node/embedding-worker.d.ts +16 -0
- package/dist/node/embedding-worker.d.ts.map +1 -0
- package/dist/node/embedding-worker.js +100 -0
- package/dist/node/embedding-worker.js.map +7 -0
- package/dist/node/embedding.d.ts +91 -8
- package/dist/node/embedding.d.ts.map +1 -1
- package/dist/node/git.d.ts +47 -0
- package/dist/node/git.d.ts.map +1 -0
- package/dist/node/gradient.d.ts +19 -1
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/index.d.ts +9 -6
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +13029 -10885
- package/dist/node/index.js.map +4 -4
- package/dist/node/lat-reader.d.ts +1 -1
- package/dist/node/lat-reader.d.ts.map +1 -1
- package/dist/node/ltm.d.ts.map +1 -1
- package/dist/node/markdown.d.ts +11 -0
- package/dist/node/markdown.d.ts.map +1 -1
- package/dist/node/prompt.d.ts +1 -1
- package/dist/node/prompt.d.ts.map +1 -1
- package/dist/node/recall.d.ts +53 -0
- package/dist/node/recall.d.ts.map +1 -1
- package/dist/node/search.d.ts +29 -0
- package/dist/node/search.d.ts.map +1 -1
- package/dist/node/temporal.d.ts +2 -0
- package/dist/node/temporal.d.ts.map +1 -1
- package/dist/node/types.d.ts +15 -0
- package/dist/node/types.d.ts.map +1 -1
- package/dist/node/worker-model.d.ts +12 -9
- package/dist/node/worker-model.d.ts.map +1 -1
- package/dist/types/agents-file.d.ts +13 -1
- package/dist/types/agents-file.d.ts.map +1 -1
- package/dist/types/config.d.ts +20 -1
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/data.d.ts +174 -0
- package/dist/types/data.d.ts.map +1 -0
- package/dist/types/db.d.ts +65 -0
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +49 -6
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/embedding-vendor.d.ts +66 -0
- package/dist/types/embedding-vendor.d.ts.map +1 -0
- package/dist/types/embedding-worker-types.d.ts +66 -0
- package/dist/types/embedding-worker-types.d.ts.map +1 -0
- package/dist/types/embedding-worker.d.ts +16 -0
- package/dist/types/embedding-worker.d.ts.map +1 -0
- package/dist/types/embedding.d.ts +91 -8
- package/dist/types/embedding.d.ts.map +1 -1
- package/dist/types/git.d.ts +47 -0
- package/dist/types/git.d.ts.map +1 -0
- package/dist/types/gradient.d.ts +19 -1
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/index.d.ts +9 -6
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/lat-reader.d.ts +1 -1
- package/dist/types/lat-reader.d.ts.map +1 -1
- package/dist/types/ltm.d.ts.map +1 -1
- package/dist/types/markdown.d.ts +11 -0
- package/dist/types/markdown.d.ts.map +1 -1
- package/dist/types/prompt.d.ts +1 -1
- package/dist/types/prompt.d.ts.map +1 -1
- package/dist/types/recall.d.ts +53 -0
- package/dist/types/recall.d.ts.map +1 -1
- package/dist/types/search.d.ts +29 -0
- package/dist/types/search.d.ts.map +1 -1
- package/dist/types/temporal.d.ts +2 -0
- package/dist/types/temporal.d.ts.map +1 -1
- package/dist/types/types.d.ts +15 -0
- package/dist/types/types.d.ts.map +1 -1
- package/dist/types/worker-model.d.ts +12 -9
- package/dist/types/worker-model.d.ts.map +1 -1
- package/package.json +5 -2
- package/src/agents-file.ts +87 -4
- package/src/config.ts +68 -5
- package/src/curator.ts +2 -2
- package/src/data.ts +768 -0
- package/src/db.ts +386 -7
- package/src/distillation.ts +178 -35
- package/src/embedding-vendor.ts +102 -0
- package/src/embedding-worker-types.ts +82 -0
- package/src/embedding-worker.ts +185 -0
- package/src/embedding.ts +607 -61
- package/src/git.ts +144 -0
- package/src/gradient.ts +174 -17
- package/src/index.ts +20 -0
- package/src/lat-reader.ts +5 -11
- package/src/ltm.ts +17 -44
- package/src/markdown.ts +15 -0
- package/src/prompt.ts +1 -2
- package/src/recall.ts +401 -70
- package/src/search.ts +71 -1
- package/src/temporal.ts +42 -35
- package/src/types.ts +15 -0
- package/src/worker-model.ts +14 -9
package/src/recall.ts
CHANGED
|
@@ -23,10 +23,10 @@ import {
|
|
|
23
23
|
expandQuery,
|
|
24
24
|
filterTerms,
|
|
25
25
|
ftsQuery,
|
|
26
|
-
ftsQueryOr,
|
|
27
26
|
reciprocalRankFusion,
|
|
27
|
+
runRelaxedSearch,
|
|
28
28
|
} from "./search";
|
|
29
|
-
import {
|
|
29
|
+
import { inline } from "./markdown";
|
|
30
30
|
|
|
31
31
|
// ---------------------------------------------------------------------------
|
|
32
32
|
// Types
|
|
@@ -49,6 +49,8 @@ export type RecallInput = {
|
|
|
49
49
|
query: string;
|
|
50
50
|
/** Narrow the search surface. Defaults to `"all"`. */
|
|
51
51
|
scope?: RecallScope;
|
|
52
|
+
/** Fetch full content of a specific result by its source-prefixed ID (e.g. "k:xxx", "d:xxx"). */
|
|
53
|
+
id?: string;
|
|
52
54
|
/** Project root — used by all scoring paths. */
|
|
53
55
|
projectPath: string;
|
|
54
56
|
/** Current session ID — required when `scope === "session"`. */
|
|
@@ -64,7 +66,7 @@ export type RecallInput = {
|
|
|
64
66
|
/** Result of a full recall run — markdown-formatted string for the LLM. */
|
|
65
67
|
export type RecallResult = string;
|
|
66
68
|
|
|
67
|
-
type TaggedResult =
|
|
69
|
+
export type TaggedResult =
|
|
68
70
|
| { source: "knowledge"; item: ltm.ScoredKnowledgeEntry }
|
|
69
71
|
| {
|
|
70
72
|
source: "cross-knowledge";
|
|
@@ -75,6 +77,8 @@ type TaggedResult =
|
|
|
75
77
|
| { source: "temporal"; item: temporal.ScoredTemporalMessage }
|
|
76
78
|
| { source: "lat-section"; item: latReader.ScoredLatSection };
|
|
77
79
|
|
|
80
|
+
export type ScoredTaggedResult = { item: TaggedResult; score: number };
|
|
81
|
+
|
|
78
82
|
// ---------------------------------------------------------------------------
|
|
79
83
|
// Tagged result helpers (used by exact-match boost + formatting)
|
|
80
84
|
// ---------------------------------------------------------------------------
|
|
@@ -149,8 +153,6 @@ function searchDistillationsScored(input: {
|
|
|
149
153
|
}): ScoredDistillation[] {
|
|
150
154
|
const pid = ensureProject(input.projectPath);
|
|
151
155
|
const limit = input.limit ?? 10;
|
|
152
|
-
const q = ftsQuery(input.query);
|
|
153
|
-
if (q === EMPTY_QUERY) return [];
|
|
154
156
|
|
|
155
157
|
const ftsSQL = input.sessionID
|
|
156
158
|
? `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, d.c_norm, rank
|
|
@@ -165,21 +167,14 @@ function searchDistillationsScored(input: {
|
|
|
165
167
|
WHERE distillation_fts MATCH ?
|
|
166
168
|
AND d.project_id = ?
|
|
167
169
|
ORDER BY rank LIMIT ?`;
|
|
168
|
-
const params = input.sessionID
|
|
169
|
-
? [q, pid, input.sessionID, limit]
|
|
170
|
-
: [q, pid, limit];
|
|
171
170
|
|
|
172
171
|
try {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
const paramsOr = input.sessionID
|
|
180
|
-
? [qOr, pid, input.sessionID, limit]
|
|
181
|
-
: [qOr, pid, limit];
|
|
182
|
-
return db().query(ftsSQL).all(...paramsOr) as ScoredDistillation[];
|
|
172
|
+
return runRelaxedSearch(input.query, (matchExpr) => {
|
|
173
|
+
const params = input.sessionID
|
|
174
|
+
? [matchExpr, pid, input.sessionID, limit]
|
|
175
|
+
: [matchExpr, pid, limit];
|
|
176
|
+
return db().query(ftsSQL).all(...params) as ScoredDistillation[];
|
|
177
|
+
});
|
|
183
178
|
} catch {
|
|
184
179
|
// FTS5 failed — fall back to LIKE search with synthetic rank
|
|
185
180
|
return searchDistillationsLike({
|
|
@@ -195,69 +190,264 @@ function searchDistillationsScored(input: {
|
|
|
195
190
|
// Result formatting
|
|
196
191
|
// ---------------------------------------------------------------------------
|
|
197
192
|
|
|
193
|
+
/** Default formatting config used when no overrides are provided. */
|
|
194
|
+
const DEFAULT_FORMAT_CONFIG = {
|
|
195
|
+
charBudget: 8000,
|
|
196
|
+
relevanceFloor: 0.15,
|
|
197
|
+
maxResults: 15,
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
type FormatConfig = typeof DEFAULT_FORMAT_CONFIG;
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Truncate text at a sentence boundary within maxChars.
|
|
204
|
+
*
|
|
205
|
+
* Walks backwards from the budget limit looking for sentence-ending
|
|
206
|
+
* punctuation (. ! ?) followed by whitespace or end-of-string.
|
|
207
|
+
* Only searches the back half of the budget to avoid cutting too short.
|
|
208
|
+
* Falls back to word boundary if no sentence end is found.
|
|
209
|
+
*/
|
|
210
|
+
function truncateAtSentence(text: string, maxChars: number): string {
|
|
211
|
+
if (text.length <= maxChars) return text;
|
|
212
|
+
|
|
213
|
+
// Search backwards from maxChars for a sentence boundary
|
|
214
|
+
const minPos = Math.floor(maxChars * 0.5);
|
|
215
|
+
for (let i = maxChars - 1; i >= minPos; i--) {
|
|
216
|
+
if (
|
|
217
|
+
(text[i] === "." || text[i] === "!" || text[i] === "?") &&
|
|
218
|
+
(i + 1 >= text.length || /\s/.test(text[i + 1]))
|
|
219
|
+
) {
|
|
220
|
+
return text.slice(0, i + 1);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// No sentence boundary — fall back to word boundary
|
|
225
|
+
const slice = text.slice(0, maxChars);
|
|
226
|
+
const lastSpace = slice.lastIndexOf(" ");
|
|
227
|
+
if (lastSpace > minPos) return text.slice(0, lastSpace) + "...";
|
|
228
|
+
return slice + "...";
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Source-type weights for budget allocation. Higher = more space. */
|
|
232
|
+
const SOURCE_WEIGHT: Record<TaggedResult["source"], number> = {
|
|
233
|
+
knowledge: 1.0,
|
|
234
|
+
"cross-knowledge": 1.0,
|
|
235
|
+
"lat-section": 0.9,
|
|
236
|
+
distillation: 0.8,
|
|
237
|
+
temporal: 0.5,
|
|
238
|
+
};
|
|
239
|
+
|
|
240
|
+
/** Tier multipliers for budget allocation. */
|
|
241
|
+
const TIER_MULTIPLIERS = [3.0, 1.5, 0.7] as const;
|
|
242
|
+
|
|
243
|
+
/** Human-readable tier labels. */
|
|
244
|
+
const TIER_NAMES = ["Strong Matches", "Supporting", "Peripheral"] as const;
|
|
245
|
+
|
|
246
|
+
/** Source display order within a tier. */
|
|
247
|
+
const SOURCE_ORDER: Record<TaggedResult["source"], number> = {
|
|
248
|
+
knowledge: 0,
|
|
249
|
+
"cross-knowledge": 1,
|
|
250
|
+
"lat-section": 2,
|
|
251
|
+
distillation: 3,
|
|
252
|
+
temporal: 4,
|
|
253
|
+
};
|
|
254
|
+
|
|
255
|
+
/** Human-readable source group labels for sub-headers. */
|
|
256
|
+
const SOURCE_LABELS: Record<TaggedResult["source"], string> = {
|
|
257
|
+
knowledge: "Knowledge",
|
|
258
|
+
"cross-knowledge": "Cross-Project",
|
|
259
|
+
"lat-section": "Reference",
|
|
260
|
+
distillation: "Distilled",
|
|
261
|
+
temporal: "Conversation",
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
/** Format a relative age string from a timestamp. */
|
|
265
|
+
function relativeAge(createdAt: number): string {
|
|
266
|
+
const diffMs = Date.now() - createdAt;
|
|
267
|
+
const mins = Math.floor(diffMs / 60_000);
|
|
268
|
+
if (mins < 60) return `${mins}m ago`;
|
|
269
|
+
const hours = Math.floor(mins / 60);
|
|
270
|
+
if (hours < 24) return `${hours}h ago`;
|
|
271
|
+
const days = Math.floor(hours / 24);
|
|
272
|
+
return `${days}d ago`;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
type TieredResult = ScoredTaggedResult & {
|
|
276
|
+
tier: 0 | 1 | 2;
|
|
277
|
+
charBudget: number;
|
|
278
|
+
};
|
|
279
|
+
|
|
198
280
|
function formatFusedResults(
|
|
199
|
-
results:
|
|
200
|
-
|
|
281
|
+
results: ScoredTaggedResult[],
|
|
282
|
+
config: FormatConfig,
|
|
201
283
|
): string {
|
|
202
284
|
if (!results.length) return "No results found for this query.";
|
|
203
285
|
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
286
|
+
const totalFound = results.length;
|
|
287
|
+
const topScore = results[0].score;
|
|
288
|
+
const scoreFloor = topScore * config.relevanceFloor;
|
|
289
|
+
|
|
290
|
+
// Step 1: Score-based cutoff + hard cap. Always keep at least 3.
|
|
291
|
+
let kept = results.filter((r) => r.score >= scoreFloor);
|
|
292
|
+
kept = kept.slice(0, config.maxResults);
|
|
293
|
+
if (kept.length < 3) kept = results.slice(0, Math.min(3, results.length));
|
|
294
|
+
|
|
295
|
+
// Step 2: Assign tiers based on relative score.
|
|
296
|
+
const tiered: TieredResult[] = kept.map((r) => ({
|
|
297
|
+
...r,
|
|
298
|
+
tier:
|
|
299
|
+
r.score >= topScore * 0.6 ? 0 : r.score >= topScore * 0.3 ? 1 : 2,
|
|
300
|
+
charBudget: 0, // computed next
|
|
301
|
+
}));
|
|
302
|
+
|
|
303
|
+
// Step 3: Compute per-result char budgets proportional to weight.
|
|
304
|
+
const rawWeights = tiered.map(
|
|
305
|
+
(r) => SOURCE_WEIGHT[r.item.source] * TIER_MULTIPLIERS[r.tier],
|
|
306
|
+
);
|
|
307
|
+
const totalWeight = rawWeights.reduce((a, b) => a + b, 0);
|
|
308
|
+
for (let i = 0; i < tiered.length; i++) {
|
|
309
|
+
tiered[i].charBudget = Math.max(
|
|
310
|
+
80,
|
|
311
|
+
Math.min(
|
|
312
|
+
1200,
|
|
313
|
+
Math.floor((config.charBudget * rawWeights[i]) / totalWeight),
|
|
314
|
+
),
|
|
315
|
+
);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
// Step 4+5: Build markdown output grouped by tier, then by source.
|
|
319
|
+
const lowScore = kept[kept.length - 1].score;
|
|
320
|
+
const lines: string[] = [];
|
|
321
|
+
|
|
322
|
+
lines.push(`## Recall Results`);
|
|
323
|
+
lines.push(``);
|
|
324
|
+
lines.push(
|
|
325
|
+
`Found ${totalFound} results, showing top ${kept.length} (score range: ${topScore.toFixed(3)}–${lowScore.toFixed(3)}).`,
|
|
326
|
+
);
|
|
327
|
+
|
|
328
|
+
for (const tierIdx of [0, 1, 2] as const) {
|
|
329
|
+
const tierResults = tiered.filter((r) => r.tier === tierIdx);
|
|
330
|
+
if (!tierResults.length) continue;
|
|
331
|
+
|
|
332
|
+
// Sort by source order within tier
|
|
333
|
+
tierResults.sort(
|
|
334
|
+
(a, b) => SOURCE_ORDER[a.item.source] - SOURCE_ORDER[b.item.source],
|
|
335
|
+
);
|
|
336
|
+
|
|
337
|
+
lines.push(``);
|
|
338
|
+
lines.push(`### ${TIER_NAMES[tierIdx]}`);
|
|
339
|
+
|
|
340
|
+
// Group by source type for sub-headers
|
|
341
|
+
let currentSource: TaggedResult["source"] | null = null;
|
|
342
|
+
|
|
343
|
+
for (const r of tierResults) {
|
|
344
|
+
if (r.item.source !== currentSource) {
|
|
345
|
+
currentSource = r.item.source;
|
|
346
|
+
lines.push(``);
|
|
347
|
+
lines.push(`#### ${SOURCE_LABELS[currentSource]}`);
|
|
248
348
|
}
|
|
349
|
+
|
|
350
|
+
const line = renderResultLine(r.item, r.charBudget);
|
|
351
|
+
lines.push(line);
|
|
249
352
|
}
|
|
250
|
-
}
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// Footer
|
|
356
|
+
const anyTruncated = tiered.some(
|
|
357
|
+
(r) => getFullContentLength(r.item) > r.charBudget,
|
|
358
|
+
);
|
|
359
|
+
lines.push(``);
|
|
360
|
+
lines.push(`---`);
|
|
361
|
+
if (anyTruncated) {
|
|
362
|
+
lines.push(
|
|
363
|
+
`*${kept.length} of ${totalFound} results shown. Use recall with id parameter to see full content of truncated results.*`,
|
|
364
|
+
);
|
|
365
|
+
} else {
|
|
366
|
+
lines.push(`*${kept.length} of ${totalFound} results shown.*`);
|
|
367
|
+
}
|
|
251
368
|
|
|
252
|
-
return
|
|
369
|
+
return lines.join("\n");
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** Get the full content length of a tagged result (before truncation). */
|
|
373
|
+
function getFullContentLength(tagged: TaggedResult): number {
|
|
374
|
+
switch (tagged.source) {
|
|
375
|
+
case "knowledge":
|
|
376
|
+
case "cross-knowledge":
|
|
377
|
+
return tagged.item.title.length + tagged.item.content.length + 4; // **: :
|
|
378
|
+
case "distillation":
|
|
379
|
+
return tagged.item.observations.length;
|
|
380
|
+
case "temporal":
|
|
381
|
+
return tagged.item.content.length;
|
|
382
|
+
case "lat-section":
|
|
383
|
+
return tagged.item.heading.length + tagged.item.content.length;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** Render a single result as a markdown list item line. */
|
|
388
|
+
function renderResultLine(tagged: TaggedResult, charBudget: number): string {
|
|
389
|
+
const id = taggedResultKey(tagged);
|
|
390
|
+
|
|
391
|
+
switch (tagged.source) {
|
|
392
|
+
case "knowledge": {
|
|
393
|
+
const k = tagged.item;
|
|
394
|
+
const titlePart = `**${inline(k.title)}**: `;
|
|
395
|
+
const contentBudget = Math.max(40, charBudget - titlePart.length);
|
|
396
|
+
const content = truncateAtSentence(inline(k.content), contentBudget);
|
|
397
|
+
const wasTruncated = inline(k.content).length > contentBudget;
|
|
398
|
+
return `- ${titlePart}${content}${wasTruncated ? ` (${id})` : ""}`;
|
|
399
|
+
}
|
|
400
|
+
case "cross-knowledge": {
|
|
401
|
+
const k = tagged.item;
|
|
402
|
+
const titlePart = `**${inline(k.title)}** (from: ${tagged.projectLabel}): `;
|
|
403
|
+
const contentBudget = Math.max(40, charBudget - titlePart.length);
|
|
404
|
+
const content = truncateAtSentence(inline(k.content), contentBudget);
|
|
405
|
+
const wasTruncated = inline(k.content).length > contentBudget;
|
|
406
|
+
return `- ${titlePart}${content}${wasTruncated ? ` (${id})` : ""}`;
|
|
407
|
+
}
|
|
408
|
+
case "distillation": {
|
|
409
|
+
const d = tagged.item;
|
|
410
|
+
const fullText = inline(d.observations);
|
|
411
|
+
const content = truncateAtSentence(fullText, charBudget);
|
|
412
|
+
const wasTruncated = fullText.length > charBudget;
|
|
413
|
+
return `- ${content}${wasTruncated ? ` (${id})` : ""}`;
|
|
414
|
+
}
|
|
415
|
+
case "temporal": {
|
|
416
|
+
const m = tagged.item;
|
|
417
|
+
const prefix = `(${m.role}, ${relativeAge(m.created_at)}) `;
|
|
418
|
+
const contentBudget = Math.max(40, charBudget - prefix.length);
|
|
419
|
+
const fullText = inline(m.content);
|
|
420
|
+
const content = truncateAtSentence(fullText, contentBudget);
|
|
421
|
+
const wasTruncated = fullText.length > contentBudget;
|
|
422
|
+
return `- ${prefix}${content}${wasTruncated ? ` (${id})` : ""}`;
|
|
423
|
+
}
|
|
424
|
+
case "lat-section": {
|
|
425
|
+
const s = tagged.item;
|
|
426
|
+
const heading = `**${inline(s.file)} \u00A7 ${inline(s.heading)}**: `;
|
|
427
|
+
const contentBudget = Math.max(40, charBudget - heading.length);
|
|
428
|
+
const fullText = s.first_paragraph
|
|
429
|
+
? inline(s.first_paragraph)
|
|
430
|
+
: inline(s.content);
|
|
431
|
+
const content = truncateAtSentence(fullText, contentBudget);
|
|
432
|
+
const wasTruncated = fullText.length > contentBudget;
|
|
433
|
+
return `- ${heading}${content}${wasTruncated ? ` (${id})` : ""}`;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
253
436
|
}
|
|
254
437
|
|
|
255
438
|
// ---------------------------------------------------------------------------
|
|
256
439
|
// Main entry point
|
|
257
440
|
// ---------------------------------------------------------------------------
|
|
258
441
|
|
|
259
|
-
/**
|
|
260
|
-
|
|
442
|
+
/**
|
|
443
|
+
* Search every relevant source, fuse with RRF, and return raw scored results.
|
|
444
|
+
*
|
|
445
|
+
* This is the search+fusion core shared by `runRecall()` (LLM-formatted) and
|
|
446
|
+
* direct consumers like the web UI that need access to the raw result items.
|
|
447
|
+
*/
|
|
448
|
+
export async function searchRecall(
|
|
449
|
+
input: RecallInput,
|
|
450
|
+
): Promise<ScoredTaggedResult[]> {
|
|
261
451
|
const {
|
|
262
452
|
query,
|
|
263
453
|
scope = "all",
|
|
@@ -272,7 +462,7 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
|
|
|
272
462
|
|
|
273
463
|
// Short-circuit vague queries — stopwords-only would match everything.
|
|
274
464
|
if (ftsQuery(query) === EMPTY_QUERY) {
|
|
275
|
-
return
|
|
465
|
+
return [];
|
|
276
466
|
}
|
|
277
467
|
|
|
278
468
|
// Optional query expansion: generate alternative phrasings via LLM.
|
|
@@ -431,6 +621,36 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
|
|
|
431
621
|
});
|
|
432
622
|
}
|
|
433
623
|
}
|
|
624
|
+
|
|
625
|
+
// Temporal vector search (undistilled messages only)
|
|
626
|
+
if (scope !== "knowledge") {
|
|
627
|
+
const pid = ensureProject(projectPath);
|
|
628
|
+
const temporalVectorHits = embedding.vectorSearchTemporal(
|
|
629
|
+
queryVec,
|
|
630
|
+
pid,
|
|
631
|
+
limit,
|
|
632
|
+
);
|
|
633
|
+
const temporalVectorTagged: TaggedResult[] = temporalVectorHits
|
|
634
|
+
.map((hit): TaggedResult | null => {
|
|
635
|
+
const row = db()
|
|
636
|
+
.query(
|
|
637
|
+
"SELECT id, project_id, session_id, role, content, tokens, distilled, created_at, metadata FROM temporal_messages WHERE id = ?",
|
|
638
|
+
)
|
|
639
|
+
.get(hit.id) as temporal.TemporalMessage | null;
|
|
640
|
+
if (!row) return null;
|
|
641
|
+
return {
|
|
642
|
+
source: "temporal",
|
|
643
|
+
item: { ...row, rank: -hit.similarity },
|
|
644
|
+
};
|
|
645
|
+
})
|
|
646
|
+
.filter((r): r is TaggedResult => r !== null);
|
|
647
|
+
if (temporalVectorTagged.length) {
|
|
648
|
+
allRrfLists.push({
|
|
649
|
+
items: temporalVectorTagged,
|
|
650
|
+
key: (r) => `t:${r.item.id}`,
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
}
|
|
434
654
|
} catch (err) {
|
|
435
655
|
log.info("recall: vector search failed:", err);
|
|
436
656
|
}
|
|
@@ -567,7 +787,117 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
|
|
|
567
787
|
}
|
|
568
788
|
|
|
569
789
|
const fused = reciprocalRankFusion<TaggedResult>(allRrfLists);
|
|
570
|
-
|
|
790
|
+
|
|
791
|
+
// Cap output: return at most 3x the per-source limit. With 7+ RRF sources
|
|
792
|
+
// each contributing up to `limit` items, uncapped output can be huge (89+
|
|
793
|
+
// results for broad OR fallbacks). The top-scoring items after RRF fusion
|
|
794
|
+
// are the ones that appeared in multiple lists — capping preserves those
|
|
795
|
+
// while dropping the long tail of single-list noise.
|
|
796
|
+
const maxResults = limit * 3;
|
|
797
|
+
return fused.slice(0, maxResults);
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
// ---------------------------------------------------------------------------
|
|
801
|
+
// Recall by ID — fetch full untruncated content of a specific result
|
|
802
|
+
// ---------------------------------------------------------------------------
|
|
803
|
+
|
|
804
|
+
/**
|
|
805
|
+
* Fetch the full content of a single result by its source-prefixed ID.
|
|
806
|
+
*
|
|
807
|
+
* IDs use the format `prefix:uuid` where prefix is one of:
|
|
808
|
+
* k: (knowledge), xk: (cross-knowledge), d: (distillation),
|
|
809
|
+
* t: (temporal), lat: (lat-section).
|
|
810
|
+
*/
|
|
811
|
+
export function recallById(id: string): string {
|
|
812
|
+
const colonIdx = id.indexOf(":");
|
|
813
|
+
if (colonIdx < 1) return `No entry found for id: ${id}`;
|
|
814
|
+
|
|
815
|
+
const prefix = id.slice(0, colonIdx);
|
|
816
|
+
const rawId = id.slice(colonIdx + 1);
|
|
817
|
+
|
|
818
|
+
switch (prefix) {
|
|
819
|
+
case "k":
|
|
820
|
+
case "xk": {
|
|
821
|
+
const entry = ltm.get(rawId);
|
|
822
|
+
if (!entry) return `No entry found for id: ${id}`;
|
|
823
|
+
return [
|
|
824
|
+
`## Recall Detail: ${id}`,
|
|
825
|
+
``,
|
|
826
|
+
`#### Knowledge`,
|
|
827
|
+
`- **${inline(entry.title)}** (${entry.category}): ${inline(entry.content)}`,
|
|
828
|
+
].join("\n");
|
|
829
|
+
}
|
|
830
|
+
case "d": {
|
|
831
|
+
const row = db()
|
|
832
|
+
.query(
|
|
833
|
+
"SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE id = ?",
|
|
834
|
+
)
|
|
835
|
+
.get(rawId) as Distillation | null;
|
|
836
|
+
if (!row) return `No entry found for id: ${id}`;
|
|
837
|
+
return [
|
|
838
|
+
`## Recall Detail: ${id}`,
|
|
839
|
+
``,
|
|
840
|
+
`#### Distilled`,
|
|
841
|
+
`${inline(row.observations)}`,
|
|
842
|
+
].join("\n");
|
|
843
|
+
}
|
|
844
|
+
case "t": {
|
|
845
|
+
const row = db()
|
|
846
|
+
.query(
|
|
847
|
+
"SELECT id, project_id, session_id, role, content, tokens, distilled, created_at, metadata FROM temporal_messages WHERE id = ?",
|
|
848
|
+
)
|
|
849
|
+
.get(rawId) as temporal.TemporalMessage | null;
|
|
850
|
+
if (!row) return `No entry found for id: ${id}`;
|
|
851
|
+
return [
|
|
852
|
+
`## Recall Detail: ${id}`,
|
|
853
|
+
``,
|
|
854
|
+
`#### Conversation`,
|
|
855
|
+
`(${row.role}, ${relativeAge(row.created_at)}, session: ${row.session_id.slice(0, 8)})`,
|
|
856
|
+
``,
|
|
857
|
+
`${inline(row.content)}`,
|
|
858
|
+
].join("\n");
|
|
859
|
+
}
|
|
860
|
+
case "lat": {
|
|
861
|
+
const row = db()
|
|
862
|
+
.query(
|
|
863
|
+
"SELECT id, project_id, file, heading, depth, content, content_hash, first_paragraph, updated_at FROM lat_sections WHERE id = ?",
|
|
864
|
+
)
|
|
865
|
+
.get(rawId) as latReader.LatSection | null;
|
|
866
|
+
if (!row) return `No entry found for id: ${id}`;
|
|
867
|
+
return [
|
|
868
|
+
`## Recall Detail: ${id}`,
|
|
869
|
+
``,
|
|
870
|
+
`#### Reference`,
|
|
871
|
+
`**${inline(row.file)} \u00A7 ${inline(row.heading)}**`,
|
|
872
|
+
``,
|
|
873
|
+
`${inline(row.content)}`,
|
|
874
|
+
].join("\n");
|
|
875
|
+
}
|
|
876
|
+
default:
|
|
877
|
+
return `Unknown source prefix "${prefix}" in id: ${id}`;
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
/** Full recall run: search every relevant source, fuse with RRF, format as markdown. */
|
|
882
|
+
export async function runRecall(input: RecallInput): Promise<RecallResult> {
|
|
883
|
+
// ID-based detail retrieval — bypass search entirely.
|
|
884
|
+
if (input.id) {
|
|
885
|
+
return recallById(input.id);
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
// Short-circuit vague queries — stopwords-only would match everything.
|
|
889
|
+
if (ftsQuery(input.query) === EMPTY_QUERY) {
|
|
890
|
+
return "Query too vague — try using specific keywords, file names, or technical terms.";
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
const fused = await searchRecall(input);
|
|
894
|
+
const recallCfg = input.searchConfig?.recall;
|
|
895
|
+
return formatFusedResults(fused, {
|
|
896
|
+
charBudget: recallCfg?.charBudget ?? DEFAULT_FORMAT_CONFIG.charBudget,
|
|
897
|
+
relevanceFloor:
|
|
898
|
+
recallCfg?.relevanceFloor ?? DEFAULT_FORMAT_CONFIG.relevanceFloor,
|
|
899
|
+
maxResults: recallCfg?.maxResults ?? DEFAULT_FORMAT_CONFIG.maxResults,
|
|
900
|
+
});
|
|
571
901
|
}
|
|
572
902
|
|
|
573
903
|
/** Standard tool description reused verbatim by each host adapter. */
|
|
@@ -579,4 +909,5 @@ export const RECALL_PARAM_DESCRIPTIONS = {
|
|
|
579
909
|
query: "What to search for — be specific. Include keywords, file names, or concepts.",
|
|
580
910
|
scope:
|
|
581
911
|
"Search scope: 'all' (default) searches everything, 'session' searches current session only, 'project' searches all sessions in this project, 'knowledge' searches only long-term knowledge.",
|
|
912
|
+
id: "Fetch full content of a specific result by its source-prefixed ID (e.g. 'k:abc123', 'd:abc123'). IDs are shown on truncated results in recall output. When id is provided, query is ignored.",
|
|
582
913
|
};
|
package/src/search.ts
CHANGED
|
@@ -173,6 +173,76 @@ export function ftsQueryOr(raw: string): string {
|
|
|
173
173
|
return terms.map((w) => `${w}*`).join(" OR ");
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
+
/**
|
|
177
|
+
* Build a cascade of progressively relaxed FTS5 queries.
|
|
178
|
+
*
|
|
179
|
+
* For N terms, produces up to (N - minTerms) queries, each dropping one more
|
|
180
|
+
* term (least significant first — shortest terms dropped first as a rough
|
|
181
|
+
* proxy for specificity). The final entry is always the full OR query.
|
|
182
|
+
*
|
|
183
|
+
* Example for 6 terms with minTerms=3:
|
|
184
|
+
* [0] 5-of-6 AND (drop shortest term)
|
|
185
|
+
* [1] 4-of-6 AND
|
|
186
|
+
* [2] 3-of-6 AND
|
|
187
|
+
* [3] full OR (all 6 terms)
|
|
188
|
+
*
|
|
189
|
+
* For ≤ minTerms terms, returns just the OR query (no intermediate steps).
|
|
190
|
+
* Callers should try each query in order, stopping at the first that returns
|
|
191
|
+
* results. This avoids the AND→OR cliff that produces massive low-quality
|
|
192
|
+
* result sets.
|
|
193
|
+
*/
|
|
194
|
+
export function ftsQueryRelaxed(raw: string, minTerms = 3): string[] {
|
|
195
|
+
const terms = filterTerms(raw);
|
|
196
|
+
if (!terms.length) return [EMPTY_QUERY];
|
|
197
|
+
|
|
198
|
+
const orQuery = terms.map((w) => `${w}*`).join(" OR ");
|
|
199
|
+
|
|
200
|
+
// Not enough terms for progressive relaxation — just OR.
|
|
201
|
+
if (terms.length <= minTerms) return [orQuery];
|
|
202
|
+
|
|
203
|
+
// Sort by length ascending — shortest (least specific) terms dropped first.
|
|
204
|
+
const ranked = [...terms].sort((a, b) => a.length - b.length);
|
|
205
|
+
|
|
206
|
+
const cascade: string[] = [];
|
|
207
|
+
for (let drop = 1; drop <= terms.length - minTerms; drop++) {
|
|
208
|
+
const kept = ranked.slice(drop);
|
|
209
|
+
cascade.push(kept.map((w) => `${w}*`).join(" "));
|
|
210
|
+
}
|
|
211
|
+
cascade.push(orQuery);
|
|
212
|
+
return cascade;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Run a search function through the relaxed cascade, stopping at the first
|
|
217
|
+
* query that produces results. Falls back through progressively looser AND
|
|
218
|
+
* queries before trying full OR.
|
|
219
|
+
*
|
|
220
|
+
* @param raw The original query string
|
|
221
|
+
* @param runner A function that takes an FTS5 MATCH expression and returns results
|
|
222
|
+
* @returns The results from the first cascade step that produced matches
|
|
223
|
+
*/
|
|
224
|
+
export function runRelaxedSearch<T>(
|
|
225
|
+
raw: string,
|
|
226
|
+
runner: (matchExpr: string) => T[],
|
|
227
|
+
): T[] {
|
|
228
|
+
// First try exact AND (all terms)
|
|
229
|
+
const q = ftsQuery(raw);
|
|
230
|
+
if (q === EMPTY_QUERY) return [];
|
|
231
|
+
|
|
232
|
+
const andResults = runner(q);
|
|
233
|
+
if (andResults.length) return andResults;
|
|
234
|
+
|
|
235
|
+
// Try progressively relaxed queries
|
|
236
|
+
const cascade = ftsQueryRelaxed(raw);
|
|
237
|
+
for (const relaxed of cascade) {
|
|
238
|
+
if (relaxed === EMPTY_QUERY) continue;
|
|
239
|
+
const results = runner(relaxed);
|
|
240
|
+
if (results.length) return results;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
return [];
|
|
244
|
+
}
|
|
245
|
+
|
|
176
246
|
// ---------------------------------------------------------------------------
|
|
177
247
|
// Term extraction (Phase 3)
|
|
178
248
|
// ---------------------------------------------------------------------------
|
|
@@ -335,7 +405,7 @@ export async function expandQuery(
|
|
|
335
405
|
llm.prompt(
|
|
336
406
|
QUERY_EXPANSION_SYSTEM,
|
|
337
407
|
`Input: "${query}"`,
|
|
338
|
-
{ model, workerID: "lore-query-expand", thinking: false, urgent: true, sessionID },
|
|
408
|
+
{ model, workerID: "lore-query-expand", thinking: false, urgent: true, sessionID, maxTokens: 256 },
|
|
339
409
|
),
|
|
340
410
|
new Promise<null>((resolve) => setTimeout(() => resolve(null), TIMEOUT_MS)),
|
|
341
411
|
]);
|