@loreai/core 0.16.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +11 -0
  2. package/dist/bun/agents-file.d.ts +13 -1
  3. package/dist/bun/agents-file.d.ts.map +1 -1
  4. package/dist/bun/config.d.ts +20 -1
  5. package/dist/bun/config.d.ts.map +1 -1
  6. package/dist/bun/data.d.ts +174 -0
  7. package/dist/bun/data.d.ts.map +1 -0
  8. package/dist/bun/db.d.ts +65 -0
  9. package/dist/bun/db.d.ts.map +1 -1
  10. package/dist/bun/distillation.d.ts +49 -6
  11. package/dist/bun/distillation.d.ts.map +1 -1
  12. package/dist/bun/embedding-vendor.d.ts +66 -0
  13. package/dist/bun/embedding-vendor.d.ts.map +1 -0
  14. package/dist/bun/embedding-worker-types.d.ts +66 -0
  15. package/dist/bun/embedding-worker-types.d.ts.map +1 -0
  16. package/dist/bun/embedding-worker.d.ts +16 -0
  17. package/dist/bun/embedding-worker.d.ts.map +1 -0
  18. package/dist/bun/embedding-worker.js +100 -0
  19. package/dist/bun/embedding-worker.js.map +7 -0
  20. package/dist/bun/embedding.d.ts +91 -8
  21. package/dist/bun/embedding.d.ts.map +1 -1
  22. package/dist/bun/git.d.ts +47 -0
  23. package/dist/bun/git.d.ts.map +1 -0
  24. package/dist/bun/gradient.d.ts +19 -1
  25. package/dist/bun/gradient.d.ts.map +1 -1
  26. package/dist/bun/index.d.ts +9 -6
  27. package/dist/bun/index.d.ts.map +1 -1
  28. package/dist/bun/index.js +13029 -10885
  29. package/dist/bun/index.js.map +4 -4
  30. package/dist/bun/lat-reader.d.ts +1 -1
  31. package/dist/bun/lat-reader.d.ts.map +1 -1
  32. package/dist/bun/ltm.d.ts.map +1 -1
  33. package/dist/bun/markdown.d.ts +11 -0
  34. package/dist/bun/markdown.d.ts.map +1 -1
  35. package/dist/bun/prompt.d.ts +1 -1
  36. package/dist/bun/prompt.d.ts.map +1 -1
  37. package/dist/bun/recall.d.ts +53 -0
  38. package/dist/bun/recall.d.ts.map +1 -1
  39. package/dist/bun/search.d.ts +29 -0
  40. package/dist/bun/search.d.ts.map +1 -1
  41. package/dist/bun/temporal.d.ts +2 -0
  42. package/dist/bun/temporal.d.ts.map +1 -1
  43. package/dist/bun/types.d.ts +15 -0
  44. package/dist/bun/types.d.ts.map +1 -1
  45. package/dist/bun/worker-model.d.ts +12 -9
  46. package/dist/bun/worker-model.d.ts.map +1 -1
  47. package/dist/node/agents-file.d.ts +13 -1
  48. package/dist/node/agents-file.d.ts.map +1 -1
  49. package/dist/node/config.d.ts +20 -1
  50. package/dist/node/config.d.ts.map +1 -1
  51. package/dist/node/data.d.ts +174 -0
  52. package/dist/node/data.d.ts.map +1 -0
  53. package/dist/node/db.d.ts +65 -0
  54. package/dist/node/db.d.ts.map +1 -1
  55. package/dist/node/distillation.d.ts +49 -6
  56. package/dist/node/distillation.d.ts.map +1 -1
  57. package/dist/node/embedding-vendor.d.ts +66 -0
  58. package/dist/node/embedding-vendor.d.ts.map +1 -0
  59. package/dist/node/embedding-worker-types.d.ts +66 -0
  60. package/dist/node/embedding-worker-types.d.ts.map +1 -0
  61. package/dist/node/embedding-worker.d.ts +16 -0
  62. package/dist/node/embedding-worker.d.ts.map +1 -0
  63. package/dist/node/embedding-worker.js +100 -0
  64. package/dist/node/embedding-worker.js.map +7 -0
  65. package/dist/node/embedding.d.ts +91 -8
  66. package/dist/node/embedding.d.ts.map +1 -1
  67. package/dist/node/git.d.ts +47 -0
  68. package/dist/node/git.d.ts.map +1 -0
  69. package/dist/node/gradient.d.ts +19 -1
  70. package/dist/node/gradient.d.ts.map +1 -1
  71. package/dist/node/index.d.ts +9 -6
  72. package/dist/node/index.d.ts.map +1 -1
  73. package/dist/node/index.js +13029 -10885
  74. package/dist/node/index.js.map +4 -4
  75. package/dist/node/lat-reader.d.ts +1 -1
  76. package/dist/node/lat-reader.d.ts.map +1 -1
  77. package/dist/node/ltm.d.ts.map +1 -1
  78. package/dist/node/markdown.d.ts +11 -0
  79. package/dist/node/markdown.d.ts.map +1 -1
  80. package/dist/node/prompt.d.ts +1 -1
  81. package/dist/node/prompt.d.ts.map +1 -1
  82. package/dist/node/recall.d.ts +53 -0
  83. package/dist/node/recall.d.ts.map +1 -1
  84. package/dist/node/search.d.ts +29 -0
  85. package/dist/node/search.d.ts.map +1 -1
  86. package/dist/node/temporal.d.ts +2 -0
  87. package/dist/node/temporal.d.ts.map +1 -1
  88. package/dist/node/types.d.ts +15 -0
  89. package/dist/node/types.d.ts.map +1 -1
  90. package/dist/node/worker-model.d.ts +12 -9
  91. package/dist/node/worker-model.d.ts.map +1 -1
  92. package/dist/types/agents-file.d.ts +13 -1
  93. package/dist/types/agents-file.d.ts.map +1 -1
  94. package/dist/types/config.d.ts +20 -1
  95. package/dist/types/config.d.ts.map +1 -1
  96. package/dist/types/data.d.ts +174 -0
  97. package/dist/types/data.d.ts.map +1 -0
  98. package/dist/types/db.d.ts +65 -0
  99. package/dist/types/db.d.ts.map +1 -1
  100. package/dist/types/distillation.d.ts +49 -6
  101. package/dist/types/distillation.d.ts.map +1 -1
  102. package/dist/types/embedding-vendor.d.ts +66 -0
  103. package/dist/types/embedding-vendor.d.ts.map +1 -0
  104. package/dist/types/embedding-worker-types.d.ts +66 -0
  105. package/dist/types/embedding-worker-types.d.ts.map +1 -0
  106. package/dist/types/embedding-worker.d.ts +16 -0
  107. package/dist/types/embedding-worker.d.ts.map +1 -0
  108. package/dist/types/embedding.d.ts +91 -8
  109. package/dist/types/embedding.d.ts.map +1 -1
  110. package/dist/types/git.d.ts +47 -0
  111. package/dist/types/git.d.ts.map +1 -0
  112. package/dist/types/gradient.d.ts +19 -1
  113. package/dist/types/gradient.d.ts.map +1 -1
  114. package/dist/types/index.d.ts +9 -6
  115. package/dist/types/index.d.ts.map +1 -1
  116. package/dist/types/lat-reader.d.ts +1 -1
  117. package/dist/types/lat-reader.d.ts.map +1 -1
  118. package/dist/types/ltm.d.ts.map +1 -1
  119. package/dist/types/markdown.d.ts +11 -0
  120. package/dist/types/markdown.d.ts.map +1 -1
  121. package/dist/types/prompt.d.ts +1 -1
  122. package/dist/types/prompt.d.ts.map +1 -1
  123. package/dist/types/recall.d.ts +53 -0
  124. package/dist/types/recall.d.ts.map +1 -1
  125. package/dist/types/search.d.ts +29 -0
  126. package/dist/types/search.d.ts.map +1 -1
  127. package/dist/types/temporal.d.ts +2 -0
  128. package/dist/types/temporal.d.ts.map +1 -1
  129. package/dist/types/types.d.ts +15 -0
  130. package/dist/types/types.d.ts.map +1 -1
  131. package/dist/types/worker-model.d.ts +12 -9
  132. package/dist/types/worker-model.d.ts.map +1 -1
  133. package/package.json +5 -2
  134. package/src/agents-file.ts +87 -4
  135. package/src/config.ts +68 -5
  136. package/src/curator.ts +2 -2
  137. package/src/data.ts +768 -0
  138. package/src/db.ts +386 -7
  139. package/src/distillation.ts +178 -35
  140. package/src/embedding-vendor.ts +102 -0
  141. package/src/embedding-worker-types.ts +82 -0
  142. package/src/embedding-worker.ts +185 -0
  143. package/src/embedding.ts +607 -61
  144. package/src/git.ts +144 -0
  145. package/src/gradient.ts +174 -17
  146. package/src/index.ts +20 -0
  147. package/src/lat-reader.ts +5 -11
  148. package/src/ltm.ts +17 -44
  149. package/src/markdown.ts +15 -0
  150. package/src/prompt.ts +1 -2
  151. package/src/recall.ts +401 -70
  152. package/src/search.ts +71 -1
  153. package/src/temporal.ts +42 -35
  154. package/src/types.ts +15 -0
  155. package/src/worker-model.ts +14 -9
package/src/recall.ts CHANGED
@@ -23,10 +23,10 @@ import {
23
23
  expandQuery,
24
24
  filterTerms,
25
25
  ftsQuery,
26
- ftsQueryOr,
27
26
  reciprocalRankFusion,
27
+ runRelaxedSearch,
28
28
  } from "./search";
29
- import { h, inline, lip, liph, p, root, serialize, t, ul } from "./markdown";
29
+ import { inline } from "./markdown";
30
30
 
31
31
  // ---------------------------------------------------------------------------
32
32
  // Types
@@ -49,6 +49,8 @@ export type RecallInput = {
49
49
  query: string;
50
50
  /** Narrow the search surface. Defaults to `"all"`. */
51
51
  scope?: RecallScope;
52
+ /** Fetch full content of a specific result by its source-prefixed ID (e.g. "k:xxx", "d:xxx"). */
53
+ id?: string;
52
54
  /** Project root — used by all scoring paths. */
53
55
  projectPath: string;
54
56
  /** Current session ID — required when `scope === "session"`. */
@@ -64,7 +66,7 @@ export type RecallInput = {
64
66
  /** Result of a full recall run — markdown-formatted string for the LLM. */
65
67
  export type RecallResult = string;
66
68
 
67
- type TaggedResult =
69
+ export type TaggedResult =
68
70
  | { source: "knowledge"; item: ltm.ScoredKnowledgeEntry }
69
71
  | {
70
72
  source: "cross-knowledge";
@@ -75,6 +77,8 @@ type TaggedResult =
75
77
  | { source: "temporal"; item: temporal.ScoredTemporalMessage }
76
78
  | { source: "lat-section"; item: latReader.ScoredLatSection };
77
79
 
80
+ export type ScoredTaggedResult = { item: TaggedResult; score: number };
81
+
78
82
  // ---------------------------------------------------------------------------
79
83
  // Tagged result helpers (used by exact-match boost + formatting)
80
84
  // ---------------------------------------------------------------------------
@@ -149,8 +153,6 @@ function searchDistillationsScored(input: {
149
153
  }): ScoredDistillation[] {
150
154
  const pid = ensureProject(input.projectPath);
151
155
  const limit = input.limit ?? 10;
152
- const q = ftsQuery(input.query);
153
- if (q === EMPTY_QUERY) return [];
154
156
 
155
157
  const ftsSQL = input.sessionID
156
158
  ? `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, d.c_norm, rank
@@ -165,21 +167,14 @@ function searchDistillationsScored(input: {
165
167
  WHERE distillation_fts MATCH ?
166
168
  AND d.project_id = ?
167
169
  ORDER BY rank LIMIT ?`;
168
- const params = input.sessionID
169
- ? [q, pid, input.sessionID, limit]
170
- : [q, pid, limit];
171
170
 
172
171
  try {
173
- const results = db().query(ftsSQL).all(...params) as ScoredDistillation[];
174
- if (results.length) return results;
175
-
176
- // AND returned nothing — try OR fallback
177
- const qOr = ftsQueryOr(input.query);
178
- if (qOr === EMPTY_QUERY) return [];
179
- const paramsOr = input.sessionID
180
- ? [qOr, pid, input.sessionID, limit]
181
- : [qOr, pid, limit];
182
- return db().query(ftsSQL).all(...paramsOr) as ScoredDistillation[];
172
+ return runRelaxedSearch(input.query, (matchExpr) => {
173
+ const params = input.sessionID
174
+ ? [matchExpr, pid, input.sessionID, limit]
175
+ : [matchExpr, pid, limit];
176
+ return db().query(ftsSQL).all(...params) as ScoredDistillation[];
177
+ });
183
178
  } catch {
184
179
  // FTS5 failed — fall back to LIKE search with synthetic rank
185
180
  return searchDistillationsLike({
@@ -195,69 +190,264 @@ function searchDistillationsScored(input: {
195
190
  // Result formatting
196
191
  // ---------------------------------------------------------------------------
197
192
 
193
+ /** Default formatting config used when no overrides are provided. */
194
+ const DEFAULT_FORMAT_CONFIG = {
195
+ charBudget: 8000,
196
+ relevanceFloor: 0.15,
197
+ maxResults: 15,
198
+ };
199
+
200
+ type FormatConfig = typeof DEFAULT_FORMAT_CONFIG;
201
+
202
+ /**
203
+ * Truncate text at a sentence boundary within maxChars.
204
+ *
205
+ * Walks backwards from the budget limit looking for sentence-ending
206
+ * punctuation (. ! ?) followed by whitespace or end-of-string.
207
+ * Only searches the back half of the budget to avoid cutting too short.
208
+ * Falls back to word boundary if no sentence end is found.
209
+ */
210
+ function truncateAtSentence(text: string, maxChars: number): string {
211
+ if (text.length <= maxChars) return text;
212
+
213
+ // Search backwards from maxChars for a sentence boundary
214
+ const minPos = Math.floor(maxChars * 0.5);
215
+ for (let i = maxChars - 1; i >= minPos; i--) {
216
+ if (
217
+ (text[i] === "." || text[i] === "!" || text[i] === "?") &&
218
+ (i + 1 >= text.length || /\s/.test(text[i + 1]))
219
+ ) {
220
+ return text.slice(0, i + 1);
221
+ }
222
+ }
223
+
224
+ // No sentence boundary — fall back to word boundary
225
+ const slice = text.slice(0, maxChars);
226
+ const lastSpace = slice.lastIndexOf(" ");
227
+ if (lastSpace > minPos) return text.slice(0, lastSpace) + "...";
228
+ return slice + "...";
229
+ }
230
+
231
+ /** Source-type weights for budget allocation. Higher = more space. */
232
+ const SOURCE_WEIGHT: Record<TaggedResult["source"], number> = {
233
+ knowledge: 1.0,
234
+ "cross-knowledge": 1.0,
235
+ "lat-section": 0.9,
236
+ distillation: 0.8,
237
+ temporal: 0.5,
238
+ };
239
+
240
+ /** Tier multipliers for budget allocation. */
241
+ const TIER_MULTIPLIERS = [3.0, 1.5, 0.7] as const;
242
+
243
+ /** Human-readable tier labels. */
244
+ const TIER_NAMES = ["Strong Matches", "Supporting", "Peripheral"] as const;
245
+
246
+ /** Source display order within a tier. */
247
+ const SOURCE_ORDER: Record<TaggedResult["source"], number> = {
248
+ knowledge: 0,
249
+ "cross-knowledge": 1,
250
+ "lat-section": 2,
251
+ distillation: 3,
252
+ temporal: 4,
253
+ };
254
+
255
+ /** Human-readable source group labels for sub-headers. */
256
+ const SOURCE_LABELS: Record<TaggedResult["source"], string> = {
257
+ knowledge: "Knowledge",
258
+ "cross-knowledge": "Cross-Project",
259
+ "lat-section": "Reference",
260
+ distillation: "Distilled",
261
+ temporal: "Conversation",
262
+ };
263
+
264
+ /** Format a relative age string from a timestamp. */
265
+ function relativeAge(createdAt: number): string {
266
+ const diffMs = Date.now() - createdAt;
267
+ const mins = Math.floor(diffMs / 60_000);
268
+ if (mins < 60) return `${mins}m ago`;
269
+ const hours = Math.floor(mins / 60);
270
+ if (hours < 24) return `${hours}h ago`;
271
+ const days = Math.floor(hours / 24);
272
+ return `${days}d ago`;
273
+ }
274
+
275
+ type TieredResult = ScoredTaggedResult & {
276
+ tier: 0 | 1 | 2;
277
+ charBudget: number;
278
+ };
279
+
198
280
  function formatFusedResults(
199
- results: Array<{ item: TaggedResult; score: number }>,
200
- maxResults: number,
281
+ results: ScoredTaggedResult[],
282
+ config: FormatConfig,
201
283
  ): string {
202
284
  if (!results.length) return "No results found for this query.";
203
285
 
204
- const items = results.slice(0, maxResults).map(({ item: tagged }) => {
205
- switch (tagged.source) {
206
- case "knowledge": {
207
- const k = tagged.item;
208
- return liph(
209
- t(
210
- `**[knowledge/${k.category}]** ${inline(k.title)}: ${inline(k.content)}`,
211
- ),
212
- );
213
- }
214
- case "cross-knowledge": {
215
- const k = tagged.item;
216
- return liph(
217
- t(
218
- `**[knowledge/${k.category} from: ${tagged.projectLabel}]** ${inline(k.title)}: ${inline(k.content)}`,
219
- ),
220
- );
221
- }
222
- case "distillation": {
223
- const d = tagged.item;
224
- const preview =
225
- d.observations.length > 500
226
- ? d.observations.slice(0, 500) + "..."
227
- : d.observations;
228
- return lip(`**[distilled]** ${inline(preview)}`);
229
- }
230
- case "temporal": {
231
- const m = tagged.item;
232
- const preview =
233
- m.content.length > 500 ? m.content.slice(0, 500) + "..." : m.content;
234
- return lip(
235
- `**[temporal/${m.role}]** (session: ${m.session_id.slice(0, 8)}...) ${inline(preview)}`,
236
- );
237
- }
238
- case "lat-section": {
239
- const s = tagged.item;
240
- const preview = s.first_paragraph
241
- ? inline(s.first_paragraph)
242
- : inline(
243
- s.content.length > 300 ? s.content.slice(0, 300) + "..." : s.content,
244
- );
245
- return liph(
246
- t(`**[lat.md/${s.file}]** ${inline(s.heading)}: ${preview}`),
247
- );
286
+ const totalFound = results.length;
287
+ const topScore = results[0].score;
288
+ const scoreFloor = topScore * config.relevanceFloor;
289
+
290
+ // Step 1: Score-based cutoff + hard cap. Always keep at least 3.
291
+ let kept = results.filter((r) => r.score >= scoreFloor);
292
+ kept = kept.slice(0, config.maxResults);
293
+ if (kept.length < 3) kept = results.slice(0, Math.min(3, results.length));
294
+
295
+ // Step 2: Assign tiers based on relative score.
296
+ const tiered: TieredResult[] = kept.map((r) => ({
297
+ ...r,
298
+ tier:
299
+ r.score >= topScore * 0.6 ? 0 : r.score >= topScore * 0.3 ? 1 : 2,
300
+ charBudget: 0, // computed next
301
+ }));
302
+
303
+ // Step 3: Compute per-result char budgets proportional to weight.
304
+ const rawWeights = tiered.map(
305
+ (r) => SOURCE_WEIGHT[r.item.source] * TIER_MULTIPLIERS[r.tier],
306
+ );
307
+ const totalWeight = rawWeights.reduce((a, b) => a + b, 0);
308
+ for (let i = 0; i < tiered.length; i++) {
309
+ tiered[i].charBudget = Math.max(
310
+ 80,
311
+ Math.min(
312
+ 1200,
313
+ Math.floor((config.charBudget * rawWeights[i]) / totalWeight),
314
+ ),
315
+ );
316
+ }
317
+
318
+ // Step 4+5: Build markdown output grouped by tier, then by source.
319
+ const lowScore = kept[kept.length - 1].score;
320
+ const lines: string[] = [];
321
+
322
+ lines.push(`## Recall Results`);
323
+ lines.push(``);
324
+ lines.push(
325
+ `Found ${totalFound} results, showing top ${kept.length} (score range: ${topScore.toFixed(3)}–${lowScore.toFixed(3)}).`,
326
+ );
327
+
328
+ for (const tierIdx of [0, 1, 2] as const) {
329
+ const tierResults = tiered.filter((r) => r.tier === tierIdx);
330
+ if (!tierResults.length) continue;
331
+
332
+ // Sort by source order within tier
333
+ tierResults.sort(
334
+ (a, b) => SOURCE_ORDER[a.item.source] - SOURCE_ORDER[b.item.source],
335
+ );
336
+
337
+ lines.push(``);
338
+ lines.push(`### ${TIER_NAMES[tierIdx]}`);
339
+
340
+ // Group by source type for sub-headers
341
+ let currentSource: TaggedResult["source"] | null = null;
342
+
343
+ for (const r of tierResults) {
344
+ if (r.item.source !== currentSource) {
345
+ currentSource = r.item.source;
346
+ lines.push(``);
347
+ lines.push(`#### ${SOURCE_LABELS[currentSource]}`);
248
348
  }
349
+
350
+ const line = renderResultLine(r.item, r.charBudget);
351
+ lines.push(line);
249
352
  }
250
- });
353
+ }
354
+
355
+ // Footer
356
+ const anyTruncated = tiered.some(
357
+ (r) => getFullContentLength(r.item) > r.charBudget,
358
+ );
359
+ lines.push(``);
360
+ lines.push(`---`);
361
+ if (anyTruncated) {
362
+ lines.push(
363
+ `*${kept.length} of ${totalFound} results shown. Use recall with id parameter to see full content of truncated results.*`,
364
+ );
365
+ } else {
366
+ lines.push(`*${kept.length} of ${totalFound} results shown.*`);
367
+ }
251
368
 
252
- return serialize(root(h(2, "Recall Results"), ul(items)));
369
+ return lines.join("\n");
370
+ }
371
+
372
+ /** Get the full content length of a tagged result (before truncation). */
373
+ function getFullContentLength(tagged: TaggedResult): number {
374
+ switch (tagged.source) {
375
+ case "knowledge":
376
+ case "cross-knowledge":
377
+ return tagged.item.title.length + tagged.item.content.length + 4; // **: :
378
+ case "distillation":
379
+ return tagged.item.observations.length;
380
+ case "temporal":
381
+ return tagged.item.content.length;
382
+ case "lat-section":
383
+ return tagged.item.heading.length + tagged.item.content.length;
384
+ }
385
+ }
386
+
387
+ /** Render a single result as a markdown list item line. */
388
+ function renderResultLine(tagged: TaggedResult, charBudget: number): string {
389
+ const id = taggedResultKey(tagged);
390
+
391
+ switch (tagged.source) {
392
+ case "knowledge": {
393
+ const k = tagged.item;
394
+ const titlePart = `**${inline(k.title)}**: `;
395
+ const contentBudget = Math.max(40, charBudget - titlePart.length);
396
+ const content = truncateAtSentence(inline(k.content), contentBudget);
397
+ const wasTruncated = inline(k.content).length > contentBudget;
398
+ return `- ${titlePart}${content}${wasTruncated ? ` (${id})` : ""}`;
399
+ }
400
+ case "cross-knowledge": {
401
+ const k = tagged.item;
402
+ const titlePart = `**${inline(k.title)}** (from: ${tagged.projectLabel}): `;
403
+ const contentBudget = Math.max(40, charBudget - titlePart.length);
404
+ const content = truncateAtSentence(inline(k.content), contentBudget);
405
+ const wasTruncated = inline(k.content).length > contentBudget;
406
+ return `- ${titlePart}${content}${wasTruncated ? ` (${id})` : ""}`;
407
+ }
408
+ case "distillation": {
409
+ const d = tagged.item;
410
+ const fullText = inline(d.observations);
411
+ const content = truncateAtSentence(fullText, charBudget);
412
+ const wasTruncated = fullText.length > charBudget;
413
+ return `- ${content}${wasTruncated ? ` (${id})` : ""}`;
414
+ }
415
+ case "temporal": {
416
+ const m = tagged.item;
417
+ const prefix = `(${m.role}, ${relativeAge(m.created_at)}) `;
418
+ const contentBudget = Math.max(40, charBudget - prefix.length);
419
+ const fullText = inline(m.content);
420
+ const content = truncateAtSentence(fullText, contentBudget);
421
+ const wasTruncated = fullText.length > contentBudget;
422
+ return `- ${prefix}${content}${wasTruncated ? ` (${id})` : ""}`;
423
+ }
424
+ case "lat-section": {
425
+ const s = tagged.item;
426
+ const heading = `**${inline(s.file)} \u00A7 ${inline(s.heading)}**: `;
427
+ const contentBudget = Math.max(40, charBudget - heading.length);
428
+ const fullText = s.first_paragraph
429
+ ? inline(s.first_paragraph)
430
+ : inline(s.content);
431
+ const content = truncateAtSentence(fullText, contentBudget);
432
+ const wasTruncated = fullText.length > contentBudget;
433
+ return `- ${heading}${content}${wasTruncated ? ` (${id})` : ""}`;
434
+ }
435
+ }
253
436
  }
254
437
 
255
438
  // ---------------------------------------------------------------------------
256
439
  // Main entry point
257
440
  // ---------------------------------------------------------------------------
258
441
 
259
- /** Full recall run: search every relevant source, fuse with RRF, format as markdown. */
260
- export async function runRecall(input: RecallInput): Promise<RecallResult> {
442
+ /**
443
+ * Search every relevant source, fuse with RRF, and return raw scored results.
444
+ *
445
+ * This is the search+fusion core shared by `runRecall()` (LLM-formatted) and
446
+ * direct consumers like the web UI that need access to the raw result items.
447
+ */
448
+ export async function searchRecall(
449
+ input: RecallInput,
450
+ ): Promise<ScoredTaggedResult[]> {
261
451
  const {
262
452
  query,
263
453
  scope = "all",
@@ -272,7 +462,7 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
272
462
 
273
463
  // Short-circuit vague queries — stopwords-only would match everything.
274
464
  if (ftsQuery(query) === EMPTY_QUERY) {
275
- return "Query too vague — try using specific keywords, file names, or technical terms.";
465
+ return [];
276
466
  }
277
467
 
278
468
  // Optional query expansion: generate alternative phrasings via LLM.
@@ -431,6 +621,36 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
431
621
  });
432
622
  }
433
623
  }
624
+
625
+ // Temporal vector search (undistilled messages only)
626
+ if (scope !== "knowledge") {
627
+ const pid = ensureProject(projectPath);
628
+ const temporalVectorHits = embedding.vectorSearchTemporal(
629
+ queryVec,
630
+ pid,
631
+ limit,
632
+ );
633
+ const temporalVectorTagged: TaggedResult[] = temporalVectorHits
634
+ .map((hit): TaggedResult | null => {
635
+ const row = db()
636
+ .query(
637
+ "SELECT id, project_id, session_id, role, content, tokens, distilled, created_at, metadata FROM temporal_messages WHERE id = ?",
638
+ )
639
+ .get(hit.id) as temporal.TemporalMessage | null;
640
+ if (!row) return null;
641
+ return {
642
+ source: "temporal",
643
+ item: { ...row, rank: -hit.similarity },
644
+ };
645
+ })
646
+ .filter((r): r is TaggedResult => r !== null);
647
+ if (temporalVectorTagged.length) {
648
+ allRrfLists.push({
649
+ items: temporalVectorTagged,
650
+ key: (r) => `t:${r.item.id}`,
651
+ });
652
+ }
653
+ }
434
654
  } catch (err) {
435
655
  log.info("recall: vector search failed:", err);
436
656
  }
@@ -567,7 +787,117 @@ export async function runRecall(input: RecallInput): Promise<RecallResult> {
567
787
  }
568
788
 
569
789
  const fused = reciprocalRankFusion<TaggedResult>(allRrfLists);
570
- return formatFusedResults(fused, 20);
790
+
791
+ // Cap output: return at most 3x the per-source limit. With 7+ RRF sources
792
+ // each contributing up to `limit` items, uncapped output can be huge (89+
793
+ // results for broad OR fallbacks). The top-scoring items after RRF fusion
794
+ // are the ones that appeared in multiple lists — capping preserves those
795
+ // while dropping the long tail of single-list noise.
796
+ const maxResults = limit * 3;
797
+ return fused.slice(0, maxResults);
798
+ }
799
+
800
+ // ---------------------------------------------------------------------------
801
+ // Recall by ID — fetch full untruncated content of a specific result
802
+ // ---------------------------------------------------------------------------
803
+
804
+ /**
805
+ * Fetch the full content of a single result by its source-prefixed ID.
806
+ *
807
+ * IDs use the format `prefix:uuid` where prefix is one of:
808
+ * k: (knowledge), xk: (cross-knowledge), d: (distillation),
809
+ * t: (temporal), lat: (lat-section).
810
+ */
811
+ export function recallById(id: string): string {
812
+ const colonIdx = id.indexOf(":");
813
+ if (colonIdx < 1) return `No entry found for id: ${id}`;
814
+
815
+ const prefix = id.slice(0, colonIdx);
816
+ const rawId = id.slice(colonIdx + 1);
817
+
818
+ switch (prefix) {
819
+ case "k":
820
+ case "xk": {
821
+ const entry = ltm.get(rawId);
822
+ if (!entry) return `No entry found for id: ${id}`;
823
+ return [
824
+ `## Recall Detail: ${id}`,
825
+ ``,
826
+ `#### Knowledge`,
827
+ `- **${inline(entry.title)}** (${entry.category}): ${inline(entry.content)}`,
828
+ ].join("\n");
829
+ }
830
+ case "d": {
831
+ const row = db()
832
+ .query(
833
+ "SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE id = ?",
834
+ )
835
+ .get(rawId) as Distillation | null;
836
+ if (!row) return `No entry found for id: ${id}`;
837
+ return [
838
+ `## Recall Detail: ${id}`,
839
+ ``,
840
+ `#### Distilled`,
841
+ `${inline(row.observations)}`,
842
+ ].join("\n");
843
+ }
844
+ case "t": {
845
+ const row = db()
846
+ .query(
847
+ "SELECT id, project_id, session_id, role, content, tokens, distilled, created_at, metadata FROM temporal_messages WHERE id = ?",
848
+ )
849
+ .get(rawId) as temporal.TemporalMessage | null;
850
+ if (!row) return `No entry found for id: ${id}`;
851
+ return [
852
+ `## Recall Detail: ${id}`,
853
+ ``,
854
+ `#### Conversation`,
855
+ `(${row.role}, ${relativeAge(row.created_at)}, session: ${row.session_id.slice(0, 8)})`,
856
+ ``,
857
+ `${inline(row.content)}`,
858
+ ].join("\n");
859
+ }
860
+ case "lat": {
861
+ const row = db()
862
+ .query(
863
+ "SELECT id, project_id, file, heading, depth, content, content_hash, first_paragraph, updated_at FROM lat_sections WHERE id = ?",
864
+ )
865
+ .get(rawId) as latReader.LatSection | null;
866
+ if (!row) return `No entry found for id: ${id}`;
867
+ return [
868
+ `## Recall Detail: ${id}`,
869
+ ``,
870
+ `#### Reference`,
871
+ `**${inline(row.file)} \u00A7 ${inline(row.heading)}**`,
872
+ ``,
873
+ `${inline(row.content)}`,
874
+ ].join("\n");
875
+ }
876
+ default:
877
+ return `Unknown source prefix "${prefix}" in id: ${id}`;
878
+ }
879
+ }
880
+
881
+ /** Full recall run: search every relevant source, fuse with RRF, format as markdown. */
882
+ export async function runRecall(input: RecallInput): Promise<RecallResult> {
883
+ // ID-based detail retrieval — bypass search entirely.
884
+ if (input.id) {
885
+ return recallById(input.id);
886
+ }
887
+
888
+ // Short-circuit vague queries — stopwords-only would match everything.
889
+ if (ftsQuery(input.query) === EMPTY_QUERY) {
890
+ return "Query too vague — try using specific keywords, file names, or technical terms.";
891
+ }
892
+
893
+ const fused = await searchRecall(input);
894
+ const recallCfg = input.searchConfig?.recall;
895
+ return formatFusedResults(fused, {
896
+ charBudget: recallCfg?.charBudget ?? DEFAULT_FORMAT_CONFIG.charBudget,
897
+ relevanceFloor:
898
+ recallCfg?.relevanceFloor ?? DEFAULT_FORMAT_CONFIG.relevanceFloor,
899
+ maxResults: recallCfg?.maxResults ?? DEFAULT_FORMAT_CONFIG.maxResults,
900
+ });
571
901
  }
572
902
 
573
903
  /** Standard tool description reused verbatim by each host adapter. */
@@ -579,4 +909,5 @@ export const RECALL_PARAM_DESCRIPTIONS = {
579
909
  query: "What to search for — be specific. Include keywords, file names, or concepts.",
580
910
  scope:
581
911
  "Search scope: 'all' (default) searches everything, 'session' searches current session only, 'project' searches all sessions in this project, 'knowledge' searches only long-term knowledge.",
912
+ id: "Fetch full content of a specific result by its source-prefixed ID (e.g. 'k:abc123', 'd:abc123'). IDs are shown on truncated results in recall output. When id is provided, query is ignored.",
582
913
  };
package/src/search.ts CHANGED
@@ -173,6 +173,76 @@ export function ftsQueryOr(raw: string): string {
173
173
  return terms.map((w) => `${w}*`).join(" OR ");
174
174
  }
175
175
 
176
+ /**
177
+ * Build a cascade of progressively relaxed FTS5 queries.
178
+ *
179
+ * For N terms, produces up to (N - minTerms) queries, each dropping one more
180
+ * term (least significant first — shortest terms dropped first as a rough
181
+ * proxy for specificity). The final entry is always the full OR query.
182
+ *
183
+ * Example for 6 terms with minTerms=3:
184
+ * [0] 5-of-6 AND (drop shortest term)
185
+ * [1] 4-of-6 AND
186
+ * [2] 3-of-6 AND
187
+ * [3] full OR (all 6 terms)
188
+ *
189
+ * For ≤ minTerms terms, returns just the OR query (no intermediate steps).
190
+ * Callers should try each query in order, stopping at the first that returns
191
+ * results. This avoids the AND→OR cliff that produces massive low-quality
192
+ * result sets.
193
+ */
194
+ export function ftsQueryRelaxed(raw: string, minTerms = 3): string[] {
195
+ const terms = filterTerms(raw);
196
+ if (!terms.length) return [EMPTY_QUERY];
197
+
198
+ const orQuery = terms.map((w) => `${w}*`).join(" OR ");
199
+
200
+ // Not enough terms for progressive relaxation — just OR.
201
+ if (terms.length <= minTerms) return [orQuery];
202
+
203
+ // Sort by length ascending — shortest (least specific) terms dropped first.
204
+ const ranked = [...terms].sort((a, b) => a.length - b.length);
205
+
206
+ const cascade: string[] = [];
207
+ for (let drop = 1; drop <= terms.length - minTerms; drop++) {
208
+ const kept = ranked.slice(drop);
209
+ cascade.push(kept.map((w) => `${w}*`).join(" "));
210
+ }
211
+ cascade.push(orQuery);
212
+ return cascade;
213
+ }
214
+
215
+ /**
216
+ * Run a search function through the relaxed cascade, stopping at the first
217
+ * query that produces results. Falls back through progressively looser AND
218
+ * queries before trying full OR.
219
+ *
220
+ * @param raw The original query string
221
+ * @param runner A function that takes an FTS5 MATCH expression and returns results
222
+ * @returns The results from the first cascade step that produced matches
223
+ */
224
+ export function runRelaxedSearch<T>(
225
+ raw: string,
226
+ runner: (matchExpr: string) => T[],
227
+ ): T[] {
228
+ // First try exact AND (all terms)
229
+ const q = ftsQuery(raw);
230
+ if (q === EMPTY_QUERY) return [];
231
+
232
+ const andResults = runner(q);
233
+ if (andResults.length) return andResults;
234
+
235
+ // Try progressively relaxed queries
236
+ const cascade = ftsQueryRelaxed(raw);
237
+ for (const relaxed of cascade) {
238
+ if (relaxed === EMPTY_QUERY) continue;
239
+ const results = runner(relaxed);
240
+ if (results.length) return results;
241
+ }
242
+
243
+ return [];
244
+ }
245
+
176
246
  // ---------------------------------------------------------------------------
177
247
  // Term extraction (Phase 3)
178
248
  // ---------------------------------------------------------------------------
@@ -335,7 +405,7 @@ export async function expandQuery(
335
405
  llm.prompt(
336
406
  QUERY_EXPANSION_SYSTEM,
337
407
  `Input: "${query}"`,
338
- { model, workerID: "lore-query-expand", thinking: false, urgent: true, sessionID },
408
+ { model, workerID: "lore-query-expand", thinking: false, urgent: true, sessionID, maxTokens: 256 },
339
409
  ),
340
410
  new Promise<null>((resolve) => setTimeout(() => resolve(null), TIMEOUT_MS)),
341
411
  ]);