om-memory-system 4.24.0-next.147 → 4.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/dist/adapters/opencode/user-prompt.js +6 -1
  2. package/dist/config.d.ts +9 -0
  3. package/dist/config.js +5 -44
  4. package/dist/core/retrieval.js +1 -1
  5. package/dist/services/chat/chat-answer.d.ts +56 -0
  6. package/dist/services/chat/chat-answer.js +88 -0
  7. package/dist/services/chat/chat-plan.d.ts +60 -0
  8. package/dist/services/chat/chat-plan.js +92 -0
  9. package/dist/services/chat/chat-retrieve.d.ts +36 -0
  10. package/dist/services/chat/chat-retrieve.js +82 -0
  11. package/dist/services/chat/chat-routes.d.ts +12 -0
  12. package/dist/services/chat/chat-routes.js +145 -0
  13. package/dist/services/chat/chat-runtime.d.ts +21 -0
  14. package/dist/services/chat/chat-runtime.js +104 -0
  15. package/dist/services/chat/chat-service.d.ts +42 -0
  16. package/dist/services/chat/chat-service.js +130 -0
  17. package/dist/services/chat/chat-status.d.ts +16 -0
  18. package/dist/services/chat/chat-status.js +23 -0
  19. package/dist/services/chat/chat-store.d.ts +60 -0
  20. package/dist/services/chat/chat-store.js +352 -0
  21. package/dist/services/chat/chat-types.d.ts +59 -0
  22. package/dist/services/chat/chat-types.js +8 -0
  23. package/dist/services/client.d.ts +1 -0
  24. package/dist/services/client.js +8 -3
  25. package/dist/services/context.d.ts +3 -1
  26. package/dist/services/context.js +4 -2
  27. package/dist/services/global-config-writer.js +8 -1
  28. package/dist/services/privacy.js +2 -2
  29. package/dist/services/settings-snapshot.js +1 -0
  30. package/dist/services/turso/exact-scan.d.ts +6 -0
  31. package/dist/services/turso/exact-scan.js +23 -0
  32. package/dist/services/turso/keyword-search.d.ts +35 -0
  33. package/dist/services/turso/keyword-search.js +153 -0
  34. package/dist/services/turso/rank-fusion.d.ts +34 -0
  35. package/dist/services/turso/rank-fusion.js +45 -0
  36. package/dist/services/turso/tag-match.d.ts +7 -0
  37. package/dist/services/turso/tag-match.js +29 -0
  38. package/dist/services/turso/types.d.ts +2 -0
  39. package/dist/services/turso/vector-search.d.ts +3 -3
  40. package/dist/services/turso/vector-search.js +23 -39
  41. package/dist/services/user-profile/profile-context.d.ts +6 -1
  42. package/dist/services/user-profile/profile-context.js +43 -7
  43. package/dist/services/user-profile/profile-relevance.d.ts +37 -0
  44. package/dist/services/user-profile/profile-relevance.js +75 -0
  45. package/dist/services/web-route-modules.d.ts +8 -0
  46. package/dist/services/web-route-modules.js +15 -0
  47. package/dist/services/web-server.js +5 -4
  48. package/dist/utils/keyword-boost.d.ts +17 -0
  49. package/dist/utils/keyword-boost.js +32 -0
  50. package/dist/utils/reranker-settings.d.ts +13 -0
  51. package/dist/utils/reranker-settings.js +48 -0
  52. package/dist/web/assets/index-CjfE2byA.js +126 -0
  53. package/dist/web/assets/index-SK0gAreR.css +2 -0
  54. package/dist/web/index.html +2 -2
  55. package/package.json +1 -1
  56. package/dist/web/assets/index-B_2GCxJJ.js +0 -127
  57. package/dist/web/assets/index-JAZ-QY-Z.css +0 -2
@@ -0,0 +1,153 @@
1
+ /**
2
+ * Exact-word matching for memory search (openspec change
3
+ * hook-search-keywords-and-profile). The pure parts take their settings as
4
+ * arguments: do not import CONFIG here, many tests stub src/config.js.
5
+ */
6
+ import { log } from "../logger.js";
7
+ const MAX_TERMS = 24;
8
+ // Words that carry no search value. With the OR query below, one of them
9
+ // would pull most of a shard in as candidates.
10
+ const STOP_WORDS = new Set([
11
+ "the",
12
+ "and",
13
+ "or",
14
+ "not",
15
+ "of",
16
+ "to",
17
+ "in",
18
+ "on",
19
+ "at",
20
+ "by",
21
+ "for",
22
+ "is",
23
+ "it",
24
+ "an",
25
+ "be",
26
+ "are",
27
+ "was",
28
+ "we",
29
+ "i",
30
+ "me",
31
+ "my",
32
+ "you",
33
+ "do",
34
+ "did",
35
+ "does",
36
+ "why",
37
+ "how",
38
+ "what",
39
+ "with",
40
+ "this",
41
+ "that",
42
+ ]);
43
+ /**
44
+ * FTS5 query for the words of a prompt: lower-case terms from letters and
45
+ * digits only ("ADR-023" gives `adr` and `023`), each in quotes, joined by OR.
46
+ * Quoting and the character filter keep prompt text from becoming index
47
+ * syntax. Returns null when no usable term is left.
48
+ */
49
+ export function buildKeywordQuery(queryText) {
50
+ const terms = new Set();
51
+ for (const word of (queryText ?? "").toLowerCase().split(/[^\p{L}\p{N}]+/u)) {
52
+ if (word.length < 2 || STOP_WORDS.has(word))
53
+ continue;
54
+ terms.add(word);
55
+ if (terms.size === MAX_TERMS)
56
+ break;
57
+ }
58
+ if (terms.size === 0)
59
+ return null;
60
+ return [...terms].map((term) => `"${term}"`).join(" OR ");
61
+ }
62
+ /**
63
+ * Similarity after the keyword boost: `base + keywordBoost × score ÷ best`,
64
+ * where `score` is the memory's match strength and `best` the strongest in the
65
+ * same search. The share is capped at 1 so the boost never passes
66
+ * `keywordBoost`, and the result never passes 1.
67
+ */
68
+ export function applyKeywordBoost(base, score, bestScore, keywordBoost) {
69
+ if (!(keywordBoost > 0) || !(score > 0) || !(bestScore > 0))
70
+ return base;
71
+ return Math.min(1, base + keywordBoost * Math.min(1, score / bestScore));
72
+ }
73
+ // The index is derived data: external content (no second copy of the text),
74
+ // kept in step by triggers, and rebuilt whenever it is missing or short.
75
+ const INDEX_OBJECTS = ["memories_fts", "memories_fts_ai", "memories_fts_ad", "memories_fts_au"];
76
+ const INDEX_SETUP = [
77
+ `CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
78
+ content, tags, content='memories', content_rowid='rowid', tokenize='unicode61'
79
+ )`,
80
+ `CREATE TRIGGER IF NOT EXISTS memories_fts_ai AFTER INSERT ON memories BEGIN
81
+ INSERT INTO memories_fts(rowid, content, tags) VALUES (new.rowid, new.content, new.tags);
82
+ END`,
83
+ `CREATE TRIGGER IF NOT EXISTS memories_fts_ad AFTER DELETE ON memories BEGIN
84
+ INSERT INTO memories_fts(memories_fts, rowid, content, tags)
85
+ VALUES ('delete', old.rowid, old.content, old.tags);
86
+ END`,
87
+ `CREATE TRIGGER IF NOT EXISTS memories_fts_au AFTER UPDATE OF content, tags ON memories BEGIN
88
+ INSERT INTO memories_fts(memories_fts, rowid, content, tags)
89
+ VALUES ('delete', old.rowid, old.content, old.tags);
90
+ INSERT INTO memories_fts(rowid, content, tags) VALUES (new.rowid, new.content, new.tags);
91
+ END`,
92
+ ];
93
+ async function indexIsCurrent(db) {
94
+ const names = INDEX_OBJECTS.map(() => "?").join(",");
95
+ const present = await db.all(`SELECT name FROM sqlite_schema WHERE name IN (${names})`, [
96
+ ...INDEX_OBJECTS,
97
+ ]);
98
+ if (present.length !== INDEX_OBJECTS.length)
99
+ return false;
100
+ // COUNT(*) on an external-content table reads the memories table, so the
101
+ // number of documents the index really holds comes from its docsize table.
102
+ const row = await db.get(`SELECT (SELECT COUNT(*) FROM memories_fts_docsize) AS indexed,
103
+ (SELECT COUNT(*) FROM memories) AS stored`);
104
+ return Number(row?.indexed) === Number(row?.stored);
105
+ }
106
+ /** Create or rebuild the keyword index of a shard. Idempotent. */
107
+ export async function ensureKeywordIndex(db) {
108
+ if (await indexIsCurrent(db))
109
+ return;
110
+ // One transaction: a failed rebuild leaves no triggers behind that would
111
+ // write to a broken index on every save.
112
+ await db.transaction("write", async (tx) => {
113
+ for (const sql of INDEX_SETUP)
114
+ await tx.execute(sql);
115
+ await tx.execute(`INSERT INTO memories_fts(memories_fts) VALUES ('rebuild')`);
116
+ });
117
+ }
118
+ /**
119
+ * Memories of one shard that contain a word of the prompt, strongest first.
120
+ * Returns null when there is nothing to match or the index fails: search then
121
+ * continues with vector and tag results. The log holds a code and an error
122
+ * name only, never the prompt or memory text.
123
+ */
124
+ export async function searchKeywords(db, queryText, containerTag, limit) {
125
+ const match = buildKeywordQuery(queryText);
126
+ if (!match)
127
+ return null;
128
+ try {
129
+ await ensureKeywordIndex(db);
130
+ const rows = await db.all(`SELECT m.id AS id, -bm25(memories_fts) AS score
131
+ FROM memories_fts JOIN memories m ON m.rowid = memories_fts.rowid
132
+ WHERE memories_fts MATCH ?${containerTag === "" ? "" : " AND m.container_tag = ?"}
133
+ ORDER BY score DESC LIMIT ?`, containerTag === "" ? [match, limit] : [match, containerTag, limit]);
134
+ const scores = new Map(rows.map((row) => [String(row.id), Number(row.score)]));
135
+ return { scores, best: Math.max(0, ...scores.values()) };
136
+ }
137
+ catch (error) {
138
+ log("Keyword search failed; using vector results", {
139
+ code: "keyword_index_failed",
140
+ error: error instanceof Error ? error.name : "unknown",
141
+ });
142
+ return null;
143
+ }
144
+ }
145
+ /**
146
+ * `base` with the boost for memory `id`; `base` itself when it has no keyword hit
147
+ * or the mode is `rrf` (rank fusion keeps each memory's own similarity).
148
+ */
149
+ export function boostedSimilarity(base, hits, id, keywordBoost, mode = "boost") {
150
+ if (mode === "rrf")
151
+ return base;
152
+ return applyKeywordBoost(base, hits?.scores.get(id) ?? 0, hits?.best ?? 0, keywordBoost);
153
+ }
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Reciprocal rank fusion of the meaning list and the word list (openspec change
3
+ * hook-search-keywords-and-profile, design decision 9). Pure: the threshold and
4
+ * the limits are passed in, so do not import CONFIG here.
5
+ */
6
+ /** The published constant of reciprocal rank fusion. */
7
+ export declare const RRF_K = 60;
8
+ /** Same size as the word-hit limit of one shard search (max(limit × 4, 32), limit 10). */
9
+ export declare const RRF_WORD_LIMIT = 40;
10
+ export interface FusionRow {
11
+ id: string;
12
+ similarity: number;
13
+ /** Match strength of a word hit; missing or 0 means the memory has no word match. */
14
+ keywordScore?: number;
15
+ }
16
+ export interface FusionOptions {
17
+ /** Lowest similarity for the meaning list. */
18
+ threshold: number;
19
+ limit: number;
20
+ wordLimit?: number;
21
+ }
22
+ /**
23
+ * Merge two ranked lists by rank only. The meaning list holds the rows at or
24
+ * above the threshold, ranked by similarity. The word list holds the rows with
25
+ * a word match, ranked by match strength. A row that is only in the word list
26
+ * can be returned although its similarity is below the threshold. Ties break by
27
+ * id so repeated searches agree. Each row keeps its own similarity.
28
+ */
29
+ export declare function rankFusion<T extends FusionRow>(rows: T[], options: FusionOptions): T[];
30
+ /**
31
+ * Fuse the candidates of all shards and drop the internal word score, so the
32
+ * results have the same shape as in `boost` mode.
33
+ */
34
+ export declare function fuseShardResults<T extends FusionRow>(rows: T[], threshold: number, limit: number): Omit<T, "keywordScore">[];
@@ -0,0 +1,45 @@
1
+ /**
2
+ * Reciprocal rank fusion of the meaning list and the word list (openspec change
3
+ * hook-search-keywords-and-profile, design decision 9). Pure: the threshold and
4
+ * the limits are passed in, so do not import CONFIG here.
5
+ */
6
+ /** The published constant of reciprocal rank fusion. */
7
+ export const RRF_K = 60;
8
+ /** Same size as the word-hit limit of one shard search (max(limit × 4, 32), limit 10). */
9
+ export const RRF_WORD_LIMIT = 40;
10
+ const byIdAfter = (compare) => (a, b) => compare(a, b) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
11
+ /**
12
+ * Merge two ranked lists by rank only. The meaning list holds the rows at or
13
+ * above the threshold, ranked by similarity. The word list holds the rows with
14
+ * a word match, ranked by match strength. A row that is only in the word list
15
+ * can be returned although its similarity is below the threshold. Ties break by
16
+ * id so repeated searches agree. Each row keeps its own similarity.
17
+ */
18
+ export function rankFusion(rows, options) {
19
+ const meaning = rows
20
+ .filter((row) => row.similarity >= options.threshold)
21
+ .sort(byIdAfter((a, b) => b.similarity - a.similarity));
22
+ const words = rows
23
+ .filter((row) => (row.keywordScore ?? 0) > 0)
24
+ .sort(byIdAfter((a, b) => (b.keywordScore ?? 0) - (a.keywordScore ?? 0)))
25
+ .slice(0, options.wordLimit ?? RRF_WORD_LIMIT);
26
+ const fused = new Map();
27
+ for (const list of [meaning, words]) {
28
+ list.forEach((row, index) => {
29
+ const entry = fused.get(row.id) ?? { row, score: 0 };
30
+ entry.score += 1 / (RRF_K + index + 1);
31
+ fused.set(row.id, entry);
32
+ });
33
+ }
34
+ return [...fused.values()]
35
+ .sort((a, b) => b.score - a.score || (a.row.id < b.row.id ? -1 : a.row.id > b.row.id ? 1 : 0))
36
+ .slice(0, Math.max(0, options.limit))
37
+ .map((entry) => entry.row);
38
+ }
39
+ /**
40
+ * Fuse the candidates of all shards and drop the internal word score, so the
41
+ * results have the same shape as in `boost` mode.
42
+ */
43
+ export function fuseShardResults(rows, threshold, limit) {
44
+ return rankFusion(rows, { threshold, limit }).map(({ keywordScore: _word, ...rest }) => rest);
45
+ }
@@ -0,0 +1,7 @@
1
+ /**
2
+ * Share of query words that name one of a memory's tags, from 0 to 1. A word
3
+ * matches when it equals a whole tag or one part of a tag ("code" matches
4
+ * "claude-code"). Plain substring matching let short tags such as "ui" match
5
+ * words like "build" or "guide", which pushed weak results up.
6
+ */
7
+ export declare function tagMatchBoost(queryText: string | undefined, tagsCsv: string): number;
@@ -0,0 +1,29 @@
1
+ // Scripts written without spaces between words. A query in one of these is a
2
+ // single token, so a tag can only be found inside it.
3
+ const UNSPACED_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}]/u;
4
+ function tokens(text) {
5
+ return text
6
+ .toLowerCase()
7
+ .split(/[^\p{L}\p{N}]+/u)
8
+ .filter((token) => token.length > 0);
9
+ }
10
+ /**
11
+ * Share of query words that name one of a memory's tags, from 0 to 1. A word
12
+ * matches when it equals a whole tag or one part of a tag ("code" matches
13
+ * "claude-code"). Plain substring matching let short tags such as "ui" match
14
+ * words like "build" or "guide", which pushed weak results up.
15
+ */
16
+ export function tagMatchBoost(queryText, tagsCsv) {
17
+ const tags = tagsCsv
18
+ .split(",")
19
+ .map((tag) => tag.trim().toLowerCase())
20
+ .filter((tag) => tag.length > 0);
21
+ // A one-character word counts only when it is a whole tag ("C language", tag "c").
22
+ const words = tokens(queryText ?? "").filter((word) => word.length > 1 || tags.includes(word));
23
+ if (words.length === 0 || tags.length === 0)
24
+ return 0;
25
+ const tagParts = new Set(tags.flatMap((tag) => [tag, ...tokens(tag)]));
26
+ const unspacedTags = [...tagParts].filter((part) => UNSPACED_SCRIPT.test(part));
27
+ const matches = words.filter((word) => tagParts.has(word) || unspacedTags.some((tag) => word.includes(tag))).length;
28
+ return matches / words.length;
29
+ }
@@ -30,6 +30,8 @@ export interface SearchResult {
30
30
  id: string;
31
31
  memory: string;
32
32
  similarity: number;
33
+ /** Internal: match strength of a word hit. Removed before results leave search. */
34
+ keywordScore?: number;
33
35
  createdAt?: number;
34
36
  tags?: string[];
35
37
  type?: string;
@@ -1,13 +1,13 @@
1
1
  import type { MemoryRecord, SearchResult, ShardInfo } from "./types.js";
2
+ import type { KeywordMode } from "../../utils/keyword-boost.js";
2
3
  import type { TursoDb } from "./turso-db.js";
3
4
  import type { Transaction } from "@libsql/client";
4
5
  export declare class TursoVectorSearch {
5
6
  insertVectorInTransaction(tx: Transaction, record: MemoryRecord): Promise<void>;
6
7
  insertVector(db: TursoDb, record: MemoryRecord): Promise<void>;
7
- searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string): Promise<SearchResult[]>;
8
+ searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode): Promise<SearchResult[]>;
8
9
  private searchKind;
9
- private exactScanKind;
10
- searchAcrossShards(shards: ShardInfo[], queryVector: Float32Array, containerTag: string, limit: number, similarityThreshold: number, queryText?: string): Promise<{
10
+ searchAcrossShards(shards: ShardInfo[], queryVector: Float32Array, containerTag: string, limit: number, similarityThreshold: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode): Promise<{
11
11
  results: SearchResult[];
12
12
  warnings: string[];
13
13
  }>;
@@ -1,6 +1,10 @@
1
1
  import { tursoConnectionManager } from "./connection-manager.js";
2
2
  import { log } from "../logger.js";
3
3
  import { distanceToSimilarity, vectorToJson } from "./vector-utils.js";
4
+ import { tagMatchBoost } from "./tag-match.js";
5
+ import { exactScanKind } from "./exact-scan.js";
6
+ import { boostedSimilarity, searchKeywords } from "./keyword-search.js";
7
+ import { fuseShardResults } from "./rank-fusion.js";
4
8
  function parseMetadata(value) {
5
9
  if (!value || typeof value !== "string")
6
10
  return undefined;
@@ -88,7 +92,7 @@ export class TursoVectorSearch {
88
92
  ) VALUES (?, ?, vector32(?), NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
89
93
  `, [record.id, record.content, contentVector, ...commonArgs]);
90
94
  }
91
- async searchInShard(shard, queryVector, containerTag, limit, queryText) {
95
+ async searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost = 0, keywordMode = "boost") {
92
96
  const db = await tursoConnectionManager.getConnection(shard.dbPath);
93
97
  const queryJson = vectorToJson(queryVector);
94
98
  // Over-fetch aggressively when filtering by container_tag after ANN,
@@ -101,6 +105,11 @@ export class TursoVectorSearch {
101
105
  candidateIds.add(result.id);
102
106
  for (const result of tagsResults)
103
107
  candidateIds.add(result.id);
108
+ const kwHits = keywordBoost > 0
109
+ ? await searchKeywords(db, queryText, containerTag, Math.max(limit * 4, 32))
110
+ : null;
111
+ for (const id of kwHits?.scores.keys() ?? [])
112
+ candidateIds.add(id);
104
113
  const ids = Array.from(candidateIds);
105
114
  if (ids.length === 0)
106
115
  return [];
@@ -132,30 +141,21 @@ export class TursoVectorSearch {
132
141
  `, containerTag === ""
133
142
  ? [queryJson, queryJson, ...ids]
134
143
  : [queryJson, queryJson, ...ids, containerTag]);
135
- const queryWords = queryText
136
- ? queryText
137
- .toLowerCase()
138
- .split(/[\s,]+/)
139
- .filter((word) => word.length > 1)
140
- : [];
141
144
  const hydratedResults = rows.map((row) => {
142
145
  const contentSim = distanceToSimilarity(Number(row.content_dist));
143
146
  const tagsSim = row.tags_dist == null || row.tags_dist === undefined
144
147
  ? 0
145
148
  : distanceToSimilarity(Number(row.tags_dist));
146
149
  const memoryTagsStr = String(row.tags || "");
147
- const memoryTags = memoryTagsStr.split(",").map((tag) => tag.trim().toLowerCase());
148
- let exactMatchBoost = 0;
149
- if (queryWords.length > 0 && memoryTags.length > 0) {
150
- const matches = queryWords.filter((word) => memoryTags.some((tag) => tag.includes(word) || word.includes(tag))).length;
151
- exactMatchBoost = matches / Math.max(queryWords.length, 1);
152
- }
150
+ const exactMatchBoost = tagMatchBoost(queryText, memoryTagsStr);
153
151
  const finalTagsSim = Math.max(tagsSim, exactMatchBoost);
154
- const similarity = contentSim * 0.6 + finalTagsSim * 0.4;
152
+ const base = contentSim * 0.6 + finalTagsSim * 0.4;
153
+ const similarity = boostedSimilarity(base, kwHits, String(row.id), keywordBoost, keywordMode);
155
154
  return {
156
155
  id: String(row.id),
157
156
  memory: String(row.content),
158
157
  similarity,
158
+ keywordScore: kwHits?.scores.get(String(row.id)),
159
159
  createdAt: Number(row.created_at),
160
160
  tags: memoryTagsStr ? memoryTagsStr.split(",") : [],
161
161
  type: row.type ? String(row.type) : undefined,
@@ -171,7 +171,9 @@ export class TursoVectorSearch {
171
171
  };
172
172
  });
173
173
  hydratedResults.sort((a, b) => b.similarity - a.similarity);
174
- return hydratedResults.slice(0, Math.max(0, limit));
174
+ // Rank fusion needs the word-only candidates too; it applies the limit itself.
175
+ const fusing = keywordMode === "rrf" && keywordBoost > 0;
176
+ return fusing ? hydratedResults : hydratedResults.slice(0, Math.max(0, limit));
175
177
  }
176
178
  async searchKind(db, queryJson, k, containerTag, indexName, columnName) {
177
179
  try {
@@ -198,35 +200,14 @@ export class TursoVectorSearch {
198
200
  indexName,
199
201
  error: String(error),
200
202
  });
201
- return this.exactScanKind(db, queryJson, k, containerTag, columnName);
203
+ return exactScanKind(db, queryJson, k, containerTag, columnName);
202
204
  }
203
205
  }
204
- async exactScanKind(db, queryJson, k, containerTag, columnName) {
205
- const rows = await db.all(containerTag === ""
206
- ? `
207
- SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist
208
- FROM memories m
209
- WHERE m.${columnName} IS NOT NULL
210
- ORDER BY dist ASC
211
- LIMIT ?
212
- `
213
- : `
214
- SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist
215
- FROM memories m
216
- WHERE m.${columnName} IS NOT NULL AND m.container_tag = ?
217
- ORDER BY dist ASC
218
- LIMIT ?
219
- `, containerTag === "" ? [queryJson, k] : [queryJson, containerTag, k]);
220
- return rows.map((row) => ({
221
- id: String(row.id),
222
- similarity: distanceToSimilarity(Number(row.dist)),
223
- }));
224
- }
225
- async searchAcrossShards(shards, queryVector, containerTag, limit, similarityThreshold, queryText) {
206
+ async searchAcrossShards(shards, queryVector, containerTag, limit, similarityThreshold, queryText, keywordBoost = 0, keywordMode = "boost") {
226
207
  const shardErrors = [];
227
208
  const shardPromises = shards.map(async (shard) => {
228
209
  try {
229
- return await this.searchInShard(shard, queryVector, containerTag, limit, queryText);
210
+ return await this.searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost, keywordMode);
230
211
  }
231
212
  catch (error) {
232
213
  const message = error instanceof Error ? error.message : String(error);
@@ -253,6 +234,9 @@ export class TursoVectorSearch {
253
234
  });
254
235
  }
255
236
  const allResults = resultsArray.flat();
237
+ if (keywordMode === "rrf" && keywordBoost > 0) {
238
+ return { results: fuseShardResults(allResults, similarityThreshold, limit), warnings };
239
+ }
256
240
  allResults.sort((a, b) => b.similarity - a.similarity);
257
241
  return {
258
242
  results: allResults
@@ -1 +1,6 @@
1
- export declare function getUserProfileContext(userId: string): Promise<string | null>;
1
+ /**
2
+ * The profile block for the prompt. With `queryVector`, half of each section
3
+ * (rounded up) follows the prompt; without it the block is the same for every
4
+ * prompt, as before.
5
+ */
6
+ export declare function getUserProfileContext(userId: string, queryVector?: ArrayLike<number>): Promise<string | null>;
@@ -2,6 +2,7 @@ import { userProfileManager } from "./user-profile-manager.js";
2
2
  import { CONFIG } from "../../config.js";
3
3
  import { sortProfileItems } from "../../utils/profile.js";
4
4
  import { log } from "../logger.js";
5
+ import { selectProfileItems } from "./profile-relevance.js";
5
6
  function dedupByCategory(items, topN) {
6
7
  if (items.length <= topN)
7
8
  return items;
@@ -33,7 +34,12 @@ function escapeContextText(value) {
33
34
  .replace(/</g, "&lt;")
34
35
  .replace(/>/g, "&gt;");
35
36
  }
36
- export async function getUserProfileContext(userId) {
37
+ /**
38
+ * The profile block for the prompt. With `queryVector`, half of each section
39
+ * (rounded up) follows the prompt; without it the block is the same for every
40
+ * prompt, as before.
41
+ */
42
+ export async function getUserProfileContext(userId, queryVector) {
37
43
  const profile = await userProfileManager.getActiveProfile(userId);
38
44
  if (!profile) {
39
45
  return null;
@@ -60,9 +66,35 @@ export async function getUserProfileContext(userId) {
60
66
  const sortedWfs = profileData.workflows.length > 0
61
67
  ? sortProfileItems(profileData.workflows, "frequency")
62
68
  : [];
63
- const topPrefs = dedupByCategory(scoreByRecency(sortedPrefs.slice(0, injectPrefs * 2)), injectPrefs);
64
- const topPats = dedupByCategory(sortedPats, injectPats);
65
- const topWfs = sortedWfs.slice(0, injectWfs);
69
+ // The prompt must be about as close to an item as it is to a memory.
70
+ const minCloseness = CONFIG.similarityThreshold;
71
+ const prefs = selectProfileItems({
72
+ baseline: dedupByCategory(scoreByRecency(sortedPrefs.slice(0, injectPrefs * 2)), injectPrefs),
73
+ candidates: sortedPrefs,
74
+ count: injectPrefs,
75
+ queryVector,
76
+ minCloseness,
77
+ onePerCategory: true,
78
+ });
79
+ const pats = selectProfileItems({
80
+ baseline: dedupByCategory(sortedPats, injectPats),
81
+ candidates: sortedPats,
82
+ count: injectPats,
83
+ queryVector,
84
+ minCloseness,
85
+ onePerCategory: true,
86
+ });
87
+ const wfs = selectProfileItems({
88
+ baseline: sortedWfs.slice(0, injectWfs),
89
+ candidates: sortedWfs,
90
+ count: injectWfs,
91
+ queryVector,
92
+ minCloseness,
93
+ onePerCategory: false,
94
+ });
95
+ const topPrefs = prefs.items;
96
+ const topPats = pats.items;
97
+ const topWfs = wfs.items;
66
98
  if (topPrefs.length > 0) {
67
99
  parts.push("User Preferences:");
68
100
  topPrefs.forEach((pref) => {
@@ -96,9 +128,13 @@ export async function getUserProfileContext(userId) {
96
128
  const text = parts.join("\n");
97
129
  if (topPrefs.length + topPats.length + topWfs.length > 0) {
98
130
  log("profile inject", {
99
- prefs: topPrefs.map((p) => `[${p.category}] ${p.description}`.substring(0, 80)),
100
- pats: topPats.map((p) => `[${p.category}] ${p.description}`.substring(0, 80)),
101
- wfs: topWfs.map((w) => w.description.substring(0, 80)),
131
+ // Counts only: the log never holds the prompt.
132
+ relevanceSlots: {
133
+ prefs: prefs.relevanceSlots,
134
+ pats: pats.relevanceSlots,
135
+ wfs: wfs.relevanceSlots,
136
+ },
137
+ counts: { prefs: topPrefs.length, pats: topPats.length, wfs: topWfs.length },
102
138
  });
103
139
  }
104
140
  return text;
@@ -0,0 +1,37 @@
1
+ /**
2
+ * Choose profile items for the injected block by relevance to the prompt
3
+ * (openspec change hook-search-keywords-and-profile). Pure functions: the
4
+ * vector, the items, the counts, and the minimum closeness are arguments, so
5
+ * callers and tests need no store and no CONFIG.
6
+ */
7
+ export interface VectorItem {
8
+ category?: string;
9
+ anchor?: number[];
10
+ centroid?: number[];
11
+ }
12
+ export interface SelectProfileItemsInput<T extends VectorItem> {
13
+ /** Today's selection for the section, in today's order. */
14
+ baseline: T[];
15
+ /** Every eligible item, in today's order. Relevance may pick from all of them. */
16
+ candidates: T[];
17
+ /** Injected items for the section. */
18
+ count: number;
19
+ /** The embedded prompt. Without it the section keeps today's selection. */
20
+ queryVector: ArrayLike<number> | undefined;
21
+ /** Items with a lower cosine similarity never take a relevance slot. */
22
+ minCloseness: number;
23
+ /** Sections that group by category keep one item per category. */
24
+ onePerCategory: boolean;
25
+ }
26
+ export interface SelectProfileItemsResult<T> {
27
+ items: T[];
28
+ /** How many items came in through relevance, for the log. */
29
+ relevanceSlots: number;
30
+ }
31
+ export declare function cosineSimilarity(a: ArrayLike<number>, b: ArrayLike<number>): number;
32
+ /**
33
+ * Half of the slots, rounded up, go to the items closest to the prompt. The
34
+ * other slots keep today's order. With no prompt vector, or no item close
35
+ * enough, the result is `baseline` unchanged.
36
+ */
37
+ export declare function selectProfileItems<T extends VectorItem>(input: SelectProfileItemsInput<T>): SelectProfileItemsResult<T>;
@@ -0,0 +1,75 @@
1
+ /**
2
+ * Choose profile items for the injected block by relevance to the prompt
3
+ * (openspec change hook-search-keywords-and-profile). Pure functions: the
4
+ * vector, the items, the counts, and the minimum closeness are arguments, so
5
+ * callers and tests need no store and no CONFIG.
6
+ */
7
+ export function cosineSimilarity(a, b) {
8
+ if (a.length === 0 || a.length !== b.length)
9
+ return 0;
10
+ let dot = 0;
11
+ let normA = 0;
12
+ let normB = 0;
13
+ for (let i = 0; i < a.length; i++) {
14
+ const x = a[i];
15
+ const y = b[i];
16
+ dot += x * y;
17
+ normA += x * x;
18
+ normB += y * y;
19
+ }
20
+ if (normA === 0 || normB === 0)
21
+ return 0;
22
+ return dot / Math.sqrt(normA * normB);
23
+ }
24
+ /**
25
+ * Half of the slots, rounded up, go to the items closest to the prompt. The
26
+ * other slots keep today's order. With no prompt vector, or no item close
27
+ * enough, the result is `baseline` unchanged.
28
+ */
29
+ export function selectProfileItems(input) {
30
+ const { baseline, candidates, count, queryVector, minCloseness, onePerCategory } = input;
31
+ const unchanged = { items: baseline, relevanceSlots: 0 };
32
+ if (!queryVector || queryVector.length === 0 || count <= 0)
33
+ return unchanged;
34
+ const ranked = candidates
35
+ .map((item) => {
36
+ const vector = item.anchor ?? item.centroid;
37
+ return { item, closeness: vector ? cosineSimilarity(queryVector, vector) : 0 };
38
+ })
39
+ .filter((entry) => entry.closeness >= minCloseness && entry.closeness > 0)
40
+ .sort((a, b) => b.closeness - a.closeness);
41
+ const chosen = [];
42
+ const categories = new Set();
43
+ const take = (item) => {
44
+ if (chosen.includes(item))
45
+ return false;
46
+ if (onePerCategory && categories.has(item.category))
47
+ return false;
48
+ chosen.push(item);
49
+ categories.add(item.category);
50
+ return true;
51
+ };
52
+ const relevanceCap = Math.ceil(count / 2);
53
+ let relevanceSlots = 0;
54
+ for (const { item } of ranked) {
55
+ if (relevanceSlots >= relevanceCap)
56
+ break;
57
+ if (take(item))
58
+ relevanceSlots++;
59
+ }
60
+ if (relevanceSlots === 0)
61
+ return unchanged;
62
+ for (const item of baseline) {
63
+ if (chosen.length >= count)
64
+ break;
65
+ take(item);
66
+ }
67
+ // A skipped duplicate can leave a slot empty; fill it from the rest of the
68
+ // eligible items so the section keeps its count.
69
+ for (const item of candidates) {
70
+ if (chosen.length >= count)
71
+ break;
72
+ take(item);
73
+ }
74
+ return { items: chosen, relevanceSlots };
75
+ }
@@ -0,0 +1,8 @@
1
+ import type { ChatRuntimeConfig } from "./chat/chat-runtime.js";
2
+ import { type RouteResult } from "./chat/chat-routes.js";
3
+ /**
4
+ * Route modules that live outside web-server.ts, which sits at its size
5
+ * ceiling. Each returns undefined for paths it does not own. The request body
6
+ * is read once, and only by a route that needs it.
7
+ */
8
+ export declare function routeModules(path: string, method: string, url: URL, req: Request, config: ChatRuntimeConfig): Promise<RouteResult | undefined>;
@@ -0,0 +1,15 @@
1
+ import { routeChat } from "./chat/chat-routes.js";
2
+ import { routeProfileCleanup } from "./profile-cleanup-routes.js";
3
+ /**
4
+ * Route modules that live outside web-server.ts, which sits at its size
5
+ * ceiling. Each returns undefined for paths it does not own. The request body
6
+ * is read once, and only by a route that needs it.
7
+ */
8
+ export async function routeModules(path, method, url, req, config) {
9
+ let body;
10
+ const readBody = () => (body ??= req.json().catch(() => ({})));
11
+ const cleanup = await routeProfileCleanup(path, method, url, readBody);
12
+ if (cleanup !== undefined)
13
+ return { status: 200, body: cleanup };
14
+ return routeChat(path, method, readBody, config);
15
+ }