om-memory-system 4.24.0-next.147 → 4.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/opencode/user-prompt.js +6 -1
- package/dist/config.d.ts +9 -0
- package/dist/config.js +5 -44
- package/dist/core/retrieval.js +1 -1
- package/dist/services/chat/chat-answer.d.ts +56 -0
- package/dist/services/chat/chat-answer.js +88 -0
- package/dist/services/chat/chat-plan.d.ts +60 -0
- package/dist/services/chat/chat-plan.js +92 -0
- package/dist/services/chat/chat-retrieve.d.ts +36 -0
- package/dist/services/chat/chat-retrieve.js +82 -0
- package/dist/services/chat/chat-routes.d.ts +12 -0
- package/dist/services/chat/chat-routes.js +145 -0
- package/dist/services/chat/chat-runtime.d.ts +21 -0
- package/dist/services/chat/chat-runtime.js +104 -0
- package/dist/services/chat/chat-service.d.ts +42 -0
- package/dist/services/chat/chat-service.js +130 -0
- package/dist/services/chat/chat-status.d.ts +16 -0
- package/dist/services/chat/chat-status.js +23 -0
- package/dist/services/chat/chat-store.d.ts +60 -0
- package/dist/services/chat/chat-store.js +352 -0
- package/dist/services/chat/chat-types.d.ts +59 -0
- package/dist/services/chat/chat-types.js +8 -0
- package/dist/services/client.d.ts +1 -0
- package/dist/services/client.js +8 -3
- package/dist/services/context.d.ts +3 -1
- package/dist/services/context.js +4 -2
- package/dist/services/global-config-writer.js +8 -1
- package/dist/services/privacy.js +2 -2
- package/dist/services/settings-snapshot.js +1 -0
- package/dist/services/turso/exact-scan.d.ts +6 -0
- package/dist/services/turso/exact-scan.js +23 -0
- package/dist/services/turso/keyword-search.d.ts +35 -0
- package/dist/services/turso/keyword-search.js +153 -0
- package/dist/services/turso/rank-fusion.d.ts +34 -0
- package/dist/services/turso/rank-fusion.js +45 -0
- package/dist/services/turso/tag-match.d.ts +7 -0
- package/dist/services/turso/tag-match.js +29 -0
- package/dist/services/turso/types.d.ts +2 -0
- package/dist/services/turso/vector-search.d.ts +3 -3
- package/dist/services/turso/vector-search.js +23 -39
- package/dist/services/user-profile/profile-context.d.ts +6 -1
- package/dist/services/user-profile/profile-context.js +43 -7
- package/dist/services/user-profile/profile-relevance.d.ts +37 -0
- package/dist/services/user-profile/profile-relevance.js +75 -0
- package/dist/services/web-route-modules.d.ts +8 -0
- package/dist/services/web-route-modules.js +15 -0
- package/dist/services/web-server.js +5 -4
- package/dist/utils/keyword-boost.d.ts +17 -0
- package/dist/utils/keyword-boost.js +32 -0
- package/dist/utils/reranker-settings.d.ts +13 -0
- package/dist/utils/reranker-settings.js +48 -0
- package/dist/web/assets/index-CjfE2byA.js +126 -0
- package/dist/web/assets/index-SK0gAreR.css +2 -0
- package/dist/web/index.html +2 -2
- package/package.json +1 -1
- package/dist/web/assets/index-B_2GCxJJ.js +0 -127
- package/dist/web/assets/index-JAZ-QY-Z.css +0 -2
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Exact-word matching for memory search (openspec change
|
|
3
|
+
* hook-search-keywords-and-profile). The pure parts take their settings as
|
|
4
|
+
* arguments: do not import CONFIG here, many tests stub src/config.js.
|
|
5
|
+
*/
|
|
6
|
+
import { log } from "../logger.js";
|
|
7
|
+
const MAX_TERMS = 24;
|
|
8
|
+
// Words that carry no search value. With the OR query below, one of them
|
|
9
|
+
// would pull most of a shard in as candidates.
|
|
10
|
+
const STOP_WORDS = new Set([
|
|
11
|
+
"the",
|
|
12
|
+
"and",
|
|
13
|
+
"or",
|
|
14
|
+
"not",
|
|
15
|
+
"of",
|
|
16
|
+
"to",
|
|
17
|
+
"in",
|
|
18
|
+
"on",
|
|
19
|
+
"at",
|
|
20
|
+
"by",
|
|
21
|
+
"for",
|
|
22
|
+
"is",
|
|
23
|
+
"it",
|
|
24
|
+
"an",
|
|
25
|
+
"be",
|
|
26
|
+
"are",
|
|
27
|
+
"was",
|
|
28
|
+
"we",
|
|
29
|
+
"i",
|
|
30
|
+
"me",
|
|
31
|
+
"my",
|
|
32
|
+
"you",
|
|
33
|
+
"do",
|
|
34
|
+
"did",
|
|
35
|
+
"does",
|
|
36
|
+
"why",
|
|
37
|
+
"how",
|
|
38
|
+
"what",
|
|
39
|
+
"with",
|
|
40
|
+
"this",
|
|
41
|
+
"that",
|
|
42
|
+
]);
|
|
43
|
+
/**
|
|
44
|
+
* FTS5 query for the words of a prompt: lower-case terms from letters and
|
|
45
|
+
* digits only ("ADR-023" gives `adr` and `023`), each in quotes, joined by OR.
|
|
46
|
+
* Quoting and the character filter keep prompt text from becoming index
|
|
47
|
+
* syntax. Returns null when no usable term is left.
|
|
48
|
+
*/
|
|
49
|
+
export function buildKeywordQuery(queryText) {
|
|
50
|
+
const terms = new Set();
|
|
51
|
+
for (const word of (queryText ?? "").toLowerCase().split(/[^\p{L}\p{N}]+/u)) {
|
|
52
|
+
if (word.length < 2 || STOP_WORDS.has(word))
|
|
53
|
+
continue;
|
|
54
|
+
terms.add(word);
|
|
55
|
+
if (terms.size === MAX_TERMS)
|
|
56
|
+
break;
|
|
57
|
+
}
|
|
58
|
+
if (terms.size === 0)
|
|
59
|
+
return null;
|
|
60
|
+
return [...terms].map((term) => `"${term}"`).join(" OR ");
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Similarity after the keyword boost: `base + keywordBoost × score ÷ best`,
|
|
64
|
+
* where `score` is the memory's match strength and `best` the strongest in the
|
|
65
|
+
* same search. The share is capped at 1 so the boost never passes
|
|
66
|
+
* `keywordBoost`, and the result never passes 1.
|
|
67
|
+
*/
|
|
68
|
+
export function applyKeywordBoost(base, score, bestScore, keywordBoost) {
|
|
69
|
+
if (!(keywordBoost > 0) || !(score > 0) || !(bestScore > 0))
|
|
70
|
+
return base;
|
|
71
|
+
return Math.min(1, base + keywordBoost * Math.min(1, score / bestScore));
|
|
72
|
+
}
|
|
73
|
+
// The index is derived data: external content (no second copy of the text),
|
|
74
|
+
// kept in step by triggers, and rebuilt whenever it is missing or short.
|
|
75
|
+
const INDEX_OBJECTS = ["memories_fts", "memories_fts_ai", "memories_fts_ad", "memories_fts_au"];
|
|
76
|
+
const INDEX_SETUP = [
|
|
77
|
+
`CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
|
|
78
|
+
content, tags, content='memories', content_rowid='rowid', tokenize='unicode61'
|
|
79
|
+
)`,
|
|
80
|
+
`CREATE TRIGGER IF NOT EXISTS memories_fts_ai AFTER INSERT ON memories BEGIN
|
|
81
|
+
INSERT INTO memories_fts(rowid, content, tags) VALUES (new.rowid, new.content, new.tags);
|
|
82
|
+
END`,
|
|
83
|
+
`CREATE TRIGGER IF NOT EXISTS memories_fts_ad AFTER DELETE ON memories BEGIN
|
|
84
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, tags)
|
|
85
|
+
VALUES ('delete', old.rowid, old.content, old.tags);
|
|
86
|
+
END`,
|
|
87
|
+
`CREATE TRIGGER IF NOT EXISTS memories_fts_au AFTER UPDATE OF content, tags ON memories BEGIN
|
|
88
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, tags)
|
|
89
|
+
VALUES ('delete', old.rowid, old.content, old.tags);
|
|
90
|
+
INSERT INTO memories_fts(rowid, content, tags) VALUES (new.rowid, new.content, new.tags);
|
|
91
|
+
END`,
|
|
92
|
+
];
|
|
93
|
+
async function indexIsCurrent(db) {
|
|
94
|
+
const names = INDEX_OBJECTS.map(() => "?").join(",");
|
|
95
|
+
const present = await db.all(`SELECT name FROM sqlite_schema WHERE name IN (${names})`, [
|
|
96
|
+
...INDEX_OBJECTS,
|
|
97
|
+
]);
|
|
98
|
+
if (present.length !== INDEX_OBJECTS.length)
|
|
99
|
+
return false;
|
|
100
|
+
// COUNT(*) on an external-content table reads the memories table, so the
|
|
101
|
+
// number of documents the index really holds comes from its docsize table.
|
|
102
|
+
const row = await db.get(`SELECT (SELECT COUNT(*) FROM memories_fts_docsize) AS indexed,
|
|
103
|
+
(SELECT COUNT(*) FROM memories) AS stored`);
|
|
104
|
+
return Number(row?.indexed) === Number(row?.stored);
|
|
105
|
+
}
|
|
106
|
+
/** Create or rebuild the keyword index of a shard. Idempotent. */
|
|
107
|
+
export async function ensureKeywordIndex(db) {
|
|
108
|
+
if (await indexIsCurrent(db))
|
|
109
|
+
return;
|
|
110
|
+
// One transaction: a failed rebuild leaves no triggers behind that would
|
|
111
|
+
// write to a broken index on every save.
|
|
112
|
+
await db.transaction("write", async (tx) => {
|
|
113
|
+
for (const sql of INDEX_SETUP)
|
|
114
|
+
await tx.execute(sql);
|
|
115
|
+
await tx.execute(`INSERT INTO memories_fts(memories_fts) VALUES ('rebuild')`);
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Memories of one shard that contain a word of the prompt, strongest first.
|
|
120
|
+
* Returns null when there is nothing to match or the index fails: search then
|
|
121
|
+
* continues with vector and tag results. The log holds a code and an error
|
|
122
|
+
* name only, never the prompt or memory text.
|
|
123
|
+
*/
|
|
124
|
+
export async function searchKeywords(db, queryText, containerTag, limit) {
|
|
125
|
+
const match = buildKeywordQuery(queryText);
|
|
126
|
+
if (!match)
|
|
127
|
+
return null;
|
|
128
|
+
try {
|
|
129
|
+
await ensureKeywordIndex(db);
|
|
130
|
+
const rows = await db.all(`SELECT m.id AS id, -bm25(memories_fts) AS score
|
|
131
|
+
FROM memories_fts JOIN memories m ON m.rowid = memories_fts.rowid
|
|
132
|
+
WHERE memories_fts MATCH ?${containerTag === "" ? "" : " AND m.container_tag = ?"}
|
|
133
|
+
ORDER BY score DESC LIMIT ?`, containerTag === "" ? [match, limit] : [match, containerTag, limit]);
|
|
134
|
+
const scores = new Map(rows.map((row) => [String(row.id), Number(row.score)]));
|
|
135
|
+
return { scores, best: Math.max(0, ...scores.values()) };
|
|
136
|
+
}
|
|
137
|
+
catch (error) {
|
|
138
|
+
log("Keyword search failed; using vector results", {
|
|
139
|
+
code: "keyword_index_failed",
|
|
140
|
+
error: error instanceof Error ? error.name : "unknown",
|
|
141
|
+
});
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* `base` with the boost for memory `id`; `base` itself when it has no keyword hit
|
|
147
|
+
* or the mode is `rrf` (rank fusion keeps each memory's own similarity).
|
|
148
|
+
*/
|
|
149
|
+
export function boostedSimilarity(base, hits, id, keywordBoost, mode = "boost") {
|
|
150
|
+
if (mode === "rrf")
|
|
151
|
+
return base;
|
|
152
|
+
return applyKeywordBoost(base, hits?.scores.get(id) ?? 0, hits?.best ?? 0, keywordBoost);
|
|
153
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reciprocal rank fusion of the meaning list and the word list (openspec change
|
|
3
|
+
* hook-search-keywords-and-profile, design decision 9). Pure: the threshold and
|
|
4
|
+
* the limits are passed in, so do not import CONFIG here.
|
|
5
|
+
*/
|
|
6
|
+
/** The published constant of reciprocal rank fusion. */
|
|
7
|
+
export declare const RRF_K = 60;
|
|
8
|
+
/** Same size as the word-hit limit of one shard search (max(limit × 4, 32), limit 10). */
|
|
9
|
+
export declare const RRF_WORD_LIMIT = 40;
|
|
10
|
+
export interface FusionRow {
|
|
11
|
+
id: string;
|
|
12
|
+
similarity: number;
|
|
13
|
+
/** Match strength of a word hit; missing or 0 means the memory has no word match. */
|
|
14
|
+
keywordScore?: number;
|
|
15
|
+
}
|
|
16
|
+
export interface FusionOptions {
|
|
17
|
+
/** Lowest similarity for the meaning list. */
|
|
18
|
+
threshold: number;
|
|
19
|
+
limit: number;
|
|
20
|
+
wordLimit?: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Merge two ranked lists by rank only. The meaning list holds the rows at or
|
|
24
|
+
* above the threshold, ranked by similarity. The word list holds the rows with
|
|
25
|
+
* a word match, ranked by match strength. A row that is only in the word list
|
|
26
|
+
* can be returned although its similarity is below the threshold. Ties break by
|
|
27
|
+
* id so repeated searches agree. Each row keeps its own similarity.
|
|
28
|
+
*/
|
|
29
|
+
export declare function rankFusion<T extends FusionRow>(rows: T[], options: FusionOptions): T[];
|
|
30
|
+
/**
|
|
31
|
+
* Fuse the candidates of all shards and drop the internal word score, so the
|
|
32
|
+
* results have the same shape as in `boost` mode.
|
|
33
|
+
*/
|
|
34
|
+
export declare function fuseShardResults<T extends FusionRow>(rows: T[], threshold: number, limit: number): Omit<T, "keywordScore">[];
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reciprocal rank fusion of the meaning list and the word list (openspec change
|
|
3
|
+
* hook-search-keywords-and-profile, design decision 9). Pure: the threshold and
|
|
4
|
+
* the limits are passed in, so do not import CONFIG here.
|
|
5
|
+
*/
|
|
6
|
+
/** The published constant of reciprocal rank fusion. */
|
|
7
|
+
export const RRF_K = 60;
|
|
8
|
+
/** Same size as the word-hit limit of one shard search (max(limit × 4, 32), limit 10). */
|
|
9
|
+
export const RRF_WORD_LIMIT = 40;
|
|
10
|
+
const byIdAfter = (compare) => (a, b) => compare(a, b) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
11
|
+
/**
|
|
12
|
+
* Merge two ranked lists by rank only. The meaning list holds the rows at or
|
|
13
|
+
* above the threshold, ranked by similarity. The word list holds the rows with
|
|
14
|
+
* a word match, ranked by match strength. A row that is only in the word list
|
|
15
|
+
* can be returned although its similarity is below the threshold. Ties break by
|
|
16
|
+
* id so repeated searches agree. Each row keeps its own similarity.
|
|
17
|
+
*/
|
|
18
|
+
export function rankFusion(rows, options) {
|
|
19
|
+
const meaning = rows
|
|
20
|
+
.filter((row) => row.similarity >= options.threshold)
|
|
21
|
+
.sort(byIdAfter((a, b) => b.similarity - a.similarity));
|
|
22
|
+
const words = rows
|
|
23
|
+
.filter((row) => (row.keywordScore ?? 0) > 0)
|
|
24
|
+
.sort(byIdAfter((a, b) => (b.keywordScore ?? 0) - (a.keywordScore ?? 0)))
|
|
25
|
+
.slice(0, options.wordLimit ?? RRF_WORD_LIMIT);
|
|
26
|
+
const fused = new Map();
|
|
27
|
+
for (const list of [meaning, words]) {
|
|
28
|
+
list.forEach((row, index) => {
|
|
29
|
+
const entry = fused.get(row.id) ?? { row, score: 0 };
|
|
30
|
+
entry.score += 1 / (RRF_K + index + 1);
|
|
31
|
+
fused.set(row.id, entry);
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
return [...fused.values()]
|
|
35
|
+
.sort((a, b) => b.score - a.score || (a.row.id < b.row.id ? -1 : a.row.id > b.row.id ? 1 : 0))
|
|
36
|
+
.slice(0, Math.max(0, options.limit))
|
|
37
|
+
.map((entry) => entry.row);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Fuse the candidates of all shards and drop the internal word score, so the
|
|
41
|
+
* results have the same shape as in `boost` mode.
|
|
42
|
+
*/
|
|
43
|
+
export function fuseShardResults(rows, threshold, limit) {
|
|
44
|
+
return rankFusion(rows, { threshold, limit }).map(({ keywordScore: _word, ...rest }) => rest);
|
|
45
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Share of query words that name one of a memory's tags, from 0 to 1. A word
|
|
3
|
+
* matches when it equals a whole tag or one part of a tag ("code" matches
|
|
4
|
+
* "claude-code"). Plain substring matching let short tags such as "ui" match
|
|
5
|
+
* words like "build" or "guide", which pushed weak results up.
|
|
6
|
+
*/
|
|
7
|
+
export declare function tagMatchBoost(queryText: string | undefined, tagsCsv: string): number;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Scripts written without spaces between words. A query in one of these is a
|
|
2
|
+
// single token, so a tag can only be found inside it.
|
|
3
|
+
const UNSPACED_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}]/u;
|
|
4
|
+
function tokens(text) {
|
|
5
|
+
return text
|
|
6
|
+
.toLowerCase()
|
|
7
|
+
.split(/[^\p{L}\p{N}]+/u)
|
|
8
|
+
.filter((token) => token.length > 0);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Share of query words that name one of a memory's tags, from 0 to 1. A word
|
|
12
|
+
* matches when it equals a whole tag or one part of a tag ("code" matches
|
|
13
|
+
* "claude-code"). Plain substring matching let short tags such as "ui" match
|
|
14
|
+
* words like "build" or "guide", which pushed weak results up.
|
|
15
|
+
*/
|
|
16
|
+
export function tagMatchBoost(queryText, tagsCsv) {
|
|
17
|
+
const tags = tagsCsv
|
|
18
|
+
.split(",")
|
|
19
|
+
.map((tag) => tag.trim().toLowerCase())
|
|
20
|
+
.filter((tag) => tag.length > 0);
|
|
21
|
+
// A one-character word counts only when it is a whole tag ("C language", tag "c").
|
|
22
|
+
const words = tokens(queryText ?? "").filter((word) => word.length > 1 || tags.includes(word));
|
|
23
|
+
if (words.length === 0 || tags.length === 0)
|
|
24
|
+
return 0;
|
|
25
|
+
const tagParts = new Set(tags.flatMap((tag) => [tag, ...tokens(tag)]));
|
|
26
|
+
const unspacedTags = [...tagParts].filter((part) => UNSPACED_SCRIPT.test(part));
|
|
27
|
+
const matches = words.filter((word) => tagParts.has(word) || unspacedTags.some((tag) => word.includes(tag))).length;
|
|
28
|
+
return matches / words.length;
|
|
29
|
+
}
|
|
@@ -30,6 +30,8 @@ export interface SearchResult {
|
|
|
30
30
|
id: string;
|
|
31
31
|
memory: string;
|
|
32
32
|
similarity: number;
|
|
33
|
+
/** Internal: match strength of a word hit. Removed before results leave search. */
|
|
34
|
+
keywordScore?: number;
|
|
33
35
|
createdAt?: number;
|
|
34
36
|
tags?: string[];
|
|
35
37
|
type?: string;
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import type { MemoryRecord, SearchResult, ShardInfo } from "./types.js";
|
|
2
|
+
import type { KeywordMode } from "../../utils/keyword-boost.js";
|
|
2
3
|
import type { TursoDb } from "./turso-db.js";
|
|
3
4
|
import type { Transaction } from "@libsql/client";
|
|
4
5
|
export declare class TursoVectorSearch {
|
|
5
6
|
insertVectorInTransaction(tx: Transaction, record: MemoryRecord): Promise<void>;
|
|
6
7
|
insertVector(db: TursoDb, record: MemoryRecord): Promise<void>;
|
|
7
|
-
searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string): Promise<SearchResult[]>;
|
|
8
|
+
searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode): Promise<SearchResult[]>;
|
|
8
9
|
private searchKind;
|
|
9
|
-
|
|
10
|
-
searchAcrossShards(shards: ShardInfo[], queryVector: Float32Array, containerTag: string, limit: number, similarityThreshold: number, queryText?: string): Promise<{
|
|
10
|
+
searchAcrossShards(shards: ShardInfo[], queryVector: Float32Array, containerTag: string, limit: number, similarityThreshold: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode): Promise<{
|
|
11
11
|
results: SearchResult[];
|
|
12
12
|
warnings: string[];
|
|
13
13
|
}>;
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { tursoConnectionManager } from "./connection-manager.js";
|
|
2
2
|
import { log } from "../logger.js";
|
|
3
3
|
import { distanceToSimilarity, vectorToJson } from "./vector-utils.js";
|
|
4
|
+
import { tagMatchBoost } from "./tag-match.js";
|
|
5
|
+
import { exactScanKind } from "./exact-scan.js";
|
|
6
|
+
import { boostedSimilarity, searchKeywords } from "./keyword-search.js";
|
|
7
|
+
import { fuseShardResults } from "./rank-fusion.js";
|
|
4
8
|
function parseMetadata(value) {
|
|
5
9
|
if (!value || typeof value !== "string")
|
|
6
10
|
return undefined;
|
|
@@ -88,7 +92,7 @@ export class TursoVectorSearch {
|
|
|
88
92
|
) VALUES (?, ?, vector32(?), NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
89
93
|
`, [record.id, record.content, contentVector, ...commonArgs]);
|
|
90
94
|
}
|
|
91
|
-
async searchInShard(shard, queryVector, containerTag, limit, queryText) {
|
|
95
|
+
async searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost = 0, keywordMode = "boost") {
|
|
92
96
|
const db = await tursoConnectionManager.getConnection(shard.dbPath);
|
|
93
97
|
const queryJson = vectorToJson(queryVector);
|
|
94
98
|
// Over-fetch aggressively when filtering by container_tag after ANN,
|
|
@@ -101,6 +105,11 @@ export class TursoVectorSearch {
|
|
|
101
105
|
candidateIds.add(result.id);
|
|
102
106
|
for (const result of tagsResults)
|
|
103
107
|
candidateIds.add(result.id);
|
|
108
|
+
const kwHits = keywordBoost > 0
|
|
109
|
+
? await searchKeywords(db, queryText, containerTag, Math.max(limit * 4, 32))
|
|
110
|
+
: null;
|
|
111
|
+
for (const id of kwHits?.scores.keys() ?? [])
|
|
112
|
+
candidateIds.add(id);
|
|
104
113
|
const ids = Array.from(candidateIds);
|
|
105
114
|
if (ids.length === 0)
|
|
106
115
|
return [];
|
|
@@ -132,30 +141,21 @@ export class TursoVectorSearch {
|
|
|
132
141
|
`, containerTag === ""
|
|
133
142
|
? [queryJson, queryJson, ...ids]
|
|
134
143
|
: [queryJson, queryJson, ...ids, containerTag]);
|
|
135
|
-
const queryWords = queryText
|
|
136
|
-
? queryText
|
|
137
|
-
.toLowerCase()
|
|
138
|
-
.split(/[\s,]+/)
|
|
139
|
-
.filter((word) => word.length > 1)
|
|
140
|
-
: [];
|
|
141
144
|
const hydratedResults = rows.map((row) => {
|
|
142
145
|
const contentSim = distanceToSimilarity(Number(row.content_dist));
|
|
143
146
|
const tagsSim = row.tags_dist == null || row.tags_dist === undefined
|
|
144
147
|
? 0
|
|
145
148
|
: distanceToSimilarity(Number(row.tags_dist));
|
|
146
149
|
const memoryTagsStr = String(row.tags || "");
|
|
147
|
-
const
|
|
148
|
-
let exactMatchBoost = 0;
|
|
149
|
-
if (queryWords.length > 0 && memoryTags.length > 0) {
|
|
150
|
-
const matches = queryWords.filter((word) => memoryTags.some((tag) => tag.includes(word) || word.includes(tag))).length;
|
|
151
|
-
exactMatchBoost = matches / Math.max(queryWords.length, 1);
|
|
152
|
-
}
|
|
150
|
+
const exactMatchBoost = tagMatchBoost(queryText, memoryTagsStr);
|
|
153
151
|
const finalTagsSim = Math.max(tagsSim, exactMatchBoost);
|
|
154
|
-
const
|
|
152
|
+
const base = contentSim * 0.6 + finalTagsSim * 0.4;
|
|
153
|
+
const similarity = boostedSimilarity(base, kwHits, String(row.id), keywordBoost, keywordMode);
|
|
155
154
|
return {
|
|
156
155
|
id: String(row.id),
|
|
157
156
|
memory: String(row.content),
|
|
158
157
|
similarity,
|
|
158
|
+
keywordScore: kwHits?.scores.get(String(row.id)),
|
|
159
159
|
createdAt: Number(row.created_at),
|
|
160
160
|
tags: memoryTagsStr ? memoryTagsStr.split(",") : [],
|
|
161
161
|
type: row.type ? String(row.type) : undefined,
|
|
@@ -171,7 +171,9 @@ export class TursoVectorSearch {
|
|
|
171
171
|
};
|
|
172
172
|
});
|
|
173
173
|
hydratedResults.sort((a, b) => b.similarity - a.similarity);
|
|
174
|
-
|
|
174
|
+
// Rank fusion needs the word-only candidates too; it applies the limit itself.
|
|
175
|
+
const fusing = keywordMode === "rrf" && keywordBoost > 0;
|
|
176
|
+
return fusing ? hydratedResults : hydratedResults.slice(0, Math.max(0, limit));
|
|
175
177
|
}
|
|
176
178
|
async searchKind(db, queryJson, k, containerTag, indexName, columnName) {
|
|
177
179
|
try {
|
|
@@ -198,35 +200,14 @@ export class TursoVectorSearch {
|
|
|
198
200
|
indexName,
|
|
199
201
|
error: String(error),
|
|
200
202
|
});
|
|
201
|
-
return
|
|
203
|
+
return exactScanKind(db, queryJson, k, containerTag, columnName);
|
|
202
204
|
}
|
|
203
205
|
}
|
|
204
|
-
async
|
|
205
|
-
const rows = await db.all(containerTag === ""
|
|
206
|
-
? `
|
|
207
|
-
SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist
|
|
208
|
-
FROM memories m
|
|
209
|
-
WHERE m.${columnName} IS NOT NULL
|
|
210
|
-
ORDER BY dist ASC
|
|
211
|
-
LIMIT ?
|
|
212
|
-
`
|
|
213
|
-
: `
|
|
214
|
-
SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist
|
|
215
|
-
FROM memories m
|
|
216
|
-
WHERE m.${columnName} IS NOT NULL AND m.container_tag = ?
|
|
217
|
-
ORDER BY dist ASC
|
|
218
|
-
LIMIT ?
|
|
219
|
-
`, containerTag === "" ? [queryJson, k] : [queryJson, containerTag, k]);
|
|
220
|
-
return rows.map((row) => ({
|
|
221
|
-
id: String(row.id),
|
|
222
|
-
similarity: distanceToSimilarity(Number(row.dist)),
|
|
223
|
-
}));
|
|
224
|
-
}
|
|
225
|
-
async searchAcrossShards(shards, queryVector, containerTag, limit, similarityThreshold, queryText) {
|
|
206
|
+
async searchAcrossShards(shards, queryVector, containerTag, limit, similarityThreshold, queryText, keywordBoost = 0, keywordMode = "boost") {
|
|
226
207
|
const shardErrors = [];
|
|
227
208
|
const shardPromises = shards.map(async (shard) => {
|
|
228
209
|
try {
|
|
229
|
-
return await this.searchInShard(shard, queryVector, containerTag, limit, queryText);
|
|
210
|
+
return await this.searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost, keywordMode);
|
|
230
211
|
}
|
|
231
212
|
catch (error) {
|
|
232
213
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -253,6 +234,9 @@ export class TursoVectorSearch {
|
|
|
253
234
|
});
|
|
254
235
|
}
|
|
255
236
|
const allResults = resultsArray.flat();
|
|
237
|
+
if (keywordMode === "rrf" && keywordBoost > 0) {
|
|
238
|
+
return { results: fuseShardResults(allResults, similarityThreshold, limit), warnings };
|
|
239
|
+
}
|
|
256
240
|
allResults.sort((a, b) => b.similarity - a.similarity);
|
|
257
241
|
return {
|
|
258
242
|
results: allResults
|
|
@@ -1 +1,6 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* The profile block for the prompt. With `queryVector`, half of each section
|
|
3
|
+
* (rounded up) follows the prompt; without it the block is the same for every
|
|
4
|
+
* prompt, as before.
|
|
5
|
+
*/
|
|
6
|
+
export declare function getUserProfileContext(userId: string, queryVector?: ArrayLike<number>): Promise<string | null>;
|
|
@@ -2,6 +2,7 @@ import { userProfileManager } from "./user-profile-manager.js";
|
|
|
2
2
|
import { CONFIG } from "../../config.js";
|
|
3
3
|
import { sortProfileItems } from "../../utils/profile.js";
|
|
4
4
|
import { log } from "../logger.js";
|
|
5
|
+
import { selectProfileItems } from "./profile-relevance.js";
|
|
5
6
|
function dedupByCategory(items, topN) {
|
|
6
7
|
if (items.length <= topN)
|
|
7
8
|
return items;
|
|
@@ -33,7 +34,12 @@ function escapeContextText(value) {
|
|
|
33
34
|
.replace(/</g, "<")
|
|
34
35
|
.replace(/>/g, ">");
|
|
35
36
|
}
|
|
36
|
-
|
|
37
|
+
/**
|
|
38
|
+
* The profile block for the prompt. With `queryVector`, half of each section
|
|
39
|
+
* (rounded up) follows the prompt; without it the block is the same for every
|
|
40
|
+
* prompt, as before.
|
|
41
|
+
*/
|
|
42
|
+
export async function getUserProfileContext(userId, queryVector) {
|
|
37
43
|
const profile = await userProfileManager.getActiveProfile(userId);
|
|
38
44
|
if (!profile) {
|
|
39
45
|
return null;
|
|
@@ -60,9 +66,35 @@ export async function getUserProfileContext(userId) {
|
|
|
60
66
|
const sortedWfs = profileData.workflows.length > 0
|
|
61
67
|
? sortProfileItems(profileData.workflows, "frequency")
|
|
62
68
|
: [];
|
|
63
|
-
|
|
64
|
-
const
|
|
65
|
-
const
|
|
69
|
+
// The prompt must be about as close to an item as it is to a memory.
|
|
70
|
+
const minCloseness = CONFIG.similarityThreshold;
|
|
71
|
+
const prefs = selectProfileItems({
|
|
72
|
+
baseline: dedupByCategory(scoreByRecency(sortedPrefs.slice(0, injectPrefs * 2)), injectPrefs),
|
|
73
|
+
candidates: sortedPrefs,
|
|
74
|
+
count: injectPrefs,
|
|
75
|
+
queryVector,
|
|
76
|
+
minCloseness,
|
|
77
|
+
onePerCategory: true,
|
|
78
|
+
});
|
|
79
|
+
const pats = selectProfileItems({
|
|
80
|
+
baseline: dedupByCategory(sortedPats, injectPats),
|
|
81
|
+
candidates: sortedPats,
|
|
82
|
+
count: injectPats,
|
|
83
|
+
queryVector,
|
|
84
|
+
minCloseness,
|
|
85
|
+
onePerCategory: true,
|
|
86
|
+
});
|
|
87
|
+
const wfs = selectProfileItems({
|
|
88
|
+
baseline: sortedWfs.slice(0, injectWfs),
|
|
89
|
+
candidates: sortedWfs,
|
|
90
|
+
count: injectWfs,
|
|
91
|
+
queryVector,
|
|
92
|
+
minCloseness,
|
|
93
|
+
onePerCategory: false,
|
|
94
|
+
});
|
|
95
|
+
const topPrefs = prefs.items;
|
|
96
|
+
const topPats = pats.items;
|
|
97
|
+
const topWfs = wfs.items;
|
|
66
98
|
if (topPrefs.length > 0) {
|
|
67
99
|
parts.push("User Preferences:");
|
|
68
100
|
topPrefs.forEach((pref) => {
|
|
@@ -96,9 +128,13 @@ export async function getUserProfileContext(userId) {
|
|
|
96
128
|
const text = parts.join("\n");
|
|
97
129
|
if (topPrefs.length + topPats.length + topWfs.length > 0) {
|
|
98
130
|
log("profile inject", {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
131
|
+
// Counts only: the log never holds the prompt.
|
|
132
|
+
relevanceSlots: {
|
|
133
|
+
prefs: prefs.relevanceSlots,
|
|
134
|
+
pats: pats.relevanceSlots,
|
|
135
|
+
wfs: wfs.relevanceSlots,
|
|
136
|
+
},
|
|
137
|
+
counts: { prefs: topPrefs.length, pats: topPats.length, wfs: topWfs.length },
|
|
102
138
|
});
|
|
103
139
|
}
|
|
104
140
|
return text;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Choose profile items for the injected block by relevance to the prompt
|
|
3
|
+
* (openspec change hook-search-keywords-and-profile). Pure functions: the
|
|
4
|
+
* vector, the items, the counts, and the minimum closeness are arguments, so
|
|
5
|
+
* callers and tests need no store and no CONFIG.
|
|
6
|
+
*/
|
|
7
|
+
export interface VectorItem {
|
|
8
|
+
category?: string;
|
|
9
|
+
anchor?: number[];
|
|
10
|
+
centroid?: number[];
|
|
11
|
+
}
|
|
12
|
+
export interface SelectProfileItemsInput<T extends VectorItem> {
|
|
13
|
+
/** Today's selection for the section, in today's order. */
|
|
14
|
+
baseline: T[];
|
|
15
|
+
/** Every eligible item, in today's order. Relevance may pick from all of them. */
|
|
16
|
+
candidates: T[];
|
|
17
|
+
/** Injected items for the section. */
|
|
18
|
+
count: number;
|
|
19
|
+
/** The embedded prompt. Without it the section keeps today's selection. */
|
|
20
|
+
queryVector: ArrayLike<number> | undefined;
|
|
21
|
+
/** Items with a lower cosine similarity never take a relevance slot. */
|
|
22
|
+
minCloseness: number;
|
|
23
|
+
/** Sections that group by category keep one item per category. */
|
|
24
|
+
onePerCategory: boolean;
|
|
25
|
+
}
|
|
26
|
+
export interface SelectProfileItemsResult<T> {
|
|
27
|
+
items: T[];
|
|
28
|
+
/** How many items came in through relevance, for the log. */
|
|
29
|
+
relevanceSlots: number;
|
|
30
|
+
}
|
|
31
|
+
export declare function cosineSimilarity(a: ArrayLike<number>, b: ArrayLike<number>): number;
|
|
32
|
+
/**
|
|
33
|
+
* Half of the slots, rounded up, go to the items closest to the prompt. The
|
|
34
|
+
* other slots keep today's order. With no prompt vector, or no item close
|
|
35
|
+
* enough, the result is `baseline` unchanged.
|
|
36
|
+
*/
|
|
37
|
+
export declare function selectProfileItems<T extends VectorItem>(input: SelectProfileItemsInput<T>): SelectProfileItemsResult<T>;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Choose profile items for the injected block by relevance to the prompt
|
|
3
|
+
* (openspec change hook-search-keywords-and-profile). Pure functions: the
|
|
4
|
+
* vector, the items, the counts, and the minimum closeness are arguments, so
|
|
5
|
+
* callers and tests need no store and no CONFIG.
|
|
6
|
+
*/
|
|
7
|
+
export function cosineSimilarity(a, b) {
|
|
8
|
+
if (a.length === 0 || a.length !== b.length)
|
|
9
|
+
return 0;
|
|
10
|
+
let dot = 0;
|
|
11
|
+
let normA = 0;
|
|
12
|
+
let normB = 0;
|
|
13
|
+
for (let i = 0; i < a.length; i++) {
|
|
14
|
+
const x = a[i];
|
|
15
|
+
const y = b[i];
|
|
16
|
+
dot += x * y;
|
|
17
|
+
normA += x * x;
|
|
18
|
+
normB += y * y;
|
|
19
|
+
}
|
|
20
|
+
if (normA === 0 || normB === 0)
|
|
21
|
+
return 0;
|
|
22
|
+
return dot / Math.sqrt(normA * normB);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Half of the slots, rounded up, go to the items closest to the prompt. The
|
|
26
|
+
* other slots keep today's order. With no prompt vector, or no item close
|
|
27
|
+
* enough, the result is `baseline` unchanged.
|
|
28
|
+
*/
|
|
29
|
+
export function selectProfileItems(input) {
|
|
30
|
+
const { baseline, candidates, count, queryVector, minCloseness, onePerCategory } = input;
|
|
31
|
+
const unchanged = { items: baseline, relevanceSlots: 0 };
|
|
32
|
+
if (!queryVector || queryVector.length === 0 || count <= 0)
|
|
33
|
+
return unchanged;
|
|
34
|
+
const ranked = candidates
|
|
35
|
+
.map((item) => {
|
|
36
|
+
const vector = item.anchor ?? item.centroid;
|
|
37
|
+
return { item, closeness: vector ? cosineSimilarity(queryVector, vector) : 0 };
|
|
38
|
+
})
|
|
39
|
+
.filter((entry) => entry.closeness >= minCloseness && entry.closeness > 0)
|
|
40
|
+
.sort((a, b) => b.closeness - a.closeness);
|
|
41
|
+
const chosen = [];
|
|
42
|
+
const categories = new Set();
|
|
43
|
+
const take = (item) => {
|
|
44
|
+
if (chosen.includes(item))
|
|
45
|
+
return false;
|
|
46
|
+
if (onePerCategory && categories.has(item.category))
|
|
47
|
+
return false;
|
|
48
|
+
chosen.push(item);
|
|
49
|
+
categories.add(item.category);
|
|
50
|
+
return true;
|
|
51
|
+
};
|
|
52
|
+
const relevanceCap = Math.ceil(count / 2);
|
|
53
|
+
let relevanceSlots = 0;
|
|
54
|
+
for (const { item } of ranked) {
|
|
55
|
+
if (relevanceSlots >= relevanceCap)
|
|
56
|
+
break;
|
|
57
|
+
if (take(item))
|
|
58
|
+
relevanceSlots++;
|
|
59
|
+
}
|
|
60
|
+
if (relevanceSlots === 0)
|
|
61
|
+
return unchanged;
|
|
62
|
+
for (const item of baseline) {
|
|
63
|
+
if (chosen.length >= count)
|
|
64
|
+
break;
|
|
65
|
+
take(item);
|
|
66
|
+
}
|
|
67
|
+
// A skipped duplicate can leave a slot empty; fill it from the rest of the
|
|
68
|
+
// eligible items so the section keeps its count.
|
|
69
|
+
for (const item of candidates) {
|
|
70
|
+
if (chosen.length >= count)
|
|
71
|
+
break;
|
|
72
|
+
take(item);
|
|
73
|
+
}
|
|
74
|
+
return { items: chosen, relevanceSlots };
|
|
75
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ChatRuntimeConfig } from "./chat/chat-runtime.js";
|
|
2
|
+
import { type RouteResult } from "./chat/chat-routes.js";
|
|
3
|
+
/**
|
|
4
|
+
* Route modules that live outside web-server.ts, which sits at its size
|
|
5
|
+
* ceiling. Each returns undefined for paths it does not own. The request body
|
|
6
|
+
* is read once, and only by a route that needs it.
|
|
7
|
+
*/
|
|
8
|
+
export declare function routeModules(path: string, method: string, url: URL, req: Request, config: ChatRuntimeConfig): Promise<RouteResult | undefined>;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { routeChat } from "./chat/chat-routes.js";
|
|
2
|
+
import { routeProfileCleanup } from "./profile-cleanup-routes.js";
|
|
3
|
+
/**
|
|
4
|
+
* Route modules that live outside web-server.ts, which sits at its size
|
|
5
|
+
* ceiling. Each returns undefined for paths it does not own. The request body
|
|
6
|
+
* is read once, and only by a route that needs it.
|
|
7
|
+
*/
|
|
8
|
+
export async function routeModules(path, method, url, req, config) {
|
|
9
|
+
let body;
|
|
10
|
+
const readBody = () => (body ??= req.json().catch(() => ({})));
|
|
11
|
+
const cleanup = await routeProfileCleanup(path, method, url, readBody);
|
|
12
|
+
if (cleanup !== undefined)
|
|
13
|
+
return { status: 200, body: cleanup };
|
|
14
|
+
return routeChat(path, method, readBody, config);
|
|
15
|
+
}
|