hippo-memory 3.0.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/api/context/context-plan.js +2 -4
- package/dist/api/context/context-select.js +12 -7
- package/dist/api/detached.d.ts +6 -0
- package/dist/api/detached.js +16 -0
- package/dist/api/embeddings.d.ts +4 -3
- package/dist/api/embeddings.js +4 -4
- package/dist/api/memories/promote.js +2 -1
- package/dist/api/memories/remember-local.d.ts +5 -4
- package/dist/api/memories/remember-local.js +13 -10
- package/dist/api/recall/rank-stages.js +2 -2
- package/dist/api/session-end.js +1 -0
- package/dist/api/sharing.js +9 -7
- package/dist/api/sleep/auto-sleep.d.ts +1 -1
- package/dist/api/sleep/auto-sleep.js +5 -3
- package/dist/api/sleep/learn.js +5 -2
- package/dist/api/sleep/sleep-run.js +4 -2
- package/dist/api/types.d.ts +4 -0
- package/dist/base/memory/entry.js +2 -2
- package/dist/base/memory/markdown-delimiters.d.ts +15 -0
- package/dist/base/memory/markdown-delimiters.js +94 -0
- package/dist/base/memory/memory-audit.js +4 -4
- package/dist/base/memory/memory-quality.d.ts +12 -0
- package/dist/base/memory/memory-quality.js +92 -3
- package/dist/base/ports/store/types.d.ts +2 -1
- package/dist/base/rank/prompt-recall.d.ts +16 -3
- package/dist/base/rank/prompt-recall.js +47 -7
- package/dist/base/util/detached-set.d.ts +8 -0
- package/dist/base/util/detached-set.js +35 -0
- package/dist/base/util/tokenize.d.ts +29 -0
- package/dist/base/util/tokenize.js +68 -5
- package/dist/base/util/version.d.ts +1 -1
- package/dist/base/util/version.js +1 -1
- package/dist/db/meta.d.ts +1 -0
- package/dist/db/meta.js +1 -0
- package/dist/db/store-runtime.d.ts +10 -3
- package/dist/db/store-runtime.js +19 -3
- package/dist/domain/consolidate/conflicts.js +2 -2
- package/dist/domain/learn/forward-claim-detector.js +4 -2
- package/dist/domain/learn/invalidation.js +5 -2
- package/dist/domain/search/bm25.js +3 -3
- package/dist/domain/search/detached-work.d.ts +6 -0
- package/dist/domain/search/detached-work.js +2 -0
- package/dist/domain/search/embed-memories.d.ts +4 -4
- package/dist/domain/search/embed-memories.js +4 -5
- package/dist/domain/sharing/global-sync.d.ts +8 -2
- package/dist/domain/sharing/global-sync.js +5 -5
- package/dist/domain/sharing/share.d.ts +4 -0
- package/dist/domain/sharing/share.js +2 -2
- package/dist/entry/eval-scripts.d.ts +1 -0
- package/dist/entry/sleep.d.ts +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/server.d.ts +1 -0
- package/dist/store/rules/windows.d.ts +3 -0
- package/dist/store/rules/windows.js +3 -0
- package/dist/store/sqlite/audit/audit-maintenance.js +16 -13
- package/dist/store/sqlite/recall/candidates.js +1 -0
- package/dist/store/sqlite/recall/search-rows.d.ts +2 -2
- package/dist/store/sqlite/recall/search-rows.js +62 -26
- package/dist/store/sqlite/runtime/server-runtime.d.ts +6 -1
- package/dist/store/sqlite/runtime/server-runtime.js +15 -3
- package/dist/store/sqlite/runtime/worker-jobs.js +2 -2
- package/dist/store/sqlite/sharing/copy-rows.d.ts +1 -0
- package/dist/store/sqlite/sharing/copy-rows.js +2 -1
- package/dist/surface/capture/command.d.ts +8 -1
- package/dist/surface/capture/command.js +38 -8
- package/dist/surface/capture/compact.d.ts +3 -3
- package/dist/surface/capture/compact.js +4 -4
- package/dist/surface/capture/extract.d.ts +10 -0
- package/dist/surface/capture/extract.js +199 -26
- package/dist/surface/capture/session-texts.js +11 -11
- package/dist/surface/cli/api-context.d.ts +5 -0
- package/dist/surface/cli/api-context.js +12 -0
- package/dist/surface/cli/goals.js +6 -8
- package/dist/surface/cli/hook-runtime.js +2 -2
- package/dist/surface/cli/memory-import.js +20 -15
- package/dist/surface/mcp/protocol.d.ts +6 -0
- package/dist/surface/mcp/protocol.js +1 -0
- package/dist/surface/mcp/stdio.js +11 -5
- package/dist/surface/open-store.d.ts +2 -2
- package/dist/surface/open-store.js +11 -7
- package/dist/surface/server/auth.d.ts +1 -1
- package/dist/surface/server/auth.js +13 -2
- package/dist/surface/server/boot.js +3 -2
- package/dist/surface/server/dashboard-listener.js +2 -2
- package/dist/surface/server/lifecycle.d.ts +2 -0
- package/dist/surface/server/lifecycle.js +4 -2
- package/dist/surface/server/mcp-http.js +1 -0
- package/dist/surface/server/types.d.ts +3 -0
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { COMPACTION_MEMORY_TAG, COMPACTION_SOURCE_PREFIX, DAG_LEVEL, } from './entry.js';
|
|
2
|
+
import { blankCodeSpans, strongMarkers } from './markdown-delimiters.js';
|
|
2
3
|
import { heldTexts } from '../util/same-text.js';
|
|
3
4
|
import { MEMORY_QUALITY_STOPWORDS } from '../util/stopwords.js';
|
|
5
|
+
import { CJK_SCRIPTS } from '../util/tokenize.js';
|
|
6
|
+
/** Bump when a certain defect is added, so the next daily run or sleep cleans each store once more. */
|
|
7
|
+
export const QUALITY_GATE_VERSION = 2;
|
|
4
8
|
/** Text shorter than this holds no fact worth keeping. */
|
|
5
9
|
export const MIN_MEMORY_CHARS = 10;
|
|
6
10
|
/** Text shorter than this must name a path, number, name or code to be kept. */
|
|
@@ -9,11 +13,12 @@ const SPECIFIC_DETAIL_CHARS = 40;
|
|
|
9
13
|
const FRAGMENT_MAX_CHARS = 50;
|
|
10
14
|
/** Fewer substantive words than this leave a memory too vague to act on. */
|
|
11
15
|
export const MIN_SUBSTANTIVE_WORDS = 2;
|
|
12
|
-
|
|
16
|
+
// Without ー, which alone says nothing, and the Han script's radicals, which are not letters.
|
|
17
|
+
const CJK_LETTER = new RegExp(String.raw `(?=\p{L})[${CJK_SCRIPTS}]`, 'gu');
|
|
13
18
|
const CHAT_ACRONYMS = /^(?:TODO|FYI|LGTM|IIRC|IMO|IMHO|FWIW|TBD|BTW|ASAP|AFAIK|WIP|NB|PS)$/;
|
|
14
19
|
/** Counts spaced words plus CJK letters without lowering mixed-script counts. */
|
|
15
20
|
export function substantiveWordCount(text) {
|
|
16
|
-
const cjkLetterCount = (text.match(
|
|
21
|
+
const cjkLetterCount = (text.match(CJK_LETTER) ?? []).length;
|
|
17
22
|
const spacedCount = text
|
|
18
23
|
.toLowerCase()
|
|
19
24
|
.split(/\s+/)
|
|
@@ -79,6 +84,84 @@ const POSSIBLE_CUT_ENDING = new RegExp(String.raw `\b(?:needs to|depends on|requ
|
|
|
79
84
|
function isBareCondition(bare) {
|
|
80
85
|
return !/[,;]|\bthen\b/i.test(bare) && !RULE_WORD.test(bare) && bare.split(/\s+/).length <= 6;
|
|
81
86
|
}
|
|
87
|
+
// A blanked code span reads as a letter that is no roman numeral, so it is never a list marker.
|
|
88
|
+
const CODE_AS_LETTER = 'q';
|
|
89
|
+
const LIST_MARKER_START = /[\s(,;:]/;
|
|
90
|
+
const ROMAN_NUMERAL = /^m{0,3}(cm|cd|d?c{0,3})(xc|xl|l?x{0,3})(ix|iv|v?i{0,3})$/i;
|
|
91
|
+
const ROMAN_MARKER_START = /[(,;]/;
|
|
92
|
+
const ROMAN_AFTER_WORD = /(?:^|[^A-Za-z])(?:option|step|item)\s+$/i;
|
|
93
|
+
const BOLD_CLOSE_NEXT = /[\s.,;:!?)\]"'’”—–]/;
|
|
94
|
+
const SENTENCE_MARK = /[.!?;:,]/;
|
|
95
|
+
const WORD_CODE_OR_CLOSER = /[\p{L}\p{N}`)\]]/u;
|
|
96
|
+
const DASH_OR_SPACE = /[\s—–]/;
|
|
97
|
+
const CLOSING_QUOTE = /['"’”]/;
|
|
98
|
+
export const QUOTE_MARK = /['"‘’“”]/;
|
|
99
|
+
// A "**" that only closes (not after a word), a dual-role one after a sentence mark with no closer,
|
|
100
|
+
// or a bare one opening the text shows the text began inside a bold span.
|
|
101
|
+
function opensInsideBold(text) {
|
|
102
|
+
const [first, ...later] = strongMarkers(blankCodeSpans(text, '`'));
|
|
103
|
+
if (first === undefined)
|
|
104
|
+
return false;
|
|
105
|
+
const next = text[first.index + 2] ?? ' ';
|
|
106
|
+
if (first.close && !first.open) {
|
|
107
|
+
// After a word, code span or closing bracket, "2** (n + 1)" and "(n + 1)** 2" are too often literal to be certain.
|
|
108
|
+
if (WORD_CODE_OR_CLOSER.test(text[first.index - 1] ?? ''))
|
|
109
|
+
return false;
|
|
110
|
+
return BOLD_CLOSE_NEXT.test(next);
|
|
111
|
+
}
|
|
112
|
+
// A quoted "**", "(**.env**)" and "foo**bar**" can open too: only "done.**— then" with no closer is a cut.
|
|
113
|
+
if (first.close) {
|
|
114
|
+
const closerFollows = later.some((marker) => marker.close);
|
|
115
|
+
return SENTENCE_MARK.test(text[first.index - 1]) && DASH_OR_SPACE.test(next) && !closerFollows;
|
|
116
|
+
}
|
|
117
|
+
return text.slice(0, first.index).trim() === '' && /^\s/.test(text.slice(first.index + 2));
|
|
118
|
+
}
|
|
119
|
+
/** "1)", "a)" and "ii)" mark list items; so does a longer valid roman numeral in a label place. */
|
|
120
|
+
function isListMarker(text, close) {
|
|
121
|
+
let start = close;
|
|
122
|
+
while (start > 0 && /[A-Za-z0-9]/.test(text[start - 1]))
|
|
123
|
+
start--;
|
|
124
|
+
const token = text.slice(start, close);
|
|
125
|
+
const before = text[start - 1] ?? '';
|
|
126
|
+
if (token.length <= 3)
|
|
127
|
+
return start === 0 || LIST_MARKER_START.test(before);
|
|
128
|
+
if (!ROMAN_NUMERAL.test(token))
|
|
129
|
+
return false;
|
|
130
|
+
return (start === 0 ||
|
|
131
|
+
ROMAN_MARKER_START.test(before) ||
|
|
132
|
+
ROMAN_AFTER_WORD.test(text.slice(Math.max(0, start - 12), start)));
|
|
133
|
+
}
|
|
134
|
+
// A ")" after a letter, a closing quote or "letter." closes an aside; a quoted ")" does not.
|
|
135
|
+
function closesAside(text, i) {
|
|
136
|
+
const before = text[i - 1] ?? '';
|
|
137
|
+
if (/[A-Za-z]/.test(before))
|
|
138
|
+
return !isListMarker(text, i);
|
|
139
|
+
if (CLOSING_QUOTE.test(before))
|
|
140
|
+
return !QUOTE_MARK.test(text[i + 1] ?? '');
|
|
141
|
+
return before === '.' && /[A-Za-z]/.test(text[i - 2] ?? '');
|
|
142
|
+
}
|
|
143
|
+
/** The offset of the ")" that closes an aside the text began inside, or -1: a ")" before any "(" or
|
|
144
|
+
* open "[". Code spans do not count. */
|
|
145
|
+
export function asideCloseOffset(text) {
|
|
146
|
+
const blanked = blankCodeSpans(text, CODE_AS_LETTER);
|
|
147
|
+
let brackets = 0;
|
|
148
|
+
for (let i = 0; i < blanked.length; i++) {
|
|
149
|
+
if (blanked[i] === '(')
|
|
150
|
+
return -1;
|
|
151
|
+
if (blanked[i] === '[')
|
|
152
|
+
brackets++;
|
|
153
|
+
else if (blanked[i] === ']')
|
|
154
|
+
brackets = Math.max(0, brackets - 1);
|
|
155
|
+
else if (blanked[i] === ')' && brackets === 0 && closesAside(blanked, i))
|
|
156
|
+
return i;
|
|
157
|
+
}
|
|
158
|
+
return -1;
|
|
159
|
+
}
|
|
160
|
+
/** Whether `text` starts inside a bold span or an aside, as rows the old capture segmenter cut do.
|
|
161
|
+
* Repair reads this; admission does not, since capture drops a sentence the split cut out of an aside. */
|
|
162
|
+
export function startsInsideCut(text) {
|
|
163
|
+
return opensInsideBold(text) || asideCloseOffset(text) >= 0;
|
|
164
|
+
}
|
|
82
165
|
function isDanglingAssertion(text) {
|
|
83
166
|
const bare = bareText(text);
|
|
84
167
|
// Only a lowercase start is a cut: "To rebuild the index, run ..." opens an instruction.
|
|
@@ -192,6 +275,12 @@ export function certainDefect(content) {
|
|
|
192
275
|
const { reason } = assessAutomaticMemory(content);
|
|
193
276
|
return isCertainReason(reason) ? reason : null;
|
|
194
277
|
}
|
|
278
|
+
/** The defect repair and the audit find in a stored text: admission's, but a start inside a bold span
|
|
279
|
+
* or an aside outranks an uncertain reason. */
|
|
280
|
+
export function storedDefect(text) {
|
|
281
|
+
const { reason } = assessAutomaticMemory(text);
|
|
282
|
+
return !isCertainReason(reason) && startsInsideCut(text.trim()) ? 'sentence-fragment' : reason;
|
|
283
|
+
}
|
|
195
284
|
// Watch failures ('autolearn') and tool failures keep a fixed "Command 'x' failed" format by
|
|
196
285
|
// design, so they are not judged.
|
|
197
286
|
const AUTOMATIC_SOURCES = new Set(['capture', 'git-learn', 'git', 'consolidation']);
|
|
@@ -235,7 +324,7 @@ export function automaticDefect(entry) {
|
|
|
235
324
|
if (!isAutomaticEntry(entry))
|
|
236
325
|
return null;
|
|
237
326
|
const parts = bundleParts(entry);
|
|
238
|
-
const reasons = (parts.length > 0 ? parts : [entry.content.trim()]).map(
|
|
327
|
+
const reasons = (parts.length > 0 ? parts : [entry.content.trim()]).map(storedDefect);
|
|
239
328
|
return reasons.every((reason) => reason !== null) ? (reasons[0] ?? null) : null;
|
|
240
329
|
}
|
|
241
330
|
/**
|
|
@@ -112,7 +112,8 @@ export interface AuditEvent {
|
|
|
112
112
|
targetId: string | null;
|
|
113
113
|
metadata: JsonObject;
|
|
114
114
|
}
|
|
115
|
-
|
|
115
|
+
/** A stored text the capture dedup reads; `pinned` is optional so an adapter that omits it still fits. */
|
|
116
|
+
export type HeldText = Pick<MemoryEntry, 'content' | 'source' | 'originProject'> & Partial<Pick<MemoryEntry, 'pinned'>>;
|
|
116
117
|
/** One project name, or every name a project's rows carry. */
|
|
117
118
|
export type OriginFilter = string | readonly string[];
|
|
118
119
|
/** The api_keys fields a tenant, revocation, rank or self-service ownership check reads. */
|
|
@@ -5,9 +5,17 @@ export interface PromptRecallGate {
|
|
|
5
5
|
minShared: number;
|
|
6
6
|
maxItems: number;
|
|
7
7
|
}
|
|
8
|
-
/** Distinct content tokens: longer than 2 chars, not a stop word. */
|
|
8
|
+
/** Distinct content tokens: longer than 2 chars or CJK, not a stop word. */
|
|
9
9
|
export declare function contentTokens(text: string): Set<string>;
|
|
10
10
|
export declare function promptTokens(prompt: string): Set<string>;
|
|
11
|
+
/** `contentTokens` of a memory, plus each one-letter CJK token of `prompt` it holds inside a run, so
|
|
12
|
+
* that letter matches as `hippo recall` matches it while the memory's other letters stay out of the
|
|
13
|
+
* overlap's denominator. */
|
|
14
|
+
export declare function memoryTokens(content: string, prompt: ReadonlySet<string>): Set<string>;
|
|
15
|
+
/** The terms a store searches for the prompt: its tokens, with each CJK run as its letter
|
|
16
|
+
* trigrams, which the trigram index can look up across the store, in place of bigrams, which it
|
|
17
|
+
* cannot. */
|
|
18
|
+
export declare function promptSearchTerms(prompt: string): string[];
|
|
11
19
|
export interface OverlapScore {
|
|
12
20
|
score: number;
|
|
13
21
|
shared: number;
|
|
@@ -31,6 +39,11 @@ export declare function rarestPromptTerms(terms: Iterable<string>, docCount: (te
|
|
|
31
39
|
* `mode`. */
|
|
32
40
|
export declare function ftsTermParts(term: string): string[];
|
|
33
41
|
/** The rarest terms as a space-joined FTS query, given each part's document count; a term counts as its rarest part, an
|
|
34
|
-
* upper bound on rows holding the whole term, and a term with no parts counts 0, so it is dropped.
|
|
35
|
-
|
|
42
|
+
* upper bound on rows holding the whole term, and a term with no parts counts 0, so it is dropped. A CJK term counts
|
|
43
|
+
* by `cjkDocCount`, a trigram index's count, in the same pick, and a run under 3 letters that counts 0 follows the
|
|
44
|
+
* pick; without one, CJK terms take `spreadCjkTerms` slots. */
|
|
45
|
+
export declare function rarestFtsQuery(terms: readonly string[], docCount: (part: string) => number, maxTerms?: number, cjkDocCount?: (term: string) => number): string;
|
|
46
|
+
/** Up to `maxTerms` of `terms` when no index counts their CJK terms: those keep half the slots, at even
|
|
47
|
+
* steps through the prompt, so neither a long run of words nor the prompt's opening letters starve them. */
|
|
48
|
+
export declare function spreadCjkTerms(terms: readonly string[], maxTerms: number): string[];
|
|
36
49
|
//# sourceMappingURL=prompt-recall.d.ts.map
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
/** Recall gated on the hook prompt, not the five newest memories (pure, no I/O). */
|
|
2
|
-
import {
|
|
2
|
+
import { cjkIndexTerms, hasCjk, partitionCjkTerms, tokenizeMin } from '../util/tokenize.js';
|
|
3
3
|
import { MEMORY_QUALITY_STOPWORDS } from '../util/stopwords.js';
|
|
4
4
|
// Latency bound, not tuned: fixed in the prereg regardless of gate config.
|
|
5
5
|
const PROMPT_RECALL_MAX_CHARS = 4000;
|
|
6
|
-
/** Distinct content tokens: longer than 2 chars, not a stop word. */
|
|
6
|
+
/** Distinct content tokens: longer than 2 chars or CJK, not a stop word. */
|
|
7
7
|
export function contentTokens(text) {
|
|
8
8
|
const out = new Set();
|
|
9
|
-
for (const t of
|
|
10
|
-
if (
|
|
9
|
+
for (const t of tokenizeMin(text, 3)) {
|
|
10
|
+
if (!MEMORY_QUALITY_STOPWORDS.has(t))
|
|
11
11
|
out.add(t);
|
|
12
12
|
}
|
|
13
13
|
return out;
|
|
@@ -15,6 +15,23 @@ export function contentTokens(text) {
|
|
|
15
15
|
export function promptTokens(prompt) {
|
|
16
16
|
return contentTokens(prompt.slice(0, PROMPT_RECALL_MAX_CHARS));
|
|
17
17
|
}
|
|
18
|
+
/** `contentTokens` of a memory, plus each one-letter CJK token of `prompt` it holds inside a run, so
|
|
19
|
+
* that letter matches as `hippo recall` matches it while the memory's other letters stay out of the
|
|
20
|
+
* overlap's denominator. */
|
|
21
|
+
export function memoryTokens(content, prompt) {
|
|
22
|
+
const out = contentTokens(content);
|
|
23
|
+
for (const t of prompt)
|
|
24
|
+
if ([...t].length === 1 && hasCjk(t) && content.includes(t))
|
|
25
|
+
out.add(t);
|
|
26
|
+
return out;
|
|
27
|
+
}
|
|
28
|
+
/** The terms a store searches for the prompt: its tokens, with each CJK run as its letter
|
|
29
|
+
* trigrams, which the trigram index can look up across the store, in place of bigrams, which it
|
|
30
|
+
* cannot. */
|
|
31
|
+
export function promptSearchTerms(prompt) {
|
|
32
|
+
const text = prompt.slice(0, PROMPT_RECALL_MAX_CHARS);
|
|
33
|
+
return [...partitionCjkTerms(contentTokens(text)).words, ...cjkIndexTerms(text)];
|
|
34
|
+
}
|
|
18
35
|
/** Overlap of two token sets under `metric`. 0 when either set is empty. */
|
|
19
36
|
export function scoreOverlap(p, m, metric) {
|
|
20
37
|
if (p.size === 0 || m.size === 0)
|
|
@@ -61,12 +78,35 @@ export function ftsTermParts(term) {
|
|
|
61
78
|
return term.split(/[^\p{L}\p{N}]+/u).filter(Boolean);
|
|
62
79
|
}
|
|
63
80
|
/** The rarest terms as a space-joined FTS query, given each part's document count; a term counts as its rarest part, an
|
|
64
|
-
* upper bound on rows holding the whole term, and a term with no parts counts 0, so it is dropped.
|
|
65
|
-
|
|
81
|
+
* upper bound on rows holding the whole term, and a term with no parts counts 0, so it is dropped. A CJK term counts
|
|
82
|
+
* by `cjkDocCount`, a trigram index's count, in the same pick, and a run under 3 letters that counts 0 follows the
|
|
83
|
+
* pick; without one, CJK terms take `spreadCjkTerms` slots. */
|
|
84
|
+
export function rarestFtsQuery(terms, docCount, maxTerms = RAREST_TERM_COUNT, cjkDocCount) {
|
|
66
85
|
const termCount = (t) => {
|
|
67
86
|
const parts = ftsTermParts(t);
|
|
68
87
|
return parts.length === 0 ? 0 : Math.min(...parts.map(docCount));
|
|
69
88
|
};
|
|
70
|
-
|
|
89
|
+
const { words, cjk } = partitionCjkTerms(terms);
|
|
90
|
+
if (cjkDocCount === undefined) {
|
|
91
|
+
return spreadCjkTerms([...rarestPromptTerms(words, termCount, maxTerms), ...cjk], maxTerms).join(' ');
|
|
92
|
+
}
|
|
93
|
+
const counts = new Map([
|
|
94
|
+
...words.map((t) => [t, termCount(t)]),
|
|
95
|
+
...cjk.map((t) => [t, cjkDocCount(t)]),
|
|
96
|
+
]);
|
|
97
|
+
const ranked = rarestPromptTerms(counts.keys(), (t) => counts.get(t) ?? 0, maxTerms);
|
|
98
|
+
// A run under 3 letters can sit in a row with no trigram that shows it, so a 0 cannot drop it.
|
|
99
|
+
const unproven = [...new Set(cjk)].filter((t) => counts.get(t) === 0 && [...t].length < 3);
|
|
100
|
+
return [...ranked, ...unproven].slice(0, maxTerms).join(' ');
|
|
101
|
+
}
|
|
102
|
+
/** Up to `maxTerms` of `terms` when no index counts their CJK terms: those keep half the slots, at even
|
|
103
|
+
* steps through the prompt, so neither a long run of words nor the prompt's opening letters starve them. */
|
|
104
|
+
export function spreadCjkTerms(terms, maxTerms) {
|
|
105
|
+
// SHORTCUT: even steps, not rarest first; a store with a trigram index ranks CJK terms by its counts.
|
|
106
|
+
const { words, cjk } = partitionCjkTerms(terms);
|
|
107
|
+
const cjkSlots = Math.min(cjk.length, Math.max(Math.ceil(maxTerms / 2), maxTerms - words.length));
|
|
108
|
+
const step = cjkSlots > 1 ? (cjk.length - 1) / (cjkSlots - 1) : 0;
|
|
109
|
+
const kept = new Set(Array.from({ length: cjkSlots }, (_, i) => Math.round(i * step)));
|
|
110
|
+
return [...words.slice(0, maxTerms - cjkSlots), ...cjk.filter((_, k) => kept.has(k))];
|
|
71
111
|
}
|
|
72
112
|
//# sourceMappingURL=prompt-recall.js.map
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare class DetachedSet {
|
|
2
|
+
#private;
|
|
3
|
+
/** Keeps `work` until settle(); a rejection nothing caught is logged here, never dropped. */
|
|
4
|
+
detach(work: Promise<unknown>): void;
|
|
5
|
+
/** Waits for every kept promise, work detached meanwhile included, up to `withinMs`; returns how many still run. */
|
|
6
|
+
settle(withinMs?: number): Promise<number>;
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=detached-set.d.ts.map
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// Work a caller starts and does not wait on, kept by whoever owns the stores it writes to, so that
|
|
2
|
+
// owner settles it before closing them.
|
|
3
|
+
import { errorFields, errorMessage, log } from './log.js';
|
|
4
|
+
export class DetachedSet {
|
|
5
|
+
#work = new Set();
|
|
6
|
+
/** Keeps `work` until settle(); a rejection nothing caught is logged here, never dropped. */
|
|
7
|
+
detach(work) {
|
|
8
|
+
const kept = work
|
|
9
|
+
.then(() => undefined, (err) => {
|
|
10
|
+
log.error(`detached work failed: ${errorMessage(err)}`, errorFields(err));
|
|
11
|
+
})
|
|
12
|
+
.finally(() => this.#work.delete(kept));
|
|
13
|
+
this.#work.add(kept);
|
|
14
|
+
}
|
|
15
|
+
/** Waits for every kept promise, work detached meanwhile included, up to `withinMs`; returns how many still run. */
|
|
16
|
+
async settle(withinMs) {
|
|
17
|
+
const drained = this.#drain();
|
|
18
|
+
if (withinMs === undefined) {
|
|
19
|
+
await drained;
|
|
20
|
+
return 0;
|
|
21
|
+
}
|
|
22
|
+
let bound;
|
|
23
|
+
const timedOut = new Promise((resolve) => {
|
|
24
|
+
bound = setTimeout(resolve, withinMs);
|
|
25
|
+
});
|
|
26
|
+
await Promise.race([drained, timedOut]);
|
|
27
|
+
clearTimeout(bound);
|
|
28
|
+
return this.#work.size;
|
|
29
|
+
}
|
|
30
|
+
async #drain() {
|
|
31
|
+
while (this.#work.size > 0)
|
|
32
|
+
await Promise.all(this.#work);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=detached-set.js.map
|
|
@@ -1,4 +1,33 @@
|
|
|
1
|
+
/** Han, kana and Hangul, as the body of a `u`-flag character class. */
|
|
2
|
+
export declare const CJK_SCRIPTS: string;
|
|
3
|
+
/** CJK_SCRIPTS and ー, the prolonged-sound mark, a Common-script letter inside katakana words. */
|
|
4
|
+
export declare const CJK_LETTERS: string;
|
|
5
|
+
/** Whether `text` holds a Chinese, Japanese or Korean letter. */
|
|
6
|
+
export declare function hasCjk(text: string): boolean;
|
|
7
|
+
/** The overlapping `n`-letter pieces of each CJK run in `text`; a run of `n` or fewer letters is one
|
|
8
|
+
* piece. */
|
|
9
|
+
export declare function cjkGrams(text: string, n: number): string[];
|
|
10
|
+
/** Each CJK run's distinct letter trigrams, a shorter run whole: the trigram index's terms. */
|
|
11
|
+
export declare function cjkIndexTerms(text: string): string[];
|
|
12
|
+
/** `terms` split by the index that can find them: the word index holds a whole CJK run as one
|
|
13
|
+
* token, so CJK terms skip it for the substring indexes. */
|
|
14
|
+
export declare function partitionCjkTerms(terms: Iterable<string>): {
|
|
15
|
+
words: string[];
|
|
16
|
+
cjk: string[];
|
|
17
|
+
};
|
|
18
|
+
/** `word` with each CJK run in it replaced by the run's letter bigrams: those scripts put no space
|
|
19
|
+
* between words, and a bigram, unlike a dictionary split, matches the same way in query and text. */
|
|
20
|
+
export declare function splitCjkRuns(word: string): string[];
|
|
21
|
+
/** Lower-cased words of `text` (runs of letters, marks, digits and `_`) of 2 or more characters; a
|
|
22
|
+
* CJK run becomes its letter bigrams, a lone CJK letter itself. */
|
|
1
23
|
export declare function tokenize(text: string): string[];
|
|
24
|
+
/** `tokenize` for the text a query is matched against: each letter of a longer CJK run is added,
|
|
25
|
+
* so a one-letter query, which `tokenize` keeps whole, matches inside a run, while a longer query,
|
|
26
|
+
* all bigrams, does not match single letters. */
|
|
27
|
+
export declare function indexTokenize(text: string): string[];
|
|
28
|
+
/** `tokenize` with words shorter than `minLength` dropped; CJK tokens, never longer than 2, are
|
|
29
|
+
* kept. */
|
|
30
|
+
export declare function tokenizeMin(text: string, minLength: number): string[];
|
|
2
31
|
/** Jaccard overlap of two token sets; two empty sets count as identical. */
|
|
3
32
|
export declare function jaccardSets(a: ReadonlySet<string>, b: ReadonlySet<string>): number;
|
|
4
33
|
/** Jaccard overlap of the two texts' token sets. */
|
|
@@ -1,9 +1,72 @@
|
|
|
1
|
+
/** Han, kana and Hangul, as the body of a `u`-flag character class. */
|
|
2
|
+
export const CJK_SCRIPTS = String.raw `\p{sc=Han}\p{sc=Hira}\p{sc=Kana}\p{sc=Hang}`;
|
|
3
|
+
/** CJK_SCRIPTS and ー, the prolonged-sound mark, a Common-script letter inside katakana words. */
|
|
4
|
+
export const CJK_LETTERS = `${CJK_SCRIPTS}ー`;
|
|
5
|
+
const HAS_CJK = new RegExp(`[${CJK_LETTERS}]`, 'u');
|
|
6
|
+
// Letters, marks and digits of every script make words, so Cyrillic, Greek or Arabic text is not
|
|
7
|
+
// cut into spaces and `café` stays one word.
|
|
8
|
+
const NOT_WORD = new RegExp(String.raw `[^\p{L}\p{M}\p{N}_\s${CJK_LETTERS}]`, 'gu');
|
|
9
|
+
const CJK_RUN = new RegExp(`[${CJK_LETTERS}]+`, 'gu');
|
|
10
|
+
const CJK_RUN_SPLIT = new RegExp(`([${CJK_LETTERS}]+)`, 'u');
|
|
11
|
+
/** Whether `text` holds a Chinese, Japanese or Korean letter. */
|
|
12
|
+
export function hasCjk(text) {
|
|
13
|
+
return HAS_CJK.test(text);
|
|
14
|
+
}
|
|
15
|
+
/** The overlapping `n`-letter pieces of each CJK run in `text`; a run of `n` or fewer letters is one
|
|
16
|
+
* piece. */
|
|
17
|
+
export function cjkGrams(text, n) {
|
|
18
|
+
return (text.match(CJK_RUN) ?? []).flatMap((run) => {
|
|
19
|
+
const letters = [...run];
|
|
20
|
+
if (letters.length <= n)
|
|
21
|
+
return [run];
|
|
22
|
+
return letters.slice(n - 1).map((_, i) => letters.slice(i, i + n).join(''));
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
/** Each CJK run's distinct letter trigrams, a shorter run whole: the trigram index's terms. */
|
|
26
|
+
export function cjkIndexTerms(text) {
|
|
27
|
+
return [...new Set(cjkGrams(text, 3))];
|
|
28
|
+
}
|
|
29
|
+
/** `terms` split by the index that can find them: the word index holds a whole CJK run as one
|
|
30
|
+
* token, so CJK terms skip it for the substring indexes. */
|
|
31
|
+
export function partitionCjkTerms(terms) {
|
|
32
|
+
const words = [];
|
|
33
|
+
const cjk = [];
|
|
34
|
+
for (const term of terms)
|
|
35
|
+
(HAS_CJK.test(term) ? cjk : words).push(term);
|
|
36
|
+
return { words, cjk };
|
|
37
|
+
}
|
|
38
|
+
/** `word` with each CJK run in it replaced by the run's letter bigrams: those scripts put no space
|
|
39
|
+
* between words, and a bigram, unlike a dictionary split, matches the same way in query and text. */
|
|
40
|
+
export function splitCjkRuns(word) {
|
|
41
|
+
if (!HAS_CJK.test(word))
|
|
42
|
+
return [word];
|
|
43
|
+
return word
|
|
44
|
+
.split(CJK_RUN_SPLIT)
|
|
45
|
+
.flatMap((part, i) => (i % 2 === 1 ? cjkGrams(part, 2) : part === '' ? [] : [part]));
|
|
46
|
+
}
|
|
47
|
+
/** Lower-cased words of `text` (runs of letters, marks, digits and `_`) of 2 or more characters; a
|
|
48
|
+
* CJK run becomes its letter bigrams, a lone CJK letter itself. */
|
|
1
49
|
export function tokenize(text) {
|
|
2
|
-
return text
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
50
|
+
return tokenizeMin(text, 2);
|
|
51
|
+
}
|
|
52
|
+
/** `tokenize` for the text a query is matched against: each letter of a longer CJK run is added,
|
|
53
|
+
* so a one-letter query, which `tokenize` keeps whole, matches inside a run, while a longer query,
|
|
54
|
+
* all bigrams, does not match single letters. */
|
|
55
|
+
export function indexTokenize(text) {
|
|
56
|
+
const tokens = tokenize(text);
|
|
57
|
+
if (!HAS_CJK.test(text))
|
|
58
|
+
return tokens;
|
|
59
|
+
const letters = (text.match(CJK_RUN) ?? []).flatMap((run) => {
|
|
60
|
+
const runLetters = [...run];
|
|
61
|
+
return runLetters.length > 1 ? runLetters : [];
|
|
62
|
+
});
|
|
63
|
+
return [...tokens, ...letters];
|
|
64
|
+
}
|
|
65
|
+
/** `tokenize` with words shorter than `minLength` dropped; CJK tokens, never longer than 2, are
|
|
66
|
+
* kept. */
|
|
67
|
+
export function tokenizeMin(text, minLength) {
|
|
68
|
+
const words = text.toLowerCase().replace(NOT_WORD, ' ').split(/\s+/);
|
|
69
|
+
return (HAS_CJK.test(text) ? words.flatMap(splitCjkRuns) : words).filter((t) => t.length >= minLength || HAS_CJK.test(t));
|
|
7
70
|
}
|
|
8
71
|
/** Jaccard overlap of two token sets; two empty sets count as identical. */
|
|
9
72
|
export function jaccardSets(a, b) {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Single source of truth for the package version, written by scripts/sync-version.mjs; never
|
|
3
3
|
* edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
|
|
4
4
|
* package.json read: the published bundle may lack package.json on a path ESM can resolve. */
|
|
5
|
-
export declare const PACKAGE_VERSION = "3.
|
|
5
|
+
export declare const PACKAGE_VERSION = "3.1.1";
|
|
6
6
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by
|
|
7
7
|
* hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
|
|
8
8
|
export declare const EXPIRING_KEYS_MIN_BINARY = "1.64.0";
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Single source of truth for the package version, written by scripts/sync-version.mjs; never
|
|
3
3
|
* edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
|
|
4
4
|
* package.json read: the published bundle may lack package.json on a path ESM can resolve. */
|
|
5
|
-
export const PACKAGE_VERSION = '3.
|
|
5
|
+
export const PACKAGE_VERSION = '3.1.1';
|
|
6
6
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by
|
|
7
7
|
* hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
|
|
8
8
|
export const EXPIRING_KEYS_MIN_BINARY = '1.64.0';
|
package/dist/db/meta.d.ts
CHANGED
|
@@ -17,6 +17,7 @@ export declare const META_KEY: {
|
|
|
17
17
|
readonly sleepCount: "sleep_count";
|
|
18
18
|
readonly legacyBootstrapCompleted: "legacy_bootstrap_completed";
|
|
19
19
|
readonly qualityRepairAuto: "quality_repair_auto";
|
|
20
|
+
readonly qualityRepairGate: "quality_repair_gate";
|
|
20
21
|
readonly projectRepairAuto: "project_repair_auto";
|
|
21
22
|
};
|
|
22
23
|
export declare function ensureMetaTable(db: DatabaseSyncLike): void;
|
package/dist/db/meta.js
CHANGED
|
@@ -24,6 +24,7 @@ export const META_KEY = {
|
|
|
24
24
|
sleepCount: 'sleep_count',
|
|
25
25
|
legacyBootstrapCompleted: 'legacy_bootstrap_completed',
|
|
26
26
|
qualityRepairAuto: 'quality_repair_auto',
|
|
27
|
+
qualityRepairGate: 'quality_repair_gate',
|
|
27
28
|
projectRepairAuto: 'project_repair_auto',
|
|
28
29
|
};
|
|
29
30
|
export function ensureMetaTable(db) {
|
|
@@ -37,9 +37,16 @@ export declare class StoreRuntime {
|
|
|
37
37
|
giveIdle(poolKey: string, db: DatabaseSyncLike): boolean;
|
|
38
38
|
/** Closes every idle handle, so a held connection can be the store's last. */
|
|
39
39
|
closeIdle(): void;
|
|
40
|
-
/**
|
|
41
|
-
|
|
42
|
-
|
|
40
|
+
/** Keeps work nothing waits on, such as a command's embed, until dispose settles it. */
|
|
41
|
+
detach(work: Promise<unknown>): void;
|
|
42
|
+
/** Waits for the detached work, up to `withinMs` when given, for a root that closes its stores
|
|
43
|
+
* itself; resolves with how many were still running. */
|
|
44
|
+
settle(withinMs?: number): Promise<number>;
|
|
45
|
+
/** Settles the detached work, which may still write through the stores, then runs closeNow. */
|
|
46
|
+
dispose(): Promise<void>;
|
|
47
|
+
/** Runs the held writes, then closes the open sessions and the idle handles, each step even if
|
|
48
|
+
* the one before throws; it skips detached work, so only a root that cannot wait calls it. */
|
|
49
|
+
closeNow(): void;
|
|
43
50
|
}
|
|
44
51
|
export {};
|
|
45
52
|
//# sourceMappingURL=store-runtime.d.ts.map
|
package/dist/db/store-runtime.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import * as path from 'node:path';
|
|
6
6
|
import { errorFields, errorMessage, log } from '../base/util/log.js';
|
|
7
7
|
import { HeldWrites } from './held-writes.js';
|
|
8
|
+
import { DetachedSet } from '../base/util/detached-set.js';
|
|
8
9
|
// Per pool key: a server answers several requests at once, and each closing session hands its
|
|
9
10
|
// handle back.
|
|
10
11
|
const IDLE_PER_KEY = 4;
|
|
@@ -20,6 +21,7 @@ export class StoreRuntime {
|
|
|
20
21
|
#idle = new Map();
|
|
21
22
|
#background = new Set();
|
|
22
23
|
#stats = new Map();
|
|
24
|
+
#detached = new DetachedSet();
|
|
23
25
|
/** Whether `step` has run to its end on the store at `key`. */
|
|
24
26
|
hasRun(key, step) {
|
|
25
27
|
return this.#settled.get(key)?.has(step) ?? false;
|
|
@@ -96,9 +98,23 @@ export class StoreRuntime {
|
|
|
96
98
|
closeQuietly(db);
|
|
97
99
|
this.#idle.clear();
|
|
98
100
|
}
|
|
99
|
-
/**
|
|
100
|
-
|
|
101
|
-
|
|
101
|
+
/** Keeps work nothing waits on, such as a command's embed, until dispose settles it. */
|
|
102
|
+
detach(work) {
|
|
103
|
+
this.#detached.detach(work);
|
|
104
|
+
}
|
|
105
|
+
/** Waits for the detached work, up to `withinMs` when given, for a root that closes its stores
|
|
106
|
+
* itself; resolves with how many were still running. */
|
|
107
|
+
settle(withinMs) {
|
|
108
|
+
return this.#detached.settle(withinMs);
|
|
109
|
+
}
|
|
110
|
+
/** Settles the detached work, which may still write through the stores, then runs closeNow. */
|
|
111
|
+
async dispose() {
|
|
112
|
+
await this.settle();
|
|
113
|
+
this.closeNow();
|
|
114
|
+
}
|
|
115
|
+
/** Runs the held writes, then closes the open sessions and the idle handles, each step even if
|
|
116
|
+
* the one before throws; it skips detached work, so only a root that cannot wait calls it. */
|
|
117
|
+
closeNow() {
|
|
102
118
|
try {
|
|
103
119
|
this.held.flush();
|
|
104
120
|
}
|
|
@@ -4,7 +4,7 @@ import { jaccardMinShared, overlapPartners } from './overlap-index.js';
|
|
|
4
4
|
import { isQuarantineScope } from '../trust/quarantine.js';
|
|
5
5
|
import { isPersonalScope } from '../../base/scope/recall-scope.js';
|
|
6
6
|
import { CONFLICT_STOPWORDS } from '../../base/util/stopwords.js';
|
|
7
|
-
import { jaccardSets,
|
|
7
|
+
import { jaccardSets, tokenizeMin } from '../../base/util/tokenize.js';
|
|
8
8
|
import { DECAY_THRESHOLD } from './decay.js';
|
|
9
9
|
import { BadRequestError } from '../../base/errors/api-errors.js';
|
|
10
10
|
// Gate contradictions on content overlap, not shared tags (`feedback`/`policy` are too coarse). The
|
|
@@ -144,7 +144,7 @@ function describeConflict(a, b) {
|
|
|
144
144
|
};
|
|
145
145
|
}
|
|
146
146
|
function distinctiveTokens(text) {
|
|
147
|
-
return new Set(
|
|
147
|
+
return new Set(tokenizeMin(text, 3).filter((t) => !CONFLICT_STOPWORDS.has(t)));
|
|
148
148
|
}
|
|
149
149
|
function openingWindow(text) {
|
|
150
150
|
return text.split(/\s+/).slice(0, POLARITY_WINDOW_WORDS).join(' ');
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* noise dominates hint UX; `recall_autodebias_hint_no_class_match` audits are the telemetry.
|
|
5
5
|
*/
|
|
6
6
|
import { FORWARD_CLAIM_STOPWORDS } from '../../base/util/stopwords.js';
|
|
7
|
+
import { CJK_LETTERS } from '../../base/util/tokenize.js';
|
|
7
8
|
/**
|
|
8
9
|
* Each pattern is deliberately narrow (add one only on audit evidence); word boundaries are
|
|
9
10
|
* mandatory so `ETA` never matches inside `BETAtesting`.
|
|
@@ -49,11 +50,12 @@ export function detectForwardClaim(queryText) {
|
|
|
49
50
|
}
|
|
50
51
|
return null;
|
|
51
52
|
}
|
|
53
|
+
const NOT_CLASS_WORD = new RegExp(String.raw `[^a-z0-9\s_${CJK_LETTERS}-]`, 'gu');
|
|
52
54
|
function extractClassQueryTokens(queryText) {
|
|
53
55
|
return (queryText
|
|
54
56
|
.toLowerCase()
|
|
55
|
-
// Strip everything that's not alphanum
|
|
56
|
-
.replace(
|
|
57
|
+
// Strip everything that's not alphanum, CJK, whitespace, hyphen or underscore.
|
|
58
|
+
.replace(NOT_CLASS_WORD, ' ')
|
|
57
59
|
// Split on whitespace, hyphen and underscore, matching the class-tag split in resolveClass.
|
|
58
60
|
.split(/[\s_-]+/)
|
|
59
61
|
.filter((t) => t.length >= 3 && !FORWARD_CLAIM_STOPWORDS.has(t) && !/^\d+$/.test(t)));
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { INVALIDATION_STOPWORDS } from '../../base/util/stopwords.js';
|
|
2
|
+
import { CJK_LETTERS, hasCjk, splitCjkRuns } from '../../base/util/tokenize.js';
|
|
2
3
|
import { READ_PAGE_ROWS } from '../../base/util/pages.js';
|
|
3
4
|
const HEADLINE_CHARS = 60;
|
|
4
5
|
// The share of a row's tokens that must match the target's before the row counts as mentioning it.
|
|
@@ -170,12 +171,14 @@ function weakenEntry(entry) {
|
|
|
170
171
|
entry.tags.push('invalidated');
|
|
171
172
|
}
|
|
172
173
|
}
|
|
174
|
+
const NOT_INVALIDATION_WORD = new RegExp(String.raw `[^a-z0-9\s_.${CJK_LETTERS}-]`, 'gu');
|
|
173
175
|
function invalidationTokenize(text) {
|
|
174
176
|
return text
|
|
175
177
|
.toLowerCase()
|
|
176
|
-
.replace(
|
|
178
|
+
.replace(NOT_INVALIDATION_WORD, ' ')
|
|
177
179
|
.split(/\s+/)
|
|
178
|
-
.
|
|
180
|
+
.flatMap(splitCjkRuns)
|
|
181
|
+
.filter((t) => (t.length >= 2 || hasCjk(t)) && !INVALIDATION_STOPWORDS.has(t));
|
|
179
182
|
}
|
|
180
183
|
function matchScore(fromTokens, contentTokens) {
|
|
181
184
|
if (fromTokens.length === 0)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { indexTokenize } from '../../base/util/tokenize.js';
|
|
2
2
|
const BM25_K1 = 1.5;
|
|
3
3
|
const BM25_B = 0.75;
|
|
4
4
|
/** The text BM25 indexes for an entry: its content followed by its tags. */
|
|
@@ -6,7 +6,7 @@ export function entryText(entry) {
|
|
|
6
6
|
return `${entry.content} ${entry.tags.join(' ')}`;
|
|
7
7
|
}
|
|
8
8
|
export function buildCorpus(texts) {
|
|
9
|
-
const docs = texts.map(
|
|
9
|
+
const docs = texts.map((text) => indexTokenize(text));
|
|
10
10
|
const N = docs.length;
|
|
11
11
|
const df = new Map();
|
|
12
12
|
let totalLen = 0;
|
|
@@ -44,7 +44,7 @@ export function bm25Score(corpus, docIdx, queryTerms) {
|
|
|
44
44
|
}
|
|
45
45
|
/** Query terms that appear in the entry's indexed text, in query order. */
|
|
46
46
|
export function matchedQueryTerms(queryTerms, entry) {
|
|
47
|
-
const docTerms = new Set(
|
|
47
|
+
const docTerms = new Set(indexTokenize(entryText(entry)));
|
|
48
48
|
const matched = [];
|
|
49
49
|
for (const t of queryTerms)
|
|
50
50
|
if (docTerms.has(t))
|