hippo-memory 3.1.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/context/context-plan.js +2 -4
- package/dist/api/context/context-select.js +12 -7
- package/dist/api/recall/rank-stages.js +2 -2
- package/dist/base/memory/entry.js +2 -2
- package/dist/base/memory/markdown-delimiters.d.ts +15 -0
- package/dist/base/memory/markdown-delimiters.js +94 -0
- package/dist/base/memory/memory-audit.js +4 -4
- package/dist/base/memory/memory-quality.d.ts +12 -0
- package/dist/base/memory/memory-quality.js +92 -3
- package/dist/base/ports/store/types.d.ts +2 -1
- package/dist/base/rank/prompt-recall.d.ts +16 -3
- package/dist/base/rank/prompt-recall.js +47 -7
- package/dist/base/util/tokenize.d.ts +29 -0
- package/dist/base/util/tokenize.js +68 -5
- package/dist/base/util/version.d.ts +1 -1
- package/dist/base/util/version.js +1 -1
- package/dist/db/meta.d.ts +1 -0
- package/dist/db/meta.js +1 -0
- package/dist/domain/consolidate/conflicts.js +2 -2
- package/dist/domain/learn/forward-claim-detector.js +4 -2
- package/dist/domain/learn/invalidation.js +5 -2
- package/dist/domain/search/bm25.js +3 -3
- package/dist/store/rules/windows.d.ts +3 -0
- package/dist/store/rules/windows.js +3 -0
- package/dist/store/sqlite/audit/audit-maintenance.js +16 -13
- package/dist/store/sqlite/recall/candidates.js +1 -0
- package/dist/store/sqlite/recall/search-rows.d.ts +2 -2
- package/dist/store/sqlite/recall/search-rows.js +62 -26
- package/dist/store/sqlite/sharing/copy-rows.d.ts +1 -0
- package/dist/store/sqlite/sharing/copy-rows.js +2 -1
- package/dist/surface/capture/command.d.ts +8 -1
- package/dist/surface/capture/command.js +38 -8
- package/dist/surface/capture/extract.d.ts +10 -0
- package/dist/surface/capture/extract.js +199 -26
- package/dist/surface/capture/session-texts.js +8 -6
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
|
@@ -1,9 +1,72 @@
|
|
|
1
|
+
/** Han, kana and Hangul, as the body of a `u`-flag character class. */
|
|
2
|
+
export const CJK_SCRIPTS = String.raw `\p{sc=Han}\p{sc=Hira}\p{sc=Kana}\p{sc=Hang}`;
|
|
3
|
+
/** CJK_SCRIPTS and ー, the prolonged-sound mark, a Common-script letter inside katakana words. */
|
|
4
|
+
export const CJK_LETTERS = `${CJK_SCRIPTS}ー`;
|
|
5
|
+
const HAS_CJK = new RegExp(`[${CJK_LETTERS}]`, 'u');
|
|
6
|
+
// Letters, marks and digits of every script make words, so Cyrillic, Greek or Arabic text is not
|
|
7
|
+
// cut into spaces and `café` stays one word.
|
|
8
|
+
const NOT_WORD = new RegExp(String.raw `[^\p{L}\p{M}\p{N}_\s${CJK_LETTERS}]`, 'gu');
|
|
9
|
+
const CJK_RUN = new RegExp(`[${CJK_LETTERS}]+`, 'gu');
|
|
10
|
+
const CJK_RUN_SPLIT = new RegExp(`([${CJK_LETTERS}]+)`, 'u');
|
|
11
|
+
/** Whether `text` holds a Chinese, Japanese or Korean letter. */
|
|
12
|
+
export function hasCjk(text) {
|
|
13
|
+
return HAS_CJK.test(text);
|
|
14
|
+
}
|
|
15
|
+
/** The overlapping `n`-letter pieces of each CJK run in `text`; a run of `n` or fewer letters is one
|
|
16
|
+
* piece. */
|
|
17
|
+
export function cjkGrams(text, n) {
|
|
18
|
+
return (text.match(CJK_RUN) ?? []).flatMap((run) => {
|
|
19
|
+
const letters = [...run];
|
|
20
|
+
if (letters.length <= n)
|
|
21
|
+
return [run];
|
|
22
|
+
return letters.slice(n - 1).map((_, i) => letters.slice(i, i + n).join(''));
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
/** Each CJK run's distinct letter trigrams, a shorter run whole: the trigram index's terms. */
|
|
26
|
+
export function cjkIndexTerms(text) {
|
|
27
|
+
return [...new Set(cjkGrams(text, 3))];
|
|
28
|
+
}
|
|
29
|
+
/** `terms` split by the index that can find them: the word index holds a whole CJK run as one
|
|
30
|
+
* token, so CJK terms skip it for the substring indexes. */
|
|
31
|
+
export function partitionCjkTerms(terms) {
|
|
32
|
+
const words = [];
|
|
33
|
+
const cjk = [];
|
|
34
|
+
for (const term of terms)
|
|
35
|
+
(HAS_CJK.test(term) ? cjk : words).push(term);
|
|
36
|
+
return { words, cjk };
|
|
37
|
+
}
|
|
38
|
+
/** `word` with each CJK run in it replaced by the run's letter bigrams: those scripts put no space
|
|
39
|
+
* between words, and a bigram, unlike a dictionary split, matches the same way in query and text. */
|
|
40
|
+
export function splitCjkRuns(word) {
|
|
41
|
+
if (!HAS_CJK.test(word))
|
|
42
|
+
return [word];
|
|
43
|
+
return word
|
|
44
|
+
.split(CJK_RUN_SPLIT)
|
|
45
|
+
.flatMap((part, i) => (i % 2 === 1 ? cjkGrams(part, 2) : part === '' ? [] : [part]));
|
|
46
|
+
}
|
|
47
|
+
/** Lower-cased words of `text` (runs of letters, marks, digits and `_`) of 2 or more characters; a
|
|
48
|
+
* CJK run becomes its letter bigrams, a lone CJK letter itself. */
|
|
1
49
|
export function tokenize(text) {
|
|
2
|
-
return text
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
50
|
+
return tokenizeMin(text, 2);
|
|
51
|
+
}
|
|
52
|
+
/** `tokenize` for the text a query is matched against: each letter of a longer CJK run is added,
|
|
53
|
+
* so a one-letter query, which `tokenize` keeps whole, matches inside a run, while a longer query,
|
|
54
|
+
* all bigrams, does not match single letters. */
|
|
55
|
+
export function indexTokenize(text) {
|
|
56
|
+
const tokens = tokenize(text);
|
|
57
|
+
if (!HAS_CJK.test(text))
|
|
58
|
+
return tokens;
|
|
59
|
+
const letters = (text.match(CJK_RUN) ?? []).flatMap((run) => {
|
|
60
|
+
const runLetters = [...run];
|
|
61
|
+
return runLetters.length > 1 ? runLetters : [];
|
|
62
|
+
});
|
|
63
|
+
return [...tokens, ...letters];
|
|
64
|
+
}
|
|
65
|
+
/** `tokenize` with words shorter than `minLength` dropped; CJK tokens, never longer than 2, are
|
|
66
|
+
* kept. */
|
|
67
|
+
export function tokenizeMin(text, minLength) {
|
|
68
|
+
const words = text.toLowerCase().replace(NOT_WORD, ' ').split(/\s+/);
|
|
69
|
+
return (HAS_CJK.test(text) ? words.flatMap(splitCjkRuns) : words).filter((t) => t.length >= minLength || HAS_CJK.test(t));
|
|
7
70
|
}
|
|
8
71
|
/** Jaccard overlap of two token sets; two empty sets count as identical. */
|
|
9
72
|
export function jaccardSets(a, b) {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Single source of truth for the package version, written by scripts/sync-version.mjs; never
|
|
3
3
|
* edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
|
|
4
4
|
* package.json read: the published bundle may lack package.json on a path ESM can resolve. */
|
|
5
|
-
export declare const PACKAGE_VERSION = "3.1.
|
|
5
|
+
export declare const PACKAGE_VERSION = "3.1.1";
|
|
6
6
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by
|
|
7
7
|
* hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
|
|
8
8
|
export declare const EXPIRING_KEYS_MIN_BINARY = "1.64.0";
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Single source of truth for the package version, written by scripts/sync-version.mjs; never
|
|
3
3
|
* edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
|
|
4
4
|
* package.json read: the published bundle may lack package.json on a path ESM can resolve. */
|
|
5
|
-
export const PACKAGE_VERSION = '3.1.
|
|
5
|
+
export const PACKAGE_VERSION = '3.1.1';
|
|
6
6
|
/** The floor a store takes on its first expiring key: the first release with schema v53, set by
|
|
7
7
|
* hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
|
|
8
8
|
export const EXPIRING_KEYS_MIN_BINARY = '1.64.0';
|
package/dist/db/meta.d.ts
CHANGED
|
@@ -17,6 +17,7 @@ export declare const META_KEY: {
|
|
|
17
17
|
readonly sleepCount: "sleep_count";
|
|
18
18
|
readonly legacyBootstrapCompleted: "legacy_bootstrap_completed";
|
|
19
19
|
readonly qualityRepairAuto: "quality_repair_auto";
|
|
20
|
+
readonly qualityRepairGate: "quality_repair_gate";
|
|
20
21
|
readonly projectRepairAuto: "project_repair_auto";
|
|
21
22
|
};
|
|
22
23
|
export declare function ensureMetaTable(db: DatabaseSyncLike): void;
|
package/dist/db/meta.js
CHANGED
|
@@ -24,6 +24,7 @@ export const META_KEY = {
|
|
|
24
24
|
sleepCount: 'sleep_count',
|
|
25
25
|
legacyBootstrapCompleted: 'legacy_bootstrap_completed',
|
|
26
26
|
qualityRepairAuto: 'quality_repair_auto',
|
|
27
|
+
qualityRepairGate: 'quality_repair_gate',
|
|
27
28
|
projectRepairAuto: 'project_repair_auto',
|
|
28
29
|
};
|
|
29
30
|
export function ensureMetaTable(db) {
|
|
@@ -4,7 +4,7 @@ import { jaccardMinShared, overlapPartners } from './overlap-index.js';
|
|
|
4
4
|
import { isQuarantineScope } from '../trust/quarantine.js';
|
|
5
5
|
import { isPersonalScope } from '../../base/scope/recall-scope.js';
|
|
6
6
|
import { CONFLICT_STOPWORDS } from '../../base/util/stopwords.js';
|
|
7
|
-
import { jaccardSets,
|
|
7
|
+
import { jaccardSets, tokenizeMin } from '../../base/util/tokenize.js';
|
|
8
8
|
import { DECAY_THRESHOLD } from './decay.js';
|
|
9
9
|
import { BadRequestError } from '../../base/errors/api-errors.js';
|
|
10
10
|
// Gate contradictions on content overlap, not shared tags (`feedback`/`policy` are too coarse). The
|
|
@@ -144,7 +144,7 @@ function describeConflict(a, b) {
|
|
|
144
144
|
};
|
|
145
145
|
}
|
|
146
146
|
function distinctiveTokens(text) {
|
|
147
|
-
return new Set(
|
|
147
|
+
return new Set(tokenizeMin(text, 3).filter((t) => !CONFLICT_STOPWORDS.has(t)));
|
|
148
148
|
}
|
|
149
149
|
function openingWindow(text) {
|
|
150
150
|
return text.split(/\s+/).slice(0, POLARITY_WINDOW_WORDS).join(' ');
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* noise dominates hint UX; `recall_autodebias_hint_no_class_match` audits are the telemetry.
|
|
5
5
|
*/
|
|
6
6
|
import { FORWARD_CLAIM_STOPWORDS } from '../../base/util/stopwords.js';
|
|
7
|
+
import { CJK_LETTERS } from '../../base/util/tokenize.js';
|
|
7
8
|
/**
|
|
8
9
|
* Each pattern is deliberately narrow (add one only on audit evidence); word boundaries are
|
|
9
10
|
* mandatory so `ETA` never matches inside `BETAtesting`.
|
|
@@ -49,11 +50,12 @@ export function detectForwardClaim(queryText) {
|
|
|
49
50
|
}
|
|
50
51
|
return null;
|
|
51
52
|
}
|
|
53
|
+
const NOT_CLASS_WORD = new RegExp(String.raw `[^a-z0-9\s_${CJK_LETTERS}-]`, 'gu');
|
|
52
54
|
function extractClassQueryTokens(queryText) {
|
|
53
55
|
return (queryText
|
|
54
56
|
.toLowerCase()
|
|
55
|
-
// Strip everything that's not alphanum
|
|
56
|
-
.replace(
|
|
57
|
+
// Strip everything that's not alphanum, CJK, whitespace, hyphen or underscore.
|
|
58
|
+
.replace(NOT_CLASS_WORD, ' ')
|
|
57
59
|
// Split on whitespace, hyphen and underscore, matching the class-tag split in resolveClass.
|
|
58
60
|
.split(/[\s_-]+/)
|
|
59
61
|
.filter((t) => t.length >= 3 && !FORWARD_CLAIM_STOPWORDS.has(t) && !/^\d+$/.test(t)));
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { INVALIDATION_STOPWORDS } from '../../base/util/stopwords.js';
|
|
2
|
+
import { CJK_LETTERS, hasCjk, splitCjkRuns } from '../../base/util/tokenize.js';
|
|
2
3
|
import { READ_PAGE_ROWS } from '../../base/util/pages.js';
|
|
3
4
|
const HEADLINE_CHARS = 60;
|
|
4
5
|
// The share of a row's tokens that must match the target's before the row counts as mentioning it.
|
|
@@ -170,12 +171,14 @@ function weakenEntry(entry) {
|
|
|
170
171
|
entry.tags.push('invalidated');
|
|
171
172
|
}
|
|
172
173
|
}
|
|
174
|
+
const NOT_INVALIDATION_WORD = new RegExp(String.raw `[^a-z0-9\s_.${CJK_LETTERS}-]`, 'gu');
|
|
173
175
|
function invalidationTokenize(text) {
|
|
174
176
|
return text
|
|
175
177
|
.toLowerCase()
|
|
176
|
-
.replace(
|
|
178
|
+
.replace(NOT_INVALIDATION_WORD, ' ')
|
|
177
179
|
.split(/\s+/)
|
|
178
|
-
.
|
|
180
|
+
.flatMap(splitCjkRuns)
|
|
181
|
+
.filter((t) => (t.length >= 2 || hasCjk(t)) && !INVALIDATION_STOPWORDS.has(t));
|
|
179
182
|
}
|
|
180
183
|
function matchScore(fromTokens, contentTokens) {
|
|
181
184
|
if (fromTokens.length === 0)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { indexTokenize } from '../../base/util/tokenize.js';
|
|
2
2
|
const BM25_K1 = 1.5;
|
|
3
3
|
const BM25_B = 0.75;
|
|
4
4
|
/** The text BM25 indexes for an entry: its content followed by its tags. */
|
|
@@ -6,7 +6,7 @@ export function entryText(entry) {
|
|
|
6
6
|
return `${entry.content} ${entry.tags.join(' ')}`;
|
|
7
7
|
}
|
|
8
8
|
export function buildCorpus(texts) {
|
|
9
|
-
const docs = texts.map(
|
|
9
|
+
const docs = texts.map((text) => indexTokenize(text));
|
|
10
10
|
const N = docs.length;
|
|
11
11
|
const df = new Map();
|
|
12
12
|
let totalLen = 0;
|
|
@@ -44,7 +44,7 @@ export function bm25Score(corpus, docIdx, queryTerms) {
|
|
|
44
44
|
}
|
|
45
45
|
/** Query terms that appear in the entry's indexed text, in query order. */
|
|
46
46
|
export function matchedQueryTerms(queryTerms, entry) {
|
|
47
|
-
const docTerms = new Set(
|
|
47
|
+
const docTerms = new Set(indexTokenize(entryText(entry)));
|
|
48
48
|
const matched = [];
|
|
49
49
|
for (const t of queryTerms)
|
|
50
50
|
if (docTerms.has(t))
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
/** Newest admitted rows LIKE tests for a term under 3 characters, which no trigram lookup can
|
|
2
2
|
* answer. */
|
|
3
3
|
export declare const LIKE_WINDOW_ROWS = 2000;
|
|
4
|
+
/** Most terms LIKE tests on each of those rows: the trigram index looks up every longer term over
|
|
5
|
+
* the whole store, so a long CJK query's 1,000s of bigrams need not all be tested here. */
|
|
6
|
+
export declare const LIKE_WINDOW_TERMS = 64;
|
|
4
7
|
/** How many of a tenant's newest rows schemaFit reads; older rows no longer move the fit. */
|
|
5
8
|
export declare const SCHEMA_FIT_WINDOW_ROWS = 2000;
|
|
6
9
|
/** Events of one completed session a sleep reads to render its trace, the newest kept. */
|
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
/** Newest admitted rows LIKE tests for a term under 3 characters, which no trigram lookup can
|
|
4
4
|
* answer. */
|
|
5
5
|
export const LIKE_WINDOW_ROWS = 2000;
|
|
6
|
+
/** Most terms LIKE tests on each of those rows: the trigram index looks up every longer term over
|
|
7
|
+
* the whole store, so a long CJK query's 1,000s of bigrams need not all be tested here. */
|
|
8
|
+
export const LIKE_WINDOW_TERMS = 64;
|
|
6
9
|
/** How many of a tenant's newest rows schemaFit reads; older rows no longer move the fit. */
|
|
7
10
|
export const SCHEMA_FIT_WINDOW_ROWS = 2000;
|
|
8
11
|
/** Events of one completed session a sleep reads to render its trace, the newest kept. */
|
|
@@ -17,7 +17,8 @@ import { purgeMirrorBestEffort } from '../folder-files/mirrors.js';
|
|
|
17
17
|
import { openStore } from '../runtime/handles.js';
|
|
18
18
|
import { answered, bound, skipEmpty } from '../runtime/bound.js';
|
|
19
19
|
import { READ_PAGE_ROWS } from '../../../base/util/pages.js';
|
|
20
|
-
import { getMeta, pragmaUserVersion, setMeta, META_KEY } from '../../../db/meta.js';
|
|
20
|
+
import { getMeta, getMetaNumber, pragmaUserVersion, setMeta, META_KEY } from '../../../db/meta.js';
|
|
21
|
+
import { QUALITY_GATE_VERSION } from '../../../base/memory/memory-quality.js';
|
|
21
22
|
// Quality repair's issue list: which rows are defects and which of those a protection keeps.
|
|
22
23
|
/** The issues among `entries` in their order, each defect kept when `protect` names the row. */
|
|
23
24
|
function repairIssues(entries, parents, backing, protect) {
|
|
@@ -223,8 +224,8 @@ function applyPlan(db, { run, plan, backup, rules }) {
|
|
|
223
224
|
else
|
|
224
225
|
warnings.push(`Kept ${issue.id}: the store's delete guard protects it.`);
|
|
225
226
|
}
|
|
226
|
-
|
|
227
|
-
setMeta(db,
|
|
227
|
+
for (const [key, value] of run.doneWrites ?? [])
|
|
228
|
+
setMeta(db, key, value);
|
|
228
229
|
return { ...plan, issues, appliedIds, backup, warnings };
|
|
229
230
|
});
|
|
230
231
|
}
|
|
@@ -248,21 +249,23 @@ function repairOn(db, files, run) {
|
|
|
248
249
|
}
|
|
249
250
|
return { ...result, warnings };
|
|
250
251
|
}
|
|
251
|
-
|
|
252
|
+
// The old key stays '1' so a binary from before the gate version keeps skipping the repair.
|
|
253
|
+
const DONE_WRITES = [
|
|
254
|
+
[META_KEY.qualityRepairAuto, '1'],
|
|
255
|
+
[META_KEY.qualityRepairGate, String(QUALITY_GATE_VERSION)],
|
|
256
|
+
];
|
|
257
|
+
/** Applies the repair once per gate version, so stores that predate a quality check are cleaned on
|
|
252
258
|
* upgrade with no command; null once done. `db` is opened writable without running migrations. */
|
|
253
259
|
function repairQualityOnce(db, files, tenantId, protect) {
|
|
254
|
-
if (
|
|
260
|
+
if (getMetaNumber(db, META_KEY.qualityRepairGate) >= QUALITY_GATE_VERSION)
|
|
255
261
|
return null;
|
|
256
262
|
// The apply commits the flag with the moves, so a lock taken between them cannot hide what moved.
|
|
257
|
-
const result = repairOn(db, files, {
|
|
258
|
-
tenantId,
|
|
259
|
-
protect,
|
|
260
|
-
apply: true,
|
|
261
|
-
doneKey: META_KEY.qualityRepairAuto,
|
|
262
|
-
});
|
|
263
|
+
const result = repairOn(db, files, { tenantId, protect, apply: true, doneWrites: DONE_WRITES });
|
|
263
264
|
// An unsupported schema changed nothing, so the next run tries again once a migration has run.
|
|
264
|
-
if (result.supported && result.backup === null)
|
|
265
|
-
|
|
265
|
+
if (result.supported && result.backup === null) {
|
|
266
|
+
for (const [key, value] of DONE_WRITES)
|
|
267
|
+
setMeta(db, key, value);
|
|
268
|
+
}
|
|
266
269
|
return result;
|
|
267
270
|
}
|
|
268
271
|
export function sqliteAuditMaintenance({ on, files }) {
|
|
@@ -16,7 +16,7 @@ export declare function listRecallEntries(db: DatabaseSyncLike, query: string, o
|
|
|
16
16
|
export declare function listVectorCandidateEntries(db: DatabaseSyncLike, queryVector: readonly number[], spec: VectorCandidateSpec, index?: VectorIndex): Promise<MemoryEntry[]>;
|
|
17
17
|
/** The rows of `ids` in that order; an id deleted since it was matched is skipped. */
|
|
18
18
|
export declare function entriesInOrder(db: DatabaseSyncLike, ids: readonly string[]): MemoryEntry[];
|
|
19
|
-
/** Rarest-K prompt terms for this connection's FTS
|
|
20
|
-
*
|
|
19
|
+
/** Rarest-K prompt terms for this connection's FTS indexes, as a space-joined query string: words
|
|
20
|
+
* count in the word index, CJK terms in the trigram index. Without FTS, the first 32 terms. */
|
|
21
21
|
export declare function pickRarestFtsQuery(db: DatabaseSyncLike, terms: readonly string[], maxTerms?: number): string;
|
|
22
22
|
//# sourceMappingURL=search-rows.d.ts.map
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { isFtsAvailable } from '../../../db/index.js';
|
|
2
|
-
import { tokenize } from '../../../base/util/tokenize.js';
|
|
2
|
+
import { cjkIndexTerms, partitionCjkTerms, tokenize } from '../../../base/util/tokenize.js';
|
|
3
3
|
import { recallScopeFilter } from '../../../base/scope/recall-scope.js';
|
|
4
4
|
import { originInSql, scopeAdmitSql } from '../runtime/rule-sql.js';
|
|
5
|
-
import { ftsTermParts, RAREST_TERM_COUNT, rarestFtsQuery, } from '../../../base/rank/prompt-recall.js';
|
|
5
|
+
import { ftsTermParts, RAREST_TERM_COUNT, rarestFtsQuery, spreadCjkTerms, } from '../../../base/rank/prompt-recall.js';
|
|
6
6
|
import { errorMessage, log, errorFields } from '../../../base/util/log.js';
|
|
7
7
|
import { topVectorMatches } from '../../../db/vector-match.js';
|
|
8
8
|
import { MEMORY_SELECT_COLUMNS, MEMORY_SEARCH_COLUMNS, rowToEntry, } from '../runtime/rows.js';
|
|
@@ -14,7 +14,7 @@ import { placeholders } from '../../../db/sql.js';
|
|
|
14
14
|
import { VECTOR_CANDIDATE_LIMIT, } from '../../../base/ports/store/vectors.js';
|
|
15
15
|
import { ALL_TENANTS } from '../../../base/util/tenant.js';
|
|
16
16
|
import { tenantAnd } from '../runtime/tenant.js';
|
|
17
|
-
import { LIKE_WINDOW_ROWS } from '../../rules/windows.js';
|
|
17
|
+
import { LIKE_WINDOW_ROWS, LIKE_WINDOW_TERMS } from '../../rules/windows.js';
|
|
18
18
|
/** The recall scope rule for a table column prefix (`m.` or none); `passesScopeFilterForRecall`
|
|
19
19
|
* in recall-scope.ts is its JS twin. */
|
|
20
20
|
function recallScopeClause(col, scopeFilter) {
|
|
@@ -55,18 +55,40 @@ function listSearchRows(db, { query, limit, tenantId, options = {} }) {
|
|
|
55
55
|
// LIMIT here too, so every candidate path honours the caller's cap.
|
|
56
56
|
return listAllCandidates(db, p, limit);
|
|
57
57
|
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
const rows = listLikeCandidates(db, terms, p, limit);
|
|
58
|
+
// The run's trigrams reach the trigram index, which the 2-letter tokens cannot.
|
|
59
|
+
const { words, cjk } = partitionCjkTerms(terms);
|
|
60
|
+
const rows = interleaveRows(words.length > 0 ? listWordCandidates(db, words, p, limit) : [], cjk.length > 0
|
|
61
|
+
? listLikeCandidates(db, [...new Set([...cjk, ...cjkIndexTerms(query)])], p, limit)
|
|
62
|
+
: [], limit);
|
|
64
63
|
if (rows.length > 0)
|
|
65
64
|
return rows;
|
|
66
65
|
// LIMIT the full-store fallback too: RecallResult reports the scorer window, so an unbounded
|
|
67
66
|
// fallback would misstate the candidate-pool size.
|
|
68
67
|
return listAllCandidates(db, p, limit);
|
|
69
68
|
}
|
|
69
|
+
/** FTS5 matches for `terms`, else their substring matches. */
|
|
70
|
+
function listWordCandidates(db, terms, p, limit) {
|
|
71
|
+
if (isFtsAvailable(db)) {
|
|
72
|
+
const rows = listFtsCandidates(db, terms, p, limit);
|
|
73
|
+
if (rows.length > 0)
|
|
74
|
+
return rows;
|
|
75
|
+
}
|
|
76
|
+
return listLikeCandidates(db, terms, p, limit);
|
|
77
|
+
}
|
|
78
|
+
/** `a` and `b` taken in turn, each row once, up to `limit`; with one list empty, the other as it
|
|
79
|
+
* is. */
|
|
80
|
+
function interleaveRows(a, b, limit) {
|
|
81
|
+
if (a.length === 0 || b.length === 0)
|
|
82
|
+
return a.length === 0 ? b : a;
|
|
83
|
+
const out = new Map();
|
|
84
|
+
for (let i = 0; out.size < limit && i < Math.max(a.length, b.length); i++) {
|
|
85
|
+
for (const row of [a[i], b[i]]) {
|
|
86
|
+
if (row !== undefined && out.size < limit)
|
|
87
|
+
out.set(row.id, row);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return [...out.values()];
|
|
91
|
+
}
|
|
70
92
|
function searchPredicates(tenantId, scopeFilter, includeSuperseded, originProjects) {
|
|
71
93
|
// One tenant reads through the composite idx_memories_tenant_created.
|
|
72
94
|
const one = tenantId !== ALL_TENANTS;
|
|
@@ -164,21 +186,18 @@ function trigramArm(terms, p) {
|
|
|
164
186
|
/** LIKE over the newest admitted rows: the bound is what keeps a short term from reading the
|
|
165
187
|
* whole store. */
|
|
166
188
|
function windowArm(terms, p, limit) {
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
const likes = terms.flatMap((term) => {
|
|
171
|
-
const like = `%${escapeLike(term)}%`;
|
|
172
|
-
return [like, like];
|
|
173
|
-
});
|
|
189
|
+
// One JSON array, not an OR per term: a long CJK query has 1,000s of terms, past SQLite's
|
|
190
|
+
// expression depth.
|
|
191
|
+
const likes = JSON.stringify(terms.slice(0, LIKE_WINDOW_TERMS).map((term) => `%${escapeLike(term)}%`));
|
|
174
192
|
// idx_memories_tenant_created serves a tenant's newest rows; with no tenant only the table's own
|
|
175
193
|
// order needs no sort.
|
|
176
194
|
const newestFirst = p.tenantParams.length > 0 ? 'created DESC' : 'rowid DESC';
|
|
177
195
|
return {
|
|
178
196
|
sql: `SELECT * FROM (SELECT ${MEMORY_SELECT_COLUMNS}, updated_at ` +
|
|
179
197
|
`FROM memories${p.tenantOnlyPredicate}${p.archivedClauseTenantOnly}${p.plainScope.sql}${p.currentNoAlias} ` +
|
|
180
|
-
`ORDER BY ${newestFirst} LIMIT ?) WHERE (
|
|
181
|
-
|
|
198
|
+
`ORDER BY ${newestFirst} LIMIT ?) WHERE EXISTS (SELECT 1 FROM json_each(?) j WHERE ` +
|
|
199
|
+
`LOWER(content) LIKE j.value ESCAPE '\\' OR LOWER(tags_json) LIKE j.value ESCAPE '\\')`,
|
|
200
|
+
params: [...p.tenantParams, ...p.scopeParams, Math.max(limit, LIKE_WINDOW_ROWS), likes],
|
|
182
201
|
};
|
|
183
202
|
}
|
|
184
203
|
/** The arms' rows, each once, most recently updated first. */
|
|
@@ -232,28 +251,45 @@ export function entriesInOrder(db, ids) {
|
|
|
232
251
|
const byId = new Map(rows.map((r) => [r.id, rowToEntry(r)]));
|
|
233
252
|
return ids.flatMap((id) => byId.get(id) ?? []);
|
|
234
253
|
}
|
|
235
|
-
// The vocabulary
|
|
254
|
+
// The vocabulary views are temp tables, so they live and die with their connection.
|
|
236
255
|
const vocabReady = new WeakSet();
|
|
237
256
|
function ensureRarestVocab(db) {
|
|
238
257
|
if (vocabReady.has(db))
|
|
239
258
|
return;
|
|
240
259
|
db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS temp.z1_rarest_vocab USING fts5vocab(main, 'memories_fts', 'row')`);
|
|
260
|
+
db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS temp.z1_trigram_vocab USING fts5vocab(main, 'memories_trigram', 'row')`);
|
|
241
261
|
vocabReady.add(db);
|
|
242
262
|
}
|
|
243
|
-
/** Rarest-K prompt terms for this connection's FTS
|
|
244
|
-
*
|
|
263
|
+
/** Rarest-K prompt terms for this connection's FTS indexes, as a space-joined query string: words
|
|
264
|
+
* count in the word index, CJK terms in the trigram index. Without FTS, the first 32 terms. */
|
|
245
265
|
export function pickRarestFtsQuery(db, terms, maxTerms = RAREST_TERM_COUNT) {
|
|
246
266
|
// The LIKE path has no bm25 ranking to bound, so it keeps the pre-rarest 32-term query.
|
|
247
267
|
if (!isFtsAvailable(db))
|
|
248
|
-
return terms
|
|
268
|
+
return spreadCjkTerms(terms, 32).join(' ');
|
|
249
269
|
ensureRarestVocab(db);
|
|
250
|
-
const
|
|
270
|
+
const { words, cjk } = partitionCjkTerms(terms);
|
|
271
|
+
const wordCounts = wordDocCounts(db, words);
|
|
272
|
+
const cjkCounts = trigramDocCounts(db, cjk);
|
|
273
|
+
return rarestFtsQuery(terms, (part) => wordCounts.get(part) ?? 0, maxTerms, (term) => cjkCounts.get(term) ?? 0);
|
|
274
|
+
}
|
|
275
|
+
function wordDocCounts(db, words) {
|
|
276
|
+
const vocab = Array.from(new Set(words.flatMap(ftsTermParts)));
|
|
251
277
|
if (vocab.length === 0)
|
|
252
|
-
return
|
|
278
|
+
return new Map();
|
|
253
279
|
// One cached shape per term count: an IN list over a virtual table needs one mark per term to
|
|
254
280
|
// keep its lookups.
|
|
255
281
|
const rows = cachedStatement(db, `SELECT term, doc FROM temp.z1_rarest_vocab WHERE term IN (${placeholders(vocab.length)})`).all(...vocab);
|
|
256
|
-
|
|
257
|
-
|
|
282
|
+
return new Map(rows.map((r) => [r.term, r.doc]));
|
|
283
|
+
}
|
|
284
|
+
/** Rows holding each CJK term by the trigram index: a trigram's own count, and for a shorter run
|
|
285
|
+
* the summed counts of the trigrams it opens, a rarity estimate, not an exact row count. */
|
|
286
|
+
function trigramDocCounts(db, cjk) {
|
|
287
|
+
// SHORTCUT: prefix seeks only, as a suffix count scans the whole vocabulary; a 0 keeps a short run.
|
|
288
|
+
if (cjk.length === 0)
|
|
289
|
+
return new Map();
|
|
290
|
+
// CROSS JOIN keeps the term list outside, so each term is one range seek into the vocabulary.
|
|
291
|
+
const rows = cachedStatement(db, `SELECT j.value AS term, SUM(v.doc) AS doc FROM json_each(?) j CROSS JOIN temp.z1_trigram_vocab v ` +
|
|
292
|
+
`ON v.term >= j.value AND v.term < j.value || char(1114111) GROUP BY j.value`).all(JSON.stringify(cjk));
|
|
293
|
+
return new Map(rows.map((r) => [r.term, r.doc]));
|
|
258
294
|
}
|
|
259
295
|
//# sourceMappingURL=search-rows.js.map
|
|
@@ -33,7 +33,8 @@ const MAY_HOLD_COPY_SQL = `(${bareTextSql('content')} IN (SELECT value FROM json
|
|
|
33
33
|
function copyChunk(db, read, texts, phrases) {
|
|
34
34
|
const tenant = tenantAnd(read.tenantId, phrases ? TENANT_IS : 'tenant_id = ?');
|
|
35
35
|
const bares = JSON.stringify([...new Set(texts.map(bareText))]);
|
|
36
|
-
return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project FROM memories
|
|
36
|
+
return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project, pinned FROM memories ` +
|
|
37
|
+
`WHERE ${phrases ? IN_PHRASE_HITS : '1'}` +
|
|
37
38
|
`${tenant.sql}${read.filter.sql} AND ${MAY_HOLD_COPY_SQL}`).all(...(phrases ? [phrases.join(' OR ')] : []), ...tenant.params, ...read.filter.params, bares, bares);
|
|
38
39
|
}
|
|
39
40
|
/** The rows `read` admits that may hold one of `texts`, a superset of the copies heldTextKeys
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { type MemoryEntry } from '../../base/memory/entry.js';
|
|
2
2
|
import type { HippoConfig } from '../../base/config/config.js';
|
|
3
3
|
import { type ProjectRef } from '../../base/scope/project-identity.js';
|
|
4
|
-
import { type ExtractedItem } from './extract.js';
|
|
4
|
+
import { type BoldTwin, type ExtractedItem } from './extract.js';
|
|
5
5
|
import { type SessionTurn } from './transcript.js';
|
|
6
6
|
import type { CaptureDedup } from '../../base/ports/store/capture.js';
|
|
7
|
+
import type { HeldText } from '../../base/ports/store/types.js';
|
|
7
8
|
import type { StoreGroups } from '../../base/ports/store.js';
|
|
8
9
|
export interface CaptureOptions {
|
|
9
10
|
source: 'stdin' | 'file' | 'last-session';
|
|
@@ -41,6 +42,12 @@ export interface CaptureTarget {
|
|
|
41
42
|
readonly embed: (entry: MemoryEntry) => void;
|
|
42
43
|
}
|
|
43
44
|
export declare function runCapture(target: CaptureTarget, options: CaptureOptions): Promise<void>;
|
|
45
|
+
/** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
|
|
46
|
+
* items one, and the first stays. */
|
|
47
|
+
export declare function scrubbedItems(items: readonly ExtractedItem[], scrub: (text: string) => string): ExtractedItem[];
|
|
48
|
+
/** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
|
|
49
|
+
* row: a pinned row, or one whose text is no certain defect. */
|
|
50
|
+
export declare function heldKeysOf(rows: readonly HeldText[], twins: readonly BoldTwin[]): Set<string>;
|
|
44
51
|
export interface CaptureTally {
|
|
45
52
|
captured: number;
|
|
46
53
|
skipped: number;
|
|
@@ -2,13 +2,14 @@ import * as fs from 'node:fs';
|
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import { createMemory, Layer } from '../../base/memory/entry.js';
|
|
4
4
|
import { duplicateKey, storedTextKeys } from '../../base/util/same-text.js';
|
|
5
|
+
import { isCertainReason, storedDefect } from '../../base/memory/memory-quality.js';
|
|
5
6
|
import { maskEmails, redactSecretsStrict } from '../../base/util/secret-detect.js';
|
|
6
7
|
import { projectId, projectNames } from '../../base/scope/project-identity.js';
|
|
7
8
|
import { isInitialized } from '../../host/project-identity.js';
|
|
8
9
|
import { errorMessage, log, errorFields } from '../../base/util/log.js';
|
|
9
10
|
import { teeStdStreams } from '../../host/stream-tee.js';
|
|
10
11
|
import { ALL_TENANTS } from '../../base/util/tenant.js';
|
|
11
|
-
import { extractFromTexts } from './extract.js';
|
|
12
|
+
import { boldTwins, extractFromTexts } from './extract.js';
|
|
12
13
|
import { collectSessionTurns, sessionTail, resolveLastSessionTranscript, } from './transcript.js';
|
|
13
14
|
import { PRIVATE_DIR } from '../../host/private-dir.js';
|
|
14
15
|
import { evalClock } from '../../infra/eval-switches.js';
|
|
@@ -71,14 +72,16 @@ async function runCaptureCore(target, options) {
|
|
|
71
72
|
console.log('No text to capture from.');
|
|
72
73
|
return;
|
|
73
74
|
}
|
|
74
|
-
// Scrub
|
|
75
|
-
// no secrets in memories).
|
|
76
|
-
const
|
|
75
|
+
// Scrub the texts and again each item: bold removal can rebuild an address or key the first
|
|
76
|
+
// pass could not see (AGENTS.md: no secrets in memories).
|
|
77
|
+
const scrub = (text) => maskEmails(redactSecretsStrict(text));
|
|
78
|
+
const scrubbed = texts.map(scrub);
|
|
79
|
+
const extracted = scrubbedItems(extractFromTexts(scrubbed, userTexts), scrub);
|
|
77
80
|
if (extracted.length === 0) {
|
|
78
81
|
console.log('No actionable items found in the input.');
|
|
79
82
|
return;
|
|
80
83
|
}
|
|
81
|
-
const keys = await storedKeysInView(target.store, options, extracted);
|
|
84
|
+
const keys = await storedKeysInView(target.store, options, extracted, boldTwins(scrubbed, scrub));
|
|
82
85
|
printCaptureTally(options, await captureFromCli(target, options, extracted, keys));
|
|
83
86
|
}
|
|
84
87
|
async function captureFromCli({ store, config, embed }, options, extracted, keys) {
|
|
@@ -94,12 +97,39 @@ async function captureFromCli({ store, config, embed }, options, extracted, keys
|
|
|
94
97
|
}
|
|
95
98
|
return previewCapture({ store, config }, writeOpts, extracted, keys);
|
|
96
99
|
}
|
|
100
|
+
/** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
|
|
101
|
+
* items one, and the first stays. */
|
|
102
|
+
export function scrubbedItems(items, scrub) {
|
|
103
|
+
const seen = new Set();
|
|
104
|
+
return items
|
|
105
|
+
.map((item) => ({ ...item, content: scrub(item.content) }))
|
|
106
|
+
.filter((item) => {
|
|
107
|
+
const key = duplicateKey(item.content);
|
|
108
|
+
if (seen.has(key))
|
|
109
|
+
return false;
|
|
110
|
+
seen.add(key);
|
|
111
|
+
return true;
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
/** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
|
|
115
|
+
* row: a pinned row, or one whose text is no certain defect. */
|
|
116
|
+
export function heldKeysOf(rows, twins) {
|
|
117
|
+
const keys = storedTextKeys(rows);
|
|
118
|
+
const pinned = storedTextKeys(rows.filter((row) => row.pinned));
|
|
119
|
+
const clean = twins
|
|
120
|
+
.filter(({ old }) => (isCertainReason(storedDefect(old)) ? pinned : keys).has(duplicateKey(old)))
|
|
121
|
+
.map((twin) => duplicateKey(twin.clean));
|
|
122
|
+
for (const key of clean)
|
|
123
|
+
keys.add(key);
|
|
124
|
+
return keys;
|
|
125
|
+
}
|
|
97
126
|
// Dedup only against rows this capture's reader sees: another tenant's or project's rows are
|
|
98
127
|
// hidden, so they must not stop its own copy.
|
|
99
|
-
async function storedKeysInView(store, options, extracted) {
|
|
128
|
+
async function storedKeysInView(store, options, extracted, twins) {
|
|
100
129
|
const { tenantId, ...filter } = captureDedup(options);
|
|
101
|
-
|
|
102
|
-
|
|
130
|
+
// The lookup matches stored text exactly, so it also asks for the lines as capture once stored them.
|
|
131
|
+
const texts = [...extracted.map((item) => item.content), ...twins.map((twin) => twin.old)];
|
|
132
|
+
return heldKeysOf(await store.captures.textsHolding(tenantId, texts, filter), twins);
|
|
103
133
|
}
|
|
104
134
|
/** The rows a capture's dedup reads, as data, so the store's write can read them again in its
|
|
105
135
|
* own transaction. */
|
|
@@ -8,4 +8,14 @@ export interface ExtractedItem {
|
|
|
8
8
|
/** Captures complete supported statements, each text parsed alone so a fence or heading in one never reaches the next.
|
|
9
9
|
* The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser`. */
|
|
10
10
|
export declare function extractFromTexts(texts: readonly string[], userTexts?: number): ExtractedItem[];
|
|
11
|
+
/** A text an older capture, which kept every bold marker, stored for a line, and what capture
|
|
12
|
+
* extracts from that line now. */
|
|
13
|
+
export interface BoldTwin {
|
|
14
|
+
readonly old: string;
|
|
15
|
+
readonly clean: string;
|
|
16
|
+
}
|
|
17
|
+
/** The twins of `texts`: each old item pairs, one to one, with the new item of the same span, the
|
|
18
|
+
* same category first and then the larger overlap; a pair whose texts differ. A new item that
|
|
19
|
+
* holds more or less than the old one is no twin, or a held row would block a rule it lacks. */
|
|
20
|
+
export declare function boldTwins(texts: readonly string[], scrub: (text: string) => string): BoldTwin[];
|
|
11
21
|
//# sourceMappingURL=extract.d.ts.map
|