hippo-memory 3.1.0 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/api/context/context-plan.js +2 -4
  2. package/dist/api/context/context-select.js +12 -7
  3. package/dist/api/recall/rank-stages.js +2 -2
  4. package/dist/base/memory/entry.js +2 -2
  5. package/dist/base/memory/markdown-delimiters.d.ts +15 -0
  6. package/dist/base/memory/markdown-delimiters.js +94 -0
  7. package/dist/base/memory/memory-audit.js +4 -4
  8. package/dist/base/memory/memory-quality.d.ts +12 -0
  9. package/dist/base/memory/memory-quality.js +92 -3
  10. package/dist/base/ports/store/types.d.ts +2 -1
  11. package/dist/base/rank/prompt-recall.d.ts +16 -3
  12. package/dist/base/rank/prompt-recall.js +47 -7
  13. package/dist/base/util/tokenize.d.ts +29 -0
  14. package/dist/base/util/tokenize.js +68 -5
  15. package/dist/base/util/version.d.ts +1 -1
  16. package/dist/base/util/version.js +1 -1
  17. package/dist/db/meta.d.ts +1 -0
  18. package/dist/db/meta.js +1 -0
  19. package/dist/domain/consolidate/conflicts.js +2 -2
  20. package/dist/domain/learn/forward-claim-detector.js +4 -2
  21. package/dist/domain/learn/invalidation.js +5 -2
  22. package/dist/domain/search/bm25.js +3 -3
  23. package/dist/store/rules/windows.d.ts +3 -0
  24. package/dist/store/rules/windows.js +3 -0
  25. package/dist/store/sqlite/audit/audit-maintenance.js +16 -13
  26. package/dist/store/sqlite/recall/candidates.js +1 -0
  27. package/dist/store/sqlite/recall/search-rows.d.ts +2 -2
  28. package/dist/store/sqlite/recall/search-rows.js +62 -26
  29. package/dist/store/sqlite/sharing/copy-rows.d.ts +1 -0
  30. package/dist/store/sqlite/sharing/copy-rows.js +2 -1
  31. package/dist/surface/capture/command.d.ts +8 -1
  32. package/dist/surface/capture/command.js +38 -8
  33. package/dist/surface/capture/extract.d.ts +10 -0
  34. package/dist/surface/capture/extract.js +199 -26
  35. package/dist/surface/capture/session-texts.js +8 -6
  36. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  37. package/extensions/openclaw-plugin/package.json +1 -1
  38. package/openclaw.plugin.json +1 -1
  39. package/package.json +1 -1
@@ -1,9 +1,72 @@
1
+ /** Han, kana and Hangul, as the body of a `u`-flag character class. */
2
+ export const CJK_SCRIPTS = String.raw `\p{sc=Han}\p{sc=Hira}\p{sc=Kana}\p{sc=Hang}`;
3
+ /** CJK_SCRIPTS and ー, the prolonged-sound mark, a Common-script letter inside katakana words. */
4
+ export const CJK_LETTERS = `${CJK_SCRIPTS}ー`;
5
+ const HAS_CJK = new RegExp(`[${CJK_LETTERS}]`, 'u');
6
+ // Letters, marks and digits of every script make words, so Cyrillic, Greek or Arabic text is not
7
+ // cut into spaces and `café` stays one word.
8
+ const NOT_WORD = new RegExp(String.raw `[^\p{L}\p{M}\p{N}_\s${CJK_LETTERS}]`, 'gu');
9
+ const CJK_RUN = new RegExp(`[${CJK_LETTERS}]+`, 'gu');
10
+ const CJK_RUN_SPLIT = new RegExp(`([${CJK_LETTERS}]+)`, 'u');
11
+ /** Whether `text` holds a Chinese, Japanese or Korean letter. */
12
+ export function hasCjk(text) {
13
+ return HAS_CJK.test(text);
14
+ }
15
+ /** The overlapping `n`-letter pieces of each CJK run in `text`; a run of `n` or fewer letters is one
16
+ * piece. */
17
+ export function cjkGrams(text, n) {
18
+ return (text.match(CJK_RUN) ?? []).flatMap((run) => {
19
+ const letters = [...run];
20
+ if (letters.length <= n)
21
+ return [run];
22
+ return letters.slice(n - 1).map((_, i) => letters.slice(i, i + n).join(''));
23
+ });
24
+ }
25
+ /** Each CJK run's distinct letter trigrams, a shorter run whole: the trigram index's terms. */
26
+ export function cjkIndexTerms(text) {
27
+ return [...new Set(cjkGrams(text, 3))];
28
+ }
29
+ /** `terms` split by the index that can find them: the word index holds a whole CJK run as one
30
+ * token, so CJK terms skip it for the substring indexes. */
31
+ export function partitionCjkTerms(terms) {
32
+ const words = [];
33
+ const cjk = [];
34
+ for (const term of terms)
35
+ (HAS_CJK.test(term) ? cjk : words).push(term);
36
+ return { words, cjk };
37
+ }
38
+ /** `word` with each CJK run in it replaced by the run's letter bigrams: those scripts put no space
39
+ * between words, and a bigram, unlike a dictionary split, matches the same way in query and text. */
40
+ export function splitCjkRuns(word) {
41
+ if (!HAS_CJK.test(word))
42
+ return [word];
43
+ return word
44
+ .split(CJK_RUN_SPLIT)
45
+ .flatMap((part, i) => (i % 2 === 1 ? cjkGrams(part, 2) : part === '' ? [] : [part]));
46
+ }
47
+ /** Lower-cased words of `text` (runs of letters, marks, digits and `_`) of 2 or more characters; a
48
+ * CJK run becomes its letter bigrams, a lone CJK letter itself. */
1
49
  export function tokenize(text) {
2
- return text
3
- .toLowerCase()
4
- .replace(/[^\w\s]/g, ' ')
5
- .split(/\s+/)
6
- .filter((t) => t.length > 1);
50
+ return tokenizeMin(text, 2);
51
+ }
52
+ /** `tokenize` for the text a query is matched against: each letter of a longer CJK run is added,
53
+ * so a one-letter query, which `tokenize` keeps whole, matches inside a run, while a longer query,
54
+ * all bigrams, does not match single letters. */
55
+ export function indexTokenize(text) {
56
+ const tokens = tokenize(text);
57
+ if (!HAS_CJK.test(text))
58
+ return tokens;
59
+ const letters = (text.match(CJK_RUN) ?? []).flatMap((run) => {
60
+ const runLetters = [...run];
61
+ return runLetters.length > 1 ? runLetters : [];
62
+ });
63
+ return [...tokens, ...letters];
64
+ }
65
+ /** `tokenize` with words shorter than `minLength` dropped; CJK tokens, never longer than 2, are
66
+ * kept. */
67
+ export function tokenizeMin(text, minLength) {
68
+ const words = text.toLowerCase().replace(NOT_WORD, ' ').split(/\s+/);
69
+ return (HAS_CJK.test(text) ? words.flatMap(splitCjkRuns) : words).filter((t) => t.length >= minLength || HAS_CJK.test(t));
7
70
  }
8
71
  /** Jaccard overlap of two token sets; two empty sets count as identical. */
9
72
  export function jaccardSets(a, b) {
@@ -2,7 +2,7 @@
2
2
  * Single source of truth for the package version, written by scripts/sync-version.mjs; never
3
3
  * edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
4
4
  * package.json read: the published bundle may lack package.json on a path ESM can resolve. */
5
- export declare const PACKAGE_VERSION = "3.1.0";
5
+ export declare const PACKAGE_VERSION = "3.1.1";
6
6
  /** The floor a store takes on its first expiring key: the first release with schema v53, set by
7
7
  * hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
8
8
  export declare const EXPIRING_KEYS_MIN_BINARY = "1.64.0";
@@ -2,7 +2,7 @@
2
2
  * Single source of truth for the package version, written by scripts/sync-version.mjs; never
3
3
  * edit by hand. Read by the db rollback guard, /health, MCP. A constant, not a runtime
4
4
  * package.json read: the published bundle may lack package.json on a path ESM can resolve. */
5
- export const PACKAGE_VERSION = '3.1.0';
5
+ export const PACKAGE_VERSION = '3.1.1';
6
6
  /** The floor a store takes on its first expiring key: the first release with schema v53, set by
7
7
  * hand in that release; scripts/checks/check-expiring-keys-floor.mjs gates it. */
8
8
  export const EXPIRING_KEYS_MIN_BINARY = '1.64.0';
package/dist/db/meta.d.ts CHANGED
@@ -17,6 +17,7 @@ export declare const META_KEY: {
17
17
  readonly sleepCount: "sleep_count";
18
18
  readonly legacyBootstrapCompleted: "legacy_bootstrap_completed";
19
19
  readonly qualityRepairAuto: "quality_repair_auto";
20
+ readonly qualityRepairGate: "quality_repair_gate";
20
21
  readonly projectRepairAuto: "project_repair_auto";
21
22
  };
22
23
  export declare function ensureMetaTable(db: DatabaseSyncLike): void;
package/dist/db/meta.js CHANGED
@@ -24,6 +24,7 @@ export const META_KEY = {
24
24
  sleepCount: 'sleep_count',
25
25
  legacyBootstrapCompleted: 'legacy_bootstrap_completed',
26
26
  qualityRepairAuto: 'quality_repair_auto',
27
+ qualityRepairGate: 'quality_repair_gate',
27
28
  projectRepairAuto: 'project_repair_auto',
28
29
  };
29
30
  export function ensureMetaTable(db) {
@@ -4,7 +4,7 @@ import { jaccardMinShared, overlapPartners } from './overlap-index.js';
4
4
  import { isQuarantineScope } from '../trust/quarantine.js';
5
5
  import { isPersonalScope } from '../../base/scope/recall-scope.js';
6
6
  import { CONFLICT_STOPWORDS } from '../../base/util/stopwords.js';
7
- import { jaccardSets, tokenize } from '../../base/util/tokenize.js';
7
+ import { jaccardSets, tokenizeMin } from '../../base/util/tokenize.js';
8
8
  import { DECAY_THRESHOLD } from './decay.js';
9
9
  import { BadRequestError } from '../../base/errors/api-errors.js';
10
10
  // Gate contradictions on content overlap, not shared tags (`feedback`/`policy` are too coarse). The
@@ -144,7 +144,7 @@ function describeConflict(a, b) {
144
144
  };
145
145
  }
146
146
  function distinctiveTokens(text) {
147
- return new Set(tokenize(text).filter((t) => t.length > 2 && !CONFLICT_STOPWORDS.has(t)));
147
+ return new Set(tokenizeMin(text, 3).filter((t) => !CONFLICT_STOPWORDS.has(t)));
148
148
  }
149
149
  function openingWindow(text) {
150
150
  return text.split(/\s+/).slice(0, POLARITY_WINDOW_WORDS).join(' ');
@@ -4,6 +4,7 @@
4
4
  * noise dominates hint UX; `recall_autodebias_hint_no_class_match` audits are the telemetry.
5
5
  */
6
6
  import { FORWARD_CLAIM_STOPWORDS } from '../../base/util/stopwords.js';
7
+ import { CJK_LETTERS } from '../../base/util/tokenize.js';
7
8
  /**
8
9
  * Each pattern is deliberately narrow (add one only on audit evidence); word boundaries are
9
10
  * mandatory so `ETA` never matches inside `BETAtesting`.
@@ -49,11 +50,12 @@ export function detectForwardClaim(queryText) {
49
50
  }
50
51
  return null;
51
52
  }
53
+ const NOT_CLASS_WORD = new RegExp(String.raw `[^a-z0-9\s_${CJK_LETTERS}-]`, 'gu');
52
54
  function extractClassQueryTokens(queryText) {
53
55
  return (queryText
54
56
  .toLowerCase()
55
- // Strip everything that's not alphanum or whitespace or hyphen / underscore.
56
- .replace(/[^a-z0-9\s_-]/g, ' ')
57
+ // Strip everything that's not alphanum, CJK, whitespace, hyphen or underscore.
58
+ .replace(NOT_CLASS_WORD, ' ')
57
59
  // Split on whitespace, hyphen and underscore, matching the class-tag split in resolveClass.
58
60
  .split(/[\s_-]+/)
59
61
  .filter((t) => t.length >= 3 && !FORWARD_CLAIM_STOPWORDS.has(t) && !/^\d+$/.test(t)));
@@ -1,4 +1,5 @@
1
1
  import { INVALIDATION_STOPWORDS } from '../../base/util/stopwords.js';
2
+ import { CJK_LETTERS, hasCjk, splitCjkRuns } from '../../base/util/tokenize.js';
2
3
  import { READ_PAGE_ROWS } from '../../base/util/pages.js';
3
4
  const HEADLINE_CHARS = 60;
4
5
  // The share of a row's tokens that must match the target's before the row counts as mentioning it.
@@ -170,12 +171,14 @@ function weakenEntry(entry) {
170
171
  entry.tags.push('invalidated');
171
172
  }
172
173
  }
174
+ const NOT_INVALIDATION_WORD = new RegExp(String.raw `[^a-z0-9\s_.${CJK_LETTERS}-]`, 'gu');
173
175
  function invalidationTokenize(text) {
174
176
  return text
175
177
  .toLowerCase()
176
- .replace(/[^a-z0-9\s_.-]/g, ' ')
178
+ .replace(NOT_INVALIDATION_WORD, ' ')
177
179
  .split(/\s+/)
178
- .filter((t) => t.length >= 2 && !INVALIDATION_STOPWORDS.has(t));
180
+ .flatMap(splitCjkRuns)
181
+ .filter((t) => (t.length >= 2 || hasCjk(t)) && !INVALIDATION_STOPWORDS.has(t));
179
182
  }
180
183
  function matchScore(fromTokens, contentTokens) {
181
184
  if (fromTokens.length === 0)
@@ -1,4 +1,4 @@
1
- import { tokenize } from '../../base/util/tokenize.js';
1
+ import { indexTokenize } from '../../base/util/tokenize.js';
2
2
  const BM25_K1 = 1.5;
3
3
  const BM25_B = 0.75;
4
4
  /** The text BM25 indexes for an entry: its content followed by its tags. */
@@ -6,7 +6,7 @@ export function entryText(entry) {
6
6
  return `${entry.content} ${entry.tags.join(' ')}`;
7
7
  }
8
8
  export function buildCorpus(texts) {
9
- const docs = texts.map(tokenize);
9
+ const docs = texts.map((text) => indexTokenize(text));
10
10
  const N = docs.length;
11
11
  const df = new Map();
12
12
  let totalLen = 0;
@@ -44,7 +44,7 @@ export function bm25Score(corpus, docIdx, queryTerms) {
44
44
  }
45
45
  /** Query terms that appear in the entry's indexed text, in query order. */
46
46
  export function matchedQueryTerms(queryTerms, entry) {
47
- const docTerms = new Set(tokenize(entryText(entry)));
47
+ const docTerms = new Set(indexTokenize(entryText(entry)));
48
48
  const matched = [];
49
49
  for (const t of queryTerms)
50
50
  if (docTerms.has(t))
@@ -1,6 +1,9 @@
1
1
  /** Newest admitted rows LIKE tests for a term under 3 characters, which no trigram lookup can
2
2
  * answer. */
3
3
  export declare const LIKE_WINDOW_ROWS = 2000;
4
+ /** Most terms LIKE tests on each of those rows: the trigram index looks up every longer term over
5
+ * the whole store, so a long CJK query's 1,000s of bigrams need not all be tested here. */
6
+ export declare const LIKE_WINDOW_TERMS = 64;
4
7
  /** How many of a tenant's newest rows schemaFit reads; older rows no longer move the fit. */
5
8
  export declare const SCHEMA_FIT_WINDOW_ROWS = 2000;
6
9
  /** Events of one completed session a sleep reads to render its trace, the newest kept. */
@@ -3,6 +3,9 @@
3
3
  /** Newest admitted rows LIKE tests for a term under 3 characters, which no trigram lookup can
4
4
  * answer. */
5
5
  export const LIKE_WINDOW_ROWS = 2000;
6
+ /** Most terms LIKE tests on each of those rows: the trigram index looks up every longer term over
7
+ * the whole store, so a long CJK query's 1,000s of bigrams need not all be tested here. */
8
+ export const LIKE_WINDOW_TERMS = 64;
6
9
  /** How many of a tenant's newest rows schemaFit reads; older rows no longer move the fit. */
7
10
  export const SCHEMA_FIT_WINDOW_ROWS = 2000;
8
11
  /** Events of one completed session a sleep reads to render its trace, the newest kept. */
@@ -17,7 +17,8 @@ import { purgeMirrorBestEffort } from '../folder-files/mirrors.js';
17
17
  import { openStore } from '../runtime/handles.js';
18
18
  import { answered, bound, skipEmpty } from '../runtime/bound.js';
19
19
  import { READ_PAGE_ROWS } from '../../../base/util/pages.js';
20
- import { getMeta, pragmaUserVersion, setMeta, META_KEY } from '../../../db/meta.js';
20
+ import { getMeta, getMetaNumber, pragmaUserVersion, setMeta, META_KEY } from '../../../db/meta.js';
21
+ import { QUALITY_GATE_VERSION } from '../../../base/memory/memory-quality.js';
21
22
  // Quality repair's issue list: which rows are defects and which of those a protection keeps.
22
23
  /** The issues among `entries` in their order, each defect kept when `protect` names the row. */
23
24
  function repairIssues(entries, parents, backing, protect) {
@@ -223,8 +224,8 @@ function applyPlan(db, { run, plan, backup, rules }) {
223
224
  else
224
225
  warnings.push(`Kept ${issue.id}: the store's delete guard protects it.`);
225
226
  }
226
- if (run.doneKey)
227
- setMeta(db, run.doneKey, '1');
227
+ for (const [key, value] of run.doneWrites ?? [])
228
+ setMeta(db, key, value);
228
229
  return { ...plan, issues, appliedIds, backup, warnings };
229
230
  });
230
231
  }
@@ -248,21 +249,23 @@ function repairOn(db, files, run) {
248
249
  }
249
250
  return { ...result, warnings };
250
251
  }
251
- /** Applies the repair once per store, so stores that predate the quality gate are cleaned on
252
+ // The old key stays '1' so a binary from before the gate version keeps skipping the repair.
253
+ const DONE_WRITES = [
254
+ [META_KEY.qualityRepairAuto, '1'],
255
+ [META_KEY.qualityRepairGate, String(QUALITY_GATE_VERSION)],
256
+ ];
257
+ /** Applies the repair once per gate version, so stores that predate a quality check are cleaned on
252
258
  * upgrade with no command; null once done. `db` is opened writable without running migrations. */
253
259
  function repairQualityOnce(db, files, tenantId, protect) {
254
- if (getMeta(db, META_KEY.qualityRepairAuto) === '1')
260
+ if (getMetaNumber(db, META_KEY.qualityRepairGate) >= QUALITY_GATE_VERSION)
255
261
  return null;
256
262
  // The apply commits the flag with the moves, so a lock taken between them cannot hide what moved.
257
- const result = repairOn(db, files, {
258
- tenantId,
259
- protect,
260
- apply: true,
261
- doneKey: META_KEY.qualityRepairAuto,
262
- });
263
+ const result = repairOn(db, files, { tenantId, protect, apply: true, doneWrites: DONE_WRITES });
263
264
  // An unsupported schema changed nothing, so the next run tries again once a migration has run.
264
- if (result.supported && result.backup === null)
265
- setMeta(db, META_KEY.qualityRepairAuto, '1');
265
+ if (result.supported && result.backup === null) {
266
+ for (const [key, value] of DONE_WRITES)
267
+ setMeta(db, key, value);
268
+ }
266
269
  return result;
267
270
  }
268
271
  export function sqliteAuditMaintenance({ on, files }) {
@@ -298,6 +298,7 @@ export function listTextsHolding(db, { tenantId, texts, project, admit = TEAM_VI
298
298
  content: row.content,
299
299
  source: row.source,
300
300
  originProject: row.origin_project,
301
+ pinned: row.pinned === 1,
301
302
  });
302
303
  }
303
304
  return [...byId.values()];
@@ -16,7 +16,7 @@ export declare function listRecallEntries(db: DatabaseSyncLike, query: string, o
16
16
  export declare function listVectorCandidateEntries(db: DatabaseSyncLike, queryVector: readonly number[], spec: VectorCandidateSpec, index?: VectorIndex): Promise<MemoryEntry[]>;
17
17
  /** The rows of `ids` in that order; an id deleted since it was matched is skipped. */
18
18
  export declare function entriesInOrder(db: DatabaseSyncLike, ids: readonly string[]): MemoryEntry[];
19
- /** Rarest-K prompt terms for this connection's FTS index, as a space-joined query string.
20
- * Without FTS, returns the first 32 terms, as before rarest-term selection. */
19
+ /** Rarest-K prompt terms for this connection's FTS indexes, as a space-joined query string: words
20
+ * count in the word index, CJK terms in the trigram index. Without FTS, the first 32 terms. */
21
21
  export declare function pickRarestFtsQuery(db: DatabaseSyncLike, terms: readonly string[], maxTerms?: number): string;
22
22
  //# sourceMappingURL=search-rows.d.ts.map
@@ -1,8 +1,8 @@
1
1
  import { isFtsAvailable } from '../../../db/index.js';
2
- import { tokenize } from '../../../base/util/tokenize.js';
2
+ import { cjkIndexTerms, partitionCjkTerms, tokenize } from '../../../base/util/tokenize.js';
3
3
  import { recallScopeFilter } from '../../../base/scope/recall-scope.js';
4
4
  import { originInSql, scopeAdmitSql } from '../runtime/rule-sql.js';
5
- import { ftsTermParts, RAREST_TERM_COUNT, rarestFtsQuery, } from '../../../base/rank/prompt-recall.js';
5
+ import { ftsTermParts, RAREST_TERM_COUNT, rarestFtsQuery, spreadCjkTerms, } from '../../../base/rank/prompt-recall.js';
6
6
  import { errorMessage, log, errorFields } from '../../../base/util/log.js';
7
7
  import { topVectorMatches } from '../../../db/vector-match.js';
8
8
  import { MEMORY_SELECT_COLUMNS, MEMORY_SEARCH_COLUMNS, rowToEntry, } from '../runtime/rows.js';
@@ -14,7 +14,7 @@ import { placeholders } from '../../../db/sql.js';
14
14
  import { VECTOR_CANDIDATE_LIMIT, } from '../../../base/ports/store/vectors.js';
15
15
  import { ALL_TENANTS } from '../../../base/util/tenant.js';
16
16
  import { tenantAnd } from '../runtime/tenant.js';
17
- import { LIKE_WINDOW_ROWS } from '../../rules/windows.js';
17
+ import { LIKE_WINDOW_ROWS, LIKE_WINDOW_TERMS } from '../../rules/windows.js';
18
18
  /** The recall scope rule for a table column prefix (`m.` or none); `passesScopeFilterForRecall`
19
19
  * in recall-scope.ts is its JS twin. */
20
20
  function recallScopeClause(col, scopeFilter) {
@@ -55,18 +55,40 @@ function listSearchRows(db, { query, limit, tenantId, options = {} }) {
55
55
  // LIMIT here too, so every candidate path honours the caller's cap.
56
56
  return listAllCandidates(db, p, limit);
57
57
  }
58
- if (isFtsAvailable(db)) {
59
- const rows = listFtsCandidates(db, terms, p, limit);
60
- if (rows.length > 0)
61
- return rows;
62
- }
63
- const rows = listLikeCandidates(db, terms, p, limit);
58
+ // The run's trigrams reach the trigram index, which the 2-letter tokens cannot.
59
+ const { words, cjk } = partitionCjkTerms(terms);
60
+ const rows = interleaveRows(words.length > 0 ? listWordCandidates(db, words, p, limit) : [], cjk.length > 0
61
+ ? listLikeCandidates(db, [...new Set([...cjk, ...cjkIndexTerms(query)])], p, limit)
62
+ : [], limit);
64
63
  if (rows.length > 0)
65
64
  return rows;
66
65
  // LIMIT the full-store fallback too: RecallResult reports the scorer window, so an unbounded
67
66
  // fallback would misstate the candidate-pool size.
68
67
  return listAllCandidates(db, p, limit);
69
68
  }
69
+ /** FTS5 matches for `terms`, else their substring matches. */
70
+ function listWordCandidates(db, terms, p, limit) {
71
+ if (isFtsAvailable(db)) {
72
+ const rows = listFtsCandidates(db, terms, p, limit);
73
+ if (rows.length > 0)
74
+ return rows;
75
+ }
76
+ return listLikeCandidates(db, terms, p, limit);
77
+ }
78
+ /** `a` and `b` taken in turn, each row once, up to `limit`; with one list empty, the other as it
79
+ * is. */
80
+ function interleaveRows(a, b, limit) {
81
+ if (a.length === 0 || b.length === 0)
82
+ return a.length === 0 ? b : a;
83
+ const out = new Map();
84
+ for (let i = 0; out.size < limit && i < Math.max(a.length, b.length); i++) {
85
+ for (const row of [a[i], b[i]]) {
86
+ if (row !== undefined && out.size < limit)
87
+ out.set(row.id, row);
88
+ }
89
+ }
90
+ return [...out.values()];
91
+ }
70
92
  function searchPredicates(tenantId, scopeFilter, includeSuperseded, originProjects) {
71
93
  // One tenant reads through the composite idx_memories_tenant_created.
72
94
  const one = tenantId !== ALL_TENANTS;
@@ -164,21 +186,18 @@ function trigramArm(terms, p) {
164
186
  /** LIKE over the newest admitted rows: the bound is what keeps a short term from reading the
165
187
  * whole store. */
166
188
  function windowArm(terms, p, limit) {
167
- const where = terms
168
- .map(() => `(LOWER(content) LIKE ? ESCAPE '\\' OR LOWER(tags_json) LIKE ? ESCAPE '\\')`)
169
- .join(' OR ');
170
- const likes = terms.flatMap((term) => {
171
- const like = `%${escapeLike(term)}%`;
172
- return [like, like];
173
- });
189
+ // One JSON array, not an OR per term: a long CJK query has 1,000s of terms, past SQLite's
190
+ // expression depth.
191
+ const likes = JSON.stringify(terms.slice(0, LIKE_WINDOW_TERMS).map((term) => `%${escapeLike(term)}%`));
174
192
  // idx_memories_tenant_created serves a tenant's newest rows; with no tenant only the table's own
175
193
  // order needs no sort.
176
194
  const newestFirst = p.tenantParams.length > 0 ? 'created DESC' : 'rowid DESC';
177
195
  return {
178
196
  sql: `SELECT * FROM (SELECT ${MEMORY_SELECT_COLUMNS}, updated_at ` +
179
197
  `FROM memories${p.tenantOnlyPredicate}${p.archivedClauseTenantOnly}${p.plainScope.sql}${p.currentNoAlias} ` +
180
- `ORDER BY ${newestFirst} LIMIT ?) WHERE (${where})`,
181
- params: [...p.tenantParams, ...p.scopeParams, Math.max(limit, LIKE_WINDOW_ROWS), ...likes],
198
+ `ORDER BY ${newestFirst} LIMIT ?) WHERE EXISTS (SELECT 1 FROM json_each(?) j WHERE ` +
199
+ `LOWER(content) LIKE j.value ESCAPE '\\' OR LOWER(tags_json) LIKE j.value ESCAPE '\\')`,
200
+ params: [...p.tenantParams, ...p.scopeParams, Math.max(limit, LIKE_WINDOW_ROWS), likes],
182
201
  };
183
202
  }
184
203
  /** The arms' rows, each once, most recently updated first. */
@@ -232,28 +251,45 @@ export function entriesInOrder(db, ids) {
232
251
  const byId = new Map(rows.map((r) => [r.id, rowToEntry(r)]));
233
252
  return ids.flatMap((id) => byId.get(id) ?? []);
234
253
  }
235
- // The vocabulary view is a temp table, so it lives and dies with its connection.
254
+ // The vocabulary views are temp tables, so they live and die with their connection.
236
255
  const vocabReady = new WeakSet();
237
256
  function ensureRarestVocab(db) {
238
257
  if (vocabReady.has(db))
239
258
  return;
240
259
  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS temp.z1_rarest_vocab USING fts5vocab(main, 'memories_fts', 'row')`);
260
+ db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS temp.z1_trigram_vocab USING fts5vocab(main, 'memories_trigram', 'row')`);
241
261
  vocabReady.add(db);
242
262
  }
243
- /** Rarest-K prompt terms for this connection's FTS index, as a space-joined query string.
244
- * Without FTS, returns the first 32 terms, as before rarest-term selection. */
263
+ /** Rarest-K prompt terms for this connection's FTS indexes, as a space-joined query string: words
264
+ * count in the word index, CJK terms in the trigram index. Without FTS, the first 32 terms. */
245
265
  export function pickRarestFtsQuery(db, terms, maxTerms = RAREST_TERM_COUNT) {
246
266
  // The LIKE path has no bm25 ranking to bound, so it keeps the pre-rarest 32-term query.
247
267
  if (!isFtsAvailable(db))
248
- return terms.slice(0, 32).join(' ');
268
+ return spreadCjkTerms(terms, 32).join(' ');
249
269
  ensureRarestVocab(db);
250
- const vocab = Array.from(new Set(terms.flatMap(ftsTermParts)));
270
+ const { words, cjk } = partitionCjkTerms(terms);
271
+ const wordCounts = wordDocCounts(db, words);
272
+ const cjkCounts = trigramDocCounts(db, cjk);
273
+ return rarestFtsQuery(terms, (part) => wordCounts.get(part) ?? 0, maxTerms, (term) => cjkCounts.get(term) ?? 0);
274
+ }
275
+ function wordDocCounts(db, words) {
276
+ const vocab = Array.from(new Set(words.flatMap(ftsTermParts)));
251
277
  if (vocab.length === 0)
252
- return '';
278
+ return new Map();
253
279
  // One cached shape per term count: an IN list over a virtual table needs one mark per term to
254
280
  // keep its lookups.
255
281
  const rows = cachedStatement(db, `SELECT term, doc FROM temp.z1_rarest_vocab WHERE term IN (${placeholders(vocab.length)})`).all(...vocab);
256
- const counts = new Map(rows.map((r) => [r.term, r.doc]));
257
- return rarestFtsQuery(terms, (part) => counts.get(part) ?? 0, maxTerms);
282
+ return new Map(rows.map((r) => [r.term, r.doc]));
283
+ }
284
+ /** Rows holding each CJK term by the trigram index: a trigram's own count, and for a shorter run
285
+ * the summed counts of the trigrams it opens, a rarity estimate, not an exact row count. */
286
+ function trigramDocCounts(db, cjk) {
287
+ // SHORTCUT: prefix seeks only, as a suffix count scans the whole vocabulary; a 0 keeps a short run.
288
+ if (cjk.length === 0)
289
+ return new Map();
290
+ // CROSS JOIN keeps the term list outside, so each term is one range seek into the vocabulary.
291
+ const rows = cachedStatement(db, `SELECT j.value AS term, SUM(v.doc) AS doc FROM json_each(?) j CROSS JOIN temp.z1_trigram_vocab v ` +
292
+ `ON v.term >= j.value AND v.term < j.value || char(1114111) GROUP BY j.value`).all(JSON.stringify(cjk));
293
+ return new Map(rows.map((r) => [r.term, r.doc]));
258
294
  }
259
295
  //# sourceMappingURL=search-rows.js.map
@@ -9,6 +9,7 @@ export type CopyDbRow = {
9
9
  content: string;
10
10
  source: string;
11
11
  origin_project: string | null;
12
+ pinned: number;
12
13
  };
13
14
  /** Which rows of the tenant a copy read looks at: `filter` adds predicates, each written ` AND
14
15
  * ...`. */
@@ -33,7 +33,8 @@ const MAY_HOLD_COPY_SQL = `(${bareTextSql('content')} IN (SELECT value FROM json
33
33
  function copyChunk(db, read, texts, phrases) {
34
34
  const tenant = tenantAnd(read.tenantId, phrases ? TENANT_IS : 'tenant_id = ?');
35
35
  const bares = JSON.stringify([...new Set(texts.map(bareText))]);
36
- return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project FROM memories WHERE ${phrases ? IN_PHRASE_HITS : '1'}` +
36
+ return cachedStatement(db, `SELECT id, tenant_id, content, source, origin_project, pinned FROM memories ` +
37
+ `WHERE ${phrases ? IN_PHRASE_HITS : '1'}` +
37
38
  `${tenant.sql}${read.filter.sql} AND ${MAY_HOLD_COPY_SQL}`).all(...(phrases ? [phrases.join(' OR ')] : []), ...tenant.params, ...read.filter.params, bares, bares);
38
39
  }
39
40
  /** The rows `read` admits that may hold one of `texts`, a superset of the copies heldTextKeys
@@ -1,9 +1,10 @@
1
1
  import { type MemoryEntry } from '../../base/memory/entry.js';
2
2
  import type { HippoConfig } from '../../base/config/config.js';
3
3
  import { type ProjectRef } from '../../base/scope/project-identity.js';
4
- import { type ExtractedItem } from './extract.js';
4
+ import { type BoldTwin, type ExtractedItem } from './extract.js';
5
5
  import { type SessionTurn } from './transcript.js';
6
6
  import type { CaptureDedup } from '../../base/ports/store/capture.js';
7
+ import type { HeldText } from '../../base/ports/store/types.js';
7
8
  import type { StoreGroups } from '../../base/ports/store.js';
8
9
  export interface CaptureOptions {
9
10
  source: 'stdin' | 'file' | 'last-session';
@@ -41,6 +42,12 @@ export interface CaptureTarget {
41
42
  readonly embed: (entry: MemoryEntry) => void;
42
43
  }
43
44
  export declare function runCapture(target: CaptureTarget, options: CaptureOptions): Promise<void>;
45
+ /** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
46
+ * items one, and the first stays. */
47
+ export declare function scrubbedItems(items: readonly ExtractedItem[], scrub: (text: string) => string): ExtractedItem[];
48
+ /** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
49
+ * row: a pinned row, or one whose text is no certain defect. */
50
+ export declare function heldKeysOf(rows: readonly HeldText[], twins: readonly BoldTwin[]): Set<string>;
44
51
  export interface CaptureTally {
45
52
  captured: number;
46
53
  skipped: number;
@@ -2,13 +2,14 @@ import * as fs from 'node:fs';
2
2
  import * as path from 'node:path';
3
3
  import { createMemory, Layer } from '../../base/memory/entry.js';
4
4
  import { duplicateKey, storedTextKeys } from '../../base/util/same-text.js';
5
+ import { isCertainReason, storedDefect } from '../../base/memory/memory-quality.js';
5
6
  import { maskEmails, redactSecretsStrict } from '../../base/util/secret-detect.js';
6
7
  import { projectId, projectNames } from '../../base/scope/project-identity.js';
7
8
  import { isInitialized } from '../../host/project-identity.js';
8
9
  import { errorMessage, log, errorFields } from '../../base/util/log.js';
9
10
  import { teeStdStreams } from '../../host/stream-tee.js';
10
11
  import { ALL_TENANTS } from '../../base/util/tenant.js';
11
- import { extractFromTexts } from './extract.js';
12
+ import { boldTwins, extractFromTexts } from './extract.js';
12
13
  import { collectSessionTurns, sessionTail, resolveLastSessionTranscript, } from './transcript.js';
13
14
  import { PRIVATE_DIR } from '../../host/private-dir.js';
14
15
  import { evalClock } from '../../infra/eval-switches.js';
@@ -71,14 +72,16 @@ async function runCaptureCore(target, options) {
71
72
  console.log('No text to capture from.');
72
73
  return;
73
74
  }
74
- // Scrub once here, as the snapshot fields are: every source can carry a pasted token (AGENTS.md:
75
- // no secrets in memories).
76
- const extracted = extractFromTexts(texts.map((text) => maskEmails(redactSecretsStrict(text))), userTexts);
75
+ // Scrub the texts and again each item: bold removal can rebuild an address or key the first
76
+ // pass could not see (AGENTS.md: no secrets in memories).
77
+ const scrub = (text) => maskEmails(redactSecretsStrict(text));
78
+ const scrubbed = texts.map(scrub);
79
+ const extracted = scrubbedItems(extractFromTexts(scrubbed, userTexts), scrub);
77
80
  if (extracted.length === 0) {
78
81
  console.log('No actionable items found in the input.');
79
82
  return;
80
83
  }
81
- const keys = await storedKeysInView(target.store, options, extracted);
84
+ const keys = await storedKeysInView(target.store, options, extracted, boldTwins(scrubbed, scrub));
82
85
  printCaptureTally(options, await captureFromCli(target, options, extracted, keys));
83
86
  }
84
87
  async function captureFromCli({ store, config, embed }, options, extracted, keys) {
@@ -94,12 +97,39 @@ async function captureFromCli({ store, config, embed }, options, extracted, keys
94
97
  }
95
98
  return previewCapture({ store, config }, writeOpts, extracted, keys);
96
99
  }
100
+ /** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
101
+ * items one, and the first stays. */
102
+ export function scrubbedItems(items, scrub) {
103
+ const seen = new Set();
104
+ return items
105
+ .map((item) => ({ ...item, content: scrub(item.content) }))
106
+ .filter((item) => {
107
+ const key = duplicateKey(item.content);
108
+ if (seen.has(key))
109
+ return false;
110
+ seen.add(key);
111
+ return true;
112
+ });
113
+ }
114
+ /** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
115
+ * row: a pinned row, or one whose text is no certain defect. */
116
+ export function heldKeysOf(rows, twins) {
117
+ const keys = storedTextKeys(rows);
118
+ const pinned = storedTextKeys(rows.filter((row) => row.pinned));
119
+ const clean = twins
120
+ .filter(({ old }) => (isCertainReason(storedDefect(old)) ? pinned : keys).has(duplicateKey(old)))
121
+ .map((twin) => duplicateKey(twin.clean));
122
+ for (const key of clean)
123
+ keys.add(key);
124
+ return keys;
125
+ }
97
126
  // Dedup only against rows this capture's reader sees: another tenant's or project's rows are
98
127
  // hidden, so they must not stop its own copy.
99
- async function storedKeysInView(store, options, extracted) {
128
+ async function storedKeysInView(store, options, extracted, twins) {
100
129
  const { tenantId, ...filter } = captureDedup(options);
101
- const texts = extracted.map((item) => item.content);
102
- return storedTextKeys(await store.captures.textsHolding(tenantId, texts, filter));
130
+ // The lookup matches stored text exactly, so it also asks for the lines as capture once stored them.
131
+ const texts = [...extracted.map((item) => item.content), ...twins.map((twin) => twin.old)];
132
+ return heldKeysOf(await store.captures.textsHolding(tenantId, texts, filter), twins);
103
133
  }
104
134
  /** The rows a capture's dedup reads, as data, so the store's write can read them again in its
105
135
  * own transaction. */
@@ -8,4 +8,14 @@ export interface ExtractedItem {
8
8
  /** Captures complete supported statements, each text parsed alone so a fence or heading in one never reaches the next.
9
9
  * The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser`. */
10
10
  export declare function extractFromTexts(texts: readonly string[], userTexts?: number): ExtractedItem[];
11
+ /** A text an older capture, which kept every bold marker, stored for a line, and what capture
12
+ * extracts from that line now. */
13
+ export interface BoldTwin {
14
+ readonly old: string;
15
+ readonly clean: string;
16
+ }
17
+ /** The twins of `texts`: each old item pairs, one to one, with the new item of the same span, the
18
+ * same category first and then the larger overlap; a pair whose texts differ. A new item that
19
+ * holds more or less than the old one is no twin, or a held row would block a rule it lacks. */
20
+ export declare function boldTwins(texts: readonly string[], scrub: (text: string) => string): BoldTwin[];
11
21
  //# sourceMappingURL=extract.d.ts.map