hippo-memory 3.1.0 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/api/context/context-plan.js +2 -4
  2. package/dist/api/context/context-select.js +12 -7
  3. package/dist/api/recall/rank-stages.js +2 -2
  4. package/dist/base/memory/entry.js +2 -2
  5. package/dist/base/memory/markdown-delimiters.d.ts +15 -0
  6. package/dist/base/memory/markdown-delimiters.js +94 -0
  7. package/dist/base/memory/memory-audit.js +4 -4
  8. package/dist/base/memory/memory-quality.d.ts +12 -0
  9. package/dist/base/memory/memory-quality.js +92 -3
  10. package/dist/base/ports/store/types.d.ts +2 -1
  11. package/dist/base/rank/prompt-recall.d.ts +16 -3
  12. package/dist/base/rank/prompt-recall.js +47 -7
  13. package/dist/base/util/tokenize.d.ts +29 -0
  14. package/dist/base/util/tokenize.js +68 -5
  15. package/dist/base/util/version.d.ts +1 -1
  16. package/dist/base/util/version.js +1 -1
  17. package/dist/db/meta.d.ts +1 -0
  18. package/dist/db/meta.js +1 -0
  19. package/dist/domain/consolidate/conflicts.js +2 -2
  20. package/dist/domain/learn/forward-claim-detector.js +4 -2
  21. package/dist/domain/learn/invalidation.js +5 -2
  22. package/dist/domain/search/bm25.js +3 -3
  23. package/dist/store/rules/windows.d.ts +3 -0
  24. package/dist/store/rules/windows.js +3 -0
  25. package/dist/store/sqlite/audit/audit-maintenance.js +16 -13
  26. package/dist/store/sqlite/recall/candidates.js +1 -0
  27. package/dist/store/sqlite/recall/search-rows.d.ts +2 -2
  28. package/dist/store/sqlite/recall/search-rows.js +62 -26
  29. package/dist/store/sqlite/sharing/copy-rows.d.ts +1 -0
  30. package/dist/store/sqlite/sharing/copy-rows.js +2 -1
  31. package/dist/surface/capture/command.d.ts +8 -1
  32. package/dist/surface/capture/command.js +38 -8
  33. package/dist/surface/capture/extract.d.ts +10 -0
  34. package/dist/surface/capture/extract.js +199 -26
  35. package/dist/surface/capture/session-texts.js +8 -6
  36. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  37. package/extensions/openclaw-plugin/package.json +1 -1
  38. package/openclaw.plugin.json +1 -1
  39. package/package.json +1 -1
@@ -3,7 +3,7 @@ import { BadRequestError } from '../../base/errors/api-errors.js';
3
3
  import { log } from '../../base/util/log.js';
4
4
  import { classifyOriginProject, projectId, projectNames, } from '../../base/scope/project-identity.js';
5
5
  import { DEFAULT_CONFIG } from '../../base/config/config.js';
6
- import { promptTokens } from '../../base/rank/prompt-recall.js';
6
+ import { promptSearchTerms } from '../../base/rank/prompt-recall.js';
7
7
  import { personalScopeOf } from '../../base/scope/recall-scope.js';
8
8
  import { finiteOr } from '../../domain/context/select-policy.js';
9
9
  import { strengthOf } from '../types.js';
@@ -125,9 +125,7 @@ export function planContext(ctx, opts, local) {
125
125
  /** The prompt-recall candidate request the ambient loads carry, when the prompt has any terms. */
126
126
  export function promptRecallRequest(opts, plan) {
127
127
  const { pinnedInject } = plan.config;
128
- const promptRecallTerms = plan.promptRecallPending && pinnedInject.enabled
129
- ? Array.from(promptTokens(opts.prompt ?? ''))
130
- : [];
128
+ const promptRecallTerms = plan.promptRecallPending && pinnedInject.enabled ? promptSearchTerms(opts.prompt ?? '') : [];
131
129
  return promptRecallTerms.length > 0
132
130
  ? {
133
131
  terms: promptRecallTerms,
@@ -5,7 +5,7 @@ import { DEFAULT_LOCAL_BUMP } from '../../base/rank/search-types.js';
5
5
  import { compareScoredResults } from '../../base/rank/compare.js';
6
6
  import { scopeMatch } from '../../base/rank/active-scope.js';
7
7
  import { scopeBoostFor } from '../../domain/search/boosts.js';
8
- import { promptTokens, contentTokens, gatePromptRecall } from '../../base/rank/prompt-recall.js';
8
+ import { promptTokens, memoryTokens, gatePromptRecall } from '../../base/rank/prompt-recall.js';
9
9
  import { admitWithinBudget, newestWorthSurfacing, promptRecallGateOf, reservePinned, strongestWithin, } from '../../domain/context/select-policy.js';
10
10
  const GLOBAL_DISCOUNT = 1 / DEFAULT_LOCAL_BUMP;
11
11
  // Share and promote copy a memory to the global store under a new id, so equal content is the only
@@ -53,7 +53,7 @@ export function selectPinned({ opts, plan, left, pools, admission, }) {
53
53
  const recentBudget = Math.max(0, effBudget - reservePinned(rankedPinned, effBudget));
54
54
  // Prompt recall gates the backfill on the prompt instead of recency.
55
55
  if (plan.promptRecallPending) {
56
- const candidates = () => promptRecallCandidates({ plan, pools, admit: admission.admit, rankedPinned, nowP });
56
+ const candidates = (prompt) => promptRecallCandidates({ plan, pools, admit: admission.admit, rankedPinned, nowP, prompt });
57
57
  backfillFromPrompt(opts, plan, { pinnedCfg, candidates, picked, recentBudget });
58
58
  }
59
59
  else if (plan.includeRecent > 0) {
@@ -100,7 +100,7 @@ function backfillFromPrompt(opts, plan, options) {
100
100
  const p = promptTokens(opts.prompt ?? '');
101
101
  if (p.size === 0)
102
102
  return;
103
- const candidateItems = candidates();
103
+ const candidateItems = candidates(p);
104
104
  const gated = gatePromptRecall(p, candidateItems, gate);
105
105
  plan.obs?.gated(p, candidateItems, gate, gated);
106
106
  const rows = gated.map((g) => ({
@@ -113,7 +113,7 @@ function backfillFromPrompt(opts, plan, options) {
113
113
  admitWithinBudget(rows, picked, recentBudget, rejectOverBudget(plan.obs));
114
114
  }
115
115
  // Candidates came off the ambient load's own connection (the recall request), not a fresh open.
116
- function promptRecallCandidates({ plan, pools, admit, rankedPinned, nowP, }) {
116
+ function promptRecallCandidates({ plan, pools, admit, rankedPinned, nowP, prompt, }) {
117
117
  const { obs, primaryIsGlobal } = plan;
118
118
  // A candidate carrying a pin's text would inject that memory a second time.
119
119
  const pinnedText = new Set(rankedPinned.map((r) => r.entry.content));
@@ -139,9 +139,9 @@ function promptRecallCandidates({ plan, pools, admit, rankedPinned, nowP, }) {
139
139
  const { strength } = plan;
140
140
  const [localCandidates, globalCandidates] = oneCopyPerMemory(localEligible, globalEligible, nowP, strength);
141
141
  obs?.dropMissing([...localEligible, ...globalEligible], [...localCandidates, ...globalCandidates], 'eligible', 'duplicate');
142
- return promptCandidateItems(localCandidates, globalCandidates, primaryIsGlobal);
142
+ return promptCandidateItems(localCandidates, globalCandidates, primaryIsGlobal, prompt);
143
143
  }
144
- function promptCandidateItems(localCandidates, globalCandidates, primaryIsGlobal) {
144
+ function promptCandidateItems(localCandidates, globalCandidates, primaryIsGlobal, prompt) {
145
145
  const seenCandidateIds = new Set();
146
146
  const candidateItems = [];
147
147
  // Local wins the id collision (a global row synced into the local store).
@@ -153,7 +153,12 @@ function promptCandidateItems(localCandidates, globalCandidates, primaryIsGlobal
153
153
  if (seenCandidateIds.has(e.id))
154
154
  continue;
155
155
  seenCandidateIds.add(e.id);
156
- candidateItems.push({ id: e.id, tokens: contentTokens(e.content), entry: e, isGlobal });
156
+ candidateItems.push({
157
+ id: e.id,
158
+ tokens: memoryTokens(e.content, prompt),
159
+ entry: e,
160
+ isGlobal,
161
+ });
157
162
  }
158
163
  }
159
164
  return candidateItems;
@@ -7,7 +7,7 @@ import { outcomeNudge, STRENGTH_RANK_FLOOR, STRENGTH_RANK_SPAN, } from '../../do
7
7
  import { applyReranker } from '../../domain/search/rerank.js';
8
8
  import { CONFLICT_PEER_MULTIPLIER, SALIENCE_MIN_MULTIPLIER, UTILITY_LENGTH_PENALTY_CAP, UTILITY_LENGTH_TOKENS, VALUE_MULT_MAX, VALUE_MULT_MIN, VALUE_MULT_SLOPE, } from '../../base/rank/search-types.js';
9
9
  import { isJsonNumber } from '../../base/util/json.js';
10
- import { textOverlap, tokenize as tokenizeQuery } from '../../base/util/tokenize.js';
10
+ import { indexTokenize, textOverlap, tokenize as tokenizeQuery } from '../../base/util/tokenize.js';
11
11
  import { poolScope } from './rank-search.js';
12
12
  /** The stages after the search, in order: graph expansion, re-rank, goal boost, salience, then
13
13
  * the outcome and layer filters. */
@@ -156,7 +156,7 @@ function newestOnTopicFirst(query, results) {
156
156
  for (const r of pool) {
157
157
  let hits = 0;
158
158
  if (queryTokens.size > 0) {
159
- const candTokens = new Set(tokenizeQuery(r.entry.content));
159
+ const candTokens = new Set(indexTokenize(r.entry.content));
160
160
  for (const t of queryTokens)
161
161
  if (candTokens.has(t))
162
162
  hits++;
@@ -4,7 +4,7 @@ import { DEFAULT_TENANT_ID } from '../util/tenant.js';
4
4
  import { BadRequestError } from '../errors/api-errors.js';
5
5
  import { randomUUID } from 'node:crypto';
6
6
  import { isJsonString } from '../util/json.js';
7
- import { tokenize } from '../util/tokenize.js';
7
+ import { tokenizeMin } from '../util/tokenize.js';
8
8
  import { ID_SUFFIX_CHARS } from '../util/token-text.js';
9
9
  import { calculateStrength, netWrong, PRODUCTION_STRENGTH, } from './strength.js';
10
10
  export const DEFAULT_SCHEMA_FIT = 0.5;
@@ -168,7 +168,7 @@ const TAG_OVERLAP_SCALE = 2;
168
168
  // hit.
169
169
  const CONTENT_ENOUGH_MIN_MATCHES = 5;
170
170
  function significantTokens(text) {
171
- return new Set(tokenize(text).filter((t) => t.length > FILLER_TOKEN_MAX_LENGTH));
171
+ return new Set(tokenizeMin(text, FILLER_TOKEN_MAX_LENGTH + 1));
172
172
  }
173
173
  /** computeSchemaFit against `source`, for a caller that holds counts and texts but no rows. */
174
174
  export function schemaFitFrom(content, tags, source) {
@@ -0,0 +1,15 @@
1
+ /** `text` with each code span replaced by `fill`, same length; an opener with no closer is text. */
2
+ export declare function blankCodeSpans(text: string, fill?: string): string;
3
+ /** One `**` delimiter run and the roles CommonMark gives it. */
4
+ export interface StrongMarker {
5
+ readonly index: number;
6
+ /** Left-flanking: it can open a span. */
7
+ readonly open: boolean;
8
+ /** Right-flanking: it can close a span. */
9
+ readonly close: boolean;
10
+ /** A word character on both sides, as in "2**10" and "foo**bar". */
11
+ readonly inWord: boolean;
12
+ }
13
+ /** Each `**` of `text` (code spans blanked) in order; a run after `/`, `*` or a backslash, or before `*`, is a glob. */
14
+ export declare function strongMarkers(text: string): StrongMarker[];
15
+ //# sourceMappingURL=markdown-delimiters.d.ts.map
@@ -0,0 +1,94 @@
1
+ // CommonMark's code span and "**" rules, read once for the quality gate and the capture extractor.
2
+ const PUNCTUATION = /[\p{P}\p{S}]/u;
3
+ const WORD = /[\p{L}\p{N}_]/u;
4
+ /** Every maximal backtick run of `text` as [start, end), grouped by its length, in text order. */
5
+ function runsByLength(text) {
6
+ const byLength = new Map();
7
+ for (const { 0: run, index } of text.matchAll(/`+/g)) {
8
+ const same = byLength.get(run.length) ?? [];
9
+ same.push([index, index + run.length]);
10
+ byLength.set(run.length, same);
11
+ }
12
+ return byLength;
13
+ }
14
+ /** `text` with each code span replaced by `fill`, same length; an opener with no closer is text. */
15
+ export function blankCodeSpans(text, fill = 'x') {
16
+ const byLength = runsByLength(text);
17
+ // Openers only move forward, so each length keeps a cursor that never goes back.
18
+ const cursor = new Map();
19
+ const closerEnd = (length, from) => {
20
+ const same = byLength.get(length) ?? [];
21
+ let k = cursor.get(length) ?? 0;
22
+ while (k < same.length && same[k][0] < from)
23
+ k++;
24
+ cursor.set(length, k);
25
+ return k < same.length ? same[k][1] : -1;
26
+ };
27
+ let out = '';
28
+ let copied = 0;
29
+ let at = 0;
30
+ while (at < text.length) {
31
+ if (text[at] === '\\') {
32
+ at += 2;
33
+ continue;
34
+ }
35
+ if (text[at] !== '`') {
36
+ at++;
37
+ continue;
38
+ }
39
+ let end = at;
40
+ while (text[end] === '`')
41
+ end++;
42
+ const close = closerEnd(end - at, end);
43
+ if (close < 0) {
44
+ at = end;
45
+ continue;
46
+ }
47
+ out += text.slice(copied, at) + fill.repeat(close - at);
48
+ copied = at = close;
49
+ }
50
+ return out + text.slice(copied);
51
+ }
52
+ function charBefore(text, i) {
53
+ if (i === 0)
54
+ return '';
55
+ const unit = text.charCodeAt(i - 1);
56
+ const pair = unit >= 0xdc00 && unit <= 0xdfff && i >= 2 && text.charCodeAt(i - 2) >= 0xd800;
57
+ return pair ? text.slice(i - 2, i) : text[i - 1];
58
+ }
59
+ function charAfter(text, i) {
60
+ const point = text.codePointAt(i);
61
+ return point === undefined ? '' : String.fromCodePoint(point);
62
+ }
63
+ const isSpace = (char) => char === '' || /\s/u.test(char);
64
+ const isPunctuation = (char) => char !== '' && PUNCTUATION.test(char);
65
+ /** Each `**` of `text` (code spans blanked) in order; a run after `/`, `*` or a backslash, or before `*`, is a glob. */
66
+ export function strongMarkers(text) {
67
+ const found = [];
68
+ for (const { index } of text.matchAll(/\*\*/g)) {
69
+ const before = charBefore(text, index);
70
+ const after = charAfter(text, index + 2);
71
+ if (['/', '*', '\\'].includes(before) || after === '*')
72
+ continue;
73
+ const [spaceBefore, spaceAfter] = [isSpace(before), isSpace(after)];
74
+ const [markBefore, markAfter] = [isPunctuation(before), isPunctuation(after)];
75
+ const marker = {
76
+ index,
77
+ open: !spaceAfter && (!markAfter || spaceBefore || markBefore),
78
+ close: !spaceBefore && (!markBefore || spaceAfter || markAfter),
79
+ inWord: WORD.test(before) && WORD.test(after),
80
+ };
81
+ found.push({ marker, slashed: after === '/' });
82
+ }
83
+ // "**/etc/hosts**" is bold when the next marker closes it with no space between; else it is "**/".
84
+ const bold = ({ marker, slashed }, i) => {
85
+ const next = found[i + 1];
86
+ if (!slashed)
87
+ return true;
88
+ if (next === undefined || next.slashed || !next.marker.close)
89
+ return false;
90
+ return !/\s/.test(text.slice(marker.index + 2, next.marker.index));
91
+ };
92
+ return found.filter(bold).map(({ marker }) => marker);
93
+ }
94
+ //# sourceMappingURL=markdown-delimiters.js.map
@@ -1,6 +1,6 @@
1
1
  // The memory-quality audit rules: pure functions of a row, so any store can be audited by them.
2
2
  import { canAutoDelete, isKeptForGood } from './keep-rules.js';
3
- import { assessAutomaticMemory, automaticDefect, BUNDLE_HEADER, isAutomaticEntry, isCertainReason, isFragment, isReleaseCommitNoise, isShortAndUnspecific, MIN_MEMORY_CHARS, MIN_SUBSTANTIVE_WORDS, substantiveWordCount, } from './memory-quality.js';
3
+ import { automaticDefect, BUNDLE_HEADER, isAutomaticEntry, isCertainReason, isFragment, isReleaseCommitNoise, isShortAndUnspecific, MIN_MEMORY_CHARS, MIN_SUBSTANTIVE_WORDS, storedDefect, substantiveWordCount, } from './memory-quality.js';
4
4
  import { heldTexts } from '../util/same-text.js';
5
5
  export function auditMemory(entry, backsObject = false) {
6
6
  const issue = classifyMemory(entry);
@@ -112,7 +112,7 @@ export function repairDefect(entry, parents) {
112
112
  return null;
113
113
  // Only a bundle header marks parts; a refined row keeps its lead line, so it is judged whole.
114
114
  if (!BUNDLE_HEADER.test(entry.content)) {
115
- const { reason } = assessAutomaticMemory(entry.content);
115
+ const reason = storedDefect(entry.content);
116
116
  return reason === null
117
117
  ? null
118
118
  : { reason, disposition: isCertainReason(reason) ? 'set-aside' : 'review' };
@@ -122,7 +122,7 @@ export function repairDefect(entry, parents) {
122
122
  return { reason: 'derived bundle has no safely parsed constituents', disposition: 'review' };
123
123
  }
124
124
  const reasons = texts
125
- .map((text) => assessAutomaticMemory(text).reason)
125
+ .map(storedDefect)
126
126
  .filter((reason) => reason !== null);
127
127
  if (reasons.length === 0)
128
128
  return null;
@@ -137,6 +137,6 @@ export function repairDefect(entry, parents) {
137
137
  /** The defect a store too old to repair can only report: an automatic row's text alone, for a
138
138
  * person to review. */
139
139
  export function previewDefect(row) {
140
- return isAutomaticEntry(row) ? assessAutomaticMemory(row.content).reason : null;
140
+ return isAutomaticEntry(row) ? storedDefect(row.content) : null;
141
141
  }
142
142
  //# sourceMappingURL=memory-audit.js.map
@@ -1,5 +1,7 @@
1
1
  import { type MemoryEntry } from './entry.js';
2
2
  export type AutomaticMemoryDefect = 'too-short' | 'release-activity' | 'sentence-fragment' | 'possible-fragment' | 'subjectless-outcome' | 'raw-output' | 'too-vague' | 'no-specificity';
3
+ /** Bump when a certain defect is added, so the next daily run or sleep cleans each store once more. */
4
+ export declare const QUALITY_GATE_VERSION = 2;
3
5
  /** Text shorter than this holds no fact worth keeping. */
4
6
  export declare const MIN_MEMORY_CHARS = 10;
5
7
  /** Fewer substantive words than this leave a memory too vague to act on. */
@@ -14,12 +16,22 @@ export interface AutomaticMemoryAssessment {
14
16
  export declare function substantiveWordCount(text: string): number;
15
17
  /** Preserves the auditor's existing destructive release-noise classification. */
16
18
  export declare function isReleaseCommitNoise(text: string): boolean;
19
+ export declare const QUOTE_MARK: RegExp;
20
+ /** The offset of the ")" that closes an aside the text began inside, or -1: a ")" before any "(" or
21
+ * open "[". Code spans do not count. */
22
+ export declare function asideCloseOffset(text: string): number;
23
+ /** Whether `text` starts inside a bold span or an aside, as rows the old capture segmenter cut do.
24
+ * Repair reads this; admission does not, since capture drops a sentence the split cut out of an aside. */
25
+ export declare function startsInsideCut(text: string): boolean;
17
26
  /** Names automatic-capture defects without inventing missing context. */
18
27
  export declare function assessAutomaticMemory(content: string): AutomaticMemoryAssessment;
19
28
  export declare function isCertainReason(reason: AutomaticMemoryDefect | null): boolean;
20
29
  /** The bar for sleep's derived memories and for repair: a defect the checks are sure of, else
21
30
  * null. */
22
31
  export declare function certainDefect(content: string): AutomaticMemoryDefect | null;
32
+ /** The defect repair and the audit find in a stored text: admission's, but a start inside a bold span
33
+ * or an aside outranks an uncertain reason. */
34
+ export declare function storedDefect(text: string): AutomaticMemoryDefect | null;
23
35
  export declare const BUNDLE_HEADER: RegExp;
24
36
  type Provenance = Pick<MemoryEntry, 'source' | 'confidence' | 'content' | 'extractedFrom' | 'dagLevel' | 'tags'>;
25
37
  /**
@@ -1,6 +1,10 @@
1
1
  import { COMPACTION_MEMORY_TAG, COMPACTION_SOURCE_PREFIX, DAG_LEVEL, } from './entry.js';
2
+ import { blankCodeSpans, strongMarkers } from './markdown-delimiters.js';
2
3
  import { heldTexts } from '../util/same-text.js';
3
4
  import { MEMORY_QUALITY_STOPWORDS } from '../util/stopwords.js';
5
+ import { CJK_SCRIPTS } from '../util/tokenize.js';
6
+ /** Bump when a certain defect is added, so the next daily run or sleep cleans each store once more. */
7
+ export const QUALITY_GATE_VERSION = 2;
4
8
  /** Text shorter than this holds no fact worth keeping. */
5
9
  export const MIN_MEMORY_CHARS = 10;
6
10
  /** Text shorter than this must name a path, number, name or code to be kept. */
@@ -9,11 +13,12 @@ const SPECIFIC_DETAIL_CHARS = 40;
9
13
  const FRAGMENT_MAX_CHARS = 50;
10
14
  /** Fewer substantive words than this leave a memory too vague to act on. */
11
15
  export const MIN_SUBSTANTIVE_WORDS = 2;
12
- const CJK_LETTERS = /(?=\p{L})(?:\p{Script=Han}|\p{Script=Hiragana}|\p{Script=Katakana})/gu;
16
+ // Without ー, which alone says nothing, and the Han script's radicals, which are not letters.
17
+ const CJK_LETTER = new RegExp(String.raw `(?=\p{L})[${CJK_SCRIPTS}]`, 'gu');
13
18
  const CHAT_ACRONYMS = /^(?:TODO|FYI|LGTM|IIRC|IMO|IMHO|FWIW|TBD|BTW|ASAP|AFAIK|WIP|NB|PS)$/;
14
19
  /** Counts spaced words plus CJK letters without lowering mixed-script counts. */
15
20
  export function substantiveWordCount(text) {
16
- const cjkLetterCount = (text.match(CJK_LETTERS) ?? []).length;
21
+ const cjkLetterCount = (text.match(CJK_LETTER) ?? []).length;
17
22
  const spacedCount = text
18
23
  .toLowerCase()
19
24
  .split(/\s+/)
@@ -79,6 +84,84 @@ const POSSIBLE_CUT_ENDING = new RegExp(String.raw `\b(?:needs to|depends on|requ
79
84
  function isBareCondition(bare) {
80
85
  return !/[,;]|\bthen\b/i.test(bare) && !RULE_WORD.test(bare) && bare.split(/\s+/).length <= 6;
81
86
  }
87
+ // A blanked code span reads as a letter that is no roman numeral, so it is never a list marker.
88
+ const CODE_AS_LETTER = 'q';
89
+ const LIST_MARKER_START = /[\s(,;:]/;
90
+ const ROMAN_NUMERAL = /^m{0,3}(cm|cd|d?c{0,3})(xc|xl|l?x{0,3})(ix|iv|v?i{0,3})$/i;
91
+ const ROMAN_MARKER_START = /[(,;]/;
92
+ const ROMAN_AFTER_WORD = /(?:^|[^A-Za-z])(?:option|step|item)\s+$/i;
93
+ const BOLD_CLOSE_NEXT = /[\s.,;:!?)\]"'’”—–]/;
94
+ const SENTENCE_MARK = /[.!?;:,]/;
95
+ const WORD_CODE_OR_CLOSER = /[\p{L}\p{N}`)\]]/u;
96
+ const DASH_OR_SPACE = /[\s—–]/;
97
+ const CLOSING_QUOTE = /['"’”]/;
98
+ export const QUOTE_MARK = /['"‘’“”]/;
99
+ // A "**" that only closes (not after a word), a dual-role one after a sentence mark with no closer,
100
+ // or a bare one opening the text shows the text began inside a bold span.
101
+ function opensInsideBold(text) {
102
+ const [first, ...later] = strongMarkers(blankCodeSpans(text, '`'));
103
+ if (first === undefined)
104
+ return false;
105
+ const next = text[first.index + 2] ?? ' ';
106
+ if (first.close && !first.open) {
107
+ // After a word, code span or closing bracket, "2** (n + 1)" and "(n + 1)** 2" are too often literal to be certain.
108
+ if (WORD_CODE_OR_CLOSER.test(text[first.index - 1] ?? ''))
109
+ return false;
110
+ return BOLD_CLOSE_NEXT.test(next);
111
+ }
112
+ // A quoted "**", "(**.env**)" and "foo**bar**" can open too: only "done.**— then" with no closer is a cut.
113
+ if (first.close) {
114
+ const closerFollows = later.some((marker) => marker.close);
115
+ return SENTENCE_MARK.test(text[first.index - 1]) && DASH_OR_SPACE.test(next) && !closerFollows;
116
+ }
117
+ return text.slice(0, first.index).trim() === '' && /^\s/.test(text.slice(first.index + 2));
118
+ }
119
+ /** "1)", "a)" and "ii)" mark list items; so does a longer valid roman numeral in a label place. */
120
+ function isListMarker(text, close) {
121
+ let start = close;
122
+ while (start > 0 && /[A-Za-z0-9]/.test(text[start - 1]))
123
+ start--;
124
+ const token = text.slice(start, close);
125
+ const before = text[start - 1] ?? '';
126
+ if (token.length <= 3)
127
+ return start === 0 || LIST_MARKER_START.test(before);
128
+ if (!ROMAN_NUMERAL.test(token))
129
+ return false;
130
+ return (start === 0 ||
131
+ ROMAN_MARKER_START.test(before) ||
132
+ ROMAN_AFTER_WORD.test(text.slice(Math.max(0, start - 12), start)));
133
+ }
134
+ // A ")" after a letter, a closing quote or "letter." closes an aside; a quoted ")" does not.
135
+ function closesAside(text, i) {
136
+ const before = text[i - 1] ?? '';
137
+ if (/[A-Za-z]/.test(before))
138
+ return !isListMarker(text, i);
139
+ if (CLOSING_QUOTE.test(before))
140
+ return !QUOTE_MARK.test(text[i + 1] ?? '');
141
+ return before === '.' && /[A-Za-z]/.test(text[i - 2] ?? '');
142
+ }
143
+ /** The offset of the ")" that closes an aside the text began inside, or -1: a ")" before any "(" or
144
+ * open "[". Code spans do not count. */
145
+ export function asideCloseOffset(text) {
146
+ const blanked = blankCodeSpans(text, CODE_AS_LETTER);
147
+ let brackets = 0;
148
+ for (let i = 0; i < blanked.length; i++) {
149
+ if (blanked[i] === '(')
150
+ return -1;
151
+ if (blanked[i] === '[')
152
+ brackets++;
153
+ else if (blanked[i] === ']')
154
+ brackets = Math.max(0, brackets - 1);
155
+ else if (blanked[i] === ')' && brackets === 0 && closesAside(blanked, i))
156
+ return i;
157
+ }
158
+ return -1;
159
+ }
160
+ /** Whether `text` starts inside a bold span or an aside, as rows the old capture segmenter cut do.
161
+ * Repair reads this; admission does not, since capture drops a sentence the split cut out of an aside. */
162
+ export function startsInsideCut(text) {
163
+ return opensInsideBold(text) || asideCloseOffset(text) >= 0;
164
+ }
82
165
  function isDanglingAssertion(text) {
83
166
  const bare = bareText(text);
84
167
  // Only a lowercase start is a cut: "To rebuild the index, run ..." opens an instruction.
@@ -192,6 +275,12 @@ export function certainDefect(content) {
192
275
  const { reason } = assessAutomaticMemory(content);
193
276
  return isCertainReason(reason) ? reason : null;
194
277
  }
278
+ /** The defect repair and the audit find in a stored text: admission's, but a start inside a bold span
279
+ * or an aside outranks an uncertain reason. */
280
+ export function storedDefect(text) {
281
+ const { reason } = assessAutomaticMemory(text);
282
+ return !isCertainReason(reason) && startsInsideCut(text.trim()) ? 'sentence-fragment' : reason;
283
+ }
195
284
  // Watch failures ('autolearn') and tool failures keep a fixed "Command 'x' failed" format by
196
285
  // design, so they are not judged.
197
286
  const AUTOMATIC_SOURCES = new Set(['capture', 'git-learn', 'git', 'consolidation']);
@@ -235,7 +324,7 @@ export function automaticDefect(entry) {
235
324
  if (!isAutomaticEntry(entry))
236
325
  return null;
237
326
  const parts = bundleParts(entry);
238
- const reasons = (parts.length > 0 ? parts : [entry.content.trim()]).map((text) => assessAutomaticMemory(text).reason);
327
+ const reasons = (parts.length > 0 ? parts : [entry.content.trim()]).map(storedDefect);
239
328
  return reasons.every((reason) => reason !== null) ? (reasons[0] ?? null) : null;
240
329
  }
241
330
  /**
@@ -112,7 +112,8 @@ export interface AuditEvent {
112
112
  targetId: string | null;
113
113
  metadata: JsonObject;
114
114
  }
115
- export type HeldText = Pick<MemoryEntry, 'content' | 'source' | 'originProject'>;
115
+ /** A stored text the capture dedup reads; `pinned` is optional so an adapter that omits it still fits. */
116
+ export type HeldText = Pick<MemoryEntry, 'content' | 'source' | 'originProject'> & Partial<Pick<MemoryEntry, 'pinned'>>;
116
117
  /** One project name, or every name a project's rows carry. */
117
118
  export type OriginFilter = string | readonly string[];
118
119
  /** The api_keys fields a tenant, revocation, rank or self-service ownership check reads. */
@@ -5,9 +5,17 @@ export interface PromptRecallGate {
5
5
  minShared: number;
6
6
  maxItems: number;
7
7
  }
8
- /** Distinct content tokens: longer than 2 chars, not a stop word. */
8
+ /** Distinct content tokens: longer than 2 chars or CJK, not a stop word. */
9
9
  export declare function contentTokens(text: string): Set<string>;
10
10
  export declare function promptTokens(prompt: string): Set<string>;
11
+ /** `contentTokens` of a memory, plus each one-letter CJK token of `prompt` it holds inside a run, so
12
+ * that letter matches as `hippo recall` matches it while the memory's other letters stay out of the
13
+ * overlap's denominator. */
14
+ export declare function memoryTokens(content: string, prompt: ReadonlySet<string>): Set<string>;
15
+ /** The terms a store searches for the prompt: its tokens, with each CJK run as its letter
16
+ * trigrams, which the trigram index can look up across the store, in place of bigrams, which it
17
+ * cannot. */
18
+ export declare function promptSearchTerms(prompt: string): string[];
11
19
  export interface OverlapScore {
12
20
  score: number;
13
21
  shared: number;
@@ -31,6 +39,11 @@ export declare function rarestPromptTerms(terms: Iterable<string>, docCount: (te
31
39
  * `mode`. */
32
40
  export declare function ftsTermParts(term: string): string[];
33
41
  /** The rarest terms as a space-joined FTS query, given each part's document count; a term counts as its rarest part, an
34
- * upper bound on rows holding the whole term, and a term with no parts counts 0, so it is dropped. */
35
- export declare function rarestFtsQuery(terms: readonly string[], docCount: (part: string) => number, maxTerms?: number): string;
42
+ * upper bound on rows holding the whole term, and a term with no parts counts 0, so it is dropped. A CJK term counts
43
+ * by `cjkDocCount`, a trigram index's count, in the same pick, and a run under 3 letters that counts 0 follows the
44
+ * pick; without one, CJK terms take `spreadCjkTerms` slots. */
45
+ export declare function rarestFtsQuery(terms: readonly string[], docCount: (part: string) => number, maxTerms?: number, cjkDocCount?: (term: string) => number): string;
46
+ /** Up to `maxTerms` of `terms` when no index counts their CJK terms: those keep half the slots, at even
47
+ * steps through the prompt, so neither a long run of words nor the prompt's opening letters starve them. */
48
+ export declare function spreadCjkTerms(terms: readonly string[], maxTerms: number): string[];
36
49
  //# sourceMappingURL=prompt-recall.d.ts.map
@@ -1,13 +1,13 @@
1
1
  /** Recall gated on the hook prompt, not the five newest memories (pure, no I/O). */
2
- import { tokenize } from '../util/tokenize.js';
2
+ import { cjkIndexTerms, hasCjk, partitionCjkTerms, tokenizeMin } from '../util/tokenize.js';
3
3
  import { MEMORY_QUALITY_STOPWORDS } from '../util/stopwords.js';
4
4
  // Latency bound, not tuned: fixed in the prereg regardless of gate config.
5
5
  const PROMPT_RECALL_MAX_CHARS = 4000;
6
- /** Distinct content tokens: longer than 2 chars, not a stop word. */
6
+ /** Distinct content tokens: longer than 2 chars or CJK, not a stop word. */
7
7
  export function contentTokens(text) {
8
8
  const out = new Set();
9
- for (const t of tokenize(text)) {
10
- if (t.length > 2 && !MEMORY_QUALITY_STOPWORDS.has(t))
9
+ for (const t of tokenizeMin(text, 3)) {
10
+ if (!MEMORY_QUALITY_STOPWORDS.has(t))
11
11
  out.add(t);
12
12
  }
13
13
  return out;
@@ -15,6 +15,23 @@ export function contentTokens(text) {
15
15
  export function promptTokens(prompt) {
16
16
  return contentTokens(prompt.slice(0, PROMPT_RECALL_MAX_CHARS));
17
17
  }
18
+ /** `contentTokens` of a memory, plus each one-letter CJK token of `prompt` it holds inside a run, so
19
+ * that letter matches as `hippo recall` matches it while the memory's other letters stay out of the
20
+ * overlap's denominator. */
21
+ export function memoryTokens(content, prompt) {
22
+ const out = contentTokens(content);
23
+ for (const t of prompt)
24
+ if ([...t].length === 1 && hasCjk(t) && content.includes(t))
25
+ out.add(t);
26
+ return out;
27
+ }
28
+ /** The terms a store searches for the prompt: its tokens, with each CJK run as its letter
29
+ * trigrams, which the trigram index can look up across the store, in place of bigrams, which it
30
+ * cannot. */
31
+ export function promptSearchTerms(prompt) {
32
+ const text = prompt.slice(0, PROMPT_RECALL_MAX_CHARS);
33
+ return [...partitionCjkTerms(contentTokens(text)).words, ...cjkIndexTerms(text)];
34
+ }
18
35
  /** Overlap of two token sets under `metric`. 0 when either set is empty. */
19
36
  export function scoreOverlap(p, m, metric) {
20
37
  if (p.size === 0 || m.size === 0)
@@ -61,12 +78,35 @@ export function ftsTermParts(term) {
61
78
  return term.split(/[^\p{L}\p{N}]+/u).filter(Boolean);
62
79
  }
63
80
  /** The rarest terms as a space-joined FTS query, given each part's document count; a term counts as its rarest part, an
64
- * upper bound on rows holding the whole term, and a term with no parts counts 0, so it is dropped. */
65
- export function rarestFtsQuery(terms, docCount, maxTerms = RAREST_TERM_COUNT) {
81
+ * upper bound on rows holding the whole term, and a term with no parts counts 0, so it is dropped. A CJK term counts
82
+ * by `cjkDocCount`, a trigram index's count, in the same pick, and a run under 3 letters that counts 0 follows the
83
+ * pick; without one, CJK terms take `spreadCjkTerms` slots. */
84
+ export function rarestFtsQuery(terms, docCount, maxTerms = RAREST_TERM_COUNT, cjkDocCount) {
66
85
  const termCount = (t) => {
67
86
  const parts = ftsTermParts(t);
68
87
  return parts.length === 0 ? 0 : Math.min(...parts.map(docCount));
69
88
  };
70
- return rarestPromptTerms(terms, termCount, maxTerms).join(' ');
89
+ const { words, cjk } = partitionCjkTerms(terms);
90
+ if (cjkDocCount === undefined) {
91
+ return spreadCjkTerms([...rarestPromptTerms(words, termCount, maxTerms), ...cjk], maxTerms).join(' ');
92
+ }
93
+ const counts = new Map([
94
+ ...words.map((t) => [t, termCount(t)]),
95
+ ...cjk.map((t) => [t, cjkDocCount(t)]),
96
+ ]);
97
+ const ranked = rarestPromptTerms(counts.keys(), (t) => counts.get(t) ?? 0, maxTerms);
98
+ // A run under 3 letters can sit in a row with no trigram that shows it, so a 0 cannot drop it.
99
+ const unproven = [...new Set(cjk)].filter((t) => counts.get(t) === 0 && [...t].length < 3);
100
+ return [...ranked, ...unproven].slice(0, maxTerms).join(' ');
101
+ }
102
+ /** Up to `maxTerms` of `terms` when no index counts their CJK terms: those keep half the slots, at even
103
+ * steps through the prompt, so neither a long run of words nor the prompt's opening letters starve them. */
104
+ export function spreadCjkTerms(terms, maxTerms) {
105
+ // SHORTCUT: even steps, not rarest first; a store with a trigram index ranks CJK terms by its counts.
106
+ const { words, cjk } = partitionCjkTerms(terms);
107
+ const cjkSlots = Math.min(cjk.length, Math.max(Math.ceil(maxTerms / 2), maxTerms - words.length));
108
+ const step = cjkSlots > 1 ? (cjk.length - 1) / (cjkSlots - 1) : 0;
109
+ const kept = new Set(Array.from({ length: cjkSlots }, (_, i) => Math.round(i * step)));
110
+ return [...words.slice(0, maxTerms - cjkSlots), ...cjk.filter((_, k) => kept.has(k))];
71
111
  }
72
112
  //# sourceMappingURL=prompt-recall.js.map
@@ -1,4 +1,33 @@
1
+ /** Han, kana and Hangul, as the body of a `u`-flag character class. */
2
+ export declare const CJK_SCRIPTS: string;
3
+ /** CJK_SCRIPTS and ー, the prolonged-sound mark, a Common-script letter inside katakana words. */
4
+ export declare const CJK_LETTERS: string;
5
+ /** Whether `text` holds a Chinese, Japanese or Korean letter. */
6
+ export declare function hasCjk(text: string): boolean;
7
+ /** The overlapping `n`-letter pieces of each CJK run in `text`; a run of `n` or fewer letters is one
8
+ * piece. */
9
+ export declare function cjkGrams(text: string, n: number): string[];
10
+ /** Each CJK run's distinct letter trigrams, a shorter run whole: the trigram index's terms. */
11
+ export declare function cjkIndexTerms(text: string): string[];
12
+ /** `terms` split by the index that can find them: the word index holds a whole CJK run as one
13
+ * token, so CJK terms skip it for the substring indexes. */
14
+ export declare function partitionCjkTerms(terms: Iterable<string>): {
15
+ words: string[];
16
+ cjk: string[];
17
+ };
18
+ /** `word` with each CJK run in it replaced by the run's letter bigrams: those scripts put no space
19
+ * between words, and a bigram, unlike a dictionary split, matches the same way in query and text. */
20
+ export declare function splitCjkRuns(word: string): string[];
21
+ /** Lower-cased words of `text` (runs of letters, marks, digits and `_`) of 2 or more characters; a
22
+ * CJK run becomes its letter bigrams, a lone CJK letter itself. */
1
23
  export declare function tokenize(text: string): string[];
24
+ /** `tokenize` for the text a query is matched against: each letter of a longer CJK run is added,
25
+ * so a one-letter query, which `tokenize` keeps whole, matches inside a run, while a longer query,
26
+ * all bigrams, does not match single letters. */
27
+ export declare function indexTokenize(text: string): string[];
28
+ /** `tokenize` with words shorter than `minLength` dropped; CJK tokens, never longer than 2, are
29
+ * kept. */
30
+ export declare function tokenizeMin(text: string, minLength: number): string[];
2
31
  /** Jaccard overlap of two token sets; two empty sets count as identical. */
3
32
  export declare function jaccardSets(a: ReadonlySet<string>, b: ReadonlySet<string>): number;
4
33
  /** Jaccard overlap of the two texts' token sets. */