@linxiraos/pi-mnemopi 1.1.19 → 1.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/README.md +4 -2
- package/THIRD-PARTY-NOTICES.txt +3 -40
- package/dist/types/core/beam/helpers.d.ts +0 -2
- package/dist/types/util/regex.d.ts +8 -1
- package/package.json +6 -6
- package/src/core/aaak.ts +26 -7
- package/src/core/beam/helpers.ts +1 -74
- package/src/core/beam/recall.ts +52 -119
- package/src/core/content-sanitizer.ts +1 -2
- package/src/core/shmr.ts +7 -7
- package/src/core/veracity-consolidation.ts +3 -4
- package/src/dr/recovery.ts +2 -3
- package/src/util/regex.ts +26 -4
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @linxiraos/pi-mnemopi
|
|
2
2
|
|
|
3
|
-
Local SQLite memory engine for
|
|
3
|
+
Local SQLite memory engine for omp agents.
|
|
4
4
|
|
|
5
5
|
This package is the Bun/TypeScript port of the Mnemosyne memory engine. It provides:
|
|
6
6
|
|
|
@@ -29,6 +29,8 @@ console.log(id, results[0]?.content);
|
|
|
29
29
|
memory.close();
|
|
30
30
|
```
|
|
31
31
|
|
|
32
|
+
Lexical recall matches whole tokens, declared synonyms, underscore-separated identifier parts, and other forms of a word of at least four characters: the word plus `s`, `es`, `d`, `ed`, `ing`, `er`, `ers`, `ment` or `ments`, a final `e` replaced by `ing`/`ion`/`ions`, or a final `y` replaced by `ies`/`ied`. `backup` matches `backups`, `facts` matches `fact` and `caching` matches `cache`, but `1Password` does not match `pass` or `word`, and `redis` does not match `redistribution`.
|
|
33
|
+
|
|
32
34
|
## Configuration
|
|
33
35
|
|
|
34
36
|
`Mnemopi` accepts LLM and embedding options directly. `MNEMOPI_*` environment variables remain fallbacks/defaults when the matching constructor option is omitted.
|
|
@@ -72,7 +74,7 @@ const dynamicLlm = new Mnemopi({
|
|
|
72
74
|
|
|
73
75
|
`Mnemopi` itself exposes banks directly through constructor options such as `bank`; it does not hard-code coding-agent project scoping.
|
|
74
76
|
|
|
75
|
-
The
|
|
77
|
+
The omp coding-agent wrapper adds `mnemopi.scoping` on top of those constructor options:
|
|
76
78
|
|
|
77
79
|
- `global`: one shared bank
|
|
78
80
|
- `per-project`: isolated project memory
|
package/THIRD-PARTY-NOTICES.txt
CHANGED
|
@@ -78,8 +78,8 @@ below.
|
|
|
78
78
|
uutils coreutils (https://github.com/uutils/coreutils), MIT
|
|
79
79
|
--------------------------------------------------------------------------------
|
|
80
80
|
Covers: base32, base64, basename, cat, cksum (shared checksum machinery),
|
|
81
|
-
b2sum, md5sum, sha1sum, sha224sum, sha256sum, sha384sum, sha512sum, comm,
|
|
82
|
-
date, dirname, head, hostname, ln, ls, mkdir, mktemp, mv, nproc, paste,
|
|
81
|
+
b2sum, md5sum, sha1sum, sha224sum, sha256sum, sha384sum, sha512sum, comm, cp,
|
|
82
|
+
cut, date, dirname, head, hostname, ln, ls, mkdir, mktemp, mv, nproc, paste,
|
|
83
83
|
printenv, readlink, realpath, rm, seq, sort, stat, tac, tail, tee, touch, tr,
|
|
84
84
|
truncate, uname, uniq, wc, whoami, yes.
|
|
85
85
|
|
|
@@ -293,7 +293,7 @@ MIT License
|
|
|
293
293
|
Copyright (c) 2026 Sander Land
|
|
294
294
|
(measured tokenizer vocabulary data, reconstruction model, and reference
|
|
295
295
|
implementation: https://github.com/sanderland/ctok)
|
|
296
|
-
Copyright (c) 2026 Can Bölük and the
|
|
296
|
+
Copyright (c) 2026 Can Bölük and the omp contributors
|
|
297
297
|
Copyright (c) 2026 Stencil Labs, Inc.
|
|
298
298
|
(Rust implementation and the compact binary vocabulary encoding in
|
|
299
299
|
crates/pi-natives/src/utok/claude)
|
|
@@ -335,43 +335,6 @@ This license allows the work and adaptations of it to be shared and used
|
|
|
335
335
|
commercially, as long as it is attributed to Poppy Works. The font is bundled
|
|
336
336
|
here (crates/pi-natives/src/fonts/Silver.ttf) as a CJK/Unicode bitmap fallback.
|
|
337
337
|
|
|
338
|
-
-------------------------------------------------------------------------------
|
|
339
|
-
packages/utils/src/vendor/mermaid-ascii/NOTICE
|
|
340
|
-
|
|
341
|
-
This directory contains an in-house Mermaid-diagram-to-ASCII renderer adapted
|
|
342
|
-
from beautiful-mermaid (https://github.com/lukilabs/beautiful-mermaid), used
|
|
343
|
-
under the MIT License.
|
|
344
|
-
|
|
345
|
-
Copyright (c) 2026 Craft Docs
|
|
346
|
-
|
|
347
|
-
Only the ASCII rendering pipeline is ported (flowchart/state, sequence, class,
|
|
348
|
-
ER, and xychart diagrams); the SVG renderer and its `elkjs` graph-layout
|
|
349
|
-
dependency, the browser entry point, and the SVG theme/style modules were
|
|
350
|
-
dropped. Terminal display width is reimplemented on `Bun.stringWidth`, and
|
|
351
|
-
inline label formatting (HTML tags, markdown emphasis) is reduced to plain text
|
|
352
|
-
for ASCII output. Layout and edge-routing logic is preserved faithfully so
|
|
353
|
-
ASCII output matches the upstream package.
|
|
354
|
-
|
|
355
|
-
MIT License
|
|
356
|
-
|
|
357
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
358
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
359
|
-
in the Software without restriction, including without limitation the rights
|
|
360
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
361
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
362
|
-
furnished to do so, subject to the following conditions:
|
|
363
|
-
|
|
364
|
-
The above copyright notice and this permission notice shall be included in all
|
|
365
|
-
copies or substantial portions of the Software.
|
|
366
|
-
|
|
367
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
368
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
369
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
370
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
371
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
372
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
373
|
-
SOFTWARE.
|
|
374
|
-
|
|
375
338
|
-------------------------------------------------------------------------------
|
|
376
339
|
packages/coding-agent/src/markit/NOTICE
|
|
377
340
|
|
|
@@ -30,8 +30,6 @@ export declare function parseQueryTime(queryTime?: string | Date | null): Date;
|
|
|
30
30
|
export declare function parseTimestampFast(ts: string | null | undefined, beam?: Pick<BeamMemoryState, "caches"> | null): Date | null;
|
|
31
31
|
export declare function recencyDecay(timestamp: string | null | undefined, halflifeHours?: number, now?: Date): number;
|
|
32
32
|
export declare function temporalBoost(memoryTimestamp: string | null | undefined, queryTime: Date | string, halflifeHours?: number, beam?: Pick<BeamMemoryState, "caches"> | null): number;
|
|
33
|
-
export declare function lexicalRelevance(queryTokens: readonly string[], content: string, queryLower?: string): number;
|
|
34
|
-
export declare function strictFactMatches(query: string, factText: string): boolean;
|
|
35
33
|
export declare function buildFtsQuery(query: string): string;
|
|
36
34
|
export declare function cjkLikeSearch(db: Database, query: string, k?: number, working?: boolean): Array<FtsRankResult | WorkingFtsRankResult>;
|
|
37
35
|
export declare function ftsSearch(db: Database, query: string, k?: number): FtsRankResult[];
|
|
@@ -3,8 +3,15 @@ export declare const RECALL_SYNONYMS: Readonly<Record<string, readonly string[]>
|
|
|
3
3
|
export declare function isCjkChar(ch: string): boolean;
|
|
4
4
|
export declare function hasCjk(text: string): boolean;
|
|
5
5
|
export declare const containsSpacelessCjk: typeof hasCjk;
|
|
6
|
+
/**
|
|
7
|
+
* Weak match between two forms of one word, in either direction: the longer token is the shorter
|
|
8
|
+
* one plus an inflectional or common derivational ending (`facts`/`fact`, `deploy`/`deployment`,
|
|
9
|
+
* `caching`/`cache`, `stories`/`story`). The shorter token needs at least four characters. Any
|
|
10
|
+
* other extension or fragment never matches, so `1password`, `password` and `passport` do not
|
|
11
|
+
* find `pass`, and `redis` does not find `redistribution`. This is deliberately not a stemmer.
|
|
12
|
+
*/
|
|
13
|
+
export declare function matchesWordForm(left: string, right: string): boolean;
|
|
6
14
|
export declare function recallTokens(text: string): string[];
|
|
7
|
-
export declare function factMatchTokens(text: string): Set<string>;
|
|
8
15
|
export declare function expandedQueryTokens(tokens: readonly string[]): string[];
|
|
9
16
|
export declare function minimumRecallRelevance(queryTokens: readonly string[]): number;
|
|
10
17
|
export declare function cjkFtsTerms(text: string): string[];
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@linxiraos/pi-mnemopi",
|
|
4
|
-
"version": "1.1.
|
|
5
|
-
"description": "Local SQLite memory engine for
|
|
4
|
+
"version": "1.1.22",
|
|
5
|
+
"description": "Local SQLite memory engine for omp agents",
|
|
6
6
|
"homepage": "https://linxira-os.github.io/zeta/",
|
|
7
7
|
"author": "Linxira-OS",
|
|
8
8
|
"contributors": [
|
|
@@ -39,10 +39,10 @@
|
|
|
39
39
|
"fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"@linxiraos/pi-ai": "1.1.
|
|
43
|
-
"@linxiraos/pi-catalog": "1.1.
|
|
44
|
-
"@linxiraos/pi-natives": "1.1.
|
|
45
|
-
"@linxiraos/pi-utils": "1.1.
|
|
42
|
+
"@linxiraos/pi-ai": "1.1.22",
|
|
43
|
+
"@linxiraos/pi-catalog": "1.1.22",
|
|
44
|
+
"@linxiraos/pi-natives": "1.1.22",
|
|
45
|
+
"@linxiraos/pi-utils": "1.1.22"
|
|
46
46
|
},
|
|
47
47
|
"peerDependencies": {
|
|
48
48
|
"fastembed": "2.1.0",
|
package/src/core/aaak.ts
CHANGED
|
@@ -80,9 +80,29 @@ function reverseMap<const T extends Readonly<Record<string, string>>>(source: T)
|
|
|
80
80
|
|
|
81
81
|
export const REV_CATEGORY = reverseMap(CATEGORY_MAP);
|
|
82
82
|
|
|
83
|
-
|
|
83
|
+
/**
|
|
84
|
+
* Literal pattern for `phrase` that never matches inside a longer word: an edge that is a
|
|
85
|
+
* word character must sit on a word boundary, so `complete` leaves `incomplete` and
|
|
86
|
+
* `completed` intact. Space and punctuation edges match as before.
|
|
87
|
+
*/
|
|
88
|
+
function wholeWordPattern(phrase: string): RegExp {
|
|
89
|
+
const start = /^[\p{L}\p{N}_]/u.test(phrase) ? "(?<![\\p{L}\\p{N}_])" : "";
|
|
90
|
+
const end = /[\p{L}\p{N}_]$/u.test(phrase) ? "(?![\\p{L}\\p{N}_])" : "";
|
|
91
|
+
return new RegExp(`${start}${RegExp.escape(phrase)}${end}`, "gu");
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const PHRASE_PATTERNS = Object.entries(PHRASE_MAP)
|
|
95
|
+
.sort(([left], [right]) => right.length - left.length)
|
|
96
|
+
.map(([phrase, shorthand]) => [wholeWordPattern(phrase), shorthand] as const);
|
|
84
97
|
export const REV_PHRASE = reverseMap(PHRASE_MAP);
|
|
85
98
|
|
|
99
|
+
const STATUS_SHORTHANDS = [
|
|
100
|
+
[wholeWordPattern("working correctly"), "OK"],
|
|
101
|
+
[wholeWordPattern("working"), "OK"],
|
|
102
|
+
[wholeWordPattern("complete"), "DONE"],
|
|
103
|
+
[wholeWordPattern("completed"), "DONE"],
|
|
104
|
+
] as const;
|
|
105
|
+
|
|
86
106
|
function replaceAllLiteral(text: string, pattern: string, replacement: string): string {
|
|
87
107
|
return text.replaceAll(pattern, replacement);
|
|
88
108
|
}
|
|
@@ -100,8 +120,8 @@ export function applyCategoryPrefixes(text: string): string {
|
|
|
100
120
|
|
|
101
121
|
export function applyPhrases(text: string): string {
|
|
102
122
|
let result = text;
|
|
103
|
-
for (const [
|
|
104
|
-
result =
|
|
123
|
+
for (const [pattern, shorthand] of PHRASE_PATTERNS) {
|
|
124
|
+
result = result.replace(pattern, shorthand);
|
|
105
125
|
}
|
|
106
126
|
return result;
|
|
107
127
|
}
|
|
@@ -132,10 +152,9 @@ export function encode(text: string): string {
|
|
|
132
152
|
result = applyPhrases(result);
|
|
133
153
|
result = applyStructural(result);
|
|
134
154
|
result = compactParens(result);
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
result = result.replaceAll("completed", "DONE");
|
|
155
|
+
for (const [pattern, shorthand] of STATUS_SHORTHANDS) {
|
|
156
|
+
result = result.replace(pattern, shorthand);
|
|
157
|
+
}
|
|
139
158
|
return result.trim();
|
|
140
159
|
}
|
|
141
160
|
|
package/src/core/beam/helpers.ts
CHANGED
|
@@ -1,17 +1,7 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
2
|
import { logger } from "@linxiraos/pi-utils";
|
|
3
3
|
import { generateId as generateTimedId, sha256Hex16, stableMemoryId } from "../../util/ids";
|
|
4
|
-
import {
|
|
5
|
-
cjkFtsTerms,
|
|
6
|
-
containsSpacelessCjk,
|
|
7
|
-
FACT_MATCH_STOPWORDS,
|
|
8
|
-
factMatchTokens,
|
|
9
|
-
ftsQueryTerms,
|
|
10
|
-
hasCjk,
|
|
11
|
-
isCjkChar,
|
|
12
|
-
RECALL_SYNONYMS,
|
|
13
|
-
recallTokens,
|
|
14
|
-
} from "../../util/regex";
|
|
4
|
+
import { cjkFtsTerms, containsSpacelessCjk, ftsQueryTerms, hasCjk, isCjkChar, recallTokens } from "../../util/regex";
|
|
15
5
|
import { currentEmbeddingModel, embed } from "../embeddings";
|
|
16
6
|
import { getMnemopiRuntimeOptions, mnemopiDebugEnabled, withMnemopiRuntimeOptions } from "../runtime-options";
|
|
17
7
|
import { buildExactVectorIndex, searchExactVectorIndex } from "../vector-index";
|
|
@@ -46,7 +36,6 @@ const DEFAULT_WEIGHTS: HybridWeights = [0.5, 0.3, 0.2];
|
|
|
46
36
|
const TS_CACHE_MAX = 2000;
|
|
47
37
|
const moduleTimestampCache = new Map<string, Date>();
|
|
48
38
|
|
|
49
|
-
const SPLIT_TOKEN_RE = /[_:/.-]+/g;
|
|
50
39
|
const WORD_RE = /[\p{L}\p{N}_]+/gu;
|
|
51
40
|
function envNumber(name: string, fallback: number): number {
|
|
52
41
|
const raw = process.env[name];
|
|
@@ -187,68 +176,6 @@ export function temporalBoost(
|
|
|
187
176
|
return Math.exp(-hoursDelta / halflife);
|
|
188
177
|
}
|
|
189
178
|
|
|
190
|
-
export function lexicalRelevance(queryTokens: readonly string[], content: string, queryLower = ""): number {
|
|
191
|
-
const contentLower = content.toLowerCase();
|
|
192
|
-
const queryCjk = new Set(Array.from(queryLower).filter(isCjkChar));
|
|
193
|
-
if (queryTokens.length === 0 && queryCjk.size === 0) return 0;
|
|
194
|
-
|
|
195
|
-
const contentTokens = new Set(recallTokens(contentLower));
|
|
196
|
-
for (const token of Array.from(contentTokens)) {
|
|
197
|
-
for (const part of token.split(SPLIT_TOKEN_RE)) {
|
|
198
|
-
if (part.length >= 3 && !FACT_MATCH_STOPWORDS.has(part) && !/^\d+$/.test(part)) contentTokens.add(part);
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
if (contentTokens.size === 0 && queryCjk.size === 0) return 0;
|
|
202
|
-
|
|
203
|
-
let exact = 0;
|
|
204
|
-
let partial = 0;
|
|
205
|
-
for (const token of queryTokens) {
|
|
206
|
-
if (contentTokens.has(token)) {
|
|
207
|
-
exact += 1;
|
|
208
|
-
continue;
|
|
209
|
-
}
|
|
210
|
-
const synonyms = RECALL_SYNONYMS[token] ?? [];
|
|
211
|
-
if (synonyms.some(syn => contentTokens.has(syn))) {
|
|
212
|
-
partial += 0.75;
|
|
213
|
-
continue;
|
|
214
|
-
}
|
|
215
|
-
if (
|
|
216
|
-
token.length >= 4 &&
|
|
217
|
-
Array.from(contentTokens).some(
|
|
218
|
-
contentToken => contentToken.length >= 4 && (token.includes(contentToken) || contentToken.includes(token)),
|
|
219
|
-
)
|
|
220
|
-
) {
|
|
221
|
-
partial += 0.4;
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
const fullMatch = queryLower !== "" && contentLower.includes(queryLower) ? 1 : 0;
|
|
226
|
-
let score = (exact + partial + fullMatch) / Math.max(queryTokens.length, 1);
|
|
227
|
-
if (score === 0 && queryCjk.size > 0) {
|
|
228
|
-
const contentCjk = new Set(Array.from(contentLower).filter(isCjkChar));
|
|
229
|
-
let overlap = 0;
|
|
230
|
-
for (const ch of queryCjk) if (contentCjk.has(ch)) overlap += 1;
|
|
231
|
-
score = overlap / queryCjk.size;
|
|
232
|
-
}
|
|
233
|
-
return Math.min(score, 1);
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
export function strictFactMatches(query: string, factText: string): boolean {
|
|
237
|
-
const queryLower = query.toLowerCase().trim();
|
|
238
|
-
const factLower = factText.toLowerCase().trim();
|
|
239
|
-
if (!queryLower || !factLower) return false;
|
|
240
|
-
if (factLower.includes(queryLower)) return true;
|
|
241
|
-
const queryTokens = factMatchTokens(queryLower);
|
|
242
|
-
const factTokens = factMatchTokens(factLower);
|
|
243
|
-
if (queryTokens.size === 0 || factTokens.size === 0) return false;
|
|
244
|
-
const overlap = Array.from(queryTokens).filter(token => factTokens.has(token));
|
|
245
|
-
if (overlap.length >= 2) return true;
|
|
246
|
-
const token = overlap[0];
|
|
247
|
-
if (token === undefined) return false;
|
|
248
|
-
if (token.length >= 8 && /[./:_-]/.test(token)) return true;
|
|
249
|
-
return token.length >= 5;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
179
|
export function buildFtsQuery(query: string): string {
|
|
253
180
|
return ftsQueryTerms(query).join(" OR ");
|
|
254
181
|
}
|
package/src/core/beam/recall.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { normalizedRecallWeights, temporalHalflifeHours } from "../../config";
|
|
2
|
+
import { hasCjk, matchesWordForm } from "../../util/regex";
|
|
2
3
|
import { embedQuery } from "../embeddings";
|
|
3
4
|
import { mmrRerank } from "../mmr";
|
|
4
5
|
import { adjustWeights, classifyIntent } from "../query-intent";
|
|
5
|
-
import { getSynonyms,
|
|
6
|
+
import { getSynonyms, STOP_WORDS as QUERY_STOP_WORDS } from "../synonyms";
|
|
6
7
|
import { extractTemporal } from "../temporal-parser";
|
|
7
8
|
import { cosineSimilarity } from "../vector-math";
|
|
8
9
|
import type { BeamMemoryState, RecallEnhancedOptions, RecallOptions, RecallResult } from "./types";
|
|
@@ -251,79 +252,53 @@ function factExpandedTokenGroups(query: string, content: string): string[][] {
|
|
|
251
252
|
return groups;
|
|
252
253
|
}
|
|
253
254
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
for (const contentToken of contentTokens) {
|
|
265
|
-
if (
|
|
266
|
-
contentToken.length >= 4 &&
|
|
267
|
-
token.length >= 4 &&
|
|
268
|
-
(contentToken.includes(token) || token.includes(contentToken))
|
|
269
|
-
) {
|
|
270
|
-
return true;
|
|
255
|
+
// Match whole tokens or another form of the same word; CJK text need not separate words with spaces.
|
|
256
|
+
function lexicalGroupRelevance(queryGroups: readonly (readonly string[])[], content: string): number {
|
|
257
|
+
if (queryGroups.length === 0) return 0;
|
|
258
|
+
const tokens = tokenize(content);
|
|
259
|
+
const contentTokens = new Set(tokens);
|
|
260
|
+
// Split document identifiers only; a query synonym such as data_store must stay atomic.
|
|
261
|
+
for (const token of tokens) {
|
|
262
|
+
if (!token.includes("_")) continue;
|
|
263
|
+
for (const part of token.split("_")) {
|
|
264
|
+
if (part.length > 0) contentTokens.add(part);
|
|
271
265
|
}
|
|
272
266
|
}
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
function lexicalGroupRelevance(
|
|
277
|
-
queryGroups: readonly (readonly string[])[],
|
|
278
|
-
content: string,
|
|
279
|
-
normalizedQuery: string,
|
|
280
|
-
): number {
|
|
281
|
-
if (queryGroups.length === 0) return 0;
|
|
282
|
-
const contentLower = content.toLowerCase();
|
|
283
|
-
if (queryGroups.length > 1 && normalizedQuery.length > 0 && contentLower.includes(normalizedQuery)) return 1;
|
|
284
|
-
const contentTokens = new Set(tokenize(contentLower));
|
|
285
|
-
let exact = 0;
|
|
267
|
+
const cjkContent = queryGroups.some(group => group.some(hasCjk)) ? content.toLowerCase() : null;
|
|
268
|
+
let matched = 0;
|
|
286
269
|
let partial = 0;
|
|
287
270
|
for (const group of queryGroups) {
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
break;
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
if (matched) exact += 1;
|
|
296
|
-
else {
|
|
297
|
-
for (const token of group) {
|
|
271
|
+
if (group.some(token => contentTokens.has(token) || (hasCjk(token) && cjkContent?.includes(token)))) {
|
|
272
|
+
matched += 1;
|
|
273
|
+
} else if (
|
|
274
|
+
group.some(token => {
|
|
298
275
|
for (const contentToken of contentTokens) {
|
|
299
|
-
if (
|
|
300
|
-
contentToken.length >= 4 &&
|
|
301
|
-
token.length >= 4 &&
|
|
302
|
-
(contentToken.includes(token) || token.includes(contentToken))
|
|
303
|
-
) {
|
|
304
|
-
partial += 1;
|
|
305
|
-
matched = true;
|
|
306
|
-
break;
|
|
307
|
-
}
|
|
276
|
+
if (matchesWordForm(token, contentToken)) return true;
|
|
308
277
|
}
|
|
309
|
-
|
|
310
|
-
}
|
|
278
|
+
return false;
|
|
279
|
+
})
|
|
280
|
+
) {
|
|
281
|
+
partial += 1;
|
|
311
282
|
}
|
|
312
283
|
}
|
|
313
|
-
if (queryGroups.length
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
284
|
+
if (queryGroups.length !== 1) return (matched + partial * 0.5) / queryGroups.length;
|
|
285
|
+
if (matched === 0) return partial > 0 ? 0.35 : 0;
|
|
286
|
+
const token = queryGroups[0]?.[0];
|
|
287
|
+
let count = 0;
|
|
288
|
+
if (token && hasCjk(token) && cjkContent !== null) {
|
|
289
|
+
for (
|
|
290
|
+
let offset = cjkContent.indexOf(token);
|
|
291
|
+
offset >= 0;
|
|
292
|
+
offset = cjkContent.indexOf(token, offset + token.length)
|
|
293
|
+
) {
|
|
321
294
|
count += 1;
|
|
322
|
-
offset = idx + token.length;
|
|
323
295
|
}
|
|
324
|
-
|
|
296
|
+
} else {
|
|
297
|
+
for (const contentToken of tokens) {
|
|
298
|
+
if (contentToken === token) count += 1;
|
|
299
|
+
}
|
|
325
300
|
}
|
|
326
|
-
return clamp01(
|
|
301
|
+
return clamp01(0.7 + Math.min(Math.max(count - 1, 0), 3) * 0.1);
|
|
327
302
|
}
|
|
328
303
|
|
|
329
304
|
function queryAsksCurrent(query: string): boolean {
|
|
@@ -346,45 +321,6 @@ function minimumRelevance(tokens: readonly string[]): number {
|
|
|
346
321
|
return 0.22;
|
|
347
322
|
}
|
|
348
323
|
|
|
349
|
-
function lexicalRelevance(queryTokens: readonly string[], content: string, normalizedQuery: string): number {
|
|
350
|
-
if (queryTokens.length === 0) return 0;
|
|
351
|
-
const contentLower = content.toLowerCase();
|
|
352
|
-
if (queryTokens.length > 1 && normalizedQuery.length > 0 && contentLower.includes(normalizedQuery)) return 1;
|
|
353
|
-
if (queryTokens.length === 1) {
|
|
354
|
-
const token = queryTokens[0] ?? "";
|
|
355
|
-
if (token.length === 0 || !contentLower.includes(token)) return 0;
|
|
356
|
-
let count = 0;
|
|
357
|
-
let offset = 0;
|
|
358
|
-
while (true) {
|
|
359
|
-
const idx = contentLower.indexOf(token, offset);
|
|
360
|
-
if (idx < 0) break;
|
|
361
|
-
count += 1;
|
|
362
|
-
offset = idx + token.length;
|
|
363
|
-
}
|
|
364
|
-
return clamp01(0.7 + Math.min(Math.max(count - 1, 0), 3) * 0.1);
|
|
365
|
-
}
|
|
366
|
-
const contentTokens = new Set(tokenize(contentLower));
|
|
367
|
-
let exact = 0;
|
|
368
|
-
let partial = 0;
|
|
369
|
-
for (const token of queryTokens) {
|
|
370
|
-
if (contentTokens.has(token) || contentLower.includes(token)) {
|
|
371
|
-
exact += 1;
|
|
372
|
-
continue;
|
|
373
|
-
}
|
|
374
|
-
for (const contentToken of contentTokens) {
|
|
375
|
-
if (
|
|
376
|
-
contentToken.length >= 4 &&
|
|
377
|
-
token.length >= 4 &&
|
|
378
|
-
(contentToken.includes(token) || token.includes(contentToken))
|
|
379
|
-
) {
|
|
380
|
-
partial += 1;
|
|
381
|
-
break;
|
|
382
|
-
}
|
|
383
|
-
}
|
|
384
|
-
}
|
|
385
|
-
return clamp01((exact + partial * 0.5) / queryTokens.length);
|
|
386
|
-
}
|
|
387
|
-
|
|
388
324
|
function recencyDecay(timestamp: unknown, halfLifeHours = 72): number {
|
|
389
325
|
const raw = asString(timestamp);
|
|
390
326
|
if (raw.length === 0) return 0;
|
|
@@ -468,10 +404,17 @@ function factsHaveScopeColumn(beam: BeamMemoryState): boolean {
|
|
|
468
404
|
|
|
469
405
|
function factVisibilityWhere(beam: BeamMemoryState, tableAlias: string): { where: string; params: DbValue[] } {
|
|
470
406
|
const prefix = tableAlias.length === 0 ? "" : `${tableAlias}.`;
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
407
|
+
const scope = factsHaveScopeColumn(beam)
|
|
408
|
+
? `(${prefix}session_id = ? OR ${prefix}scope = 'global')`
|
|
409
|
+
: `${prefix}session_id = ?`;
|
|
410
|
+
// A fact is a derivative of the working_memory row it was extracted from: once that row is
|
|
411
|
+
// explicitly superseded (`memory_edit invalidate`) or has expired, the fact must stop
|
|
412
|
+
// surfacing — otherwise an explicit invalidation is silently defeated at the fact layer.
|
|
413
|
+
// Correlated EXISTS probes the source's primary key (same shape as the fts_working /
|
|
414
|
+
// fts_episodes visibility predicates). Facts whose source row no longer exists stay
|
|
415
|
+
// visible: nothing contradicts them.
|
|
416
|
+
const sourceSuperseded = `NOT EXISTS (SELECT 1 FROM working_memory w WHERE w.id = ${prefix}source_msg_id AND (w.superseded_by IS NOT NULL OR (w.valid_until IS NOT NULL AND w.valid_until <= ?)))`;
|
|
417
|
+
return { where: `${scope} AND ${sourceSuperseded}`, params: [beam.sessionId, nowIso()] };
|
|
475
418
|
}
|
|
476
419
|
|
|
477
420
|
function buildWhere(
|
|
@@ -723,16 +666,12 @@ function scoreCandidate(
|
|
|
723
666
|
candidate: MemoryCandidate,
|
|
724
667
|
queryTokens: readonly string[],
|
|
725
668
|
queryGroups: readonly (readonly string[])[],
|
|
726
|
-
normalizedQueryLower: string,
|
|
727
669
|
weights: readonly [number, number, number],
|
|
728
670
|
options: RecallOptionsInternal,
|
|
729
671
|
): RecallResult | null {
|
|
730
672
|
const content = asString(candidate.row.content);
|
|
731
673
|
const searchableContent = asString(candidate.row.embed_text) || content;
|
|
732
|
-
const lexical =
|
|
733
|
-
queryGroups.length > 0
|
|
734
|
-
? lexicalGroupRelevance(queryGroups, searchableContent, normalizedQueryLower)
|
|
735
|
-
: lexicalRelevance(queryTokens, searchableContent, normalizedQueryLower);
|
|
674
|
+
const lexical = lexicalGroupRelevance(queryGroups, searchableContent);
|
|
736
675
|
const minRel = minimumRelevance(queryTokens);
|
|
737
676
|
if (lexical < minRel && candidate.signals.dense < 0.65) return null;
|
|
738
677
|
const [vecWeight, ftsWeight, importanceWeight] = weights;
|
|
@@ -993,11 +932,10 @@ export async function recall(
|
|
|
993
932
|
const useSynonyms = options.useSynonyms !== false;
|
|
994
933
|
const tokens = expandedTokens(query, useSynonyms);
|
|
995
934
|
const tokenGroups = expandedTokenGroups(query, useSynonyms);
|
|
996
|
-
const normalized = normalizeQuery(query).toLowerCase();
|
|
997
935
|
const candidates = collectMemoryCandidates(beam, query, topK, temporalOptions);
|
|
998
936
|
const scored: RecallResult[] = [];
|
|
999
937
|
for (const candidate of candidates) {
|
|
1000
|
-
const result = scoreCandidate(candidate, tokens, tokenGroups,
|
|
938
|
+
const result = scoreCandidate(candidate, tokens, tokenGroups, weights, temporalOptions);
|
|
1001
939
|
if (result !== null) scored.push(result);
|
|
1002
940
|
}
|
|
1003
941
|
scored.sort((left, right) => (right.score ?? 0) - (left.score ?? 0));
|
|
@@ -1184,7 +1122,6 @@ export function factRecall(beam: BeamMemoryState, query: string, topK = 30): Fac
|
|
|
1184
1122
|
if (rowids.length === 0) return [];
|
|
1185
1123
|
const visibility = factVisibilityWhere(beam, "");
|
|
1186
1124
|
const ranks = normalizeRanks(matched, "rowid");
|
|
1187
|
-
const normalized = normalizeQuery(query).toLowerCase();
|
|
1188
1125
|
const rows = queryAll(
|
|
1189
1126
|
beam,
|
|
1190
1127
|
`SELECT rowid, fact_id, subject, predicate, object, timestamp, confidence
|
|
@@ -1203,11 +1140,7 @@ export function factRecall(beam: BeamMemoryState, query: string, topK = 30): Fac
|
|
|
1203
1140
|
const content = object.length > 0 ? object : `${subject} ${predicate}`.trim();
|
|
1204
1141
|
const searchable = factSearchableText(subject, predicate, object);
|
|
1205
1142
|
const queryGroups = factExpandedTokenGroups(query, searchable);
|
|
1206
|
-
const
|
|
1207
|
-
const lexical =
|
|
1208
|
-
queryGroups.length > 0
|
|
1209
|
-
? lexicalGroupRelevance(queryGroups, searchable, normalized)
|
|
1210
|
-
: lexicalRelevance(queryTokens, searchable, normalized);
|
|
1143
|
+
const lexical = lexicalGroupRelevance(queryGroups, searchable);
|
|
1211
1144
|
const rank = ranks.get(asNumber(row.rowid)) ?? 0;
|
|
1212
1145
|
const result: FactRecallResult = {
|
|
1213
1146
|
id: asString(row.fact_id),
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
1
|
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
3
2
|
import { homedir } from "node:os";
|
|
4
3
|
import { join } from "node:path";
|
|
@@ -25,7 +24,7 @@ export function blobRoot(env: NodeJS.ProcessEnv = process.env): string {
|
|
|
25
24
|
}
|
|
26
25
|
|
|
27
26
|
export function computeSha256(data: Uint8Array | string): string {
|
|
28
|
-
return
|
|
27
|
+
return Bun.SHA256.hash(data, "hex");
|
|
29
28
|
}
|
|
30
29
|
|
|
31
30
|
export function isDataUri(content: string): boolean {
|
package/src/core/shmr.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { createHash } from "node:crypto";
|
|
3
2
|
import { cosineSimilarityPairs } from "@linxiraos/pi-natives";
|
|
4
3
|
import { logger } from "@linxiraos/pi-utils";
|
|
5
4
|
import * as embeddings from "./embeddings";
|
|
@@ -109,9 +108,10 @@ export function initSchema(db: Database): void {
|
|
|
109
108
|
function hashEmbedding(text: string): Vector {
|
|
110
109
|
const out = new Float32Array(EMBEDDING_DIM);
|
|
111
110
|
const words = text.toLowerCase().match(/[a-z0-9]+/g) ?? [];
|
|
111
|
+
const digest = new Uint8Array(20);
|
|
112
112
|
for (const word of words) {
|
|
113
|
-
|
|
114
|
-
const slot = digest
|
|
113
|
+
Bun.SHA1.hash(word, digest);
|
|
114
|
+
const slot = ((digest[0] << 8) | digest[1]) % EMBEDDING_DIM;
|
|
115
115
|
out[slot] = (out[slot] ?? 0) + 1;
|
|
116
116
|
}
|
|
117
117
|
return out;
|
|
@@ -335,10 +335,10 @@ export function applyBeliefs(
|
|
|
335
335
|
confidence,
|
|
336
336
|
belief.target_fact_id,
|
|
337
337
|
]);
|
|
338
|
-
const beliefId =
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
338
|
+
const beliefId = Bun.SHA256.hash(
|
|
339
|
+
`${clusterId}:${belief.subject}:${belief.predicate}:${belief.object.slice(0, 50)}`,
|
|
340
|
+
"hex",
|
|
341
|
+
).slice(0, 24);
|
|
342
342
|
const provenance = JSON.stringify(
|
|
343
343
|
cluster.map(item => item.fact_id).filter((id): id is string => typeof id === "string"),
|
|
344
344
|
);
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { createHash } from "node:crypto";
|
|
3
2
|
import { type DatabasePath, openDatabase } from "../db";
|
|
4
3
|
|
|
5
4
|
export const VERACITY_WEIGHTS = Object.freeze({
|
|
@@ -123,12 +122,12 @@ export function computeFactId(subject: string, predicate: string, object: string
|
|
|
123
122
|
if (value === "") throw new RangeError(`compute_fact_id: ${name} must be non-empty`);
|
|
124
123
|
}
|
|
125
124
|
|
|
126
|
-
const
|
|
125
|
+
const hash = new Bun.SHA256();
|
|
127
126
|
for (const value of [subject, predicate, object]) {
|
|
128
127
|
const bytes = Buffer.from(value.normalize("NFC"), "utf8");
|
|
129
|
-
|
|
128
|
+
hash.update(`${bytes.length}:`).update(bytes);
|
|
130
129
|
}
|
|
131
|
-
return `cf_${
|
|
130
|
+
return `cf_${hash.digest("hex").slice(0, 24)}`;
|
|
132
131
|
}
|
|
133
132
|
export function clampVeracity(raw: unknown, context = "veracity"): Veracity {
|
|
134
133
|
if (raw === null || raw === undefined) return "unknown";
|
package/src/dr/recovery.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { createHash } from "node:crypto";
|
|
3
2
|
import {
|
|
4
3
|
copyFileSync,
|
|
5
4
|
existsSync,
|
|
@@ -90,8 +89,8 @@ function timestampForBackup(now = new Date()): string {
|
|
|
90
89
|
return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
|
|
91
90
|
}
|
|
92
91
|
|
|
93
|
-
function sha256Hex16(bytes:
|
|
94
|
-
return
|
|
92
|
+
function sha256Hex16(bytes: Uint8Array): string {
|
|
93
|
+
return Bun.SHA256.hash(bytes, "hex").slice(0, 16);
|
|
95
94
|
}
|
|
96
95
|
|
|
97
96
|
function nextUniqueToken(): string {
|
package/src/util/regex.ts
CHANGED
|
@@ -79,6 +79,32 @@ export function hasCjk(text: string): boolean {
|
|
|
79
79
|
|
|
80
80
|
export const containsSpacelessCjk = hasCjk;
|
|
81
81
|
|
|
82
|
+
/** Endings that turn one form of a word into another: `backup`/`backups`, `deploy`/`deployment`. */
|
|
83
|
+
const WORD_FORM_ENDINGS: readonly string[] = ["s", "es", "d", "ed", "ing", "er", "ers", "ment", "ments"];
|
|
84
|
+
/** Endings that replace a final `e`: `cache`/`caching`, `create`/`creation`. */
|
|
85
|
+
const SILENT_E_ENDINGS: readonly string[] = ["ing", "ion", "ions"];
|
|
86
|
+
/** Endings that replace a final `y`: `story`/`stories`, `copy`/`copied`. */
|
|
87
|
+
const FINAL_Y_ENDINGS: readonly string[] = ["ies", "ied"];
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Weak match between two forms of one word, in either direction: the longer token is the shorter
|
|
91
|
+
* one plus an inflectional or common derivational ending (`facts`/`fact`, `deploy`/`deployment`,
|
|
92
|
+
* `caching`/`cache`, `stories`/`story`). The shorter token needs at least four characters. Any
|
|
93
|
+
* other extension or fragment never matches, so `1password`, `password` and `passport` do not
|
|
94
|
+
* find `pass`, and `redis` does not find `redistribution`. This is deliberately not a stemmer.
|
|
95
|
+
*/
|
|
96
|
+
export function matchesWordForm(left: string, right: string): boolean {
|
|
97
|
+
const [shorter, longer] = left.length <= right.length ? [left, right] : [right, left];
|
|
98
|
+
if (shorter.length < 4 || longer.length === shorter.length) return false;
|
|
99
|
+
if (longer.startsWith(shorter) && WORD_FORM_ENDINGS.includes(longer.slice(shorter.length))) return true;
|
|
100
|
+
const stem = shorter.slice(0, -1);
|
|
101
|
+
if (!longer.startsWith(stem)) return false;
|
|
102
|
+
const ending = longer.slice(stem.length);
|
|
103
|
+
if (shorter.endsWith("e")) return SILENT_E_ENDINGS.includes(ending);
|
|
104
|
+
if (shorter.endsWith("y")) return FINAL_Y_ENDINGS.includes(ending);
|
|
105
|
+
return false;
|
|
106
|
+
}
|
|
107
|
+
|
|
82
108
|
export function recallTokens(text: string): string[] {
|
|
83
109
|
RECALL_TOKEN_RE.lastIndex = 0;
|
|
84
110
|
const tokens: string[] = [];
|
|
@@ -92,10 +118,6 @@ export function recallTokens(text: string): string[] {
|
|
|
92
118
|
return tokens;
|
|
93
119
|
}
|
|
94
120
|
|
|
95
|
-
export function factMatchTokens(text: string): Set<string> {
|
|
96
|
-
return new Set(recallTokens(text));
|
|
97
|
-
}
|
|
98
|
-
|
|
99
121
|
export function expandedQueryTokens(tokens: readonly string[]): string[] {
|
|
100
122
|
const expanded: string[] = [];
|
|
101
123
|
const seen = new Set<string>();
|