agent-working-memory 0.16.1 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +321 -318
- package/dist/adapters/common.d.ts +34 -0
- package/dist/adapters/common.d.ts.map +1 -1
- package/dist/adapters/common.js +528 -444
- package/dist/adapters/common.js.map +1 -1
- package/dist/api/routes.d.ts.map +1 -1
- package/dist/api/routes.js +19 -0
- package/dist/api/routes.js.map +1 -1
- package/dist/cli/migrate.d.ts.map +1 -1
- package/dist/cli/migrate.js +31 -29
- package/dist/cli/migrate.js.map +1 -1
- package/dist/core/asked-value.d.ts +37 -0
- package/dist/core/asked-value.d.ts.map +1 -0
- package/dist/core/asked-value.js +77 -0
- package/dist/core/asked-value.js.map +1 -0
- package/dist/core/conflicts.d.ts +89 -0
- package/dist/core/conflicts.d.ts.map +1 -0
- package/dist/core/conflicts.js +267 -0
- package/dist/core/conflicts.js.map +1 -0
- package/dist/core/embeddings.d.ts +14 -2
- package/dist/core/embeddings.d.ts.map +1 -1
- package/dist/core/embeddings.js +32 -10
- package/dist/core/embeddings.js.map +1 -1
- package/dist/core/format-recall.d.ts +21 -1
- package/dist/core/format-recall.d.ts.map +1 -1
- package/dist/core/format-recall.js +60 -5
- package/dist/core/format-recall.js.map +1 -1
- package/dist/core/ml-worker.d.ts +24 -1
- package/dist/core/ml-worker.d.ts.map +1 -1
- package/dist/core/ml-worker.js +165 -4
- package/dist/core/ml-worker.js.map +1 -1
- package/dist/core/model-daemon-client.d.ts +77 -0
- package/dist/core/model-daemon-client.d.ts.map +1 -0
- package/dist/core/model-daemon-client.js +383 -0
- package/dist/core/model-daemon-client.js.map +1 -0
- package/dist/core/model-daemon-server.d.ts +71 -0
- package/dist/core/model-daemon-server.d.ts.map +1 -0
- package/dist/core/model-daemon-server.js +363 -0
- package/dist/core/model-daemon-server.js.map +1 -0
- package/dist/core/model-daemon.d.ts +137 -0
- package/dist/core/model-daemon.d.ts.map +1 -0
- package/dist/core/model-daemon.js +231 -0
- package/dist/core/model-daemon.js.map +1 -0
- package/dist/core/query-expander.d.ts +5 -0
- package/dist/core/query-expander.d.ts.map +1 -1
- package/dist/core/query-expander.js +3 -2
- package/dist/core/query-expander.js.map +1 -1
- package/dist/core/recall-config.d.ts +1 -1
- package/dist/core/recall-config.d.ts.map +1 -1
- package/dist/core/recall-config.js +5 -0
- package/dist/core/recall-config.js.map +1 -1
- package/dist/core/rerank-window.d.ts +7 -2
- package/dist/core/rerank-window.d.ts.map +1 -1
- package/dist/core/rerank-window.js +9 -4
- package/dist/core/rerank-window.js.map +1 -1
- package/dist/core/rerank2.d.ts +52 -2
- package/dist/core/rerank2.d.ts.map +1 -1
- package/dist/core/rerank2.js +60 -3
- package/dist/core/rerank2.js.map +1 -1
- package/dist/core/reranker.d.ts +19 -0
- package/dist/core/reranker.d.ts.map +1 -1
- package/dist/core/reranker.js +76 -12
- package/dist/core/reranker.js.map +1 -1
- package/dist/core/salience.d.ts +2 -0
- package/dist/core/salience.d.ts.map +1 -1
- package/dist/core/salience.js +45 -18
- package/dist/core/salience.js.map +1 -1
- package/dist/core/trust.d.ts +32 -0
- package/dist/core/trust.d.ts.map +1 -0
- package/dist/core/trust.js +66 -0
- package/dist/core/trust.js.map +1 -0
- package/dist/core/validity.d.ts +21 -0
- package/dist/core/validity.d.ts.map +1 -0
- package/dist/core/validity.js +41 -0
- package/dist/core/validity.js.map +1 -0
- package/dist/core/whoami.d.ts +3 -0
- package/dist/core/whoami.d.ts.map +1 -1
- package/dist/core/whoami.js +14 -0
- package/dist/core/whoami.js.map +1 -1
- package/dist/core/write-pipeline.d.ts +48 -0
- package/dist/core/write-pipeline.d.ts.map +1 -1
- package/dist/core/write-pipeline.js +104 -4
- package/dist/core/write-pipeline.js.map +1 -1
- package/dist/engine/activation.d.ts.map +1 -1
- package/dist/engine/activation.js +27 -4
- package/dist/engine/activation.js.map +1 -1
- package/dist/engine/consolidation.d.ts.map +1 -1
- package/dist/engine/consolidation.js +67 -5
- package/dist/engine/consolidation.js.map +1 -1
- package/dist/hooks/prime-query.d.ts +57 -0
- package/dist/hooks/prime-query.d.ts.map +1 -0
- package/dist/hooks/prime-query.js +388 -0
- package/dist/hooks/prime-query.js.map +1 -0
- package/dist/hooks/prime.d.ts +8 -0
- package/dist/hooks/prime.d.ts.map +1 -1
- package/dist/hooks/prime.js +21 -2
- package/dist/hooks/prime.js.map +1 -1
- package/dist/hooks/sidecar.d.ts +7 -0
- package/dist/hooks/sidecar.d.ts.map +1 -1
- package/dist/hooks/sidecar.js +10 -2
- package/dist/hooks/sidecar.js.map +1 -1
- package/dist/index.js +11 -3
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +148 -103
- package/dist/mcp.js.map +1 -1
- package/dist/model-daemon.d.ts +8 -0
- package/dist/model-daemon.d.ts.map +1 -0
- package/dist/model-daemon.js +45 -0
- package/dist/model-daemon.js.map +1 -0
- package/dist/storage/factory.d.ts.map +1 -1
- package/dist/storage/factory.js +5 -0
- package/dist/storage/factory.js.map +1 -1
- package/dist/storage/pglite-schema.d.ts.map +1 -1
- package/dist/storage/pglite-schema.js +176 -170
- package/dist/storage/pglite-schema.js.map +1 -1
- package/dist/storage/pglite.d.ts +5 -0
- package/dist/storage/pglite.d.ts.map +1 -1
- package/dist/storage/pglite.js +165 -153
- package/dist/storage/pglite.js.map +1 -1
- package/dist/storage/postgres.d.ts +5 -0
- package/dist/storage/postgres.d.ts.map +1 -1
- package/dist/storage/postgres.js +165 -153
- package/dist/storage/postgres.js.map +1 -1
- package/dist/storage/sqlite.d.ts +22 -0
- package/dist/storage/sqlite.d.ts.map +1 -1
- package/dist/storage/sqlite.js +400 -363
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/types/engram.d.ts +33 -0
- package/dist/types/engram.d.ts.map +1 -1
- package/dist/types/engram.js.map +1 -1
- package/package.json +106 -106
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/README.md +1 -1
- package/plugin/bin/awm-mcp-launcher.cjs +2 -2
- package/plugin/skills/awm-memory/SKILL.md +16 -5
- package/src/adapters/common.ts +881 -800
- package/src/api/routes.ts +20 -0
- package/src/cli/migrate.ts +309 -307
- package/src/core/asked-value.ts +74 -0
- package/src/core/conflicts.ts +280 -0
- package/src/core/embeddings.ts +35 -10
- package/src/core/format-recall.ts +83 -25
- package/src/core/ml-worker.ts +458 -281
- package/src/core/model-daemon-client.ts +374 -0
- package/src/core/model-daemon-server.ts +344 -0
- package/src/core/model-daemon.ts +243 -0
- package/src/core/query-expander.ts +127 -126
- package/src/core/recall-config.ts +5 -0
- package/src/core/rerank-window.ts +164 -159
- package/src/core/rerank2.ts +143 -82
- package/src/core/reranker.ts +78 -11
- package/src/core/salience.ts +45 -18
- package/src/core/trust.ts +65 -0
- package/src/core/validity.ts +47 -0
- package/src/core/whoami.ts +136 -120
- package/src/core/write-pipeline.ts +143 -4
- package/src/engine/activation.ts +27 -4
- package/src/engine/consolidation.ts +950 -887
- package/src/hooks/prime-query.ts +375 -0
- package/src/hooks/prime.ts +25 -2
- package/src/hooks/sidecar.ts +13 -3
- package/src/index.ts +255 -248
- package/src/mcp.ts +1521 -1475
- package/src/model-daemon.ts +48 -0
- package/src/storage/factory.ts +25 -19
- package/src/storage/pglite-schema.ts +199 -193
- package/src/storage/pglite.ts +1513 -1498
- package/src/storage/postgres.ts +1613 -1598
- package/src/storage/sqlite.ts +2056 -2019
- package/src/types/engram.ts +30 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* "Known subject, attribute not stored" — the near-miss that invites invention.
|
|
5
|
+
*
|
|
6
|
+
* The strongest mechanistic account of hallucination (Ferrando et al., ICLR
|
|
7
|
+
* 2025; Lindsey et al. 2025) is that a model fabricates when it RECOGNISES an
|
|
8
|
+
* entity but lacks the fact asked about it: recognition switches off the
|
|
9
|
+
* "can't answer" default. A memory layer cannot reach the model's internals,
|
|
10
|
+
* but it can stop handing the model exactly that shape of context: three
|
|
11
|
+
* memories that are plainly about Sam's Denver flight, none of which holds a
|
|
12
|
+
* departure time, read to the model as "I know this flight", and the time
|
|
13
|
+
* gets made up. AWM's abstention gate does not fire here, by design: a
|
|
14
|
+
* near-miss usually has a clear winner.
|
|
15
|
+
*
|
|
16
|
+
* So, when the query asks for a KIND of value — a time, a date, an amount, a
|
|
17
|
+
* count, a status — and no returned memory holds a value of that kind, the
|
|
18
|
+
* reply says so in one line. It costs the agent one read of the source; the
|
|
19
|
+
* alternative is a confident wrong number. Detection is deliberately simple
|
|
20
|
+
* and conservative: it only ever ADDS a line, never changes ranking, and says
|
|
21
|
+
* nothing when the query names no kind.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { valueSlots, type SlotKind } from './conflicts.js';
|
|
25
|
+
|
|
26
|
+
/** Query phrasings that ask for each kind of value. Lower-case, tested as regexes. */
|
|
27
|
+
const ASKS: Array<[SlotKind, RegExp]> = [
|
|
28
|
+
['time', /\b(what time|time does|time is|what hour|o'?clock|depart(?:s|ure)?|leaves?|arrives?|arrival|starts? at|ends? at|opens? at|closes? at)\b/],
|
|
29
|
+
['date', /\b(what day|what date|which day|which date|due|deadline|by when|expire|expires|renews?|renewal date)\b/],
|
|
30
|
+
['money', /\b(how much|price|prices|cost|costs|fee|fees|rate|rates|charge|charges|amount|invoice total|\$)/],
|
|
31
|
+
['number', /\b(how many|count|which port|port number|version|how old|what number|phone|extension|zip|room number|flight number)\b/],
|
|
32
|
+
['status', /\b(status|(?:is|are|was|were) (?:\w+ ){0,4}(?:open|closed|paid|unpaid|cancell?ed|confirmed|approved|done|live|down|blocked|overdue|pending)|still (?:open|unpaid|pending|blocked)|did (?:\w+ ){0,3}(?:pay|cancel|confirm|approve|renew|ship|deploy))\b/],
|
|
33
|
+
];
|
|
34
|
+
|
|
35
|
+
/** "when" asks for a time OR a date; either kind stored answers it. */
|
|
36
|
+
const WHEN = /\b(when)\b/;
|
|
37
|
+
|
|
38
|
+
/** The kinds of value the query asks for; empty when it names none. */
|
|
39
|
+
export function askedKinds(query: string): Set<SlotKind> {
|
|
40
|
+
const q = query.toLowerCase();
|
|
41
|
+
const out = new Set<SlotKind>();
|
|
42
|
+
for (const [kind, re] of ASKS) if (re.test(q)) out.add(kind);
|
|
43
|
+
if (WHEN.test(q)) { out.add('time'); out.add('date'); }
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** What this needs from a result. */
|
|
48
|
+
export interface ValueBearing { engram: { content: string } }
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The kinds asked for that NO returned memory holds, or an empty array. With
|
|
52
|
+
* a "when" question, a stored time or date on any result satisfies both.
|
|
53
|
+
*/
|
|
54
|
+
export function missingAskedKinds(query: string, results: ValueBearing[]): SlotKind[] {
|
|
55
|
+
const asked = askedKinds(query);
|
|
56
|
+
if (asked.size === 0 || results.length === 0) return [];
|
|
57
|
+
const held = new Set<SlotKind>();
|
|
58
|
+
for (const r of results) for (const k of valueSlots(r.engram.content).keys()) held.add(k);
|
|
59
|
+
const whenAsked = WHEN.test(query.toLowerCase());
|
|
60
|
+
if (whenAsked && (held.has('time') || held.has('date'))) { asked.delete('time'); asked.delete('date'); }
|
|
61
|
+
return [...asked].filter(k => !held.has(k));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const LABEL: Record<SlotKind, string> = {
|
|
65
|
+
time: 'a time', date: 'a date', money: 'an amount', number: 'a number', status: 'a status',
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
/** One reply line, or empty. */
|
|
69
|
+
export function missingValueNote(query: string, results: ValueBearing[]): string {
|
|
70
|
+
const missing = missingAskedKinds(query, results);
|
|
71
|
+
if (missing.length === 0) return '';
|
|
72
|
+
const what = missing.map(k => LABEL[k]).join(' or ');
|
|
73
|
+
return `\n⚠ These memories are about the subject, but none of them holds ${what}, which is what the question asks for. Do not state one from memory — read the source.`;
|
|
74
|
+
}
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
// Copyright 2026 Robert Winter / Complete Ideas
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
/**
|
|
4
|
+
* Conflict surfacing among the memories ONE recall returns.
|
|
5
|
+
*
|
|
6
|
+
* AWM keeps contradictions on purpose (edge-case test 7 requires both sides of
|
|
7
|
+
* a contradiction in the top 5): settling them is the agent's job, with the
|
|
8
|
+
* source. What it did not do, until 2026-10-19, was SAY that two returned
|
|
9
|
+
* memories disagree, or which is newer. The MWA evaluation probe showed the
|
|
10
|
+
* cost: "departs 10:15" and "now departs 8:40" came back as two undated lines,
|
|
11
|
+
* the stale one first, and nothing on the page told the model it was looking
|
|
12
|
+
* at two versions of one fact. In the BL-40 assessment "newer facts lose to
|
|
13
|
+
* older sources" cost four checks.
|
|
14
|
+
*
|
|
15
|
+
* This is a detector over the returned set only — never over the store — so it
|
|
16
|
+
* costs a handful of string comparisons per recall. It is deliberately
|
|
17
|
+
* conservative and its output is a QUESTION ("possible conflict … confirm
|
|
18
|
+
* against the source"), not a verdict: a false mark costs the agent one check,
|
|
19
|
+
* a missed one costs a wrong answer stated with confidence.
|
|
20
|
+
*
|
|
21
|
+
* Two memories are flagged when
|
|
22
|
+
* 1. they are about the same thing — related by embedding cosine, or by word
|
|
23
|
+
* overlap when an embedding is missing — and
|
|
24
|
+
* 2. they carry a value of the same kind (a time, a date, a money amount, a
|
|
25
|
+
* bare number, a status word) and those values differ, and
|
|
26
|
+
* 3. neither supersedes the other and neither is already stale (superseded,
|
|
27
|
+
* expired, not yet valid) — those are marked elsewhere.
|
|
28
|
+
*
|
|
29
|
+
* Known limit, measured in tests/core/conflicts.test.ts: "template twins" —
|
|
30
|
+
* two genuinely distinct facts that share a template ("flight out 8:40",
|
|
31
|
+
* "flight back 10:15") — are flagged. That is the precision cost of (1)+(2)
|
|
32
|
+
* without a parser, and the reason the mark is phrased as a question.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { cosineSimilarity } from './embeddings.js';
|
|
36
|
+
import { validityState } from './validity.js';
|
|
37
|
+
|
|
38
|
+
export type SlotKind = 'time' | 'date' | 'money' | 'number' | 'status';
|
|
39
|
+
|
|
40
|
+
/** What this needs from a result — any ActivationResult qualifies. */
|
|
41
|
+
export interface ConflictCandidate {
|
|
42
|
+
engram: {
|
|
43
|
+
id: string;
|
|
44
|
+
concept: string;
|
|
45
|
+
content: string;
|
|
46
|
+
embedding?: number[] | null;
|
|
47
|
+
createdAt?: Date | string | null;
|
|
48
|
+
supersededBy?: string | null;
|
|
49
|
+
supersedes?: string | null;
|
|
50
|
+
validFrom?: string | null;
|
|
51
|
+
validTo?: string | null;
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface ConflictMark {
|
|
56
|
+
/** The other memory in the pair. */
|
|
57
|
+
id: string;
|
|
58
|
+
/** True when THIS memory was written after the other one. */
|
|
59
|
+
newer: boolean;
|
|
60
|
+
/** The kind of value the two disagree on. */
|
|
61
|
+
slot: SlotKind;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const MONTHS = 'jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec';
|
|
65
|
+
const STATUS_WORDS = new Set([
|
|
66
|
+
'moved', 'cancelled', 'canceled', 'renewed', 'rejected', 'approved', 'closed', 'open', 'opened',
|
|
67
|
+
'paused', 'resumed', 'delayed', 'postponed', 'confirmed', 'unpaid', 'paid', 'overdue', 'done',
|
|
68
|
+
'blocked', 'shipped', 'deployed', 'reverted', 'merged', 'unmerged', 'live', 'offline', 'down', 'up',
|
|
69
|
+
]);
|
|
70
|
+
const STOP = new Set([
|
|
71
|
+
'the', 'and', 'for', 'with', 'that', 'this', 'from', 'per', 'now', 'was', 'were', 'are', 'has',
|
|
72
|
+
'have', 'had', 'into', 'onto', 'than', 'then', 'not', 'but', 'its', 'his', 'her', 'their', 'our',
|
|
73
|
+
'according', 'page', 'said', 'says', 'still', 'also', 'any', 'all', 'one', 'two', 'about',
|
|
74
|
+
]);
|
|
75
|
+
|
|
76
|
+
/** Normalise a clock time to HH:MM (24h). Returns null when it is not one. */
|
|
77
|
+
function clock(h: string, m: string | undefined, ampm: string | undefined): string | null {
|
|
78
|
+
let hh = Number(h);
|
|
79
|
+
const mm = m ? Number(m) : 0;
|
|
80
|
+
if (!Number.isFinite(hh) || hh > 23 || mm > 59) return null;
|
|
81
|
+
if (ampm) {
|
|
82
|
+
const p = ampm.toLowerCase();
|
|
83
|
+
if (hh > 12) return null;
|
|
84
|
+
if (p === 'pm' && hh < 12) hh += 12;
|
|
85
|
+
if (p === 'am' && hh === 12) hh = 0;
|
|
86
|
+
}
|
|
87
|
+
return `${String(hh).padStart(2, '0')}:${String(mm).padStart(2, '0')}`;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The comparable values in a text, by kind. Text matched by an earlier,
|
|
92
|
+
* more specific kind is blanked before a later kind runs, so "3:30pm" is one
|
|
93
|
+
* time rather than a time and the numbers 3 and 30, and "$49" is money, not
|
|
94
|
+
* a number.
|
|
95
|
+
*/
|
|
96
|
+
export function valueSlots(text: string): Map<SlotKind, Set<string>> {
|
|
97
|
+
const slots = new Map<SlotKind, Set<string>>();
|
|
98
|
+
const add = (k: SlotKind, v: string) => {
|
|
99
|
+
if (!slots.has(k)) slots.set(k, new Set());
|
|
100
|
+
slots.get(k)!.add(v);
|
|
101
|
+
};
|
|
102
|
+
let t = ` ${text} `;
|
|
103
|
+
const take = (re: RegExp, fn: (m: RegExpExecArray) => void) => {
|
|
104
|
+
t = t.replace(re, (...args) => {
|
|
105
|
+
const m = args as unknown as RegExpExecArray;
|
|
106
|
+
fn(m);
|
|
107
|
+
return ' ';
|
|
108
|
+
});
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
// Times: 8:40 AM, 3:30pm, 14:00, 7pm.
|
|
112
|
+
take(/\b(\d{1,2}):(\d{2})\s*(am|pm)\b/gi, m => { const c = clock(m[1], m[2], m[3]); if (c) add('time', c); });
|
|
113
|
+
take(/\b(\d{1,2})\s*(am|pm)\b/gi, m => { const c = clock(m[1], undefined, m[2]); if (c) add('time', c); });
|
|
114
|
+
take(/\b([01]?\d|2[0-3]):([0-5]\d)\b/g, m => { const c = clock(m[1], m[2], undefined); if (c) add('time', c); });
|
|
115
|
+
// Dates: 2026-01-15, Nov 14, 14 Nov, November 14th.
|
|
116
|
+
take(/\b(\d{4})-(\d{2})-(\d{2})\b/g, m => add('date', `${m[1]}-${m[2]}-${m[3]}`));
|
|
117
|
+
take(new RegExp(`\\b(${MONTHS})[a-z]*\\.?\\s+(\\d{1,2})(?:st|nd|rd|th)?\\b`, 'gi'),
|
|
118
|
+
m => add('date', `${m[1].slice(0, 3).toLowerCase()}-${Number(m[2])}`));
|
|
119
|
+
take(new RegExp(`\\b(\\d{1,2})(?:st|nd|rd|th)?\\s+(${MONTHS})[a-z]*\\b`, 'gi'),
|
|
120
|
+
m => add('date', `${m[2].slice(0, 3).toLowerCase()}-${Number(m[1])}`));
|
|
121
|
+
// A bare month after a preposition ("in October", "by March", "since
|
|
122
|
+
// November") is a date too — coarser, but "in October the club chose
|
|
123
|
+
// fantasy" against "in March … a mystery" is exactly the two-versions case
|
|
124
|
+
// (MWA, 0.18.1). Full names only and a preposition required: an unanchored
|
|
125
|
+
// match took the modal verb "may" and every passing "March" as a date, and
|
|
126
|
+
// on the real store that flagged enough false conflicts to show in the
|
|
127
|
+
// benchmark. Runs after the month-day forms, which have already been
|
|
128
|
+
// blanked, so "Nov 14" is not also counted as "nov".
|
|
129
|
+
take(/\b(?:in|by|until|till|from|since|during|before|after|early|late|mid|next|last|this|of|through)\s+(january|february|march|april|may|june|july|august|september|october|november|december)\b/gi,
|
|
130
|
+
m => add('date', m[1].slice(0, 3).toLowerCase()));
|
|
131
|
+
// Money: $1.45, $2,400, $49/month.
|
|
132
|
+
take(/\$\s?(\d[\d,]*(?:\.\d+)?)/g, m => add('money', m[1].replace(/,/g, '')));
|
|
133
|
+
// Bare numbers (ids, counts, flight numbers): shared ones do no harm, differing ones may.
|
|
134
|
+
take(/\b\d+(?:\.\d+)?\b/g, m => add('number', m[0]));
|
|
135
|
+
// Status vocabulary.
|
|
136
|
+
for (const w of t.toLowerCase().match(/[a-z]+/g) ?? []) {
|
|
137
|
+
if (STATUS_WORDS.has(w)) add('status', w);
|
|
138
|
+
}
|
|
139
|
+
return slots;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Content words for the overlap fallback: lower-case, 3+ letters, no stopwords, no digits. */
|
|
143
|
+
function words(c: ConflictCandidate): Set<string> {
|
|
144
|
+
const out = new Set<string>();
|
|
145
|
+
for (const w of `${c.engram.concept} ${c.engram.content}`.toLowerCase().match(/[a-z][a-z']{2,}/g) ?? []) {
|
|
146
|
+
if (!STOP.has(w)) out.add(w);
|
|
147
|
+
}
|
|
148
|
+
return out;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function jaccard(a: Set<string>, b: Set<string>): number {
|
|
152
|
+
if (a.size === 0 || b.size === 0) return 0;
|
|
153
|
+
let inter = 0;
|
|
154
|
+
for (const w of a) if (b.has(w)) inter++;
|
|
155
|
+
return inter / (a.size + b.size - inter);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const COSINE_RELATED = 0.80;
|
|
159
|
+
const OVERLAP_RELATED = 0.30;
|
|
160
|
+
|
|
161
|
+
/** True when the two memories look like they are about the same thing. */
|
|
162
|
+
export function related(a: ConflictCandidate, b: ConflictCandidate): boolean {
|
|
163
|
+
const ea = a.engram.embedding, eb = b.engram.embedding;
|
|
164
|
+
if (ea && eb && ea.length === eb.length && ea.length > 0) {
|
|
165
|
+
if (cosineSimilarity(ea, eb) >= COSINE_RELATED) return true;
|
|
166
|
+
}
|
|
167
|
+
return jaccard(words(a), words(b)) >= OVERLAP_RELATED;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** The first kind of value present in BOTH texts on which they disagree, or null. */
|
|
171
|
+
export function disagreement(a: ConflictCandidate, b: ConflictCandidate): SlotKind | null {
|
|
172
|
+
const sa = valueSlots(a.engram.content), sb = valueSlots(b.engram.content);
|
|
173
|
+
for (const kind of ['time', 'date', 'money', 'status', 'number'] as SlotKind[]) {
|
|
174
|
+
let va = sa.get(kind), vb = sb.get(kind);
|
|
175
|
+
if (!va || !vb) continue;
|
|
176
|
+
// Dates come at three grains (2026-01-15, nov-14, nov). When either side
|
|
177
|
+
// has a month-only value, compare both at month grain: "Nov 14" and
|
|
178
|
+
// "in November" agree; "in October" and "in March" do not.
|
|
179
|
+
if (kind === 'date' && ([...va, ...vb].some(v => /^[a-z]{3}$/.test(v)))) {
|
|
180
|
+
va = new Set([...va].map(monthOf)); vb = new Set([...vb].map(monthOf));
|
|
181
|
+
}
|
|
182
|
+
// Disagree when neither set contains the other: a memory that mentions
|
|
183
|
+
// two times, one of which the other repeats, is an elaboration, not a conflict.
|
|
184
|
+
const aInB = [...va].every(v => vb.has(v));
|
|
185
|
+
const bInA = [...vb].every(v => va.has(v));
|
|
186
|
+
if (!aInB && !bInA) return kind;
|
|
187
|
+
}
|
|
188
|
+
return null;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const MONTH_NAMES = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
|
|
192
|
+
/** The month token of any date value this file produces. */
|
|
193
|
+
function monthOf(v: string): string {
|
|
194
|
+
const iso = /^(\d{4})-(\d{2})-\d{2}$/.exec(v);
|
|
195
|
+
if (iso) return MONTH_NAMES[Number(iso[2]) - 1] ?? v;
|
|
196
|
+
return v.slice(0, 3);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function createdMs(c: ConflictCandidate): number {
|
|
200
|
+
const v = c.engram.createdAt;
|
|
201
|
+
if (!v) return NaN;
|
|
202
|
+
return v instanceof Date ? v.getTime() : Date.parse(v);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function stale(c: ConflictCandidate, now: number): boolean {
|
|
206
|
+
return !!c.engram.supersededBy || validityState(c.engram, now) !== 'current';
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Pairwise over the returned set. Returns one list of marks per input, in
|
|
211
|
+
* input order; an empty list means no conflict for that memory.
|
|
212
|
+
*/
|
|
213
|
+
export function detectConflicts(results: ConflictCandidate[], now: number = Date.now()): ConflictMark[][] {
|
|
214
|
+
const marks: ConflictMark[][] = results.map(() => []);
|
|
215
|
+
for (let i = 0; i < results.length; i++) {
|
|
216
|
+
const a = results[i];
|
|
217
|
+
if (stale(a, now)) continue;
|
|
218
|
+
for (let j = i + 1; j < results.length; j++) {
|
|
219
|
+
const b = results[j];
|
|
220
|
+
if (stale(b, now)) continue;
|
|
221
|
+
if (a.engram.supersedes === b.engram.id || b.engram.supersedes === a.engram.id) continue;
|
|
222
|
+
if (!related(a, b)) continue;
|
|
223
|
+
const slot = disagreement(a, b);
|
|
224
|
+
if (!slot) continue;
|
|
225
|
+
const ta = createdMs(a), tb = createdMs(b);
|
|
226
|
+
const aNewer = Number.isFinite(ta) && Number.isFinite(tb) ? ta > tb : false;
|
|
227
|
+
const bNewer = Number.isFinite(ta) && Number.isFinite(tb) ? tb > ta : false;
|
|
228
|
+
marks[i].push({ id: b.engram.id, newer: aNewer, slot });
|
|
229
|
+
marks[j].push({ id: a.engram.id, newer: bNewer, slot });
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return marks;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Newest first among results that are two versions of one fact — FOR HARNESSES.
|
|
237
|
+
*
|
|
238
|
+
* MWA's case: "In October the club chose fantasy / December science fiction /
|
|
239
|
+
* March a mystery" for "what did the club most recently choose". The
|
|
240
|
+
* cross-encoder cannot tell them apart, and a sort by its score put the newest
|
|
241
|
+
* last. AWM itself does NOT apply this (0.18.1): a score-epsilon fallback to the
|
|
242
|
+
* blend measured -14pp identifier s@1 (rerank2.ts), and applying this helper
|
|
243
|
+
* inside the engine measured -18pp with s@5 unchanged, because on a real store
|
|
244
|
+
* the detector also marks related memories that differ in a ticket id or a
|
|
245
|
+
* date. The marks are a cue to check the source. A harness whose policy is
|
|
246
|
+
* "newer beats older" (MWA's is) can call this on the returned results with
|
|
247
|
+
* their `conflictsWith` marks. `marks` is parallel to `results`; the returned
|
|
248
|
+
* array is new and its marks still describe the same ids.
|
|
249
|
+
*/
|
|
250
|
+
export function newestFirstAmongConflicts<T extends ConflictCandidate>(results: T[], marks: ConflictMark[][]): T[] {
|
|
251
|
+
const out = results.slice();
|
|
252
|
+
const m = marks.slice();
|
|
253
|
+
for (let guard = 0; guard < out.length * out.length; guard++) {
|
|
254
|
+
let moved = false;
|
|
255
|
+
for (let j = 1; j < out.length && !moved; j++) {
|
|
256
|
+
for (let i = 0; i < j; i++) {
|
|
257
|
+
const mk = m[j].find(x => x.id === out[i].engram.id);
|
|
258
|
+
if (mk && mk.newer) {
|
|
259
|
+
const [r] = out.splice(j, 1); const [mm] = m.splice(j, 1);
|
|
260
|
+
out.splice(i, 0, r); m.splice(i, 0, mm);
|
|
261
|
+
moved = true;
|
|
262
|
+
break;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
if (!moved) break;
|
|
267
|
+
}
|
|
268
|
+
return out;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** The id of the most recently written memory among those carrying a mark, or null. */
|
|
272
|
+
export function newestConflicting(results: ConflictCandidate[], marks: ConflictMark[][]): string | null {
|
|
273
|
+
let best: string | null = null, bestMs = -Infinity;
|
|
274
|
+
results.forEach((r, i) => {
|
|
275
|
+
if (marks[i].length === 0) return;
|
|
276
|
+
const ms = createdMs(r);
|
|
277
|
+
if (Number.isFinite(ms) && ms > bestMs) { bestMs = ms; best = r.engram.id; }
|
|
278
|
+
});
|
|
279
|
+
return best;
|
|
280
|
+
}
|
package/src/core/embeddings.ts
CHANGED
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
20
|
import { pipeline, type FeatureExtractionPipeline } from '@huggingface/transformers';
|
|
21
|
-
import { dispatchEmbed, registerInProcessHandlers } from './ml-worker.js';
|
|
21
|
+
import { dispatchEmbed, registerInferenceConfig, registerInProcessHandlers } from './ml-worker.js';
|
|
22
|
+
import type { Lane } from './model-daemon.js';
|
|
22
23
|
import { noteModelLoad } from './write-telemetry.js';
|
|
23
24
|
import { ensureModelCacheDir } from './model-cache.js';
|
|
24
25
|
|
|
@@ -26,6 +27,23 @@ const MODEL_ID = process.env.AWM_EMBED_MODEL ?? 'Xenova/bge-small-en-v1.5';
|
|
|
26
27
|
const DIMENSIONS = parseInt(process.env.AWM_EMBED_DIMS ?? '384', 10);
|
|
27
28
|
const POOLING = (process.env.AWM_EMBED_POOLING ?? 'mean') as 'cls' | 'mean';
|
|
28
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Texts per forward pass (default 8; 0 = all at once). onnxruntime-node's CPU arena grows
|
|
32
|
+
* to the largest tensor it has run and never shrinks, and attention is quadratic in the
|
|
33
|
+
* padded length: a 100-memory /memory/write-batch or a 32-engram consolidation backfill
|
|
34
|
+
* at 512 tokens set a high-water mark the process then kept for life. Chunking bounds it.
|
|
35
|
+
* A text's vector does not depend on the others in its batch (padding is masked).
|
|
36
|
+
* See reranker.ts for the measurement.
|
|
37
|
+
*/
|
|
38
|
+
const EMBED_BATCH = (() => {
|
|
39
|
+
// An explicit 0 disables chunking; anything else invalid falls back to 8 (see reranker.ts).
|
|
40
|
+
const v = Number(process.env.AWM_EMBED_BATCH?.trim() || NaN);
|
|
41
|
+
return Number.isInteger(v) && v >= 0 ? v : 8;
|
|
42
|
+
})();
|
|
43
|
+
|
|
44
|
+
/** The texts-per-pass setting, for the model daemon's background chunking. */
|
|
45
|
+
export const EMBED_BATCH_SIZE = EMBED_BATCH;
|
|
46
|
+
|
|
29
47
|
// --- In-process fallback (used by tests and crash recovery) ---
|
|
30
48
|
|
|
31
49
|
let inProcessInstance: FeatureExtractionPipeline | null = null;
|
|
@@ -45,21 +63,27 @@ async function loadInProcess(): Promise<FeatureExtractionPipeline> {
|
|
|
45
63
|
return inProcessInitPromise;
|
|
46
64
|
}
|
|
47
65
|
|
|
48
|
-
|
|
66
|
+
/** Exported with its batch size as a parameter so tests can compare settings in one process. */
|
|
67
|
+
export async function inProcessEmbed(args: { texts: string[]; pooling: 'cls' | 'mean'; dimensions: number }, batch = EMBED_BATCH): Promise<number[][]> {
|
|
49
68
|
const { texts, pooling, dimensions } = args;
|
|
50
69
|
if (texts.length === 0) return [];
|
|
51
70
|
const embedder = await loadInProcess();
|
|
52
|
-
const result = await embedder(texts, { pooling, normalize: true });
|
|
53
|
-
const data = result.data as Float32Array;
|
|
54
71
|
const vectors: number[][] = [];
|
|
55
|
-
|
|
56
|
-
|
|
72
|
+
const size = batch > 0 ? batch : texts.length;
|
|
73
|
+
for (let b = 0; b < texts.length; b += size) {
|
|
74
|
+
const chunk = texts.slice(b, b + size);
|
|
75
|
+
const result = await embedder(chunk, { pooling, normalize: true });
|
|
76
|
+
const data = result.data as Float32Array;
|
|
77
|
+
for (let i = 0; i < chunk.length; i++) {
|
|
78
|
+
vectors.push(Array.from(data.slice(i * dimensions, (i + 1) * dimensions)));
|
|
79
|
+
}
|
|
57
80
|
}
|
|
58
81
|
return vectors;
|
|
59
82
|
}
|
|
60
83
|
|
|
61
84
|
// Register the in-process handler with the pool (used in test mode and as fallback)
|
|
62
|
-
registerInProcessHandlers({ embed: inProcessEmbed });
|
|
85
|
+
registerInProcessHandlers({ embed: (args) => inProcessEmbed(args) });
|
|
86
|
+
registerInferenceConfig({ embedModel: MODEL_ID, embedDims: DIMENSIONS, embedPooling: POOLING, embedBatch: EMBED_BATCH });
|
|
63
87
|
|
|
64
88
|
// --- Public API ---
|
|
65
89
|
|
|
@@ -85,11 +109,12 @@ export async function embed(text: string): Promise<number[]> {
|
|
|
85
109
|
/**
|
|
86
110
|
* Generate embeddings for multiple texts in a batch.
|
|
87
111
|
* More efficient than calling embed() in a loop — the worker batches the
|
|
88
|
-
* tokenization + forward pass.
|
|
112
|
+
* tokenization + forward pass. `lane: 'background'` (consolidation backfill) lets the
|
|
113
|
+
* shared model daemon run interactive requests ahead of it.
|
|
89
114
|
*/
|
|
90
|
-
export async function embedBatch(texts: string[]): Promise<number[][]> {
|
|
115
|
+
export async function embedBatch(texts: string[], opts: { lane?: Lane } = {}): Promise<number[][]> {
|
|
91
116
|
if (texts.length === 0) return [];
|
|
92
|
-
return dispatchEmbed({ texts, pooling: POOLING, dimensions: DIMENSIONS });
|
|
117
|
+
return dispatchEmbed({ texts, pooling: POOLING, dimensions: DIMENSIONS }, opts.lane);
|
|
93
118
|
}
|
|
94
119
|
|
|
95
120
|
/** Get the current embedding model ID (for version tracking in stored embeddings) */
|
|
@@ -1,25 +1,83 @@
|
|
|
1
|
-
import type { ActivationResult } from '../types/engram.js';
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
1
|
+
import type { ActivationResult } from '../types/engram.js';
|
|
2
|
+
import { validityState } from './validity.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Formats one memory_recall result line for the MCP text response.
|
|
6
|
+
*
|
|
7
|
+
* Extracted out of the inline closure in mcp.ts (0.12.1) so the format itself
|
|
8
|
+
* — specifically, that every result carries its engram id — can be unit
|
|
9
|
+
* tested without booting the MCP server (mcp.ts has a top-level `await` and
|
|
10
|
+
* opens the store as a side effect of import, so it cannot be imported by a
|
|
11
|
+
* test directly).
|
|
12
|
+
*
|
|
13
|
+
* The id is placed right after the score, not at the end of the line: result
|
|
14
|
+
* bodies can be long, and a consumer scanning for `[id: ...]` shouldn't have
|
|
15
|
+
* to read past a paragraph of content to find it.
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* True when Phase 5.5 has already trimmed this engram's body.
|
|
19
|
+
*
|
|
20
|
+
* The fade marker is written INTO the content (`consolidation.ts` appends
|
|
21
|
+
* `'… [faded]'` after slicing to FADE_KEEP_CHARS), which means it is only
|
|
22
|
+
* visible to a reader who reaches the end of the body. Two things hide it
|
|
23
|
+
* there: `granularity: 'compact'`/`'auto'`, where the displayed body is a
|
|
24
|
+
* query-aware 200-char window that will rarely include the tail, and the
|
|
25
|
+
* plain length of a full body. So the marker is detected on the CONTENT and
|
|
26
|
+
* rendered in the header, next to `[valid until …]`, where a partial answer
|
|
27
|
+
* cannot be mistaken for a complete one.
|
|
28
|
+
*
|
|
29
|
+
* Matched anywhere rather than at the end because merge-on-reinforce
|
|
30
|
+
* (`AWM_REINFORCE_MERGE_CONTENT`) can append text after a fade, which would
|
|
31
|
+
* leave the marker mid-body. A false positive labels a full memory partial,
|
|
32
|
+
* which is the harmless direction.
|
|
33
|
+
*/
|
|
34
|
+
export function isFaded(content: string): boolean {
|
|
35
|
+
return content.includes('[faded]');
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function formatRecallResultLine(r: ActivationResult, index: number, now: number = Date.now()): string {
|
|
39
|
+
const body = r.summary ?? r.engram.content;
|
|
40
|
+
// Fade retains 7.1% of answer-bearing identifiers while leaving findability
|
|
41
|
+
// intact (docs/eval-gap-2026-10-11.md §8.1) — 37 of 38 faded engrams still
|
|
42
|
+
// surface by cue. That combination is the worst possible shape: the result
|
|
43
|
+
// ranks, reads as an answer, and no longer contains one. The label does not
|
|
44
|
+
// fix the retrieval, it stops the silent part.
|
|
45
|
+
const faded = isFaded(r.engram.content)
|
|
46
|
+
// Fade overwrites the content (consolidation.ts Phase 5.5); no tool can bring the
|
|
47
|
+
// rest back, so the label says what to do instead.
|
|
48
|
+
? ' [faded — body trimmed, the rest is gone; confirm against the source or a newer memory]'
|
|
49
|
+
: '';
|
|
50
|
+
const chain = r.engram.supersededBy
|
|
51
|
+
? ` ⚠ SUPERSEDED by ${r.engram.supersededBy} — treat as historical; recall/fetch the successor before relying on this.`
|
|
52
|
+
: '';
|
|
53
|
+
// Temporal validity (D8). Same decision as ranking (activation.ts Phase 8c,
|
|
54
|
+
// via core/validity.ts): a memory past its valid_to is down-ranked AND says
|
|
55
|
+
// so here — "[valid until 2026-01-15]" on a line read in March was the label
|
|
56
|
+
// that let 482 expired memories in the live store pass as current.
|
|
57
|
+
const vstate = validityState(r.engram, now);
|
|
58
|
+
const validity = vstate === 'expired'
|
|
59
|
+
? ` ⚠ EXPIRED ${r.engram.validTo} — treat as historical; confirm the current state before relying on this.`
|
|
60
|
+
: vstate === 'not-yet-valid'
|
|
61
|
+
? ` [not yet valid — from ${r.engram.validFrom}]`
|
|
62
|
+
: r.engram.validTo
|
|
63
|
+
? ` [valid until ${r.engram.validTo}]`
|
|
64
|
+
: '';
|
|
65
|
+
// The written date on every line (2026-10-19). Two versions of a fact used
|
|
66
|
+
// to arrive with nothing to say which came first; the prime hook printed an
|
|
67
|
+
// age, this format printed nothing.
|
|
68
|
+
const written = writtenDate(r.engram.createdAt);
|
|
69
|
+
const dated = written ? ` [written ${written}]` : '';
|
|
70
|
+
// Conflict mark (src/core/conflicts.ts): phrased as a question, because the
|
|
71
|
+
// detector cannot tell a correction from two facts that share a template.
|
|
72
|
+
const conflicts = (r.conflictsWith ?? []).map(c =>
|
|
73
|
+
` ⚠ POSSIBLE CONFLICT with ${c.id} on a ${c.slot} (this one is ${c.newer ? 'newer' : 'older'}; confirm against the source before stating a value)`,
|
|
74
|
+
).join('');
|
|
75
|
+
return `${index + 1}. **${r.engram.concept}** (${r.score.toFixed(3)}) [id: ${r.engram.id}]${dated}${validity}${faded}: ${body}${chain}${conflicts}`;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** YYYY-MM-DD from a Date or ISO string; empty when absent or unparseable. */
|
|
79
|
+
export function writtenDate(createdAt: Date | string | null | undefined): string {
|
|
80
|
+
if (!createdAt) return '';
|
|
81
|
+
const ms = createdAt instanceof Date ? createdAt.getTime() : Date.parse(createdAt);
|
|
82
|
+
return Number.isFinite(ms) ? new Date(ms).toISOString().slice(0, 10) : '';
|
|
83
|
+
}
|