@remind_ai/remind 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +6 -0
- package/README.md +127 -0
- package/dist/src/brain/consolidation.d.ts +3 -0
- package/dist/src/brain/consolidation.js +153 -0
- package/dist/src/brain/constants.d.ts +18 -0
- package/dist/src/brain/constants.js +26 -0
- package/dist/src/brain/encoding.d.ts +10 -0
- package/dist/src/brain/encoding.js +45 -0
- package/dist/src/brain/forgetting.d.ts +3 -0
- package/dist/src/brain/forgetting.js +48 -0
- package/dist/src/brain/index.d.ts +6 -0
- package/dist/src/brain/index.js +13 -0
- package/dist/src/brain/llm-deps.d.ts +8 -0
- package/dist/src/brain/llm-deps.js +4 -0
- package/dist/src/brain/selftest.d.ts +1 -0
- package/dist/src/brain/selftest.js +229 -0
- package/dist/src/brain/similarity.d.ts +10 -0
- package/dist/src/brain/similarity.js +65 -0
- package/dist/src/brain/temporal.d.ts +9 -0
- package/dist/src/brain/temporal.js +65 -0
- package/dist/src/brain/tiering.d.ts +9 -0
- package/dist/src/brain/tiering.js +39 -0
- package/dist/src/config.d.ts +36 -0
- package/dist/src/config.js +61 -0
- package/dist/src/core/index.d.ts +12 -0
- package/dist/src/core/index.js +31 -0
- package/dist/src/core/ingest.d.ts +2 -0
- package/dist/src/core/ingest.js +117 -0
- package/dist/src/core/memory-types.d.ts +45 -0
- package/dist/src/core/memory-types.js +104 -0
- package/dist/src/core/retrieval.d.ts +2 -0
- package/dist/src/core/retrieval.js +55 -0
- package/dist/src/core/stores/doc.store.d.ts +2 -0
- package/dist/src/core/stores/doc.store.js +74 -0
- package/dist/src/core/stores/graph.store.d.ts +2 -0
- package/dist/src/core/stores/graph.store.js +61 -0
- package/dist/src/core/stores/vector.store.d.ts +2 -0
- package/dist/src/core/stores/vector.store.js +87 -0
- package/dist/src/engine.d.ts +33 -0
- package/dist/src/engine.js +63 -0
- package/dist/src/ids.d.ts +1 -0
- package/dist/src/ids.js +6 -0
- package/dist/src/index.d.ts +19 -0
- package/dist/src/index.js +25 -0
- package/dist/src/llm.d.ts +8 -0
- package/dist/src/llm.js +68 -0
- package/dist/src/memguard/auditor.d.ts +19 -0
- package/dist/src/memguard/auditor.js +48 -0
- package/dist/src/memguard/canary.d.ts +49 -0
- package/dist/src/memguard/canary.js +73 -0
- package/dist/src/memguard/detectors/exfiltration.d.ts +3 -0
- package/dist/src/memguard/detectors/exfiltration.js +18 -0
- package/dist/src/memguard/detectors/pii.d.ts +12 -0
- package/dist/src/memguard/detectors/pii.js +32 -0
- package/dist/src/memguard/detectors/secrets.d.ts +9 -0
- package/dist/src/memguard/detectors/secrets.js +25 -0
- package/dist/src/memguard/firewall.d.ts +15 -0
- package/dist/src/memguard/firewall.js +93 -0
- package/dist/src/memguard/index.d.ts +26 -0
- package/dist/src/memguard/index.js +24 -0
- package/dist/src/memguard/output-filter.d.ts +6 -0
- package/dist/src/memguard/output-filter.js +10 -0
- package/dist/src/memguard/provenance.d.ts +10 -0
- package/dist/src/memguard/provenance.js +36 -0
- package/dist/src/memguard/trust.d.ts +27 -0
- package/dist/src/memguard/trust.js +53 -0
- package/dist/src/queue/connection.d.ts +6 -0
- package/dist/src/queue/connection.js +23 -0
- package/dist/src/queue/queues.d.ts +29 -0
- package/dist/src/queue/queues.js +64 -0
- package/dist/src/types.d.ts +97 -0
- package/dist/src/types.js +3 -0
- package/dist/src/workers/consolidate.worker.d.ts +5 -0
- package/dist/src/workers/consolidate.worker.js +15 -0
- package/dist/src/workers/forget.worker.d.ts +4 -0
- package/dist/src/workers/forget.worker.js +8 -0
- package/dist/src/workers/ingest.worker.d.ts +6 -0
- package/dist/src/workers/ingest.worker.js +36 -0
- package/package.json +46 -0
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
// REMind · L1 offline selftest · Owner: Charan
|
|
2
|
+
// Deterministic checks for the Brain — pure helpers + the full sleep cycle via fake LLM/Stores.
|
|
3
|
+
// Run: `npx tsx src/brain/selftest.ts`
|
|
4
|
+
import { cosine, cluster } from './similarity.js';
|
|
5
|
+
import { currentlyValid, activeAt } from './temporal.js';
|
|
6
|
+
import { tierFor } from './tiering.js';
|
|
7
|
+
import { consolidate } from './consolidation.js';
|
|
8
|
+
import { DAY_MS, MERGE_THRESHOLD, HOT_DAYS, WARM_DAYS } from './constants.js';
|
|
9
|
+
let failures = 0;
|
|
10
|
+
function check(name, cond) {
|
|
11
|
+
console.log(`${cond ? 'PASS' : 'FAIL'} — ${name}`);
|
|
12
|
+
if (!cond)
|
|
13
|
+
failures++;
|
|
14
|
+
}
|
|
15
|
+
function rec(id, over = {}) {
|
|
16
|
+
return {
|
|
17
|
+
id,
|
|
18
|
+
agentId: 'a1',
|
|
19
|
+
type: 'episodic',
|
|
20
|
+
content: id,
|
|
21
|
+
trust: 1,
|
|
22
|
+
confidence: 0.7,
|
|
23
|
+
provenance: { source: 'test' },
|
|
24
|
+
tier: 'hot',
|
|
25
|
+
createdAt: new Date().toISOString(),
|
|
26
|
+
...over,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
// ---------- pure: cosine ----------
|
|
30
|
+
check('cosine identical = 1', Math.abs(cosine([1, 0, 0], [1, 0, 0]) - 1) < 1e-9);
|
|
31
|
+
check('cosine orthogonal = 0', Math.abs(cosine([1, 0], [0, 1])) < 1e-9);
|
|
32
|
+
check('cosine empty = 0', cosine([], [1]) === 0);
|
|
33
|
+
// ---------- pure: clustering ----------
|
|
34
|
+
const recs = [rec('p1'), rec('p2'), rec('q1'), rec('q2'), rec('z')];
|
|
35
|
+
const vecs = new Map([
|
|
36
|
+
['p1', [1, 0, 0]],
|
|
37
|
+
['p2', [0.99, 0.02, 0]],
|
|
38
|
+
['q1', [0, 1, 0]],
|
|
39
|
+
['q2', [0.02, 0.99, 0]],
|
|
40
|
+
['z', [0, 0, 1]],
|
|
41
|
+
]);
|
|
42
|
+
const groups = cluster(recs, vecs, MERGE_THRESHOLD);
|
|
43
|
+
check('two pairs + outlier ⇒ 3 clusters', groups.length === 3);
|
|
44
|
+
check('a tight pair is grouped', groups.some((g) => g.length === 2));
|
|
45
|
+
// ---------- pure: bi-temporal validity ----------
|
|
46
|
+
const tNow = Date.now();
|
|
47
|
+
const open = rec('open', { validFrom: new Date(tNow - DAY_MS).toISOString() });
|
|
48
|
+
const closed = rec('closed', {
|
|
49
|
+
validFrom: new Date(tNow - 5 * DAY_MS).toISOString(),
|
|
50
|
+
validTo: new Date(tNow - DAY_MS).toISOString(),
|
|
51
|
+
});
|
|
52
|
+
const valid = currentlyValid([open, closed]);
|
|
53
|
+
check('currentlyValid keeps the open belief only', valid.length === 1 && valid[0].id === 'open');
|
|
54
|
+
check('activeAt inside closed window', activeAt([closed], new Date(tNow - 3 * DAY_MS).toISOString()).length === 1);
|
|
55
|
+
check('activeAt excludes after validTo', activeAt([closed], new Date(tNow).toISOString()).length === 0);
|
|
56
|
+
// ---------- pure: tiering boundaries (relative to the active constants) ----------
|
|
57
|
+
const ago = (days) => new Date(tNow - days * DAY_MS).toISOString();
|
|
58
|
+
const warmAge = (HOT_DAYS + WARM_DAYS) / 2;
|
|
59
|
+
const coldAge = WARM_DAYS * 2 + 1;
|
|
60
|
+
check('fresh ⇒ hot', tierFor(rec('h', { lastUsedAt: ago(0) }), tNow) === 'hot');
|
|
61
|
+
check('mid ⇒ warm', tierFor(rec('w', { lastUsedAt: ago(warmAge) }), tNow) === 'warm');
|
|
62
|
+
check('beyond warm ⇒ cold', tierFor(rec('c', { lastUsedAt: ago(coldAge) }), tNow) === 'cold');
|
|
63
|
+
// ---------- fakes for the sleep cycle ----------
|
|
64
|
+
function fakeStores() {
|
|
65
|
+
const records = new Map();
|
|
66
|
+
const facts = new Map();
|
|
67
|
+
const upserts = new Set();
|
|
68
|
+
const triplesWritten = [];
|
|
69
|
+
const doc = {
|
|
70
|
+
async pushFact(agentId, fact) {
|
|
71
|
+
if (!facts.has(agentId))
|
|
72
|
+
facts.set(agentId, new Set());
|
|
73
|
+
facts.get(agentId).add(fact);
|
|
74
|
+
},
|
|
75
|
+
async getFacts(agentId) {
|
|
76
|
+
return [...(facts.get(agentId) ?? [])];
|
|
77
|
+
},
|
|
78
|
+
async saveRecord(r) {
|
|
79
|
+
records.set(r.id, { ...r });
|
|
80
|
+
},
|
|
81
|
+
async getRecords(agentId, filter = {}) {
|
|
82
|
+
return [...records.values()].filter((r) => r.agentId === agentId && Object.entries(filter).every(([k, v]) => r[k] === v));
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
const vector = {
|
|
86
|
+
async upsert(r) {
|
|
87
|
+
upserts.add(r.id);
|
|
88
|
+
},
|
|
89
|
+
async search() {
|
|
90
|
+
return [];
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
const graph = {
|
|
94
|
+
async writeTriples(_a, t) {
|
|
95
|
+
triplesWritten.push(...t);
|
|
96
|
+
},
|
|
97
|
+
async fetchGraph() {
|
|
98
|
+
return '';
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
return { stores: { vector, graph, doc }, records, facts, upserts, triplesWritten };
|
|
102
|
+
}
|
|
103
|
+
function hashVec(s) {
|
|
104
|
+
let h = 0;
|
|
105
|
+
const v = [0, 0, 0];
|
|
106
|
+
for (let i = 0; i < s.length; i++) {
|
|
107
|
+
h = (h * 31 + s.charCodeAt(i)) >>> 0;
|
|
108
|
+
v[i % 3] += (h % 97) / 97;
|
|
109
|
+
}
|
|
110
|
+
return v.some((x) => x !== 0) ? v : [1, 0, 0];
|
|
111
|
+
}
|
|
112
|
+
function fakeLlm(opts) {
|
|
113
|
+
return {
|
|
114
|
+
async complete(prompt) {
|
|
115
|
+
if (/Distill/.test(prompt)) {
|
|
116
|
+
if (opts.throwOnDistill)
|
|
117
|
+
throw new Error('boom');
|
|
118
|
+
return JSON.stringify({ belief: opts.belief ?? 'A belief', triples: opts.triples ?? [] });
|
|
119
|
+
}
|
|
120
|
+
if (/higher-order insights/.test(prompt))
|
|
121
|
+
return '[]';
|
|
122
|
+
return '';
|
|
123
|
+
},
|
|
124
|
+
async classify(prompt) {
|
|
125
|
+
if (/contradict or replace/.test(prompt))
|
|
126
|
+
return opts.contradict ?? 'no';
|
|
127
|
+
return 'no'; // surprise probe
|
|
128
|
+
},
|
|
129
|
+
async embed(text) {
|
|
130
|
+
return opts.embedTable?.[text] ?? hashVec(text);
|
|
131
|
+
},
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
const allow = async () => ({ promote: true });
|
|
135
|
+
const deny = async () => ({ promote: false, reason: 'poison' });
|
|
136
|
+
async function main() {
|
|
137
|
+
// Scenario 1 — merge + promote + persist (facts, vector, triples) + episodic demotion.
|
|
138
|
+
{
|
|
139
|
+
const fx = fakeStores();
|
|
140
|
+
const llm = fakeLlm({
|
|
141
|
+
belief: 'User likes pizza',
|
|
142
|
+
triples: [{ subject: 'User', relation: 'likes', object: 'pizza' }],
|
|
143
|
+
embedTable: { pa: [1, 0, 0], pb: [0.87, 0.493, 0], 'User likes pizza': [1, 0, 0] },
|
|
144
|
+
});
|
|
145
|
+
await fx.stores.doc.saveRecord(rec('pa', { content: 'pa' }));
|
|
146
|
+
await fx.stores.doc.saveRecord(rec('pb', { content: 'pb' }));
|
|
147
|
+
const report = await consolidate('a1', fx.stores, allow, llm);
|
|
148
|
+
const factual = await fx.stores.doc.getRecords('a1', { type: 'factual' });
|
|
149
|
+
check('merge: two near-dups fold into one cluster', report.merged === 1);
|
|
150
|
+
check('merge: exactly one belief promoted', report.promoted.length === 1);
|
|
151
|
+
check('merge: belief persisted to doc store', factual.length === 1 && factual[0].content === 'User likes pizza');
|
|
152
|
+
check('merge: fact written', (await fx.stores.doc.getFacts('a1')).includes('User likes pizza'));
|
|
153
|
+
check('merge: belief vector upserted', fx.upserts.has(factual[0].id));
|
|
154
|
+
check('merge: triples written to graph', fx.triplesWritten.length >= 1);
|
|
155
|
+
const episodics = await fx.stores.doc.getRecords('a1', { type: 'episodic' });
|
|
156
|
+
check('merge: consumed episodics demoted to warm', episodics.every((e) => e.tier === 'warm'));
|
|
157
|
+
// idempotency — re-seed hot + re-run ⇒ same belief id, no duplicate.
|
|
158
|
+
await fx.stores.doc.saveRecord(rec('pa', { content: 'pa', tier: 'hot' }));
|
|
159
|
+
await fx.stores.doc.saveRecord(rec('pb', { content: 'pb', tier: 'hot' }));
|
|
160
|
+
await consolidate('a1', fx.stores, allow, llm);
|
|
161
|
+
const factual2 = await fx.stores.doc.getRecords('a1', { type: 'factual' });
|
|
162
|
+
check('idempotent: re-run does not duplicate the belief', factual2.length === 1);
|
|
163
|
+
}
|
|
164
|
+
// Scenario 2 — gate denies ⇒ nothing persisted, recorded as blocked.
|
|
165
|
+
{
|
|
166
|
+
const fx = fakeStores();
|
|
167
|
+
const llm = fakeLlm({ belief: 'User likes pizza', embedTable: { pa: [1, 0, 0], pb: [0.87, 0.493, 0] } });
|
|
168
|
+
await fx.stores.doc.saveRecord(rec('pa', { content: 'pa' }));
|
|
169
|
+
await fx.stores.doc.saveRecord(rec('pb', { content: 'pb' }));
|
|
170
|
+
const report = await consolidate('a1', fx.stores, deny, llm);
|
|
171
|
+
const factual = await fx.stores.doc.getRecords('a1', { type: 'factual' });
|
|
172
|
+
check('deny: nothing promoted', report.promoted.length === 0);
|
|
173
|
+
check('deny: candidate recorded as blocked', report.blocked.length >= 1);
|
|
174
|
+
check('deny: no belief persisted', factual.length === 0);
|
|
175
|
+
}
|
|
176
|
+
// Scenario 3 — a contradicting belief supersedes the old one (validTo closed).
|
|
177
|
+
{
|
|
178
|
+
const fx = fakeStores();
|
|
179
|
+
const llm = fakeLlm({
|
|
180
|
+
belief: 'User lives in Bangalore',
|
|
181
|
+
triples: [{ subject: 'User', relation: 'livesIn', object: 'Bangalore' }],
|
|
182
|
+
contradict: 'yes',
|
|
183
|
+
embedTable: {
|
|
184
|
+
m1: [0.9, 0.1, 0],
|
|
185
|
+
m2: [0.92, 0.08, 0],
|
|
186
|
+
'User lives in Bangalore': [0.9, 0.1, 0],
|
|
187
|
+
'User lives in Delhi': [1, 0, 0],
|
|
188
|
+
},
|
|
189
|
+
});
|
|
190
|
+
await fx.stores.doc.saveRecord({
|
|
191
|
+
...rec('delhi', { content: 'User lives in Delhi', type: 'factual', validFrom: ago(30) }),
|
|
192
|
+
triples: [{ subject: 'User', relation: 'livesIn', object: 'Delhi' }],
|
|
193
|
+
});
|
|
194
|
+
await fx.stores.doc.saveRecord(rec('m1', { content: 'm1' }));
|
|
195
|
+
await fx.stores.doc.saveRecord(rec('m2', { content: 'm2' }));
|
|
196
|
+
const report = await consolidate('a1', fx.stores, allow, llm);
|
|
197
|
+
const delhi = (await fx.stores.doc.getRecords('a1', { type: 'factual' })).find((r) => r.id === 'delhi');
|
|
198
|
+
check('supersede: new belief promoted', report.promoted.some((p) => p.content === 'User lives in Bangalore'));
|
|
199
|
+
check('supersede: old belief got a validTo', !!delhi?.validTo);
|
|
200
|
+
check('supersede: old belief no longer currently-valid', currentlyValid([delhi]).length === 0);
|
|
201
|
+
}
|
|
202
|
+
// Scenario 4 — a failing LLM call is isolated; the cycle still resolves.
|
|
203
|
+
{
|
|
204
|
+
const fx = fakeStores();
|
|
205
|
+
const llm = fakeLlm({ throwOnDistill: true });
|
|
206
|
+
await fx.stores.doc.saveRecord(rec('x1', { content: 'x1' }));
|
|
207
|
+
await fx.stores.doc.saveRecord(rec('x2', { content: 'x2' }));
|
|
208
|
+
let threw = false;
|
|
209
|
+
let report;
|
|
210
|
+
try {
|
|
211
|
+
report = await consolidate('a1', fx.stores, allow, llm);
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
threw = true;
|
|
215
|
+
}
|
|
216
|
+
check('robust: a distill failure does not crash the cycle', !threw);
|
|
217
|
+
check('robust: nothing promoted on failure', report?.promoted.length === 0);
|
|
218
|
+
check('robust: failure recorded as blocked', (report?.blocked.length ?? 0) >= 1);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
main()
|
|
222
|
+
.then(() => {
|
|
223
|
+
console.log(failures ? `\n${failures} check(s) FAILED` : '\nAll checks passed');
|
|
224
|
+
process.exit(failures ? 1 : 0);
|
|
225
|
+
})
|
|
226
|
+
.catch((err) => {
|
|
227
|
+
console.error('selftest crashed:', err);
|
|
228
|
+
process.exit(1);
|
|
229
|
+
});
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { MemoryRecord } from '../types.js';
|
|
2
|
+
export type Vec = number[];
|
|
3
|
+
/**
|
|
4
|
+
* Embed every record once into `cache` (reusing a record's own embedding when present).
|
|
5
|
+
* Pass the same `cache` across calls in a sleep cycle to avoid paying for an embedding twice.
|
|
6
|
+
*/
|
|
7
|
+
export declare function embedAll(records: MemoryRecord[], cache?: Map<string, Vec>, embedFn?: (text: string) => Promise<Vec>): Promise<Map<string, Vec>>;
|
|
8
|
+
export declare function cosine(a: Vec, b: Vec): number;
|
|
9
|
+
/** Greedy single-pass clustering; deterministic in input order. */
|
|
10
|
+
export declare function cluster(records: MemoryRecord[], vecs: Map<string, Vec>, threshold: number): MemoryRecord[][];
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// REMind · L1 embedding/similarity utils · Owner: Charan
|
|
2
|
+
// Shared cosine + greedy clustering used by encoding, consolidation and forgetting.
|
|
3
|
+
import { embed as defaultEmbed } from '../llm.js';
|
|
4
|
+
const keyOf = (r) => r.id || r.content;
|
|
5
|
+
/**
|
|
6
|
+
* Embed every record once into `cache` (reusing a record's own embedding when present).
|
|
7
|
+
* Pass the same `cache` across calls in a sleep cycle to avoid paying for an embedding twice.
|
|
8
|
+
*/
|
|
9
|
+
export async function embedAll(records, cache = new Map(), embedFn = defaultEmbed) {
|
|
10
|
+
for (const r of records) {
|
|
11
|
+
const key = keyOf(r);
|
|
12
|
+
if (cache.has(key))
|
|
13
|
+
continue;
|
|
14
|
+
cache.set(key, r.embedding ?? (await embedFn(r.content)));
|
|
15
|
+
}
|
|
16
|
+
return cache;
|
|
17
|
+
}
|
|
18
|
+
export function cosine(a, b) {
|
|
19
|
+
if (!a?.length || !b?.length || a.length !== b.length)
|
|
20
|
+
return 0;
|
|
21
|
+
let dot = 0;
|
|
22
|
+
let na = 0;
|
|
23
|
+
let nb = 0;
|
|
24
|
+
for (let i = 0; i < a.length; i++) {
|
|
25
|
+
dot += a[i] * b[i];
|
|
26
|
+
na += a[i] * a[i];
|
|
27
|
+
nb += b[i] * b[i];
|
|
28
|
+
}
|
|
29
|
+
if (na === 0 || nb === 0)
|
|
30
|
+
return 0;
|
|
31
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
32
|
+
}
|
|
33
|
+
/** Greedy single-pass clustering; deterministic in input order. */
|
|
34
|
+
export function cluster(records, vecs, threshold) {
|
|
35
|
+
const clusters = [];
|
|
36
|
+
for (const r of records) {
|
|
37
|
+
const v = vecs.get(keyOf(r));
|
|
38
|
+
if (!v) {
|
|
39
|
+
clusters.push({ centroid: [], items: [r] });
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
let best = null;
|
|
43
|
+
for (const c of clusters) {
|
|
44
|
+
const sim = cosine(v, c.centroid);
|
|
45
|
+
if (sim >= threshold && (!best || sim > best.sim))
|
|
46
|
+
best = { c, sim };
|
|
47
|
+
}
|
|
48
|
+
if (best) {
|
|
49
|
+
best.c.items.push(r);
|
|
50
|
+
best.c.centroid = recenter(best.c.centroid, v, best.c.items.length);
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
clusters.push({ centroid: v.slice(), items: [r] });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return clusters.map((c) => c.items);
|
|
57
|
+
}
|
|
58
|
+
function recenter(centroid, v, n) {
|
|
59
|
+
if (!centroid.length)
|
|
60
|
+
return v.slice();
|
|
61
|
+
const out = centroid.slice();
|
|
62
|
+
for (let i = 0; i < out.length; i++)
|
|
63
|
+
out[i] += (v[i] - out[i]) / n;
|
|
64
|
+
return out;
|
|
65
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { MemoryRecord, Stores } from '../types.js';
|
|
2
|
+
import { type Vec } from './similarity.js';
|
|
3
|
+
import { type LlmDeps } from './llm-deps.js';
|
|
4
|
+
/** Records whose valid-time window is still open. */
|
|
5
|
+
export declare function currentlyValid(records: MemoryRecord[]): MemoryRecord[];
|
|
6
|
+
/** Records whose valid-time window contains the instant `at` (ISO). */
|
|
7
|
+
export declare function activeAt(records: MemoryRecord[], at: string): MemoryRecord[];
|
|
8
|
+
/** Close the valid-time of active beliefs that `newRecord` contradicts. Returns the retired records. */
|
|
9
|
+
export declare function supersede(newRecord: MemoryRecord, existing: MemoryRecord[], stores: Stores, llm?: LlmDeps, cache?: Map<string, Vec>): Promise<MemoryRecord[]>;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// REMind · L1 temporal · Owner: Charan
|
|
2
|
+
// supersede(newRecord, existing, stores): set validTo on contradicted beliefs, validFrom on new ones; time-bounded query helpers (bi-temporal validity).
|
|
3
|
+
import { embedAll, cosine } from './similarity.js';
|
|
4
|
+
import { defaultLlm } from './llm-deps.js';
|
|
5
|
+
import { SUPERSEDE_SIM_THRESHOLD } from './constants.js';
|
|
6
|
+
const nowIso = () => new Date().toISOString();
|
|
7
|
+
const keyOf = (r) => r.id || r.content;
|
|
8
|
+
/** Records whose valid-time window is still open. */
|
|
9
|
+
export function currentlyValid(records) {
|
|
10
|
+
const t = Date.now();
|
|
11
|
+
return records.filter((r) => !r.validTo || Date.parse(r.validTo) > t);
|
|
12
|
+
}
|
|
13
|
+
/** Records whose valid-time window contains the instant `at` (ISO). */
|
|
14
|
+
export function activeAt(records, at) {
|
|
15
|
+
const t = Date.parse(at);
|
|
16
|
+
return records.filter((r) => {
|
|
17
|
+
const from = r.validFrom ? Date.parse(r.validFrom) : 0;
|
|
18
|
+
const to = r.validTo ? Date.parse(r.validTo) : Infinity;
|
|
19
|
+
return from <= t && t < to;
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
/** Close the valid-time of active beliefs that `newRecord` contradicts. Returns the retired records. */
|
|
23
|
+
export async function supersede(newRecord, existing, stores, llm = defaultLlm, cache = new Map()) {
|
|
24
|
+
if (!newRecord.validFrom)
|
|
25
|
+
newRecord.validFrom = nowIso();
|
|
26
|
+
const candidates = currentlyValid(existing).filter((r) => r.id !== newRecord.id);
|
|
27
|
+
if (!candidates.length)
|
|
28
|
+
return [];
|
|
29
|
+
// Embedding prefilter: only spend an LLM call on beliefs that are topically related,
|
|
30
|
+
// so this stays O(related) instead of O(every belief).
|
|
31
|
+
await embedAll([newRecord, ...candidates], cache, llm.embed);
|
|
32
|
+
const nv = cache.get(keyOf(newRecord));
|
|
33
|
+
const retired = [];
|
|
34
|
+
for (const old of candidates) {
|
|
35
|
+
const ov = cache.get(keyOf(old));
|
|
36
|
+
const sim = nv && ov ? cosine(nv, ov) : 1; // missing vectors ⇒ fail safe and check
|
|
37
|
+
if (!sharesSubject(newRecord, old) && sim < SUPERSEDE_SIM_THRESHOLD)
|
|
38
|
+
continue;
|
|
39
|
+
const ans = await llm.classify(`OLD belief: "${old.content}"\nNEW belief: "${newRecord.content}"\n` +
|
|
40
|
+
`Does NEW contradict or replace OLD about the same subject? Answer 'yes' or 'no'.`);
|
|
41
|
+
if (ans.startsWith('yes')) {
|
|
42
|
+
const closed = { ...old, validTo: newRecord.validFrom };
|
|
43
|
+
await stores.doc.saveRecord(closed);
|
|
44
|
+
retired.push(closed);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return retired;
|
|
48
|
+
}
|
|
49
|
+
function sharesSubject(a, b) {
|
|
50
|
+
const sa = subjects(a);
|
|
51
|
+
const sb = subjects(b);
|
|
52
|
+
if (!sa.size || !sb.size)
|
|
53
|
+
return false; // no triples ⇒ rely on the embedding prefilter
|
|
54
|
+
for (const s of sa)
|
|
55
|
+
if (sb.has(s))
|
|
56
|
+
return true;
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
function subjects(r) {
|
|
60
|
+
const out = new Set();
|
|
61
|
+
for (const t of r.triples ?? [])
|
|
62
|
+
if (t.subject)
|
|
63
|
+
out.add(t.subject.toLowerCase());
|
|
64
|
+
return out;
|
|
65
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { MemoryRecord, Stores, Tier } from '../types.js';
|
|
2
|
+
/** Tier a record should occupy given how recently it was used. */
|
|
3
|
+
export declare function tierFor(record: MemoryRecord, nowMs?: number): Tier;
|
|
4
|
+
/**
|
|
5
|
+
* Re-place records by access recency. Demotions (hot→warm→cold) always apply.
|
|
6
|
+
* A cold record only climbs back out on a strong recent-access signal (used within HOT_DAYS),
|
|
7
|
+
* so forgetting's archival decisions are not silently undone. Returns #changed.
|
|
8
|
+
*/
|
|
9
|
+
export declare function retier(agentId: string, stores: Stores): Promise<number>;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// REMind · L1 tiering · Owner: Charan
|
|
2
|
+
// retier(agentId, stores): move records hot -> warm -> cold based on access patterns.
|
|
3
|
+
import { DAY_MS, HOT_DAYS, WARM_DAYS } from './constants.js';
|
|
4
|
+
/** Tier a record should occupy given how recently it was used. */
|
|
5
|
+
export function tierFor(record, nowMs = Date.now()) {
|
|
6
|
+
const last = Date.parse(record.lastUsedAt ?? record.createdAt);
|
|
7
|
+
const ageDays = (nowMs - (Number.isNaN(last) ? nowMs : last)) / DAY_MS;
|
|
8
|
+
if (ageDays <= HOT_DAYS)
|
|
9
|
+
return 'hot';
|
|
10
|
+
if (ageDays <= WARM_DAYS)
|
|
11
|
+
return 'warm';
|
|
12
|
+
return 'cold';
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Re-place records by access recency. Demotions (hot→warm→cold) always apply.
|
|
16
|
+
* A cold record only climbs back out on a strong recent-access signal (used within HOT_DAYS),
|
|
17
|
+
* so forgetting's archival decisions are not silently undone. Returns #changed.
|
|
18
|
+
*/
|
|
19
|
+
export async function retier(agentId, stores) {
|
|
20
|
+
const records = await stores.doc.getRecords(agentId);
|
|
21
|
+
const nowMs = Date.now();
|
|
22
|
+
let changed = 0;
|
|
23
|
+
for (const r of records) {
|
|
24
|
+
const next = tierFor(r, nowMs);
|
|
25
|
+
if (next === r.tier)
|
|
26
|
+
continue;
|
|
27
|
+
if (r.tier === 'cold' && !recentlyUsed(r, nowMs))
|
|
28
|
+
continue;
|
|
29
|
+
await stores.doc.saveRecord({ ...r, tier: next });
|
|
30
|
+
changed++;
|
|
31
|
+
}
|
|
32
|
+
return changed;
|
|
33
|
+
}
|
|
34
|
+
function recentlyUsed(record, nowMs) {
|
|
35
|
+
if (!record.lastUsedAt)
|
|
36
|
+
return false;
|
|
37
|
+
const last = Date.parse(record.lastUsedAt);
|
|
38
|
+
return !Number.isNaN(last) && nowMs - last <= HOT_DAYS * DAY_MS;
|
|
39
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import 'dotenv/config';
|
|
2
|
+
export declare const config: {
|
|
3
|
+
azureOpenAI: {
|
|
4
|
+
endpoint: string;
|
|
5
|
+
apiKey: string;
|
|
6
|
+
apiVersion: string;
|
|
7
|
+
chatDeployment: string;
|
|
8
|
+
embedDeployment: string;
|
|
9
|
+
embedDimensions: number;
|
|
10
|
+
};
|
|
11
|
+
search: {
|
|
12
|
+
endpoint: string;
|
|
13
|
+
apiKey: string;
|
|
14
|
+
index: string;
|
|
15
|
+
};
|
|
16
|
+
gremlin: {
|
|
17
|
+
endpoint: string;
|
|
18
|
+
key: string;
|
|
19
|
+
database: string;
|
|
20
|
+
graph: string;
|
|
21
|
+
};
|
|
22
|
+
mongo: {
|
|
23
|
+
uri: string;
|
|
24
|
+
db: string;
|
|
25
|
+
};
|
|
26
|
+
redis: {
|
|
27
|
+
url: string;
|
|
28
|
+
};
|
|
29
|
+
sync: boolean;
|
|
30
|
+
};
|
|
31
|
+
export type Config = typeof config;
|
|
32
|
+
/**
|
|
33
|
+
* Fail fast with a clear, actionable message when a credential needed to actually run is
|
|
34
|
+
* missing — call this at process entry points (demo, server) before touching Azure.
|
|
35
|
+
*/
|
|
36
|
+
export declare function requireConfig(): void;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// REMind · Config · Owner: Yasho
|
|
2
|
+
// Loads .env and exposes the Azure / Microsoft service settings every layer reads.
|
|
3
|
+
import 'dotenv/config';
|
|
4
|
+
function opt(name, fallback = '') {
|
|
5
|
+
return process.env[name] ?? fallback;
|
|
6
|
+
}
|
|
7
|
+
export const config = {
|
|
8
|
+
azureOpenAI: {
|
|
9
|
+
endpoint: opt('AZURE_OPENAI_ENDPOINT'),
|
|
10
|
+
apiKey: opt('AZURE_OPENAI_API_KEY'),
|
|
11
|
+
apiVersion: opt('AZURE_OPENAI_API_VERSION', '2024-10-21'),
|
|
12
|
+
chatDeployment: opt('AZURE_OPENAI_CHAT_DEPLOYMENT', 'gpt-4o-mini'),
|
|
13
|
+
embedDeployment: opt('AZURE_OPENAI_EMBED_DEPLOYMENT', 'text-embedding-3-large'),
|
|
14
|
+
embedDimensions: Number(opt('AZURE_OPENAI_EMBED_DIMENSIONS', '3072')),
|
|
15
|
+
},
|
|
16
|
+
// Azure AI Search — vector store (replaces Qdrant)
|
|
17
|
+
search: {
|
|
18
|
+
endpoint: opt('AZURE_SEARCH_ENDPOINT'),
|
|
19
|
+
apiKey: opt('AZURE_SEARCH_API_KEY'),
|
|
20
|
+
index: opt('AZURE_SEARCH_INDEX', 'remind-memory'),
|
|
21
|
+
},
|
|
22
|
+
// Azure Cosmos DB for Apache Gremlin — knowledge graph (replaces Neo4j)
|
|
23
|
+
gremlin: {
|
|
24
|
+
endpoint: opt('COSMOS_GREMLIN_ENDPOINT'),
|
|
25
|
+
key: opt('COSMOS_GREMLIN_KEY'),
|
|
26
|
+
database: opt('COSMOS_GREMLIN_DB', 'remind'),
|
|
27
|
+
graph: opt('COSMOS_GREMLIN_GRAPH', 'memory'),
|
|
28
|
+
},
|
|
29
|
+
// Azure Cosmos DB for MongoDB — facts/records (Mongoose drop-in)
|
|
30
|
+
mongo: {
|
|
31
|
+
uri: opt('COSMOS_MONGO_URI'),
|
|
32
|
+
db: opt('COSMOS_MONGO_DB', 'remind'),
|
|
33
|
+
},
|
|
34
|
+
// Azure Cache for Redis — queues (Layer 0 does not use this yet)
|
|
35
|
+
redis: { url: opt('REDIS_URL') },
|
|
36
|
+
sync: opt('SYNC', '1') === '1',
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* Fail fast with a clear, actionable message when a credential needed to actually run is
|
|
40
|
+
* missing — call this at process entry points (demo, server) before touching Azure.
|
|
41
|
+
*/
|
|
42
|
+
export function requireConfig() {
|
|
43
|
+
const missing = [];
|
|
44
|
+
const need = (value, name) => {
|
|
45
|
+
if (!value)
|
|
46
|
+
missing.push(name);
|
|
47
|
+
};
|
|
48
|
+
need(config.azureOpenAI.endpoint, 'AZURE_OPENAI_ENDPOINT');
|
|
49
|
+
need(config.azureOpenAI.apiKey, 'AZURE_OPENAI_API_KEY');
|
|
50
|
+
need(config.search.endpoint, 'AZURE_SEARCH_ENDPOINT');
|
|
51
|
+
need(config.search.apiKey, 'AZURE_SEARCH_API_KEY');
|
|
52
|
+
need(config.gremlin.endpoint, 'COSMOS_GREMLIN_ENDPOINT');
|
|
53
|
+
need(config.gremlin.key, 'COSMOS_GREMLIN_KEY');
|
|
54
|
+
need(config.mongo.uri, 'COSMOS_MONGO_URI');
|
|
55
|
+
if (!config.sync)
|
|
56
|
+
need(config.redis.url, 'REDIS_URL');
|
|
57
|
+
if (missing.length > 0) {
|
|
58
|
+
throw new Error(`REMind is missing required environment variables:\n - ${missing.join('\n - ')}\n` +
|
|
59
|
+
`Copy .env.example to .env and fill these in.`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { IngestInput, MemoryEngineCore, MemoryRecord, RetrievalResult, Stores } from '../types.js';
|
|
2
|
+
import { vectorStore } from './stores/vector.store.js';
|
|
3
|
+
import { graphStore } from './stores/graph.store.js';
|
|
4
|
+
import { docStore } from './stores/doc.store.js';
|
|
5
|
+
export declare const stores: Stores;
|
|
6
|
+
export declare class CoreMemory implements MemoryEngineCore {
|
|
7
|
+
draft(input: IngestInput): Promise<MemoryRecord | null>;
|
|
8
|
+
/** Idempotent: vector/graph/doc writes key on deterministic ids. */
|
|
9
|
+
persist(record: MemoryRecord): Promise<MemoryRecord>;
|
|
10
|
+
fetchMemory(agentId: string, query: string, sourceId?: string): Promise<RetrievalResult>;
|
|
11
|
+
}
|
|
12
|
+
export { vectorStore, graphStore, docStore };
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// REMind · L0 Foundation export · Owner: Bibhas
|
|
2
|
+
// CoreMemory implements MemoryEngineCore over the three Azure stores. The single public seam of core/.
|
|
3
|
+
import { draft as draftFn } from './ingest.js';
|
|
4
|
+
import { fetchMemory as fetchFn } from './retrieval.js';
|
|
5
|
+
import { vectorStore } from './stores/vector.store.js';
|
|
6
|
+
import { graphStore } from './stores/graph.store.js';
|
|
7
|
+
import { docStore } from './stores/doc.store.js';
|
|
8
|
+
export const stores = { vector: vectorStore, graph: graphStore, doc: docStore };
|
|
9
|
+
export class CoreMemory {
|
|
10
|
+
async draft(input) {
|
|
11
|
+
return draftFn(input);
|
|
12
|
+
}
|
|
13
|
+
/** Idempotent: vector/graph/doc writes key on deterministic ids. */
|
|
14
|
+
async persist(record) {
|
|
15
|
+
if (record.type === 'factual') {
|
|
16
|
+
await stores.doc.pushFact(record.agentId, record.content);
|
|
17
|
+
}
|
|
18
|
+
else {
|
|
19
|
+
await stores.vector.upsert(record);
|
|
20
|
+
if (record.triples?.length) {
|
|
21
|
+
await stores.graph.writeTriples(record.agentId, record.triples);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
await stores.doc.saveRecord(record);
|
|
25
|
+
return record;
|
|
26
|
+
}
|
|
27
|
+
async fetchMemory(agentId, query, sourceId) {
|
|
28
|
+
return fetchFn(stores, agentId, query, sourceId);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
export { vectorStore, graphStore, docStore };
|