memtrails 0.0.0-stage → 0.2.0

package/dist/core/recall.js

@@ -0,0 +1,172 @@
+/**
+ * Recall: what a search returns, an item opened whole, and exploring around a topic.
+ *
+ * `memorySearch` embeds the query and searches dense vectors (each item's key, its chunks and its cues) and a keyword
+ * vector at once, fused by rank and grouped by item; with dates, what happened within them first. The best 30 can be
+ * reordered (`rerank`: word by word on this machine, words.ts, or any cross-encoder) and fused with the search's own order.
+ * `memoryOpen` reads one item whole. `memoryExplore` walks the concept graph (graph.ts) from a topic or from items to
+ * what is related but not found directly.
+ */
+import { pointId } from './qdrant-client.js';
+import { queryText } from './embedder.js';
+import { explore } from './graph.js';
+import { RERANK_TOP, rerankedOrder } from './reranker.js';
+import { children, cut, querySparse } from './text.js';
+import { dateSpan, LONG_SUMMARY_CHUNKS } from './writer.js';
+import { collectionKey } from '../records.js';
+/** What search may show: active items, never a throwaway run, never superseded or archived knowledge. */
+function baseFilter(kinds) {
+    return {
+        must: [{ key: 'synthetic', match: { value: false } }, { key: 'status', match: { value: 'active' } },
+            ...(kinds?.length ? [{ key: 'engine', match: { any: kinds } }] : [])],
+    };
+}
+async function items(d, uids) {
+    const got = await d.client.retrieve(uids.map((u) => pointId(u, 0)));
+    const byItem = new Map(got.map((p) => [String(p.payload.item), p.payload]));
+    return uids.flatMap((uid) => {
+        const p = byItem.get(uid);
+        if (!p)
+            return [];
+        return [{
+                id: uid, title: String(p.title ?? ''), summary: String(p.summary ?? ''), where: String(p.where ?? ''),
+                project: String(p.project ?? ''), engine: String(p.engine ?? ''), ts: Number(p.ts ?? 0), kind: String(p.kind ?? ''), score: 0,
+                ...(p.eventAt ? { eventAt: String(p.eventAt) } : {}), ...(p.startAt ? { startAt: String(p.startAt) } : {}),
+                ...(p.endAt ? { endAt: String(p.endAt) } : {}), ...(p.record ? { record: p.record } : {}),
+                ...(p.tier === 'stm' ? { kind: String(p.engine ?? 'record') } : {}),
+            }];
+    });
+}
+export const MAX_SEARCH = 200;
+/** A captured request's title shorter than this says nothing on its own. */
+const SHORT_TITLE = 24;
+/** What happened within the dates (YYYY-MM-DD, YYYY-MM or YYYY; either end may be open): an observation by when it
+ *  happened — its time overlapping the range — anything else by when it was written. The caller reads the dates out of
+ *  a question; nothing here parses words. */
+function withinDates(f, from, to) {
+    const lo = dateSpan(from)?.from ?? 0;
+    const hi = dateSpan(to)?.to ?? Number.MAX_SAFE_INTEGER;
+    return {
+        ...f,
+        must: [...(f.must ?? []), {
+                should: [
+                    { must: [{ key: 'eventFrom', range: { lte: hi } }, { key: 'eventTo', range: { gte: lo } }] },
+                    { must: [{ is_empty: { key: 'eventFrom' } }, { key: 'ts', range: { gte: lo, lte: hi } }] },
+                ],
+            }],
+    };
+}
+/** How long a search waits for its reranker before keeping its own order. */
+const RERANK_WAIT_MS = 3000;
+/** The best RERANK_TOP in the reranker's order fused with their own; with dates, the hits within them stay first — each
+ *  block is reordered on its own. */
+async function reranked(d, query, list, dated) {
+    if (!d.rerank || list.length < 2)
+        return list;
+    const top = list.slice(0, RERANK_TOP);
+    const scores = await d.rerank(query, top.map((it) => `${it.title}\n${it.summary}`), RERANK_WAIT_MS).catch(() => null);
+    if (!scores)
+        return list;
+    const score = new Map(top.map((it, i) => [it.id, scores[i]]));
+    const order = (xs) => rerankedOrder(xs, xs.map((it) => score.get(it.id)));
+    return [...order(top.filter((it) => dated.has(it.id))), ...order(top.filter((it) => !dated.has(it.id))), ...list.slice(RERANK_TOP)];
+}
+/** Up to this many asked for, memory_search ranks one fused list of twice as many (the room the two kinds had); past
+ *  it — a question that gathers (memoryGather asks for MAX_SEARCH, then 30 a follow-up) — the dated observations and
+ *  everything else are two searches of that many each, interleaved, so neither crowds the other out of the room. */
+const SINGLE_LIST_UP_TO = 20;
+export async function memorySearch(d, q) {
+    const projects = q.collections?.length ? q.collections.map(collectionKey) : undefined;
+    // Up to 200: a question that gathers ("everything she did with her family") needs the facts of many threads at once,
+    // a dated observation a line.
+    const limit = Math.max(1, Math.min(MAX_SEARCH, q.limit ?? 8));
+    const vecs = await d.embed([queryText(q.query)]);
+    const base = baseFilter(q.kinds);
+    const scoped = projects ? { ...base, must: [...(base.must ?? []), { key: 'project', match: { any: projects } }] } : base;
+    // Two searches at once: the dated observations, and everything else — up to `limit` of each. In one search an
+    // observation (one point) competed with every chunk of every long request for the same places: 78 of 200 hits on a
+    // list question, while the store held the facts it needed (measured on LoCoMo: 22 observation lines in the room
+    // against Mem0's 145).
+    const dense = vecs.map((v) => Array.from(v));
+    const sparse = querySparse([q.query]);
+    const observations = { key: 'kind', match: { value: 'observation' } };
+    const ranked = async (filter) => {
+        // A search for a few: one fused list over every kind, twice as deep as asked. Interleaving the kinds by rank cost
+        // the top of the list (all of LoCoMo, consolidated, gold evidence any@5 84.6 → 86.8 with one list); one list only as
+        // deep as asked lost the bottom (multi-hop all@40 57.8 → 44.7; twice as deep: 59.6). The interleave is for gathering.
+        if (limit <= SINGLE_LIST_UP_TO)
+            return d.client.fusedItems({ dense, sparse, filter, limit: 2 * limit });
+        const [obs, rest] = await Promise.all([
+            d.client.fusedItems({ dense, sparse, filter: { ...filter, must: [...(filter.must ?? []), observations] }, limit }),
+            d.client.fusedItems({ dense, sparse, filter: { ...filter, must_not: [...(filter.must_not ?? []), observations] }, limit }),
+        ]);
+        // Interleaved by rank, so each kind keeps its order and neither crowds out the other.
+        return Array.from({ length: Math.max(obs.length, rest.length) }, (_, i) => [rest[i], obs[i]]).flat().filter((h) => !!h);
+    };
+    const [inDates, open] = await Promise.all([q.from || q.to ? ranked(withinDates(scoped, q.from, q.to)) : [], ranked(scoped)]);
+    const dated = new Set(inDates.map((h) => h.item));
+    const merged = [...inDates, ...open.filter((h) => !dated.has(h.item))].slice(0, Math.max(open.length, inDates.length));
+    // A kept episode and its captured request hold the same text: the one found first stands for both.
+    const seen = new Set();
+    const hits = merged.filter((h) => {
+        const request = h.item.startsWith('ep:') ? h.item.slice(3) : h.item.startsWith('stm:') ? h.item.slice(4) : null;
+        if (request === null)
+            return true;
+        if (seen.has(request))
+            return false;
+        seen.add(request);
+        return true;
+    });
+    const list = await reranked(d, q.query, await items(d, hits.map((h) => h.item)), dated);
+    const score = new Map(hits.map((h) => [h.item, h.score]));
+    const matched = await passages(d, hits);
+    // A record titled by its first line, when that line is as short as "continue" or "ok", says nothing of what was found:
+    // it shows the start of its key instead, which is what matched.
+    const titled = (it) => (it.id.startsWith('stm:') && it.title.length < SHORT_TITLE ? { ...it, title: cut(it.summary.replace(/\s+/g, ' '), 100) } : it);
+    const out = list.map((it) => ({ ...titled(it), score: score.get(it.id) ?? 0, ...(matched.has(it.id) ? { match: matched.get(it.id) } : {}) }));
+    d.record?.({ ev: 'search', project: projects?.join(','), query: q.query, shown: out.map((it) => it.id) });
+    return out;
+}
+/** For each hit whose best point was a chunk of its text, that chunk — found again from the text (writer.ts chunks a
+ *  request's whole body; an item's body, and a summary too long for one key vector). */
+async function passages(d, hits) {
+    const chunked = hits.filter((h) => h.point && h.point !== pointId(h.item, 0));
+    if (!chunked.length)
+        return new Map();
+    const got = await d.client.retrieve(chunked.map((h) => pointId(h.item, 0)));
+    const byItem = new Map(got.map((p) => [String(p.payload.item), p.payload]));
+    const out = new Map();
+    for (const h of chunked) {
+        const p = byItem.get(h.item);
+        if (!p)
+            continue;
+        const body = String(p.body ?? '');
+        const summary = String(p.summary ?? '');
+        const kids = p.tier === 'stm' ? children(body) : [...children(body), ...(children(summary).length >= LONG_SUMMARY_CHUNKS ? children(summary) : [])];
+        const k = kids.findIndex((_, i) => pointId(h.item, i + 1) === h.point);
+        if (k >= 0)
+            out.set(h.item, kids[k]);
+    }
+    return out;
+}
+/** An agent exploring: from a topic (search finds the first items; the topic's concepts start the walk too) or from
+ *  items, what the concept graph reaches that search did not, and the concepts around it. */
+export async function memoryExplore(d, q) {
+    const projects = q.collections?.length ? q.collections.map(collectionKey) : undefined;
+    const base = baseFilter();
+    const filter = projects ? { ...base, must: [...(base.must ?? []), { key: 'project', match: { any: projects } }] } : base;
+    const r = await explore(d, { topic: q.topic, from: q.from, filter, projects, limit: Math.max(1, Math.min(20, q.limit ?? 10)) });
+    const [found, related] = await Promise.all([items(d, r.found.slice(0, 8)), items(d, r.related.map((x) => x.id))]);
+    const how = new Map(r.related.map((x) => [x.id, x]));
+    return { found, related: related.map((it) => ({ ...it, score: how.get(it.id).score, via: how.get(it.id).via })), concepts: r.concepts.map((c) => c.name) };
+}
+/** One item whole, or null. */
+export async function memoryOpen(d, id) {
+    const got = await d.client.retrieve([pointId(id, 0)]);
+    const p = got[0]?.payload ?? null;
+    if (p) {
+        d.used?.([id], 'opened');
+        d.record?.({ ev: 'open', item: id });
+    }
+    return p;
+}

package/dist/core/redact.d.ts

@@ -0,0 +1,2 @@
+/** Scrub one string. Never throws; returns `''` for a non-string. */
+export declare function redactString(input: unknown): string;

package/dist/core/redact.js

@@ -0,0 +1,48 @@
+/**
+ * Secrets are scrubbed from what memory sends to a language model and from what consolidation keeps: provider keys,
+ * tokens, credentials in URLs, private keys, e-mail local parts, JWTs, `key=value` secrets; home directories become `~`;
+ * a long high-entropy run last. The records you add are kept as you wrote them.
+ */
+const SECRET_PATTERNS = [
+    [/\bsk-[A-Za-z0-9_-]{16,}/g, '<redacted:sk>'],
+    [/\bsk_(?:live|test)_[A-Za-z0-9]{16,}/g, '<redacted:stripe>'],
+    [/\bpk_(?:live|test)_[A-Za-z0-9]{16,}/g, '<redacted:stripe-pub>'],
+    [/\bgithub_pat_[A-Za-z0-9_]{20,}/g, '<redacted:github>'],
+    [/\bgh[porsu]_[A-Za-z0-9]{20,}/g, '<redacted:github>'],
+    [/\bxox[abprs]-[A-Za-z0-9-]{10,}/g, '<redacted:slack>'],
+    [/\bAKIA[0-9A-Z]{16}\b/g, '<redacted:aws>'],
+    [/\bAIza[A-Za-z0-9_-]{20,}/g, '<redacted:google>'],
+    [/\b([a-z][a-z0-9+.-]*:\/\/)([^\s/:@]*):([^\s/@]+)@/gi, '$1$2:<redacted>@'],
+    [/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, 'https://hooks.slack.com/services/<redacted>'],
+    [/https:\/\/(?:canary\.|ptb\.)?discord(?:app)?\.com\/api\/webhooks\/[0-9]+\/[A-Za-z0-9_-]+/g, '<redacted:discord-webhook>'],
+    [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, '<redacted:private-key>'],
+    [/\b[A-Za-z0-9._%+-]+@([A-Za-z0-9.-]+\.[A-Za-z]{2,})\b/g, '<redacted:email>@$1'],
+    [/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g, '<redacted:jwt>'],
+    [/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{12,}/gi, '$1 <redacted>'],
+    [/([?&](?:token|key|api_key|apikey|secret|password|access_token)=)[^&\s"']+/gi, '$1<redacted>'],
+    [/\b(api[_-]?key|token|secret|password|passwd|authorization)\b(\s*[=:]\s*)[^\s&"']+/gi, '$1$2<redacted>'],
+];
+const HOME_DIR_PATTERNS = [
+    [/\/Users\/[^/\s"')\]]+/g, '~'],
+    [/\/home\/[^/\s"')\]]+/g, '~'],
+    [/[A-Za-z]:\\Users\\[^\\\s"')\]]+/gi, '~'],
+];
+/** Conservative on purpose (48+): digests and minified names are shorter. */
+const HIGH_ENTROPY = /\b[A-Za-z0-9_-]{48,}\b/g;
+/** Scrub one string. Never throws; returns `''` for a non-string. */
+export function redactString(input) {
+    if (typeof input !== 'string' || input.length === 0)
+        return '';
+    let out = input;
+    try {
+        for (const [re, replacement] of SECRET_PATTERNS)
+            out = out.replace(re, replacement);
+        for (const [re, replacement] of HOME_DIR_PATTERNS)
+            out = out.replace(re, replacement);
+        out = out.replace(HIGH_ENTROPY, '<redacted:entropy>');
+    }
+    catch {
+        /* a pathological input must not break the caller */
+    }
+    return out;
+}

package/dist/core/reranker.d.ts

@@ -0,0 +1,26 @@
+export declare const MEMORY_RERANKER: {
+    readonly model: "bge-reranker-v2-m3";
+};
+/** How many of the best candidates it reads (20 lost 4.6 points of LoCoMo multi-hop against 30). */
+export declare const RERANK_TOP = 30;
+/** What it reads of each: the title and summary, cut here. */
+export declare const RERANK_CHARS = 1500;
+export interface RerankRoute {
+    /** The API's base, e.g. `http://localhost:8081/v1` (`/rerank` is added). */
+    url: string;
+    apiKey?: string;
+    /** Default `bge-reranker-v2-m3`. */
+    model?: string;
+    gzip?: boolean;
+}
+/** The route the environment names: MEMTRAILS_RERANK_URL, MEMTRAILS_RERANK_KEY, MEMTRAILS_RERANK_MODEL; null without one. */
+export declare function resolveRerankRoute(env?: NodeJS.ProcessEnv): RerankRoute | null;
+export interface RerankerDeps {
+    route?: () => RerankRoute | null;
+    fetchImpl?: typeof fetch;
+}
+/** Each document's relevance to the query, in the documents' order — or null (no route, refused, failed, late). */
+export declare function remoteReranker(d?: RerankerDeps): (query: string, documents: string[], timeoutMs: number) => Promise<number[] | null>;
+/** The candidates in the reranker's order, fused with their own (by rank, k = 10: 0.7 to the reranker, 0.3 to the
+ *  search's own order — the measured mix); null scores leave the order as it was. */
+export declare function rerankedOrder<T>(items: T[], scores: number[] | null): T[];

package/dist/core/reranker.js

@@ -0,0 +1,67 @@
+/**
+ * An optional cross-encoder reranker on a server, for search: any service that speaks the common rerank API
+ * (`POST {url}/rerank`, `{ model, query, documents }` → `results[{ index, relevance_score }]`, as text-embeddings-
+ * inference, Infinity, Jina and Cohere do). Measured with BAAI/bge-reranker-v2-m3 over the best 30 (docs/BENCHMARKS.md):
+ * on little text it matched word by word (words.ts), which needs no server — so it is off unless you give it one.
+ *
+ * Optional by design: no route, a refusal, an error or a late answer — and the search keeps its own order.
+ */
+import { promisify } from 'node:util';
+import { gzip } from 'node:zlib';
+export const MEMORY_RERANKER = { model: 'bge-reranker-v2-m3' };
+/** How many of the best candidates it reads (20 lost 4.6 points of LoCoMo multi-hop against 30). */
+export const RERANK_TOP = 30;
+/** What it reads of each: the title and summary, cut here. */
+export const RERANK_CHARS = 1500;
+/** The route the environment names: MEMTRAILS_RERANK_URL, MEMTRAILS_RERANK_KEY, MEMTRAILS_RERANK_MODEL; null without one. */
+export function resolveRerankRoute(env = process.env) {
+    const url = env.MEMTRAILS_RERANK_URL?.trim();
+    if (!url)
+        return null;
+    return { url, apiKey: env.MEMTRAILS_RERANK_KEY?.trim() || undefined, model: env.MEMTRAILS_RERANK_MODEL?.trim() || undefined };
+}
+const compress = promisify(gzip);
+/** Each document's relevance to the query, in the documents' order — or null (no route, refused, failed, late). */
+export function remoteReranker(d = {}) {
+    const route = d.route ?? (() => resolveRerankRoute());
+    return async (query, documents, timeoutMs) => {
+        const r = route();
+        if (!r || !documents.length || timeoutMs < 100)
+            return null;
+        try {
+            const json = Buffer.from(JSON.stringify({ model: r.model ?? MEMORY_RERANKER.model, query, documents: documents.map((t) => t.slice(0, RERANK_CHARS) || ' ') }));
+            const body = r.gzip ? await compress(json) : json;
+            const res = await (d.fetchImpl ?? fetch)(`${r.url.replace(/\/+$/, '').replace(/\/rerank$/, '')}/rerank`, {
+                method: 'POST',
+                signal: AbortSignal.timeout(timeoutMs),
+                headers: { 'content-type': 'application/json', ...(r.gzip ? { 'content-encoding': 'gzip' } : {}), ...(r.apiKey ? { authorization: `Bearer ${r.apiKey}` } : {}) },
+                body: new Uint8Array(body),
+            });
+            if (!res.ok) {
+                await res.text().catch(() => '');
+                return null;
+            }
+            const reply = (await res.json());
+            const scores = new Array(documents.length).fill(Number.NaN);
+            for (const x of reply.results ?? []) {
+                if (typeof x.index === 'number' && x.index >= 0 && x.index < documents.length && typeof x.relevance_score === 'number')
+                    scores[x.index] = x.relevance_score;
+            }
+            return scores.some(Number.isNaN) ? null : scores;
+        }
+        catch {
+            return null;
+        }
+    };
+}
+/** The candidates in the reranker's order, fused with their own (by rank, k = 10: 0.7 to the reranker, 0.3 to the
+ *  search's own order — the measured mix); null scores leave the order as it was. */
+export function rerankedOrder(items, scores) {
+    if (!scores || scores.length !== items.length)
+        return items;
+    const byScore = items.map((_, i) => i).sort((a, b) => scores[b] - scores[a] || a - b);
+    const fused = new Map();
+    byScore.forEach((i, r) => fused.set(i, (fused.get(i) ?? 0) + 0.7 / (10 + r + 1)));
+    items.forEach((_, i) => fused.set(i, (fused.get(i) ?? 0) + 0.3 / (10 + i + 1)));
+    return items.map((_, i) => i).sort((a, b) => fused.get(b) - fused.get(a) || a - b).map((i) => items[i]);
+}

package/dist/core/room.d.ts

@@ -0,0 +1,20 @@
+/**
+ * A small room filled with sentences, not memories. An agent reads a few thousand characters of what memory found; a
+ * memory's first 500 of them are rarely where its answer is. Each found memory's text is cut into sentences (at any
+ * script's end marks, no word list), each sentence scored word by word against the question, and the room filled with
+ * the best sentences, a memory's place in the search counting too (docs/RESEARCH.md: held out, the answer's words in
+ * 6,000 characters 66.0 against 57.2 for Mem0's room on raw memory).
+ *
+ * Sentences are kept packed beside the memories' own words (words.ts WordStore), by their text, so each is encoded once.
+ */
+import { type WordEncoder, type WordStore } from './words.js';
+/** [start, end) of each sentence: after `.`, `!`, `?`, their full-width forms, `。` or a line break, then a space. */
+export declare function sentenceSpans(text: string): Array<[number, number]>;
+export interface RoomItem {
+    id: string;
+    head: string;
+    text: string;
+    ts: number;
+}
+/** The room: the best sentences of the first `top` memories, fitted in `chars`, each memory's head once, oldest first. */
+export declare function sentenceRoom(query: string, items: RoomItem[], encode: WordEncoder, store: WordStore, chars: number, top?: number): Promise<string>;

package/dist/core/room.js

@@ -0,0 +1,63 @@
+/**
+ * A small room filled with sentences, not memories. An agent reads a few thousand characters of what memory found; a
+ * memory's first 500 of them are rarely where its answer is. Each found memory's text is cut into sentences (at any
+ * script's end marks, no word list), each sentence scored word by word against the question, and the room filled with
+ * the best sentences, a memory's place in the search counting too (docs/RESEARCH.md: held out, the answer's words in
+ * 6,000 characters 66.0 against 57.2 for Mem0's room on raw memory).
+ *
+ * Sentences are kept packed beside the memories' own words (words.ts WordStore), by their text, so each is encoded once.
+ */
+import { maxSim, packWords } from './words.js';
+/** Sentences shorter than this say too little to be worth a place. */
+const MIN_SENTENCE = 8;
+/** A sentence longer than this is cut into pieces of it. */
+const MAX_SENTENCE = 400;
+/** A memory's date and kind, once, before its sentences. */
+const HEAD_CHARS = 20;
+/** [start, end) of each sentence: after `.`, `!`, `?`, their full-width forms, `。` or a line break, then a space. */
+export function sentenceSpans(text) {
+    const out = [];
+    const push = (a, b) => {
+        while (a < b && /\s/.test(text[a]))
+            a++;
+        while (b > a && /\s/.test(text[b - 1]))
+            b--;
+        for (let s = a; s < b; s += MAX_SENTENCE)
+            out.push([s, Math.min(b, s + MAX_SENTENCE)]);
+    };
+    const end = /[.!?。！？](?=\s|$)|\n/gu;
+    let from = 0;
+    for (let m = end.exec(text); m; m = end.exec(text)) {
+        push(from, m.index + m[0].length);
+        from = m.index + m[0].length;
+    }
+    push(from, text.length);
+    return out.filter(([a, b]) => b - a >= MIN_SENTENCE);
+}
+/** The room: the best sentences of the first `top` memories, fitted in `chars`, each memory's head once, oldest first. */
+export async function sentenceRoom(query, items, encode, store, chars, top = 30) {
+    const listed = items.slice(0, top).map((it, rank) => ({ it, rank, spans: sentenceSpans(it.text) }));
+    const texts = [...new Set(listed.flatMap((l) => l.spans.map(([a, b]) => l.it.text.slice(a, b))))];
+    const missing = texts.filter((t) => !store.get(t));
+    if (missing.length) {
+        const encoded = await encode(missing, 'document');
+        await store.put(missing.map((t, i) => [t, packWords(encoded[i])]));
+    }
+    const [q] = await encode([query], 'query');
+    const candidates = listed.flatMap((l) => l.spans.map(([a, b]) => ({
+        l, a, b, score: maxSim(q, store.get(l.it.text.slice(a, b))) * (1 + 2 / (1 + l.rank / 10)),
+    }))).sort((x, y) => y.score - x.score);
+    const picked = new Map();
+    let used = 0;
+    for (const c of candidates) {
+        const extra = c.b - c.a + 3 + (picked.has(c.l.it) ? 0 : HEAD_CHARS + c.l.it.head.length);
+        if (used + extra > chars)
+            continue;
+        picked.set(c.l.it, [...(picked.get(c.l.it) ?? []), [c.a, c.b]]);
+        used += extra;
+    }
+    return [...picked.entries()]
+        .sort(([x], [y]) => x.ts - y.ts)
+        .map(([it, spans]) => `${it.head} ${spans.sort((x, y) => x[0] - y[0]).map(([a, b]) => it.text.slice(a, b)).join(' … ')}`)
+        .join('\n');
+}

package/dist/core/sleep.d.ts

@@ -0,0 +1,44 @@
+import { type ConsolidateDeps } from './consolidate.js';
+import type { MemoryItem, UsageLog } from './store.js';
+export declare const PAIRS_PER_SLEEP = 1600;
+export declare const ARCHIVE_BELOW = 0.02;
+export declare const ARCHIVE_UNUSED_MS: number;
+export declare function priority(it: Pick<MemoryItem, 'kind' | 'importance' | 'ts'>, use: {
+    shown: number;
+    opened: number;
+}, now: number): number;
+export interface SleepDeps extends ConsolidateDeps {
+    usage: UsageLog;
+    statePath: string;
+    /** Where the last sleep is kept instead of `statePath` (a server running many memories: a row per memory). */
+    sleepState?: {
+        read(): Promise<{
+            last?: number;
+        } | null>;
+        write(state: {
+            last: number;
+        } & SleepReport): Promise<void>;
+    };
+    /** A backfill's night: fresh items looked at this many at once (1), relate asks in flight (RELATE_IN_FLIGHT), and
+     *  pairs a night asks about (PAIRS_PER_SLEEP). */
+    scanAtOnce?: number;
+    relateInFlight?: number;
+    pairsPerSleep?: number;
+    /** Answers applied at once (1, in order). */
+    applyAtOnce?: number;
+}
+export interface SleepReport {
+    looked: number;
+    merged: number;
+    superseded: number;
+    linked: number;
+    archived: number;
+    profiled: number;
+    unprofiled: number;
+}
+export declare class Sleep {
+    private readonly d;
+    constructor(d: SleepDeps);
+    private lastSleep;
+    run(): Promise<SleepReport>;
+}

package/dist/core/sleep.js

@@ -0,0 +1,199 @@
+/**
+ * Sleep — the daily pass that keeps long-term memory organised (LightMem's sleep-time update; MemoryOS's heat):
+ *
+ *   1. RELATE what was learned since the last sleep to every near neighbour (cosine >= 0.80, any knowledge kind, the
+ *      same scope) — not only the nearest one consolidation looked at: from 0.93 they merge; below, the fast model
+ *      merges the two, says one replaces the other (superseded), links them (the same subject, different facts) or
+ *      says they differ. Measured: 0.80–0.90 held 62% duplicates, outdated items and
+ *      contradictions, 38% that belong together — the 0.90 floor this pass had left them all;
+ *   2. PRIORITY for every active item = importance × recency (a half-life by kind: preferences, architecture,
+ *      references and notes do not fade; facts and pitfalls fade in a year; episodes in three months) × use (how
+ *      often recall showed it and an agent opened it — MemRL-style utility);
+ *   3. ARCHIVE what stays low and unused for half a year: out of the index, kept in the log. Nothing is forgotten
+ *      outright: everything is remembered, efficiently;
+ *   4. PROFILES (profiles.ts): for each subject enough items share, what they say gathered into one dated list —
+ *      written again when those items changed, archived when the subject no longer qualifies.
+ */
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { Consolidator, RELATED, relateBatches, SAME } from './consolidate.js';
+import { docText } from './embedder.js';
+import { pointId } from './qdrant-client.js';
+import { writeProfiles } from './profiles.js';
+import { writeLtm } from './writer.js';
+const DAY = 24 * 60 * 60_000;
+/** A fresh item's neighbours looked at, and the pairs a night asks about (a large first sleep goes on the next nights). */
+const NEIGHBOURS = 8;
+export const PAIRS_PER_SLEEP = 1600;
+const RELATE_IN_FLIGHT = 4;
+export const ARCHIVE_BELOW = 0.02;
+export const ARCHIVE_UNUSED_MS = 180 * DAY;
+const HALF_LIFE = {
+    episode: 90 * DAY, fact: 365 * DAY, pitfall: 365 * DAY,
+    decision: 540 * DAY, procedure: 540 * DAY, snippet: 540 * DAY, template: 540 * DAY,
+};
+export function priority(it, use, now) {
+    const half = HALF_LIFE[it.kind];
+    const recency = half ? 0.5 ** (Math.max(0, now - it.ts) / half) : 1;
+    return (it.importance / 5) * recency * (1 + Math.log1p(use.opened) + 0.3 * Math.log1p(use.shown));
+}
+export class Sleep {
+    d;
+    constructor(d) {
+        this.d = d;
+    }
+    async lastSleep() {
+        if (this.d.sleepState)
+            return (await this.d.sleepState.read())?.last ?? 0;
+        try {
+            return JSON.parse(readFileSync(this.d.statePath, 'utf8')).last ?? 0;
+        }
+        catch {
+            return 0;
+        }
+    }
+    async run() {
+        const now = (this.d.now ?? Date.now)();
+        const since = await this.lastSleep();
+        const r = { looked: 0, merged: 0, superseded: 0, linked: 0, archived: 0, profiled: 0, unprofiled: 0 };
+        // 1. What was learned since the last sleep, against its near neighbours, oldest first.
+        const worker = new Consolidator(this.d);
+        // Observations are add-only (consolidate.ts observe): two events are never one, and an older one stays, dated. A
+        // profile is written again from its items (4), never related to anything.
+        const knowledge = (m) => m.status === 'active' && m.kind !== 'episode' && m.kind !== 'note' && m.kind !== 'observation' && m.kind !== 'profile';
+        const fresh = [...this.d.ltm.entries()].filter(([, m]) => knowledge(m) && m.ts >= since).sort((a, b) => a[1].ts - b[1].ts).map(([uid]) => uid);
+        const pairs = [];
+        const seen = new Set();
+        let resumeAt = null;
+        const cap = this.d.pairsPerSleep ?? PAIRS_PER_SLEEP;
+        const look = async (it) => {
+            // The key vector the index already holds; embedded again only when it is not there.
+            const held = (await this.d.client.retrieve([pointId(it.uid, 0)], true))[0]?.vector;
+            const v = held ?? Array.from((await this.d.embed([docText(it.title, it.summary)]))[0]);
+            const must = [{ key: 'tier', match: { value: 'ltm' } }, { key: 'status', match: { value: 'active' } }];
+            must.push(it.scope === 'project' ? { key: 'project', match: { value: it.project } } : { key: 'scope', match: { any: ['global', 'person'] } });
+            const filter = { must, must_not: [{ key: 'kind', match: { any: ['episode', 'note', 'observation', 'profile'] } }] };
+            return (await this.d.client.neighbors(Array.from(v), filter, NEIGHBOURS)).map((n) => ({ uid: it.uid, item: n.item, score: n.score }));
+        };
+        // Oldest first, a few at once: each group's neighbours become pairs in order, and a full night stops at the next
+        // group, which the next night starts from.
+        const atOnce = Math.max(1, this.d.scanAtOnce ?? 1);
+        for (let i = 0; i < fresh.length; i += atOnce) {
+            if (this.d.aborted())
+                return r;
+            const group = fresh.slice(i, i + atOnce).map((uid) => this.d.ltm.get(uid)).filter((it) => !!it && it.status === 'active');
+            if (!group.length)
+                continue;
+            if (pairs.length >= cap) {
+                resumeAt = group[0].ts;
+                break;
+            }
+            r.looked += group.length;
+            for (const [k, near] of (await Promise.all(group.map(look))).entries()) {
+                const it = group[k];
+                for (const n of near) {
+                    const known = n.item !== it.uid && n.score >= RELATED ? this.d.ltm.get(n.item) : null;
+                    const key = [it.uid, n.item].sort().join(' ');
+                    if (!known || !knowledge(known) || seen.has(key))
+                        continue;
+                    seen.add(key);
+                    // The newer of the two is NEW, the older KNOWN: a contradiction retires the older.
+                    pairs.push(it.ts >= known.ts ? { it, known, score: n.score } : { it: known, known: it, score: n.score });
+                }
+            }
+        }
+        // The fast model reads the pairs below 0.93, several asks at once; the answers are applied in order, each to the
+        // items as they are by then (one an earlier answer retired is left alone).
+        const asked = pairs.filter((p) => p.score < SAME);
+        const answers = new Map();
+        const batches = relateBatches(asked);
+        // As many asks in flight as allowed, the next one starting as soon as any is answered; one the model did not answer
+        // stops the asking, and the next night starts from its pairs.
+        const inFlight = Math.max(1, this.d.relateInFlight ?? RELATE_IN_FLIGHT);
+        let next = 0;
+        let unanswered = false;
+        await Promise.all(Array.from({ length: Math.min(inFlight, batches.length) }, async () => {
+            while (!unanswered && next < batches.length && !this.d.aborted()) {
+                const b = batches[next++];
+                const got = await worker.relate(b);
+                if (!got) {
+                    unanswered = true;
+                    resumeAt = Math.min(resumeAt ?? Infinity, ...b.map((p) => p.it.ts));
+                    return;
+                }
+                b.forEach((p, n) => { const d = got.get(`p${n + 1}`); if (d)
+                    answers.set(p, d); });
+            }
+        }));
+        // The answers applied — in order, one at a time, unless a backfill applies several at once — each to the two items as
+        // they are by then, under their lock (one an earlier answer retired is left alone).
+        const apply = async (p) => worker.locked([p.it.uid, p.known.uid], async () => {
+            const it = this.d.ltm.get(p.it.uid);
+            const known = this.d.ltm.get(p.known.uid);
+            if (!it || !known || it.status !== 'active' || known.status !== 'active')
+                return;
+            const d = p.score >= SAME ? { relation: 'same', title: '', summary: '', body: '' } : answers.get(p);
+            if (!d || d.relation === 'different')
+                return;
+            if (d.relation === 'related') {
+                await worker.link(it, known);
+                r.linked++;
+                return;
+            }
+            if (d.relation === 'contradicts') {
+                await worker.retire(known, it.uid);
+                r.superseded++;
+                return;
+            }
+            await worker.merge(known, it, d.title && d.summary ? { title: d.title, summary: d.summary, body: d.body || known.body } : null);
+            await worker.retire(it, known.uid);
+            r.merged++;
+        });
+        let at = 0;
+        await Promise.all(Array.from({ length: Math.max(1, this.d.applyAtOnce ?? 1) }, async () => {
+            while (at < pairs.length && !this.d.aborted())
+                await apply(pairs[at++]);
+        }));
+        if (this.d.aborted())
+            return r;
+        // 2–3. Priority, and the archive.
+        const retire = [];
+        for (const [uid, m] of this.d.ltm.entries()) {
+            if (m.status !== 'active' || m.kind === 'note' || !HALF_LIFE[m.kind])
+                continue;
+            const use = this.d.usage.of(uid);
+            if (priority(m, use, now) >= ARCHIVE_BELOW || now - Math.max(m.ts, use.lastUsed) < ARCHIVE_UNUSED_MS)
+                continue;
+            const it = this.d.ltm.get(uid);
+            if (it)
+                retire.push({ ...it, status: 'archived' });
+        }
+        for (let i = 0; i < retire.length && !this.d.aborted(); i += 50) {
+            const part = retire.slice(i, i + 50);
+            await writeLtm(this.d, part);
+            this.d.ltm.put(part);
+            r.archived += part.length;
+        }
+        // 4. Profiles of what is known now; one the fast model did not answer waits for the next night.
+        if (this.d.aborted())
+            return r;
+        const p = await writeProfiles(this.d);
+        r.profiled = p.profiled;
+        r.unprofiled = p.unprofiled;
+        if (p.paused)
+            this.d.log(`memory sleep: profiles wait for the next night (${p.paused})`);
+        this.d.ltm.compact();
+        // A night that stopped early (the pair cap, the fast model away) starts the next one where it stopped.
+        if (this.d.sleepState) {
+            await this.d.ltm.flush();
+            await this.d.sleepState.write({ last: resumeAt ?? now, ...r });
+            return r;
+        }
+        try {
+            mkdirSync(dirname(this.d.statePath), { recursive: true });
+            writeFileSync(this.d.statePath, JSON.stringify({ last: resumeAt ?? now, ...r }), { mode: 0o600 });
+        }
+        catch { /* the next sleep looks at a little more */ }
+        return r;
+    }
+}