memtrails 0.0.0-stage → 0.2.0

package/dist/core/graph.js

@@ -0,0 +1,208 @@
+/**
+ * The concept graph — memory's associative index (HippoRAG's hippocampal index, 2405.14831), for an agent EXPLORING a
+ * topic or what it found, rather than for a search's top few (as a ranker feature it added ~1 point; for reaching
+ * what search does not, it is where the gain is).
+ *
+ * Nodes are long-term items and the concepts they are about (consolidation writes 3–12 per item: files, services,
+ * commands, named things), a concept being its spelling key (text.ts conceptKey). Edges:
+ *   - item ↔ concept, weighted by how specific the concept is (1 / log(2 + the items that share it));
+ *   - item ↔ item: `related` (the same subject, different facts — consolidation and sleep link them) and each seed's
+ *     nearest items by key vector, two hops out.
+ * Concepts are NOT embedded: matched and linked by embedding instead of spelling, the walk found the same needed items
+ * (195 either way) and ~0.4 more items a topic, for a point and six embeddings an item more — not worth a laptop's RAM
+ * and CPU (measured 2026-10-25). It lives in Qdrant — the concept keys on items' key points, indexed — so it grows with
+ * memory without a copy in RAM: `explore` reads the neighbourhood of its seeds and walks it with Personalized PageRank
+ * (restart 0.5, the eval's), within recall's walls (active, real, the scope asked).
+ */
+import { pointId } from './qdrant-client.js';
+import { queryText } from './embedder.js';
+import { conceptKey, fold, querySparse } from './text.js';
+/** A seed's nearest items by key vector (from cos 0.5: a small memory's "nearest" can be anything), the walk's search
+ *  hits used as seeds, the items read through concepts. */
+const KNN = 10;
+const KNN_NEAR = 0.5;
+const SEEDS = 5;
+const FOUND = 20;
+const POSTINGS = 400;
+/** A `related` link's weight, beside item ↔ concept's 1/log(2 + df). */
+const RELATED = 1.0;
+const RESTART = 0.5;
+const STEPS = 20;
+const strings = (v) => (Array.isArray(v) ? v.filter((x) => typeof x === 'string') : []);
+/** A small weighted undirected graph, walked by Personalized PageRank. */
+class Walk {
+    ids = new Map();
+    adj = [];
+    node(name) {
+        let i = this.ids.get(name);
+        if (i === undefined) {
+            i = this.adj.length;
+            this.ids.set(name, i);
+            this.adj.push(new Map());
+        }
+        return i;
+    }
+    link(a, b, w) {
+        if (a === b || !(w > 0))
+            return;
+        const i = this.node(a);
+        const j = this.node(b);
+        this.adj[i].set(j, Math.max(this.adj[i].get(j) ?? 0, w));
+        this.adj[j].set(i, Math.max(this.adj[j].get(i) ?? 0, w));
+    }
+    /** Each node's share of a walk that starts again at `seeds` (by weight) half the time. */
+    ppr(seeds) {
+        const n = this.adj.length;
+        const e = new Float64Array(n);
+        let total = 0;
+        for (const [name, w] of seeds) {
+            const i = this.ids.get(name);
+            if (i !== undefined) {
+                e[i] += w;
+                total += w;
+            }
+        }
+        if (!total)
+            return new Map();
+        for (let i = 0; i < n; i++)
+            e[i] /= total;
+        const deg = this.adj.map((m) => [...m.values()].reduce((s, w) => s + w, 0));
+        let x = Float64Array.from(e);
+        for (let step = 0; step < STEPS; step++) {
+            const next = new Float64Array(n);
+            for (let i = 0; i < n; i++) {
+                if (!x[i] || !deg[i])
+                    continue;
+                for (const [j, w] of this.adj[i])
+                    next[j] += (1 - RESTART) * x[i] * (w / deg[i]);
+            }
+            for (let i = 0; i < n; i++)
+                next[i] += RESTART * e[i];
+            x = next;
+        }
+        return new Map([...this.ids].map(([name, i]) => [name, x[i]]));
+    }
+}
+/** Walk the graph from a topic and/or items: what is related but not found directly, and the concepts around it. */
+export async function explore(d, q) {
+    const keyOnly = (extra = []) => ({
+        ...q.filter, must: [...(q.filter.must ?? []), { key: 'tier', match: { value: 'ltm' } }, { key: 'part', match: { value: 'key' } }, ...extra],
+    });
+    const shown = (p) => p.tier === 'ltm' && p.status === 'active' && p.synthetic === false && (!q.projects?.length || q.projects.includes(String(p.project)));
+    const seedItems = new Map();
+    const seedConcepts = new Map();
+    let found = [];
+    if (q.topic?.trim()) {
+        const [qv] = await d.embed([queryText(q.topic)]);
+        found = (await d.client.fusedItems({ dense: [Array.from(qv)], sparse: querySparse([q.topic]), filter: q.filter, limit: FOUND })).map((h) => h.item);
+        for (const u of found.slice(0, SEEDS))
+            seedItems.set(u, 1);
+        // The topic's own concepts: the topic named as one, and those of what search found that the topic names.
+        const topic = conceptKey(q.topic);
+        if (topic)
+            seedConcepts.set(topic, 1);
+        const said = fold(q.topic);
+        for (const p of await d.client.retrieve(found.slice(0, SEEDS).map((u) => pointId(u, 0)))) {
+            for (const k of strings(p.payload.concepts))
+                if (k.length > 2 && said.includes(k))
+                    seedConcepts.set(k, 1);
+        }
+    }
+    for (const u of q.from ?? [])
+        seedItems.set(u, 1);
+    if (!seedItems.size && !seedConcepts.size)
+        return { found, related: [], concepts: [] };
+    const g = new Walk();
+    const items = new Map();
+    // Hop 1: the seed items, their concepts and links, and their nearest items by key vector.
+    const seeds = (await d.client.retrieve([...seedItems.keys()].map((u) => pointId(u, 0)), true)).filter((p) => p.vector);
+    for (const p of seeds)
+        items.set(String(p.payload.item), p.payload);
+    const near = await d.client.nearestMany(seeds.map((p) => p.vector), keyOnly(), KNN + 1);
+    const hop1 = new Map();
+    seeds.forEach((p, i) => {
+        for (const n of near[i] ?? []) {
+            if (n.item === p.payload.item || n.score < KNN_NEAR)
+                continue;
+            if (!items.has(n.item))
+                items.set(n.item, n.payload);
+            if (!seedItems.has(n.item))
+                hop1.set(n.item, Math.max(hop1.get(n.item) ?? 0, n.score));
+            g.link(`i:${String(p.payload.item)}`, `i:${n.item}`, n.score);
+        }
+    });
+    // A second hop from the nearest of those (measured: the multi-hop item walk found more than one hop).
+    const next = (await d.client.retrieve([...hop1].sort((a, b) => b[1] - a[1]).slice(0, KNN).map(([u]) => pointId(u, 0)), true)).filter((p) => p.vector);
+    const near2 = await d.client.nearestMany(next.map((p) => p.vector), keyOnly(), KNN + 1);
+    next.forEach((p, i) => {
+        for (const n of near2[i] ?? []) {
+            if (n.item === p.payload.item || n.score < KNN_NEAR)
+                continue;
+            if (!items.has(n.item))
+                items.set(n.item, n.payload);
+            g.link(`i:${String(p.payload.item)}`, `i:${n.item}`, n.score);
+        }
+    });
+    // The concepts of what was reached and of the topic.
+    const concepts = new Set(seedConcepts.keys());
+    for (const p of items.values())
+        for (const k of strings(p.concepts))
+            concepts.add(k);
+    // Hop 2: the items holding those concepts — a concept many of them share links each one less — and the items
+    // linked to anything reached.
+    const asked = [...concepts].slice(0, 300);
+    if (asked.length) {
+        for (const p of await d.client.scroll(keyOnly([{ key: 'concepts', match: { any: asked } }]), POSTINGS)) {
+            if (!items.has(String(p.payload.item)))
+                items.set(String(p.payload.item), p.payload);
+        }
+    }
+    const linkedOut = [...new Set([...items.values()].flatMap((p) => strings(p.related)))].filter((u) => !items.has(u));
+    for (const p of await d.client.retrieve(linkedOut.map((u) => pointId(u, 0))))
+        items.set(String(p.payload.item), p.payload);
+    const df = new Map();
+    for (const p of items.values())
+        for (const k of strings(p.concepts))
+            if (concepts.has(k))
+                df.set(k, (df.get(k) ?? 0) + 1);
+    for (const [u, p] of items) {
+        if (!shown(p))
+            continue;
+        for (const k of strings(p.concepts))
+            if (concepts.has(k))
+                g.link(`i:${u}`, `c:${k}`, 1 / Math.log(2 + (df.get(k) ?? 1)));
+        for (const r of strings(p.related))
+            if (items.has(r) && shown(items.get(r)))
+                g.link(`i:${u}`, `i:${r}`, RELATED);
+    }
+    // The walk starts again half at the seed items and half at the topic's concepts (either alone when only one is).
+    const start = new Map();
+    for (const u of seedItems.keys())
+        start.set(`i:${u}`, (seedConcepts.size ? 0.5 : 1) / seedItems.size);
+    const cTotal = [...seedConcepts.values()].reduce((s, w) => s + w, 0);
+    for (const [k, w] of seedConcepts)
+        start.set(`c:${k}`, (seedItems.size ? 0.5 : 1) * (w / cTotal));
+    const x = g.ppr(start);
+    const skip = new Set([...found, ...seedItems.keys()]);
+    const seedTitles = new Map([...seedItems.keys()].map((u) => [u, String(items.get(u)?.title ?? u)]));
+    const seedKeys = new Set([...seedConcepts.keys(), ...[...seedItems.keys()].flatMap((u) => strings(items.get(u)?.concepts))]);
+    const related = [...items].filter(([u, p]) => !skip.has(u) && shown(p) && (x.get(`i:${u}`) ?? 0) > 0)
+        .map(([u, p]) => ({ id: u, score: x.get(`i:${u}`), via: via(p) }))
+        .sort((a, b) => b.score - a.score).slice(0, q.limit ?? 10);
+    /** How an item was reached: the concepts it shares with where the walk started, a link, or nearness to a seed. */
+    function via(p) {
+        const shared = strings(p.concepts).filter((k) => seedKeys.has(k)).slice(0, 3).map((k) => `"${k}"`);
+        if (shared.length)
+            return `shares ${shared.join(', ')}`;
+        const linked = strings(p.related).find((r) => seedTitles.has(r));
+        if (linked)
+            return `linked to "${seedTitles.get(linked)}"`;
+        const through = strings(p.concepts).find((k) => (x.get(`c:${k}`) ?? 0) > 0);
+        if (through)
+            return `through "${through}"`;
+        return 'near what was found';
+    }
+    const conceptScores = [...x].filter(([n, s]) => n.startsWith('c:') && s > 0).sort((a, b) => b[1] - a[1]).slice(0, 12)
+        .map(([n, score]) => ({ name: n.slice(2), score }));
+    return { found, related, concepts: conceptScores };
+}

package/dist/core/local-embedder.d.ts

@@ -0,0 +1,40 @@
+import { type WordEncoder } from './words.js';
+/** The slice of onnxruntime-node used here, typed locally so the build never needs the binding. */
+interface OrtTensor {
+    dims: readonly number[];
+    data: Float32Array;
+}
+interface OrtSession {
+    outputNames: readonly string[];
+    run(feeds: Record<string, unknown>): Promise<Record<string, OrtTensor>>;
+    release?(): Promise<void>;
+}
+export interface OrtModule {
+    InferenceSession: {
+        create(path: string, opts?: Record<string, unknown>): Promise<OrtSession>;
+    };
+    Tensor: new (type: string, data: BigInt64Array, dims: number[]) => unknown;
+}
+export interface Tokens {
+    encode(text: string): {
+        ids?: number[];
+        input_ids?: number[];
+    } | number[];
+}
+export interface LocalEmbedderOptions {
+    threads?: number;
+    batch?: number;
+    /** Tokens read of a text (the model's own limit is longer; memory's keys are short). */
+    maxTokens?: number;
+    /** Stand-ins for the two optional packages (tests). */
+    ort?: OrtModule;
+    tokenizer?: Tokens;
+}
+export interface LocalEmbedder {
+    /** Already prompted texts → unit vectors, in order: RecallDeps.embed. */
+    embed(texts: string[], timeoutMs?: number): Promise<Float32Array[]>;
+    words: WordEncoder;
+    close(): Promise<void>;
+}
+export declare function loadLocalEmbedder(dir: string, opts?: LocalEmbedderOptions): Promise<LocalEmbedder>;
+export {};

package/dist/core/local-embedder.js

@@ -0,0 +1,98 @@
+/**
+ * EmbeddingGemma on this machine, for a memory that needs no server: the same quantised model and the same prompts as
+ * the remote workers, so vectors and an index move between the two. One run gives both what the search keeps (the
+ * memory's vector) and what word by word keeps (a vector per word, the step before it is averaged: `last_hidden_state`).
+ *
+ * The model and its tokenizer are a directory you give it (onnx/model_q4.onnx, tokenizer.json, tokenizer_config.json).
+ * `onnxruntime-node` and `@huggingface/tokenizers` are optional dependencies, imported only here. One inference at a
+ * time; it works on the calling process, so a host with its own work should run it in a worker.
+ */
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { docText, queryText } from './embedder.js';
+import { WORD_LIMIT } from './words.js';
+const BOS_EOS = 2;
+function unit(v) {
+    let s = 0;
+    for (let i = 0; i < v.length; i++)
+        s += v[i] * v[i];
+    const n = Math.sqrt(s) || 1;
+    return v.map((x) => x / n);
+}
+async function importOptional(name) {
+    try {
+        const mod = (await import(name));
+        return mod.default ?? mod;
+    }
+    catch (e) {
+        throw new Error(`The local embedder needs the optional package ${name}: ${e.message}`);
+    }
+}
+export async function loadLocalEmbedder(dir, opts = {}) {
+    const ort = opts.ort ?? await importOptional('onnxruntime-node');
+    const tokenizer = opts.tokenizer ?? await (async () => {
+        const { Tokenizer } = await importOptional('@huggingface/tokenizers');
+        const json = async (f) => JSON.parse(await readFile(join(dir, f), 'utf8'));
+        return new Tokenizer(await json('tokenizer.json'), await json('tokenizer_config.json'));
+    })();
+    const session = await ort.InferenceSession.create(join(dir, 'onnx', 'model_q4.onnx'), {
+        executionProviders: ['cpu'], graphOptimizationLevel: 'all', executionMode: 'sequential', intraOpNumThreads: Math.max(1, opts.threads ?? 1),
+    });
+    const batch = Math.max(1, opts.batch ?? 8);
+    const maxTokens = opts.maxTokens ?? 512;
+    const tokens = (text, limit) => {
+        const e = tokenizer.encode(text);
+        const ids = Array.isArray(e) ? e : (e.ids ?? e.input_ids ?? []);
+        return { ids: ids.slice(0, limit), cut: ids.length > limit };
+    };
+    const run = async (encoded) => {
+        const width = Math.max(1, ...encoded.map((e) => e.length));
+        const ids = new BigInt64Array(encoded.length * width);
+        const mask = new BigInt64Array(encoded.length * width);
+        encoded.forEach((e, b) => e.forEach((id, i) => { ids[b * width + i] = BigInt(id); mask[b * width + i] = 1n; }));
+        const dims = [encoded.length, width];
+        return session.run({ input_ids: new ort.Tensor('int64', ids, dims), attention_mask: new ort.Tensor('int64', mask, dims) });
+    };
+    // One inference at a time, like lengths in one run: each text is padded to the run's longest.
+    let queue = Promise.resolve();
+    const inOrder = (job) => {
+        const next = queue.then(job, job);
+        queue = next.catch(() => undefined);
+        return next;
+    };
+    const batches = async (items, read) => {
+        const order = items.map((_, i) => i).sort((a, b) => items[a].ids.length - items[b].ids.length);
+        const results = new Array(items.length);
+        for (let i = 0; i < order.length; i += batch) {
+            const part = order.slice(i, i + batch);
+            const out = await run(part.map((j) => items[j].ids));
+            part.forEach((j, b) => { results[j] = read(out, b, items[j]); });
+        }
+        return results;
+    };
+    return {
+        embed: (texts) => inOrder(() => batches(texts.map((t) => tokens(t, maxTokens)), (out, b) => {
+            const t = out['sentence_embedding'] ?? out[session.outputNames[session.outputNames.length - 1]];
+            const d = t.dims[t.dims.length - 1];
+            return unit(Float32Array.from(t.data.subarray(b * d, (b + 1) * d)));
+        })),
+        words: (texts, kind) => inOrder(() => {
+            const prompted = texts.map((t) => (kind === 'query' ? queryText(t) : docText('', t)));
+            return batches(prompted.map((t) => tokens(t, WORD_LIMIT + BOS_EOS)), (out, b, it) => {
+                const h = out['last_hidden_state'];
+                if (!h)
+                    throw new Error('This model gives no vector per word (last_hidden_state).');
+                const [, width, dims] = h.dims;
+                // Without its first token, and its last unless the text was cut before it.
+                const words = Math.max(0, it.ids.length - (it.cut ? 1 : BOS_EOS));
+                const data = new Float32Array(words * dims);
+                for (let w = 0; w < words; w++) {
+                    const from = (b * width + w + 1) * dims;
+                    data.set(unit(Float32Array.from(h.data.subarray(from, from + dims))), w * dims);
+                }
+                return { words, dims, data };
+            });
+        }),
+        close: async () => { await queue; await session.release?.(); },
+    };
+}

package/dist/core/model.d.ts

@@ -0,0 +1,16 @@
+export declare const MODEL_REPO = "onnx-community/embeddinggemma-300m-ONNX";
+export declare const MODEL_REVISION = "5090578d9565bb06545b4552f76e6bc2c93e4a66";
+export declare const MODEL_FILES: ReadonlyArray<{
+    rel: string;
+    size: number;
+    sha256: string;
+}>;
+export declare function defaultModelDir(): string;
+export declare function modelReady(dir?: string): boolean;
+/** The model in `dir` (fetched and checked unless it is already there); the directory, ready for `modelDir`. */
+export declare function ensureModel(opts?: {
+    dir?: string;
+    log?: (line: string) => void;
+    fetchImpl?: typeof fetch;
+    signal?: AbortSignal;
+}): Promise<string>;

package/dist/core/model.js

@@ -0,0 +1,74 @@
+/**
+ * The embedding model memory runs on this machine: EmbeddingGemma-300m, 4-bit (onnx-community/embeddinggemma-300m-ONNX
+ * on Hugging Face, one pinned revision), fetched on first use into the cache (paths.ts), each file checked against its
+ * pinned size and SHA-256. 218 MB. The model is Google's, under the Gemma Terms of Use (ai.google.dev/gemma/terms);
+ * memtrails does not ship it.
+ */
+import { createHash } from 'node:crypto';
+import { createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { cacheDir } from './paths.js';
+export const MODEL_REPO = 'onnx-community/embeddinggemma-300m-ONNX';
+export const MODEL_REVISION = '5090578d9565bb06545b4552f76e6bc2c93e4a66';
+export const MODEL_FILES = [
+    { rel: 'onnx/model_q4.onnx', size: 519_322, sha256: 'ad1dfee81a70f7944b9b9d1cc6e48075b832881cf33fab2f2b248be78f3f0043' },
+    { rel: 'onnx/model_q4.onnx_data', size: 196_725_760, sha256: '599962c3143b040de2dd05e5975be3e9091dd067cacc6a8f7186e3203bab9e02' },
+    { rel: 'tokenizer.json', size: 20_323_312, sha256: '4dda02faaf32bc91031dc8c88457ac272b00c1016cc679757d1c441b248b9c47' },
+    { rel: 'tokenizer_config.json', size: 1_156_830, sha256: '3ca953eea6c3c9fcda9cf3df22949ff18b216f7c74bd6459230f3f1013953f3a' },
+];
+export function defaultModelDir() {
+    return join(cacheDir(), `embeddinggemma-300m-${MODEL_REVISION.slice(0, 8)}`);
+}
+const marker = (dir) => join(dir, '.verified');
+const stamp = MODEL_FILES.map((f) => `${f.rel}:${f.size}:${f.sha256}`).join('\n');
+export function modelReady(dir = defaultModelDir()) {
+    try {
+        return readFileSync(marker(dir), 'utf8') === stamp && MODEL_FILES.every((f) => existsSync(join(dir, f.rel)));
+    }
+    catch {
+        return false;
+    }
+}
+/** The model in `dir` (fetched and checked unless it is already there); the directory, ready for `modelDir`. */
+export async function ensureModel(opts = {}) {
+    const dir = opts.dir ?? defaultModelDir();
+    if (modelReady(dir))
+        return dir;
+    opts.log?.(`[memtrails] fetching ${MODEL_REPO} (218 MB, Gemma Terms of Use)`);
+    for (const f of MODEL_FILES) {
+        const path = join(dir, f.rel);
+        mkdirSync(dirname(path), { recursive: true });
+        const part = `${path}.part`;
+        rmSync(part, { force: true });
+        const res = await (opts.fetchImpl ?? fetch)(`https://huggingface.co/${MODEL_REPO}/resolve/${MODEL_REVISION}/${f.rel}`, { signal: opts.signal, redirect: 'follow' });
+        if (!res.ok || !res.body)
+            throw new Error(`HTTP ${res.status} for ${f.rel}`);
+        const hash = createHash('sha256');
+        let size = 0;
+        try {
+            await pipeline(Readable.fromWeb(res.body), async function* (src) {
+                for await (const chunk of src) {
+                    const b = chunk;
+                    hash.update(b);
+                    size += b.length;
+                    yield b;
+                }
+            }, createWriteStream(part));
+        }
+        catch (err) {
+            rmSync(part, { force: true });
+            throw err;
+        }
+        const digest = hash.digest('hex');
+        if (size !== f.size || digest !== f.sha256) {
+            rmSync(part, { force: true });
+            throw new Error(`${f.rel}: got ${size} bytes sha256 ${digest}, expected ${f.size} ${f.sha256}`);
+        }
+        renameSync(part, path);
+    }
+    writeFileSync(marker(dir), stamp);
+    opts.log?.('[memtrails] embedding model ready');
+    return dir;
+}

package/dist/core/paths.d.ts

@@ -0,0 +1 @@
+export declare function cacheDir(): string;

package/dist/core/paths.js

@@ -0,0 +1,7 @@
+/** Where memtrails keeps what it downloads (the Qdrant binary, the embedding model): $MEMTRAILS_CACHE, else
+ *  $XDG_CACHE_HOME/memtrails, else ~/.cache/memtrails. Shared by every memory on the machine; safe to delete. */
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+export function cacheDir() {
+    return process.env.MEMTRAILS_CACHE?.trim() || join(process.env.XDG_CACHE_HOME?.trim() || join(homedir(), '.cache'), 'memtrails');
+}

package/dist/core/profiles.d.ts

@@ -0,0 +1,30 @@
+import type { ConsolidateDeps } from './consolidate.js';
+import type { MemoryItem, MemoryScope } from './store.js';
+export declare const PROFILE_MIN_ITEMS = 3;
+export declare const PROFILE_BROAD = 0.2;
+/** The items a profile reads: the newest this many, oldest first, each line cut to PROFILE_LINE. */
+export declare const PROFILE_READ = 40;
+/** Profiles a night writes (a large first sleep goes on the next nights), and asks in flight. */
+export declare const PROFILES_PER_SLEEP = 400;
+/** What memory_search and gathering show of a profile: all of it, up to this (measured p90 1,082 characters). */
+export declare const PROFILE_SHOWN = 1500;
+export interface ProfileSubject {
+    uid: string;
+    concept: string;
+    scope: MemoryScope;
+    project: string;
+    read: MemoryItem[];
+    fingerprint: string;
+}
+export interface ProfileReport {
+    profiled: number;
+    unprofiled: number;
+    paused?: string;
+}
+/** The subjects worth a profile, per project: each concept (as conceptKey folds it, named as most items spell it) on at
+ *  least PROFILE_MIN_ITEMS items and on at most PROFILE_BROAD of the project's `sizes` (its active items). */
+export declare function profileSubjects(items: MemoryItem[], sizes: Map<string, number>): ProfileSubject[];
+/** Profiles written for the subjects whose items changed (at most `max`, several asks at once), and archived for the
+ *  subjects that no longer qualify. An ask the model did not answer stops the night's profiles; the next night asks
+ *  again, as their fingerprints still differ. */
+export declare function writeProfiles(d: ConsolidateDeps, max?: number): Promise<ProfileReport>;

package/dist/core/profiles.js

@@ -0,0 +1,147 @@
+/**
+ * Profiles — what memory knows about one subject, gathered in one place (sleep's last step).
+ *
+ * A question about a subject whose facts were told over many conversations ("when did John join the brigade, and what
+ * did he do there?") needs the dated facts of all of them, and a top 10 holds a few. For every concept enough long-term
+ * items share — at least PROFILE_MIN_ITEMS of a project's items, and on at most PROFILE_BROAD of them (not the person
+ * themselves, not the project's own name) — the fast model gathers what they say into one dated list of facts, word for
+ * word: a `profile` item, keyed and searched like any other, whose provenance is every source it read.
+ *
+ * Measured on all of LoCoMo (docs/BENCHMARKS.md): the answer fully shown in the search's top 10
+ * 59.1% → 61.7% (56 questions gained, 16 lost), dates 66.6% → 73.1%, 709 profiles for $0.11 of the fast model — shown
+ * whole (cut at 500 characters like any item, the gain was not sure). Questions over many conversations did not move.
+ *
+ *   - written again only when what it read changed (a hash of those items is its fingerprint), never from a profile;
+ *   - a subject that no longer qualifies has its profile archived;
+ *   - never related, merged or superseded by sleep or consolidation: it is written again from its items.
+ */
+import { createHash } from 'node:crypto';
+import { redactString } from './redact.js';
+import { defaultAsk } from '../llm.js';
+import { conceptKey, cut } from './text.js';
+import { writeLtm } from './writer.js';
+export const PROFILE_MIN_ITEMS = 3;
+export const PROFILE_BROAD = 0.2;
+/** The items a profile reads: the newest this many, oldest first, each line cut to PROFILE_LINE. */
+export const PROFILE_READ = 40;
+const PROFILE_LINE = 300;
+/** Profiles a night writes (a large first sleep goes on the next nights), and asks in flight. */
+export const PROFILES_PER_SLEEP = 400;
+const PROFILE_IN_FLIGHT = 8;
+/** What memory_search and gathering show of a profile: all of it, up to this (measured p90 1,082 characters). */
+export const PROFILE_SHOWN = 1500;
+const PROFILE_SYSTEM = `You write a profile: everything a person's memories say about one subject, gathered in one place, so that a
+later question about the subject finds all of it at once. You get the subject and the memories that mention it, each
+with its date. Write:
+- title: the subject and what the profile covers, at most 12 words
+- facts: one line per distinct fact about the subject, oldest first, each starting with the date it happened or was
+  said, as given. Keep every name, number, place, title and quote exactly as written; say who did what. Merge
+  repeats into one line, keep facts that differ apart. Nothing that is not in the memories. Never include secret
+  values. Never translate.`;
+const PROFILE_SCHEMA = {
+    type: 'object', additionalProperties: false, required: ['title', 'facts'],
+    properties: { title: { type: 'string' }, facts: { type: 'array', items: { type: 'string' } } },
+};
+const day = (ts) => new Date(ts).toLocaleDateString('en-GB', { day: 'numeric', month: 'long', year: 'numeric', timeZone: 'UTC' });
+const when = (it) => it.eventAt || day(it.ts);
+const hash = (s) => createHash('sha256').update(s).digest('hex');
+/** The subjects worth a profile, per project: each concept (as conceptKey folds it, named as most items spell it) on at
+ *  least PROFILE_MIN_ITEMS items and on at most PROFILE_BROAD of the project's `sizes` (its active items). */
+export function profileSubjects(items, sizes) {
+    const on = new Map();
+    for (const it of items) {
+        if (it.status !== 'active' || it.kind === 'profile')
+            continue;
+        const named = new Map();
+        for (const c of it.concepts ?? []) {
+            const key = conceptKey(c);
+            if (key && !named.has(key))
+                named.set(key, c.trim());
+        }
+        for (const [key, name] of named) {
+            const id = `${it.project}\n${key}`;
+            const s = on.get(id) ?? { project: it.project, key, names: new Map(), items: [] };
+            s.names.set(name, (s.names.get(name) ?? 0) + 1);
+            s.items.push(it);
+            on.set(id, s);
+        }
+    }
+    const out = [];
+    for (const [id, s] of on) {
+        if (s.items.length < PROFILE_MIN_ITEMS || s.items.length > PROFILE_BROAD * (sizes.get(s.project) ?? 0))
+            continue;
+        const read = [...s.items].sort((a, b) => b.ts - a.ts || a.uid.localeCompare(b.uid)).slice(0, PROFILE_READ).reverse();
+        const scopes = new Map();
+        for (const it of read)
+            scopes.set(it.scope, (scopes.get(it.scope) ?? 0) + 1);
+        out.push({
+            uid: `pf:${hash(id).slice(0, 24)}`,
+            concept: [...s.names].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0][0],
+            scope: [...scopes].sort((a, b) => b[1] - a[1])[0][0],
+            project: s.project,
+            read,
+            fingerprint: hash([s.key, ...read.map((it) => `${it.uid}\n${when(it)}\n${it.title}\n${it.summary}`)].join('\n\n')),
+        });
+    }
+    return out;
+}
+const promptOf = (s) => `SUBJECT: ${s.concept}\n\nMEMORIES:\n${s.read.map((it) => `- (${when(it)}) ${it.title}: ${cut(it.summary.replace(/\s+/g, ' '), PROFILE_LINE)}`).join('\n')}`;
+function profileItem(s, title, facts) {
+    const newest = s.read[s.read.length - 1];
+    return {
+        uid: s.uid, kind: 'profile', scope: s.scope, project: s.project, engine: newest.engine, session: null, cwd: newest.cwd,
+        where: s.concept, ts: Math.max(...s.read.map((it) => it.ts)), firstTs: Math.min(...s.read.map((it) => it.firstTs || it.ts)),
+        title: cut(title.trim() || s.concept, 120), summary: facts.join('\n'), body: '', files: [], appliesWhen: '',
+        confidence: 'curated', importance: 3, status: 'active', provenance: [...new Set(s.read.flatMap((it) => it.provenance))],
+        seen: 1, fingerprint: s.fingerprint, concepts: [s.concept],
+    };
+}
+/** Profiles written for the subjects whose items changed (at most `max`, several asks at once), and archived for the
+ *  subjects that no longer qualify. An ask the model did not answer stops the night's profiles; the next night asks
+ *  again, as their fingerprints still differ. */
+export async function writeProfiles(d, max = PROFILES_PER_SLEEP) {
+    const r = { profiled: 0, unprofiled: 0 };
+    const sizes = new Map();
+    const mapped = [];
+    for (const [uid, m] of d.ltm.entries()) {
+        if (m.status !== 'active' || m.kind === 'profile')
+            continue;
+        sizes.set(m.project, (sizes.get(m.project) ?? 0) + 1);
+        const it = m.mapped ? d.ltm.get(uid) : undefined;
+        if (it)
+            mapped.push(it);
+    }
+    const subjects = profileSubjects(mapped, sizes);
+    const wanted = new Set(subjects.map((s) => s.uid));
+    const gone = [...d.ltm.entries()].filter(([uid, m]) => m.kind === 'profile' && m.status === 'active' && !wanted.has(uid))
+        .map(([uid]) => d.ltm.get(uid)).filter((it) => !!it).map((it) => ({ ...it, status: 'archived' }));
+    if (gone.length) {
+        await writeLtm(d, gone);
+        d.ltm.put(gone);
+        r.unprofiled = gone.length;
+    }
+    const todo = subjects.filter((s) => { const m = d.ltm.meta(s.uid); return m?.status !== 'active' || m.fp !== s.fingerprint; }).slice(0, max);
+    let next = 0;
+    await Promise.all(Array.from({ length: Math.min(PROFILE_IN_FLIGHT, todo.length) }, async () => {
+        while (!r.paused && next < todo.length && !d.aborted()) {
+            const s = todo[next++];
+            const a = await (d.ask ?? defaultAsk)({
+                purpose: 'memory', system: PROFILE_SYSTEM, prompt: redactString(promptOf(s)), schema: PROFILE_SCHEMA,
+                name: 'memory_profile', maxTokens: 3000, timeoutMs: 180_000, consequence: "so this subject's profile waits for the next night",
+            });
+            if (!('answer' in a)) {
+                r.paused = 'unavailable' in a ? a.unavailable : 'busy';
+                return;
+            }
+            const answer = a.answer;
+            const facts = (Array.isArray(answer.facts) ? answer.facts : []).filter((f) => typeof f === 'string' && !!f.trim()).map((f) => f.trim());
+            if (!facts.length)
+                continue;
+            const it = profileItem(s, typeof answer.title === 'string' ? answer.title : '', facts);
+            await writeLtm(d, [it]);
+            d.ltm.put([it]);
+            r.profiled++;
+        }
+    }));
+    return r;
+}

package/dist/core/qdrant-binary.d.ts

@@ -0,0 +1,25 @@
+export declare const QDRANT_VERSION = "1.20.0";
+export interface QdrantAsset {
+    asset: string;
+    size: number;
+    sha256: string;
+    /** The executable's name inside the archive. */
+    exe: string;
+}
+/** Per `${platform}-${arch}`. Linux takes the static musl build: it runs on any distribution. */
+export declare const QDRANT_ASSETS: Readonly<Record<string, QdrantAsset>>;
+export declare function qdrantAsset(platform?: string, arch?: string): QdrantAsset | null;
+export declare function qdrantDir(): string;
+export declare function qdrantBinaryPath(a?: QdrantAsset | null): string | null;
+export declare const QDRANT_RELEASE_BASE = "https://github.com/qdrant/qdrant/releases/download/v1.20.0";
+/** The binary is in place and its archive was verified when it was unpacked. */
+export declare function qdrantReady(a?: QdrantAsset | null): boolean;
+/**
+ * Fetch, verify and unpack the binary unless it is already there. Throws on a network failure, a size or hash
+ * mismatch (the download is deleted), or a platform Qdrant does not ship for; the caller retries with backoff.
+ */
+export declare function ensureQdrant(opts?: {
+    signal?: AbortSignal;
+    log?: (line: string) => void;
+    fetchImpl?: typeof fetch;
+}): Promise<string>;