memtrails 0.0.0-stage → 0.2.0

package/dist/core/qdrant-process.js

@@ -0,0 +1,160 @@
+import { randomBytes } from 'node:crypto';
+import { closeSync, mkdirSync, openSync, statSync, renameSync, writeFileSync } from 'node:fs';
+import { createServer } from 'node:net';
+import { join, resolve } from 'node:path';
+import { spawn } from 'node:child_process';
+/** A port the OS says is free on loopback. */
+async function freePort() {
+    return new Promise((resolve, reject) => {
+        const s = createServer();
+        s.unref();
+        s.once('error', reject);
+        s.listen(0, '127.0.0.1', () => {
+            const addr = s.address();
+            const port = typeof addr === 'object' && addr ? addr.port : 0;
+            s.close(() => (port ? resolve(port) : reject(new Error('no free port'))));
+        });
+    });
+}
+/** The config file: what environment variables cannot say (gRPC off is `grpc_port: null`). */
+export function qdrantConfig(storage, port) {
+    return [
+        'log_level: WARN',
+        'telemetry_disabled: true',
+        'service:',
+        '  host: 127.0.0.1',
+        `  http_port: ${port}`,
+        '  grpc_port: null',
+        '  enable_cors: false',
+        '  max_workers: 1',
+        'storage:',
+        `  storage_path: ${JSON.stringify(join(storage, 'storage'))}`,
+        `  snapshots_path: ${JSON.stringify(join(storage, 'snapshots'))}`,
+        '  on_disk_payload: true',
+        '  performance:',
+        '    max_search_threads: 2',
+        '  optimizers:',
+        '    max_optimization_threads: 1',
+        '',
+    ].join('\n');
+}
+const LOG_MAX = 4 * 1024 * 1024;
+/** The process's log beside its data, rotated once at start when it has grown past 4 MB. */
+function openLog(dir) {
+    const path = join(dir, 'qdrant.log');
+    try {
+        if (statSync(path).size > LOG_MAX)
+            renameSync(path, `${path}.1`);
+    }
+    catch { /* none yet */ }
+    mkdirSync(dir, { recursive: true });
+    return openSync(path, 'a');
+}
+export class QdrantProcess {
+    child = null;
+    endpoint = null;
+    stopping = false;
+    deaths = [];
+    restartTimer = null;
+    opts;
+    /** `dir` may be relative: Qdrant runs inside it, so every path it is given is made absolute first. */
+    constructor(opts) {
+        this.opts = { ...opts, dir: resolve(opts.dir), binary: opts.binary.includes('/') || opts.binary.includes('\\') ? resolve(opts.binary) : opts.binary };
+    }
+    /** Where it answers, once it does; null while it starts or after it stopped. */
+    current() {
+        return this.endpoint;
+    }
+    async start() {
+        this.stopping = false;
+        if (this.endpoint && this.child && this.child.exitCode === null)
+            return this.endpoint;
+        const port = await freePort();
+        const apiKey = randomBytes(24).toString('hex');
+        mkdirSync(this.opts.dir, { recursive: true });
+        const config = join(this.opts.dir, 'config.yaml');
+        writeFileSync(config, qdrantConfig(this.opts.dir, port), { mode: 0o600 });
+        const fd = openLog(this.opts.dir);
+        let child;
+        try {
+            child = spawn(this.opts.binary, ['--config-path', config, '--disable-telemetry'], {
+                cwd: this.opts.dir,
+                stdio: ['ignore', fd, fd],
+                // Only the key: the rest is in the config file.
+                env: { ...process.env, QDRANT__SERVICE__API_KEY: apiKey, QDRANT__TELEMETRY_DISABLED: 'true' },
+            });
+        }
+        finally {
+            closeSync(fd);
+        }
+        this.child = child;
+        child.once('exit', (code, signal) => this.exited(child, code, signal));
+        const endpoint = { url: `http://127.0.0.1:${port}`, apiKey };
+        await this.ready(endpoint, child);
+        this.endpoint = endpoint;
+        this.opts.log(`[memory] Qdrant up on 127.0.0.1:${port}`);
+        return endpoint;
+    }
+    /** Answering `/readyz` within 30 s, or the start failed. */
+    async ready(e, child) {
+        const until = Date.now() + 30_000;
+        while (Date.now() < until) {
+            if (child.exitCode !== null)
+                throw new Error(`Qdrant exited (${child.exitCode}) while starting — see ${join(this.opts.dir, 'qdrant.log')}`);
+            try {
+                const r = await fetch(`${e.url}/readyz`, { headers: { 'api-key': e.apiKey }, signal: AbortSignal.timeout(1000) });
+                if (r.ok)
+                    return;
+            }
+            catch { /* not listening yet */ }
+            await new Promise((r) => setTimeout(r, 100));
+        }
+        child.kill('SIGTERM');
+        throw new Error('Qdrant did not answer within 30 s');
+    }
+    exited(child, code, signal) {
+        if (this.child !== child)
+            return;
+        this.child = null;
+        this.endpoint = null;
+        if (this.stopping)
+            return;
+        const why = `Qdrant stopped (${signal ?? code})`;
+        this.opts.log(`[memory] ${why}`);
+        const now = Date.now();
+        this.deaths = [...this.deaths.filter((t) => now - t < 10 * 60_000), now];
+        if (this.deaths.length >= 5) {
+            this.opts.onDown?.(`${why}; five times in ten minutes, so it is left stopped`);
+            return;
+        }
+        const pause = Math.min(60_000, 1000 * 2 ** (this.deaths.length - 1));
+        this.restartTimer = setTimeout(() => {
+            this.restartTimer = null;
+            this.start().catch((err) => this.opts.onDown?.(err.message));
+        }, pause);
+        this.restartTimer.unref?.();
+    }
+    stop() {
+        this.stopping = true;
+        if (this.restartTimer) {
+            clearTimeout(this.restartTimer);
+            this.restartTimer = null;
+        }
+        const c = this.child;
+        this.child = null;
+        this.endpoint = null;
+        if (c)
+            c.kill('SIGTERM');
+    }
+    /** Stop and wait for the process to be gone — before its storage is deleted. */
+    async stopAndWait(ms = 10_000) {
+        const c = this.child;
+        this.stop();
+        if (!c || c.exitCode !== null)
+            return;
+        await new Promise((resolve) => {
+            const t = setTimeout(resolve, ms);
+            c.once('exit', () => { clearTimeout(t); resolve(); });
+        });
+    }
+}

package/dist/core/rebuild.d.ts

@@ -0,0 +1,31 @@
+import type { LtmStore, MemoryItem, StmEntry, StmStore } from './store.js';
+interface RebuildDeps {
+    path: string;
+    ltm: LtmStore;
+    stm: StmStore;
+    writeLtm: (items: MemoryItem[]) => Promise<void>;
+    writeStm: (items: StmEntry[]) => Promise<void>;
+    aborted: () => boolean;
+    now?: () => number;
+    /** Batches written at once (1 when not given). The checkpoint only ever passes batches already written: it moves over
+     *  the written ones at the front, so a crash never skips one still in flight. */
+    concurrency?: number;
+}
+export declare class MemoryRebuild {
+    private readonly d;
+    private running;
+    private generation;
+    constructor(d: RebuildDeps);
+    cancel(): void;
+    settled(): Promise<void>;
+    pending(): boolean;
+    /** Written BEFORE clearing an old index. A crash at any later point leaves a resumable rebuild. */
+    begin(): void;
+    run(): Promise<void>;
+    private save;
+    /** One phase from its checkpoint: batches of `size`, up to `concurrency` written at once, the checkpoint moved over
+     *  the written batches at the front. False when cancelled. A failed batch fails the run, the checkpoint before it. */
+    private phase;
+    private resume;
+}
+export {};

package/dist/core/rebuild.js

@@ -0,0 +1,109 @@
+/** A derived index rebuild survives unavailable embeddings, a failed batch and a restart. Logs stay truth. */
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+export class MemoryRebuild {
+    d;
+    running = null;
+    generation = 0;
+    constructor(d) {
+        this.d = d;
+    }
+    cancel() { this.generation++; }
+    async settled() { await this.running?.catch(() => { }); }
+    pending() { return existsSync(this.d.path); }
+    /** Written BEFORE clearing an old index. A crash at any later point leaves a resumable rebuild. */
+    begin() { this.save({ phase: 'ltm', after: null }); }
+    run() {
+        this.running ??= this.resume(this.generation).finally(() => { this.running = null; });
+        return this.running;
+    }
+    save(checkpoint) {
+        mkdirSync(dirname(this.d.path), { recursive: true });
+        const tmp = `${this.d.path}.tmp`;
+        writeFileSync(tmp, JSON.stringify(checkpoint), { mode: 0o600 });
+        renameSync(tmp, this.d.path);
+    }
+    /** One phase from its checkpoint: batches of `size`, up to `concurrency` written at once, the checkpoint moved over
+     *  the written batches at the front. False when cancelled. A failed batch fails the run, the checkpoint before it. */
+    async phase(all, after, keep, size, write, phase, cancelled) {
+        const limit = Math.max(1, this.d.concurrency ?? 1);
+        const queue = [];
+        const advance = () => {
+            let last = null;
+            while (queue.length && queue[0].done)
+                last = queue.shift().last;
+            if (last !== null && !cancelled())
+                this.save({ phase, after: last });
+        };
+        // On a failure: the batches still in flight settle first, and the checkpoint takes every written one at the front.
+        const fail = async (err) => {
+            await Promise.allSettled(queue.map((q) => q.p));
+            advance();
+            throw err;
+        };
+        const send = async (items) => {
+            const slot = { last: items.at(-1).uid, done: false, p: Promise.resolve() };
+            slot.p = write(items).then(() => { slot.done = true; });
+            slot.p.catch(() => { }); // awaited below; a batch left in flight by a failure elsewhere is not unhandled
+            queue.push(slot);
+            for (let busy = queue.filter((q) => !q.done); busy.length >= limit; busy = queue.filter((q) => !q.done)) {
+                await Promise.race(busy.map((q) => q.p)).catch(fail);
+                advance();
+            }
+        };
+        let skip = after !== null;
+        let batch = [];
+        for (const it of all) {
+            if (cancelled())
+                return false;
+            if (skip) {
+                if (it.uid === after)
+                    skip = false;
+                continue;
+            }
+            if (!keep(it))
+                continue;
+            batch.push(it);
+            if (batch.length >= size) {
+                await send(batch);
+                batch = [];
+            }
+        }
+        if (batch.length)
+            await send(batch);
+        await Promise.all(queue.map((q) => q.p)).catch(fail);
+        advance();
+        return !cancelled();
+    }
+    async resume(generation) {
+        const cancelled = () => generation !== this.generation || this.d.aborted();
+        if (!this.pending() || cancelled())
+            return;
+        let checkpoint;
+        try {
+            checkpoint = JSON.parse(readFileSync(this.d.path, 'utf8'));
+            if (!['ltm', 'stm'].includes(checkpoint.phase) || (checkpoint.after !== null && typeof checkpoint.after !== 'string'))
+                throw new Error('invalid checkpoint');
+        }
+        catch {
+            checkpoint = { phase: 'ltm', after: null };
+        }
+        // While rebuilding, nothing else writes to the logs. Live log order therefore stays stable; if a
+        // checkpoint's item is absent (a repaired/compacted log), replay that phase instead of silently skipping records.
+        if (checkpoint.after && !(checkpoint.phase === 'ltm' ? this.d.ltm : this.d.stm).meta(checkpoint.after))
+            checkpoint.after = null;
+        if (checkpoint.phase === 'ltm') {
+            const done = await this.phase(this.d.ltm.all(), checkpoint.after, (it) => it.status === 'active', 20, this.d.writeLtm, 'ltm', cancelled);
+            if (!done)
+                return;
+            checkpoint = { phase: 'stm', after: null };
+            this.save(checkpoint);
+        }
+        // Every request the person made stays searchable, consolidated or not: the raw record is the truth, a memory a key
+        // to it (LongMemEval 2410.10813). Only throwaway runs (synthetic) stay out.
+        if (!(await this.phase(this.d.stm.all(), checkpoint.after, (it) => !it.synthetic, 32, this.d.writeStm, 'stm', cancelled)))
+            return;
+        if (!cancelled())
+            rmSync(this.d.path);
+    }
+}

package/dist/core/recall-log.d.ts

@@ -0,0 +1,29 @@
+/** A query kept up to this: the start of a long message is what the search embedded first. */
+export declare const RECALL_QUERY_CHARS = 2000;
+export declare const RECALL_KEEP_MS: number;
+export declare const RECALL_KEEP_BYTES: number;
+export type RecallEvent = {
+    ev: 'search';
+    project?: string;
+    query: string;
+    shown: string[];
+} | {
+    ev: 'open';
+    item: string;
+};
+export declare class RecallBook {
+    private readonly path;
+    private readonly now;
+    private readonly keep;
+    private chain;
+    constructor(path: string, now?: () => number, keep?: {
+        ms: number;
+        bytes: number;
+    });
+    record(e: RecallEvent): void;
+    /** The events older than `keep.ms` dropped, then the oldest until the rest fit `keep.bytes` — rewritten between
+     *  appends (a new file renamed over the old). How many were dropped; 0 on any failure, the record as it was. */
+    trim(): Promise<number>;
+    /** Every event recorded so far written (tests, shutdown). */
+    settled(): Promise<void>;
+}

package/dist/core/recall-log.js

@@ -0,0 +1,82 @@
+/**
+ * What recall showed, for what, and what was opened after it — the record a model that learns from USE reads.
+ *
+ * Learned only from what memory says and how it is laid out, two light models did not beat the first stage on the
+ * answers shown (docs/RESEARCH.md: a hetero-associative map over the memory's own pairs, a LightGBM
+ * ranker over language-free association features). What works in the literature is learned from use — links that
+ * strengthen when memories are recalled together and fade when not (HeLa-Mem, ACL 2026), rankers trained on what was
+ * opened (online learning to rank). The usage counts (store.ts UsageBook) say how often, never with what nor for what;
+ * this says both.
+ *
+ * One line per event in `recall.jsonl` beside the logs — on this machine only, cleared with the index, never sent:
+ *   {"t", "ev": "search", "project", "query", "shown": [ids]}                                   memory_search
+ *   {"t", "ev": "open",   "item"}                                                                memory_open
+ * Appended in order, a line at a time; a write that fails is dropped (recall never waits on, or fails for, its record).
+ * The ids are the logs' own (`stm:…`, long-term uids): a rebuild from the logs keeps them (only `qdrant/` is made
+ * again), so the record outlives a rebuild and stays true to it. Kept: the last RECALL_KEEP_MS, at most
+ * RECALL_KEEP_BYTES of the newest (`trim()`).
+ */
+import { appendFile, mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+/** A query kept up to this: the start of a long message is what the search embedded first. */
+export const RECALL_QUERY_CHARS = 2000;
+export const RECALL_KEEP_MS = 180 * 24 * 60 * 60_000;
+export const RECALL_KEEP_BYTES = 32 * 1024 * 1024;
+export class RecallBook {
+    path;
+    now;
+    keep;
+    chain = Promise.resolve();
+    constructor(path, now = Date.now, keep = { ms: RECALL_KEEP_MS, bytes: RECALL_KEEP_BYTES }) {
+        this.path = path;
+        this.now = now;
+        this.keep = keep;
+    }
+    record(e) {
+        const line = `${JSON.stringify({ t: this.now(), ...e, ...('query' in e ? { query: e.query.slice(0, RECALL_QUERY_CHARS) } : {}) })}\n`;
+        this.chain = this.chain
+            .then(async () => {
+            await mkdir(dirname(this.path), { recursive: true });
+            await appendFile(this.path, line, { mode: 0o600 });
+        })
+            .catch(() => { });
+    }
+    /** The events older than `keep.ms` dropped, then the oldest until the rest fit `keep.bytes` — rewritten between
+     *  appends (a new file renamed over the old). How many were dropped; 0 on any failure, the record as it was. */
+    trim() {
+        const run = this.chain.then(async () => {
+            let text;
+            try {
+                text = await readFile(this.path, 'utf8');
+            }
+            catch {
+                return 0;
+            }
+            const lines = text.split('\n').filter(Boolean);
+            const since = this.now() - this.keep.ms;
+            const recent = lines.filter((l) => { try {
+                return Number(JSON.parse(l).t) >= since;
+            }
+            catch {
+                return false;
+            } });
+            let bytes = recent.reduce((n, l) => n + Buffer.byteLength(l, 'utf8') + 1, 0);
+            let from = 0;
+            while (bytes > this.keep.bytes && from < recent.length)
+                bytes -= Buffer.byteLength(recent[from++], 'utf8') + 1;
+            const kept = recent.slice(from);
+            if (kept.length === lines.length)
+                return 0;
+            const next = `${this.path}.trim`;
+            await writeFile(next, kept.map((l) => `${l}\n`).join(''), { mode: 0o600 });
+            await rename(next, this.path);
+            return lines.length - kept.length;
+        });
+        this.chain = run.then(() => { }, () => { });
+        return run.catch(() => 0);
+    }
+    /** Every event recorded so far written (tests, shutdown). */
+    settled() {
+        return this.chain;
+    }
+}

package/dist/core/recall.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Recall: what a search returns, an item opened whole, and exploring around a topic.
+ *
+ * `memorySearch` embeds the query and searches dense vectors (each item's key, its chunks and its cues) and a keyword
+ * vector at once, fused by rank and grouped by item; with dates, what happened within them first. The best 30 can be
+ * reordered (`rerank`: word by word on this machine, words.ts, or any cross-encoder) and fused with the search's own order.
+ * `memoryOpen` reads one item whole. `memoryExplore` walks the concept graph (graph.ts) from a topic or from items to
+ * what is related but not found directly.
+ */
+import { type QdrantClient } from './qdrant-client.js';
+import type { RecallEvent } from './recall-log.js';
+export interface RecallDeps {
+    client: QdrantClient;
+    embed: (texts: string[], timeoutMs?: number) => Promise<Float32Array[]>;
+    /** What was shown / opened: the sleep pass's "use" (store.ts UsageBook). */
+    used?: (items: string[], what: 'shown' | 'opened') => void;
+    /** What each search showed, for what, and what was opened (recall-log.ts). */
+    record?: (e: RecallEvent) => void;
+    now?: () => number;
+    /** Each document's relevance to the query, or null — the search then keeps its own order. None: no reordering. */
+    rerank?: (query: string, documents: string[], timeoutMs: number) => Promise<number[] | null>;
+}
+/** Where a found item came from, when it is a record you added. */
+export interface RecordRef {
+    id: string;
+    source?: string;
+    author?: string;
+    url?: string;
+    metadata?: Record<string, string | number | boolean | null>;
+}
+export interface RecalledItem {
+    id: string;
+    title: string;
+    summary: string;
+    where: string;
+    project: string;
+    engine: string;
+    ts: number;
+    /** For a record you added: its kind (`email`, `message`, …, `record` when none was given) and `record` says which.
+     *  For what consolidation learned: decision, procedure, pitfall, fact, preference, …, observation (one dated thing a
+     *  thread told), profile (what is known about one subject), episode (a kept summary of earlier work). */
+    kind: string;
+    /** The record it is, for a record you added. */
+    record?: RecordRef;
+    /** For an observation: when it happened (YYYY-MM-DD, YYYY-MM or YYYY) — `ts` is when it was said. */
+    eventAt?: string;
+    /** For an observation of something that lasted: when it began and ended. */
+    startAt?: string;
+    endAt?: string;
+    /** The passage of its text that matched the search, when that was not its key (memory_search): a fact said deep in
+     *  a long request is shown, not only the request's start. */
+    match?: string;
+    /** The search's fused rank score. */
+    score: number;
+}
+export declare const MAX_SEARCH = 200;
+/** A search: best first, in every collection or the ones named — and with dates, what happened within them first, then
+ *  the rest. Never only what lies within them: on 64 held-out questions that name a time, the answer reached the reader
+ *  whole for 54 searched without dates, 53 with what lies within them first, 44 with only that. */
+export interface SearchQuery {
+    query: string;
+    /** Only these collections (records' `collection`). Default: all. */
+    collections?: string[];
+    /** Only these kinds (records' `kind`, or what consolidation learned: `observation`, `fact`, …). */
+    kinds?: string[];
+    limit?: number;
+    from?: string;
+    to?: string;
+}
+export declare function memorySearch(d: RecallDeps, q: SearchQuery): Promise<RecalledItem[]>;
+export interface ExploreQuery {
+    topic?: string;
+    from?: string[];
+    collections?: string[];
+    limit?: number;
+}
+export interface Explored {
+    /** What search found for the topic, best first. */
+    found: RecalledItem[];
+    /** Reached through the graph and not found by search, best first, with the way it was reached. */
+    related: Array<RecalledItem & {
+        via: string;
+    }>;
+    /** The concepts around it: where to look next. */
+    concepts: string[];
+}
+/** An agent exploring: from a topic (search finds the first items; the topic's concepts start the walk too) or from
+ *  items, what the concept graph reaches that search did not, and the concepts around it. */
+export declare function memoryExplore(d: RecallDeps, q: ExploreQuery): Promise<Explored>;
+/** One item whole, or null. */
+export declare function memoryOpen(d: RecallDeps, id: string): Promise<Record<string, unknown> | null>;