memtrails 0.3.0 → 0.4.1

package/dist/core/qdrant-process.d.ts

@@ -4,12 +4,24 @@ export interface QdrantEndpoint {
 }
 /** The config file: what environment variables cannot say (gRPC off is `grpc_port: null`). */
 export declare function qdrantConfig(storage: string, port: number): string;
+/** When this machine started, in ms: a pid written down before a reboot names some other process now. */
+export declare function bootTime(): number;
+/** The same boot as `boot` (bootTime() of then): a minute apart at most, uptime being in whole seconds on some systems. */
+export declare function sameBoot(boot: unknown): boolean;
+/** A process with this pid exists — one of another user's too (EPERM). */
+export declare function alive(pid: number): boolean;
 export declare class QdrantProcess {
     private child;
     private endpoint;
     private stopping;
     private deaths;
     private restartTimer;
+    /** A restart's start while it runs: stopAndWait waits for it, and for what it began to be gone. */
+    private restarting;
+    /** Counts stop()s: a start that sees it move while it starts gives up — before it spawns too, where `child` is null. */
+    private stops;
+    /** This instance's children that have not exited: stopAndWait waits for each, a restart's own included. */
+    private readonly children;
     private readonly opts;
     /** `dir` may be relative: Qdrant runs inside it, so every path it is given is made absolute first. */
     constructor(opts: {
@@ -20,11 +32,27 @@ export declare class QdrantProcess {
     });
     /** Where it answers, once it does; null while it starts or after it stopped. */
     current(): QdrantEndpoint | null;
+    /** Up and proven ours (owned), or it throws — and then nothing of this start keeps running or is started again: the
+     *  caller was told, and may not be there to stop it. */
     start(): Promise<QdrantEndpoint>;
-    /** Answering `/readyz` within 30 s, or the start failed. */
+    /** The pid file, and the child tracked for the host's exit; the file goes when that child does. */
+    private recordPid;
+    /** A Qdrant left running on this storage by a host that died without `close()` (a crash, a kill): stopped before
+     *  another starts on it. Only when the pid is still the process written down — this boot, its command naming this
+     *  folder's config — never an unrelated process that has the pid since. */
+    private reapLeftover;
+    /** Answering `/readyz` within 30 s, or the start failed. /readyz asks no key, so none is sent: what answers may not be
+     *  ours (owned). */
     private ready;
+    /** The port was free when picked, not when Qdrant bound it: another local process may have taken it, answered
+     *  /readyz, and would be handed the key and every document. Ours is still running, refuses a wrong key and takes
+     *  this run's — the wrong key asked first, so a process that takes any key never sees the real one. */
+    private owned;
     private exited;
+    /** Started again after a growing pause; a restart that fails counts as another death. */
+    private restartLater;
     stop(): void;
-    /** Stop and wait for the process to be gone — before its storage is deleted. */
+    /** Stop and wait for the process to be gone — before its storage is deleted. A restart that was starting one ends
+     *  first, and the Qdrant it began is waited for too: none is left running after a close. */
     stopAndWait(ms?: number): Promise<void>;
 }

package/dist/core/qdrant-process.js

@@ -1,8 +1,9 @@
+import { execFileSync, spawn } from 'node:child_process';
 import { randomBytes } from 'node:crypto';
-import { closeSync, mkdirSync, openSync, statSync, renameSync, writeFileSync } from 'node:fs';
+import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { createServer } from 'node:net';
+import { uptime } from 'node:os';
 import { join, resolve } from 'node:path';
-import { spawn } from 'node:child_process';
 /** A port the OS says is free on loopback. */
 async function freePort() {
     return new Promise((resolve, reject) => {
@@ -39,7 +40,9 @@ export function qdrantConfig(storage, port) {
     ].join('\n');
 }
 const LOG_MAX = 4 * 1024 * 1024;
-/** The process's log beside its data, rotated once at start when it has grown past 4 MB. */
+/** Beside the storage: the pid of the Qdrant running on it, with its config and the machine's boot (reapLeftover). */
+const PID_FILE = 'qdrant.pid';
+/** The process's log beside its data, rotated once at start when it has grown past 4 MB. Private: it names paths. */
 function openLog(dir) {
     const path = join(dir, 'qdrant.log');
     try {
@@ -47,8 +50,75 @@ function openLog(dir) {
             renameSync(path, `${path}.1`);
     }
     catch { /* none yet */ }
-    mkdirSync(dir, { recursive: true });
-    return openSync(path, 'a');
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    const fd = openSync(path, 'a', 0o600);
+    try {
+        chmodSync(path, 0o600);
+    }
+    catch { /* not ours to change: written all the same */ }
+    return fd;
+}
+/** When this machine started, in ms: a pid written down before a reboot names some other process now. */
+export function bootTime() {
+    return Date.now() - uptime() * 1000;
+}
+/** The same boot as `boot` (bootTime() of then): a minute apart at most, uptime being in whole seconds on some systems. */
+export function sameBoot(boot) {
+    return typeof boot === 'number' && Math.abs(bootTime() - boot) < 60_000;
+}
+/** A process with this pid exists — one of another user's too (EPERM). */
+export function alive(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code === 'EPERM';
+    }
+}
+const pause = (ms) => new Promise((r) => setTimeout(r, ms));
+/** A running process's command line, or null when it cannot be read (no /proc and no ps, Windows). */
+function commandOf(pid) {
+    try {
+        return readFileSync(`/proc/${pid}/cmdline`, 'utf8').split('\0').join(' ');
+    }
+    catch { /* not Linux */ }
+    if (process.platform === 'win32')
+        return null;
+    try {
+        return execFileSync('ps', ['-ww', '-o', 'command=', '-p', String(pid)], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 5000 });
+    }
+    catch {
+        return null;
+    }
+}
+/** What Qdrant gets of the environment: what a process needs to run and the key. None of the host's secrets, and none
+ *  of its QDRANT__* variables — they override the config (QDRANT__SERVICE__HOST=0.0.0.0 would open it to the network). */
+function childEnv(apiKey) {
+    const env = {};
+    for (const k of ['PATH', 'HOME', 'TMPDIR', 'TEMP', 'TMP', 'SystemRoot', 'USERPROFILE'])
+        if (process.env[k] !== undefined)
+            env[k] = process.env[k];
+    return { ...env, QDRANT__SERVICE__API_KEY: apiKey, QDRANT__TELEMETRY_DISABLED: 'true' };
+}
+/** Every Qdrant this process started and has not seen exit: stopped when the host exits without `close()` (an 'exit'
+ *  handler can only send the signal; a host killed outright leaves its Qdrant to the next start's reapLeftover). */
+const running = new Set();
+let exitHook = false;
+function track(child) {
+    running.add(child);
+    child.once('exit', () => running.delete(child));
+    if (exitHook)
+        return;
+    exitHook = true;
+    process.on('exit', () => { for (const c of running) {
+        try {
+            c.kill('SIGTERM');
+        }
+        catch { /* gone */ }
+    } });
 }
 export class QdrantProcess {
     child = null;
@@ -56,6 +126,12 @@ export class QdrantProcess {
     stopping = false;
     deaths = [];
     restartTimer = null;
+    /** A restart's start while it runs: stopAndWait waits for it, and for what it began to be gone. */
+    restarting = null;
+    /** Counts stop()s: a start that sees it move while it starts gives up — before it spawns too, where `child` is null. */
+    stops = 0;
+    /** This instance's children that have not exited: stopAndWait waits for each, a restart's own included. */
+    children = new Set();
     opts;
     /** `dir` may be relative: Qdrant runs inside it, so every path it is given is made absolute first. */
     constructor(opts) {
@@ -65,13 +141,19 @@ export class QdrantProcess {
     current() {
         return this.endpoint;
     }
+    /** Up and proven ours (owned), or it throws — and then nothing of this start keeps running or is started again: the
+     *  caller was told, and may not be there to stop it. */
     async start() {
         this.stopping = false;
+        const stops = this.stops;
         if (this.endpoint && this.child && this.child.exitCode === null)
             return this.endpoint;
+        mkdirSync(this.opts.dir, { recursive: true, mode: 0o700 });
+        await this.reapLeftover();
         const port = await freePort();
+        if (this.stops !== stops)
+            throw new Error('Qdrant was stopped while it started');
         const apiKey = randomBytes(24).toString('hex');
-        mkdirSync(this.opts.dir, { recursive: true });
         const config = join(this.opts.dir, 'config.yaml');
         writeFileSync(config, qdrantConfig(this.opts.dir, port), { mode: 0o600 });
         const fd = openLog(this.opts.dir);
@@ -80,46 +162,139 @@ export class QdrantProcess {
             child = spawn(this.opts.binary, ['--config-path', config, '--disable-telemetry'], {
                 cwd: this.opts.dir,
                 stdio: ['ignore', fd, fd],
-                // Only the key: the rest is in the config file.
-                env: { ...process.env, QDRANT__SERVICE__API_KEY: apiKey, QDRANT__TELEMETRY_DISABLED: 'true' },
+                env: childEnv(apiKey),
             });
         }
         finally {
             closeSync(fd);
         }
         this.child = child;
+        if (child.pid) {
+            this.children.add(child);
+            child.once('exit', () => this.children.delete(child));
+        }
+        // A binary that is missing or cannot run is an 'error', never an 'exit' — unheard, it is thrown in the host.
+        let failed = null;
+        child.on('error', (err) => { failed ??= err; });
         child.once('exit', (code, signal) => this.exited(child, code, signal));
+        if (child.pid)
+            this.recordPid(child, config);
         const endpoint = { url: `http://127.0.0.1:${port}`, apiKey };
-        await this.ready(endpoint, child);
+        try {
+            await this.ready(endpoint, child, () => failed);
+            await this.owned(endpoint, child);
+            if (this.child !== child)
+                throw new Error('Qdrant was stopped while it started');
+        }
+        catch (err) {
+            if (this.child === child)
+                this.child = null; // exited() then schedules nothing
+            if (child.pid && child.exitCode === null && child.signalCode === null)
+                child.kill('SIGTERM');
+            throw err;
+        }
         this.endpoint = endpoint;
         this.opts.log(`[memory] Qdrant up on 127.0.0.1:${port}`);
         return endpoint;
     }
-    /** Answering `/readyz` within 30 s, or the start failed. */
-    async ready(e, child) {
+    /** The pid file, and the child tracked for the host's exit; the file goes when that child does. */
+    recordPid(child, config) {
+        const file = join(this.opts.dir, PID_FILE);
+        writeFileSync(file, JSON.stringify({ pid: child.pid, config, boot: bootTime() }), { mode: 0o600 });
+        track(child);
+        child.once('exit', () => {
+            try {
+                if (JSON.parse(readFileSync(file, 'utf8')).pid === child.pid)
+                    rmSync(file, { force: true });
+            }
+            catch { /* none */ }
+        });
+    }
+    /** A Qdrant left running on this storage by a host that died without `close()` (a crash, a kill): stopped before
+     *  another starts on it. Only when the pid is still the process written down — this boot, its command naming this
+     *  folder's config — never an unrelated process that has the pid since. */
+    async reapLeftover() {
+        const file = join(this.opts.dir, PID_FILE);
+        let rec;
+        try {
+            rec = JSON.parse(readFileSync(file, 'utf8'));
+        }
+        catch {
+            return;
+        }
+        const pid = Number(rec.pid);
+        const config = join(this.opts.dir, 'config.yaml');
+        if (rec.config === config && sameBoot(rec.boot) && pid !== this.child?.pid && alive(pid) && commandOf(pid)?.includes(config)) {
+            this.opts.log(`[memory] stopping the Qdrant left running on ${this.opts.dir} (pid ${pid})`);
+            for (const [signal, ms] of [['SIGTERM', 10_000], ['SIGKILL', 2000]]) {
+                try {
+                    process.kill(pid, signal);
+                }
+                catch { /* gone */ }
+                for (const until = Date.now() + ms; alive(pid) && Date.now() < until;)
+                    await pause(50);
+                if (!alive(pid))
+                    break;
+            }
+        }
+        rmSync(file, { force: true });
+    }
+    /** Answering `/readyz` within 30 s, or the start failed. /readyz asks no key, so none is sent: what answers may not be
+     *  ours (owned). */
+    async ready(e, child, failed) {
         const until = Date.now() + 30_000;
         while (Date.now() < until) {
-            if (child.exitCode !== null)
-                throw new Error(`Qdrant exited (${child.exitCode}) while starting — see ${join(this.opts.dir, 'qdrant.log')}`);
+            const err = failed();
+            if (err)
+                throw new Error(`Qdrant could not be started (${this.opts.binary}): ${err.message}`);
+            if (child.exitCode !== null || child.signalCode !== null)
+                throw new Error(`Qdrant exited (${child.signalCode ?? child.exitCode}) while starting — see ${join(this.opts.dir, 'qdrant.log')}`);
             try {
-                const r = await fetch(`${e.url}/readyz`, { headers: { 'api-key': e.apiKey }, signal: AbortSignal.timeout(1000) });
+                const r = await fetch(`${e.url}/readyz`, { signal: AbortSignal.timeout(1000) });
+                await r.arrayBuffer().catch(() => undefined);
                 if (r.ok)
                     return;
             }
             catch { /* not listening yet */ }
-            await new Promise((r) => setTimeout(r, 100));
+            await pause(100);
         }
-        child.kill('SIGTERM');
         throw new Error('Qdrant did not answer within 30 s');
     }
+    /** The port was free when picked, not when Qdrant bound it: another local process may have taken it, answered
+     *  /readyz, and would be handed the key and every document. Ours is still running, refuses a wrong key and takes
+     *  this run's — the wrong key asked first, so a process that takes any key never sees the real one. */
+    async owned(e, child) {
+        const status = async (key) => {
+            const r = await fetch(`${e.url}/collections`, { headers: { 'api-key': key }, signal: AbortSignal.timeout(5000) });
+            await r.arrayBuffer().catch(() => undefined);
+            return r.status;
+        };
+        const gone = () => child.exitCode !== null || child.signalCode !== null;
+        const notOurs = `what answers on ${e.url} is not this memory's Qdrant`;
+        if (gone())
+            throw new Error(`${notOurs}: ours exited — see ${join(this.opts.dir, 'qdrant.log')}`);
+        const wrong = await status(randomBytes(24).toString('hex'));
+        if (wrong !== 401 && wrong !== 403)
+            throw new Error(`${notOurs}: it took a wrong key (${wrong})`);
+        const right = await status(e.apiKey);
+        if (right !== 200)
+            throw new Error(`${notOurs}: it refused this run's key (${right})`);
+        if (gone())
+            throw new Error(`${notOurs}: ours exited — see ${join(this.opts.dir, 'qdrant.log')}`);
+    }
     exited(child, code, signal) {
         if (this.child !== child)
             return;
+        // One that never came up is its start()'s to report: that start rejects, and schedules nothing.
+        const wasUp = this.endpoint !== null;
         this.child = null;
         this.endpoint = null;
-        if (this.stopping)
+        if (this.stopping || !wasUp)
             return;
-        const why = `Qdrant stopped (${signal ?? code})`;
+        this.restartLater(`Qdrant stopped (${signal ?? code})`);
+    }
+    /** Started again after a growing pause; a restart that fails counts as another death. */
+    restartLater(why) {
         this.opts.log(`[memory] ${why}`);
         const now = Date.now();
         this.deaths = [...this.deaths.filter((t) => now - t < 10 * 60_000), now];
@@ -127,15 +302,26 @@ export class QdrantProcess {
             this.opts.onDown?.(`${why}; five times in ten minutes, so it is left stopped`);
             return;
         }
-        const pause = Math.min(60_000, 1000 * 2 ** (this.deaths.length - 1));
+        const wait = Math.min(60_000, 1000 * 2 ** (this.deaths.length - 1));
         this.restartTimer = setTimeout(() => {
             this.restartTimer = null;
-            this.start().catch((err) => this.opts.onDown?.(err.message));
-        }, pause);
+            if (this.stopping)
+                return;
+            // Held until its start ends: a stop() meanwhile makes that start give up and stop what it began.
+            const run = this.start().then(() => undefined, (err) => {
+                if (this.stopping)
+                    return; // given up for a stop: nothing is down
+                this.opts.onDown?.(err.message);
+                this.restartLater(err.message);
+            }).finally(() => { if (this.restarting === run)
+                this.restarting = null; });
+            this.restarting = run;
+        }, wait);
         this.restartTimer.unref?.();
     }
     stop() {
         this.stopping = true;
+        this.stops++;
         if (this.restartTimer) {
             clearTimeout(this.restartTimer);
             this.restartTimer = null;
@@ -146,15 +332,19 @@ export class QdrantProcess {
         if (c)
             c.kill('SIGTERM');
     }
-    /** Stop and wait for the process to be gone — before its storage is deleted. */
+    /** Stop and wait for the process to be gone — before its storage is deleted. A restart that was starting one ends
+     *  first, and the Qdrant it began is waited for too: none is left running after a close. */
     async stopAndWait(ms = 10_000) {
-        const c = this.child;
+        const restart = this.restarting;
         this.stop();
-        if (!c || c.exitCode !== null)
-            return;
-        await new Promise((resolve) => {
+        const within = (p) => new Promise((resolve) => {
             const t = setTimeout(resolve, ms);
-            c.once('exit', () => { clearTimeout(t); resolve(); });
+            const done = () => { clearTimeout(t); resolve(); };
+            p.then(done, done);
         });
+        if (restart)
+            await within(restart);
+        await Promise.all([...this.children].filter((c) => c.exitCode === null && c.signalCode === null)
+            .map((c) => within(new Promise((exited) => c.once('exit', exited)))));
     }
 }

package/dist/core/rebuild.d.ts

@@ -14,13 +14,17 @@ interface RebuildDeps {
 export declare class MemoryRebuild {
     private readonly d;
     private running;
+    /** The generation `running` was started for: a run() of the same one joins it, a later one waits for it. */
+    private runningFor;
     private generation;
     constructor(d: RebuildDeps);
     cancel(): void;
     settled(): Promise<void>;
     pending(): boolean;
-    /** Written BEFORE clearing an old index. A crash at any later point leaves a resumable rebuild. */
+    /** Written BEFORE clearing an old index. A crash at any later point leaves a resumable rebuild. A run in flight is
+     *  cancelled: it wrote into the index about to be cleared, and must neither move nor delete the new checkpoint. */
     begin(): void;
+    /** Joins the run in flight for the same begin(); after a newer begin(), waits for the cancelled one to stop first. */
     run(): Promise<void>;
     private save;
     /** One phase from its checkpoint: batches of `size`, up to `concurrency` written at once, the checkpoint moved over

package/dist/core/rebuild.js

@@ -4,6 +4,8 @@ import { dirname } from 'node:path';
 export class MemoryRebuild {
     d;
     running = null;
+    /** The generation `running` was started for: a run() of the same one joins it, a later one waits for it. */
+    runningFor = -1;
     generation = 0;
     constructor(d) {
         this.d = d;
@@ -11,11 +13,24 @@ export class MemoryRebuild {
     cancel() { this.generation++; }
     async settled() { await this.running?.catch(() => { }); }
     pending() { return existsSync(this.d.path); }
-    /** Written BEFORE clearing an old index. A crash at any later point leaves a resumable rebuild. */
-    begin() { this.save({ phase: 'ltm', after: null }); }
+    /** Written BEFORE clearing an old index. A crash at any later point leaves a resumable rebuild. A run in flight is
+     *  cancelled: it wrote into the index about to be cleared, and must neither move nor delete the new checkpoint. */
+    begin() {
+        this.generation++;
+        this.save({ phase: 'ltm', after: null });
+    }
+    /** Joins the run in flight for the same begin(); after a newer begin(), waits for the cancelled one to stop first. */
     run() {
-        this.running ??= this.resume(this.generation).finally(() => { this.running = null; });
-        return this.running;
+        const generation = this.generation;
+        if (this.running && this.runningFor === generation)
+            return this.running;
+        const before = this.running?.catch(() => { });
+        const work = before ? before.then(() => this.resume(generation)) : this.resume(generation);
+        const p = work.finally(() => { if (this.running === p)
+            this.running = null; });
+        this.running = p;
+        this.runningFor = generation;
+        return p;
     }
     save(checkpoint) {
         mkdirSync(dirname(this.d.path), { recursive: true });
@@ -54,8 +69,11 @@ export class MemoryRebuild {
         let skip = after !== null;
         let batch = [];
         for (const it of all) {
-            if (cancelled())
+            if (cancelled()) {
+                // Stopped (a newer begin, a close): what is being written ends before this returns, not into the next index.
+                await Promise.allSettled(queue.map((q) => q.p));
                 return false;
+            }
             if (skip) {
                 if (it.uid === after)
                     skip = false;

package/dist/core/recall-log.d.ts

@@ -11,11 +11,14 @@ export type RecallEvent = {
     ev: 'open';
     item: string;
 };
+/** Events between trims of a book left open for long (a server): the cap holds without a restart. */
+export declare const RECALL_TRIM_EVERY = 1000;
 export declare class RecallBook {
     private readonly path;
     private readonly now;
     private readonly keep;
     private chain;
+    private appended;
     constructor(path: string, now?: () => number, keep?: {
         ms: number;
         bytes: number;

package/dist/core/recall-log.js

@@ -14,7 +14,7 @@
  * Appended in order, a line at a time; a write that fails is dropped (recall never waits on, or fails for, its record).
  * The ids are the logs' own (`stm:…`, long-term uids): a rebuild from the logs keeps them (only `qdrant/` is made
  * again), so the record outlives a rebuild and stays true to it. Kept: the last RECALL_KEEP_MS, at most
- * RECALL_KEEP_BYTES of the newest (`trim()`).
+ * RECALL_KEEP_BYTES of the newest (`trim()`: when memory opens, and every RECALL_TRIM_EVERY events).
  */
 import { appendFile, mkdir, readFile, rename, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
@@ -22,11 +22,14 @@ import { dirname } from 'node:path';
 export const RECALL_QUERY_CHARS = 2000;
 export const RECALL_KEEP_MS = 180 * 24 * 60 * 60_000;
 export const RECALL_KEEP_BYTES = 32 * 1024 * 1024;
+/** Events between trims of a book left open for long (a server): the cap holds without a restart. */
+export const RECALL_TRIM_EVERY = 1000;
 export class RecallBook {
     path;
     now;
     keep;
     chain = Promise.resolve();
+    appended = 0;
     constructor(path, now = Date.now, keep = { ms: RECALL_KEEP_MS, bytes: RECALL_KEEP_BYTES }) {
         this.path = path;
         this.now = now;
@@ -40,6 +43,8 @@ export class RecallBook {
             await appendFile(this.path, line, { mode: 0o600 });
         })
             .catch(() => { });
+        if (++this.appended % RECALL_TRIM_EVERY === 0)
+            void this.trim();
     }
     /** The events older than `keep.ms` dropped, then the oldest until the rest fit `keep.bytes` — rewritten between
      *  appends (a new file renamed over the old). How many were dropped; 0 on any failure, the record as it was. */

package/dist/core/recall.js

@@ -11,15 +11,19 @@ import { pointId } from './qdrant-client.js';
 import { queryText } from './embedder.js';
 import { explore } from './graph.js';
 import { RERANK_TOP, rerankedOrder } from './reranker.js';
-import { children, cut, querySparse } from './text.js';
+import { children, cut, flat, querySparse } from './text.js';
 import { dateSpan, LONG_SUMMARY_CHUNKS } from './writer.js';
 import { collectionKey } from '../records.js';
-/** What search may show: active items, never a throwaway run, never superseded or archived knowledge. */
+/** What search may show: active items, never a throwaway run, never superseded or archived knowledge. `kinds`: a
+ *  record's kind is its `engine` (every point of a record has `kind: episode`), a learned item's its `kind`. */
 function baseFilter(kinds) {
-    return {
-        must: [{ key: 'synthetic', match: { value: false } }, { key: 'status', match: { value: 'active' } },
-            ...(kinds?.length ? [{ key: 'engine', match: { any: kinds } }] : [])],
-    };
+    const ofKind = kinds?.length ? [{
+            should: [
+                { must: [{ key: 'tier', match: { value: 'stm' } }, { key: 'engine', match: { any: kinds } }] },
+                { must: [{ key: 'tier', match: { value: 'ltm' } }, { key: 'kind', match: { any: kinds } }] },
+            ],
+        }] : [];
+    return { must: [{ key: 'synthetic', match: { value: false } }, { key: 'status', match: { value: 'active' } }, ...ofKind] };
 }
 async function items(d, uids) {
     const got = await d.client.retrieve(uids.map((u) => pointId(u, 0)));
@@ -76,6 +80,11 @@ async function reranked(d, query, list, dated) {
  *  everything else are two searches of that many each, interleaved, so neither crowds the other out of the room. */
 const SINGLE_LIST_UP_TO = 20;
 export async function memorySearch(d, q) {
+    // A date that is not one would leave its end open: the search would quietly ignore the range it was asked for.
+    for (const [end, at] of [['from', q.from], ['to', q.to]]) {
+        if (at && !dateSpan(at))
+            throw new Error(`memory search: ${end} "${at}" is not a date (YYYY-MM-DD, YYYY-MM or YYYY)`);
+    }
     const projects = q.collections?.length ? q.collections.map(collectionKey) : undefined;
     // Up to 200: a question that gathers ("everything she did with her family") needs the facts of many threads at once,
     // a dated observation a line.
@@ -128,7 +137,7 @@ export async function memorySearch(d, q) {
     return out;
 }
 /** For each hit whose best point was a chunk of its text, that chunk — found again from the text (writer.ts chunks a
- *  request's whole body; an item's body, and a summary too long for one key vector). */
+ *  request's whole body; an item's body, and a summary too long for one key vector) — unless the key starts with it. */
 async function passages(d, hits) {
     const chunked = hits.filter((h) => h.point && h.point !== pointId(h.item, 0));
     if (!chunked.length)
@@ -144,7 +153,9 @@ async function passages(d, hits) {
         const summary = String(p.summary ?? '');
         const kids = p.tier === 'stm' ? children(body) : [...children(body), ...(children(summary).length >= LONG_SUMMARY_CHUNKS ? children(summary) : [])];
         const k = kids.findIndex((_, i) => pointId(h.item, i + 1) === h.point);
-        if (k >= 0)
+        // A chunk that is the key's own start is not a passage found elsewhere: a short record's only chunk is its whole
+        // key, the two points tie, and either may come back first.
+        if (k >= 0 && !flat(summary).startsWith(kids[k]))
             out.set(h.item, kids[k]);
     }
     return out;

package/dist/core/redact.d.ts

@@ -1,2 +1,10 @@
+/**
+ * Secrets are scrubbed from what memory sends to a language model and from what consolidation keeps: provider keys,
+ * tokens, credentials in URLs, private keys, e-mail local parts, JWTs, `key=value` secrets; home directories become `~`;
+ * a long high-entropy run last. The records you add are kept as you wrote them.
+ *
+ * The input can be anything a record held, so every pattern is bounded: no rule may take more than linear time on a
+ * hostile string (a long run of what almost matches).
+ */
 /** Scrub one string. Never throws; returns `''` for a non-string. */
 export declare function redactString(input: unknown): string;