memtrails 0.3.0 → 0.4.1

package/dist/core/redact.js

@@ -2,41 +2,80 @@
  * Secrets are scrubbed from what memory sends to a language model and from what consolidation keeps: provider keys,
  * tokens, credentials in URLs, private keys, e-mail local parts, JWTs, `key=value` secrets; home directories become `~`;
  * a long high-entropy run last. The records you add are kept as you wrote them.
+ *
+ * The input can be anything a record held, so every pattern is bounded: no rule may take more than linear time on a
+ * hostile string (a long run of what almost matches).
  */
+/**
+ * A name that says its value is a secret: `password`, `GITHUB_TOKEN`, `AWS_SECRET_ACCESS_KEY`, `PGPASSWORD`,
+ * `client_secret`, `AccountKey`. The secret word must end the name (`token_count`, `max_tokens` are not secrets), and
+ * any prefix is allowed (`_` is a word character, so `\btoken` alone misses `GITHUB_TOKEN`).
+ */
+const SECRET_NAME = String.raw `[A-Za-z0-9_.-]{0,64}?(?:api[_-]?key|apikey|(?:access|secret|private|account|signing|encryption|master|client)[_-]?key|token|secret|password|passwd|passphrase|authorization)`;
+/** `name=value`, `name: value`, `"name": "value"`, `Authorization: token X` (the scheme word is kept, its credential not). */
+const KEY_VALUE = new RegExp(String.raw `\b(${SECRET_NAME})(["']?[ \t]{0,16}[=:][ \t]{0,16})(?:(["'])((?:[^"'\\\n]|\\.){1,512})|((?:(?:Bearer|Basic|Digest|token|Bot)[ \t]{1,16})?)([^\s&"';]+))`, 'gi');
 const SECRET_PATTERNS = [
     [/\bsk-[A-Za-z0-9_-]{16,}/g, '<redacted:sk>'],
     [/\bsk_(?:live|test)_[A-Za-z0-9]{16,}/g, '<redacted:stripe>'],
     [/\bpk_(?:live|test)_[A-Za-z0-9]{16,}/g, '<redacted:stripe-pub>'],
     [/\bgithub_pat_[A-Za-z0-9_]{20,}/g, '<redacted:github>'],
     [/\bgh[porsu]_[A-Za-z0-9]{20,}/g, '<redacted:github>'],
+    [/\bglpat-[A-Za-z0-9_-]{20,}/g, '<redacted:gitlab>'],
+    [/\bhf_[A-Za-z0-9]{30,}/g, '<redacted:huggingface>'],
+    [/\bnpm_[A-Za-z0-9]{36,}/g, '<redacted:npm>'],
     [/\bxox[abprs]-[A-Za-z0-9-]{10,}/g, '<redacted:slack>'],
-    [/\bAKIA[0-9A-Z]{16}\b/g, '<redacted:aws>'],
+    [/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g, '<redacted:aws>'],
     [/\bAIza[A-Za-z0-9_-]{20,}/g, '<redacted:google>'],
-    [/\b([a-z][a-z0-9+.-]*:\/\/)([^\s/:@]*):([^\s/@]+)@/gi, '$1$2:<redacted>@'],
+    [/\bSG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}/g, '<redacted:sendgrid>'],
+    // A Telegram bot token: the bot's number, then 35 characters.
+    [/\b\d{8,10}:[A-Za-z0-9_-]{35}(?![A-Za-z0-9_-])/g, '<redacted:telegram>'],
+    // `scheme://user:password@host`: the password runs to the LAST `@` before the host, so `p@ss` and `pa/ss` go whole.
+    [/\b([a-z][a-z0-9+.-]{0,31}:\/\/)([^\s/:@]{0,256}):([^\s]{1,256})@(?=[^\s@/?#]{0,253}(?:[/?#\s]|$))/gi, '$1$2:<redacted>@'],
     [/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, 'https://hooks.slack.com/services/<redacted>'],
     [/https:\/\/(?:canary\.|ptb\.)?discord(?:app)?\.com\/api\/webhooks\/[0-9]+\/[A-Za-z0-9_-]+/g, '<redacted:discord-webhook>'],
-    [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, '<redacted:private-key>'],
-    [/\b[A-Za-z0-9._%+-]+@([A-Za-z0-9.-]+\.[A-Za-z]{2,})\b/g, '<redacted:email>@$1'],
-    [/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g, '<redacted:jwt>'],
-    [/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{12,}/gi, '$1 <redacted>'],
-    [/([?&](?:token|key|api_key|apikey|secret|password|access_token)=)[^&\s"']+/gi, '$1<redacted>'],
-    [/\b(api[_-]?key|token|secret|password|passwd|authorization)\b(\s*[=:]\s*)[^\s&"']+/gi, '$1$2<redacted>'],
+    // Local part and domain bounded as RFC 5321 bounds them: unbounded, a long run of `a.a.a…` without an `@` is quadratic.
+    [/\b[A-Za-z0-9._%+-]{1,64}@([A-Za-z0-9.-]{1,253}\.[A-Za-z]{2,63})\b/g, '<redacted:email>@$1'],
+    // Bounded: every `eyJ` in one long run is a start that scans it again. A longer JWT is caught as entropy below.
+    [/\beyJ[A-Za-z0-9_-]{8,1024}\.[A-Za-z0-9_-]{8,8192}\.[A-Za-z0-9_-]{8,}/g, '<redacted:jwt>'],
+    [/\b(Bearer|Basic)\s{1,16}[A-Za-z0-9._~+/=-]{12,}/gi, '$1 <redacted>'],
+    [/([?&](?:token|key|api_key|apikey|secret|password|access_token|sig)=)[^&\s"']+/gi, '$1<redacted>'],
 ];
 const HOME_DIR_PATTERNS = [
     [/\/Users\/[^/\s"')\]]+/g, '~'],
     [/\/home\/[^/\s"')\]]+/g, '~'],
     [/[A-Za-z]:\\Users\\[^\\\s"')\]]+/gi, '~'],
 ];
-/** Conservative on purpose (48+): digests and minified names are shorter. */
+/** Conservative on purpose (48+): digests and minified names are shorter, and a git sha (40 hex) stays. */
 const HIGH_ENTROPY = /\b[A-Za-z0-9_-]{48,}\b/g;
+const PEM_BEGIN = /-----BEGIN [A-Z ]{0,40}PRIVATE KEY-----/g;
+const PEM_END = /-----END [A-Z ]{0,40}PRIVATE KEY-----/g;
+/** Every PEM private key; one left unterminated is cut to the end. A scan, not `BEGIN[\s\S]*?END`: many BEGINs without an END made that quadratic. */
+function dropPrivateKeys(s) {
+    if (!s.includes('PRIVATE KEY-----'))
+        return s;
+    let out = '';
+    let from = 0;
+    PEM_BEGIN.lastIndex = 0;
+    for (let m = PEM_BEGIN.exec(s); m; m = PEM_BEGIN.exec(s)) {
+        PEM_END.lastIndex = m.index + m[0].length;
+        const end = PEM_END.exec(s);
+        out += `${s.slice(from, m.index)}<redacted:private-key>`;
+        if (!end)
+            return out;
+        from = PEM_BEGIN.lastIndex = end.index + end[0].length;
+    }
+    return out + s.slice(from);
+}
 /** Scrub one string. Never throws; returns `''` for a non-string. */
 export function redactString(input) {
     if (typeof input !== 'string' || input.length === 0)
         return '';
     let out = input;
     try {
+        out = dropPrivateKeys(out);
         for (const [re, replacement] of SECRET_PATTERNS)
             out = out.replace(re, replacement);
+        out = out.replace(KEY_VALUE, (_w, name, sep, quote, _q, scheme) => quote ? `${name}${sep}${quote}<redacted>` : `${name}${sep}${scheme ?? ''}<redacted>`);
         for (const [re, replacement] of HOME_DIR_PATTERNS)
             out = out.replace(re, replacement);
         out = out.replace(HIGH_ENTROPY, '<redacted:entropy>');

package/dist/core/sleep.d.ts

@@ -19,11 +19,9 @@ export interface SleepDeps extends ConsolidateDeps {
             last: number;
         } & SleepReport): Promise<void>;
     };
-    /** A backfill's night: fresh items looked at this many at once (1), relate asks in flight (RELATE_IN_FLIGHT), and
-     *  pairs a night asks about (PAIRS_PER_SLEEP). */
+    /** A backfill's night: fresh items looked at this many at once (1), and relate asks in flight (RELATE_IN_FLIGHT). */
     scanAtOnce?: number;
     relateInFlight?: number;
-    pairsPerSleep?: number;
     /** Answers applied at once (1, in order). */
     applyAtOnce?: number;
 }
@@ -35,6 +33,9 @@ export interface SleepReport {
     archived: number;
     profiled: number;
     unprofiled: number;
+    /** Set when the language model did not answer (none configured, unreachable, busy): why. What it was to read waits for
+     *  the next night. */
+    paused?: string;
 }
 export declare class Sleep {
     private readonly d;

package/dist/core/sleep.js

@@ -16,8 +16,9 @@
  */
 import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { dirname } from 'node:path';
-import { Consolidator, RELATED, relateBatches, SAME } from './consolidate.js';
+import { Consolidator, learnedAt, RELATED, relateBatches, SAME } from './consolidate.js';
 import { docText } from './embedder.js';
+import { allOrAbort } from '../llm.js';
 import { pointId } from './qdrant-client.js';
 import { writeProfiles } from './profiles.js';
 import { writeLtm } from './writer.js';
@@ -61,11 +62,13 @@ export class Sleep {
         // Observations are add-only (consolidate.ts observe): two events are never one, and an older one stays, dated. A
         // profile is written again from its items (4), never related to anything.
         const knowledge = (m) => m.status === 'active' && m.kind !== 'episode' && m.kind !== 'note' && m.kind !== 'observation' && m.kind !== 'profile';
-        const fresh = [...this.d.ltm.entries()].filter(([, m]) => knowledge(m) && m.ts >= since).sort((a, b) => a[1].ts - b[1].ts).map(([uid]) => uid);
+        // Learned since: by when it was written, not by its source's time — history imported after a sleep is new to memory.
+        const fresh = [...this.d.ltm.entries()].filter(([, m]) => knowledge(m) && learnedAt(m) >= since)
+            .sort((a, b) => learnedAt(a[1]) - learnedAt(b[1])).map(([uid]) => uid);
+        /** `at`: when the fresh item of the pair was learned — where the next night starts when this pair is not asked. */
         const pairs = [];
         const seen = new Set();
         let resumeAt = null;
-        const cap = this.d.pairsPerSleep ?? PAIRS_PER_SLEEP;
         const look = async (it) => {
             // The key vector the index already holds; embedded again only when it is not there.
             const held = (await this.d.client.retrieve([pointId(it.uid, 0)], true))[0]?.vector;
@@ -84,8 +87,8 @@ export class Sleep {
             const group = fresh.slice(i, i + atOnce).map((uid) => this.d.ltm.get(uid)).filter((it) => !!it && it.status === 'active');
             if (!group.length)
                 continue;
-            if (pairs.length >= cap) {
-                resumeAt = group[0].ts;
+            if (pairs.length >= PAIRS_PER_SLEEP) {
+                resumeAt = learnedAt(group[0]);
                 break;
             }
             r.looked += group.length;
@@ -98,7 +101,8 @@ export class Sleep {
                         continue;
                     seen.add(key);
                     // The newer of the two is NEW, the older KNOWN: a contradiction retires the older.
-                    pairs.push(it.ts >= known.ts ? { it, known, score: n.score } : { it: known, known: it, score: n.score });
+                    const at = learnedAt(it);
+                    pairs.push(it.ts >= known.ts ? { it, known, score: n.score, at } : { it: known, known: it, score: n.score, at });
                 }
             }
         }
@@ -112,13 +116,15 @@ export class Sleep {
         const inFlight = Math.max(1, this.d.relateInFlight ?? RELATE_IN_FLIGHT);
         let next = 0;
         let unanswered = false;
-        await Promise.all(Array.from({ length: Math.min(inFlight, batches.length) }, async () => {
-            while (!unanswered && next < batches.length && !this.d.aborted()) {
+        // A lane that throws stops the others, their asks too (allOrAbort).
+        await allOrAbort(Array.from({ length: Math.min(inFlight, batches.length) }, () => async (signal) => {
+            while (!unanswered && next < batches.length && !this.d.aborted() && !signal.aborted) {
                 const b = batches[next++];
-                const got = await worker.relate(b);
-                if (!got) {
+                const got = await worker.relate(b, signal);
+                if (!(got instanceof Map)) {
                     unanswered = true;
-                    resumeAt = Math.min(resumeAt ?? Infinity, ...b.map((p) => p.it.ts));
+                    r.paused ??= got.paused;
+                    resumeAt = Math.min(resumeAt ?? Infinity, ...b.map((p) => p.at));
                     return;
                 }
                 b.forEach((p, n) => { const d = got.get(`p${n + 1}`); if (d)
@@ -150,8 +156,9 @@ export class Sleep {
             r.merged++;
         });
         let at = 0;
-        await Promise.all(Array.from({ length: Math.max(1, this.d.applyAtOnce ?? 1) }, async () => {
-            while (at < pairs.length && !this.d.aborted())
+        // An answer that fails to apply stops the other lanes at their next pair.
+        await allOrAbort(Array.from({ length: Math.max(1, this.d.applyAtOnce ?? 1) }, () => async (signal) => {
+            while (at < pairs.length && !this.d.aborted() && !signal.aborted)
                 await apply(pairs[at++]);
         }));
         if (this.d.aborted())
@@ -162,10 +169,12 @@ export class Sleep {
             if (m.status !== 'active' || m.kind === 'note' || !HALF_LIFE[m.kind])
                 continue;
             const use = this.d.usage.of(uid);
-            if (priority(m, use, now) >= ARCHIVE_BELOW || now - Math.max(m.ts, use.lastUsed) < ARCHIVE_UNUSED_MS)
+            // Unused for half a year since it was learned, not since its source was written: imported history is not archived
+            // the night it arrives.
+            if (priority(m, use, now) >= ARCHIVE_BELOW || now - Math.max(m.ts, learnedAt(m), use.lastUsed) < ARCHIVE_UNUSED_MS)
                 continue;
             const it = this.d.ltm.get(uid);
-            if (it)
+            if (it && now - learnedAt(it) >= ARCHIVE_UNUSED_MS)
                 retire.push({ ...it, status: 'archived' });
         }
         for (let i = 0; i < retire.length && !this.d.aborted(); i += 50) {
@@ -180,8 +189,10 @@ export class Sleep {
         const p = await writeProfiles(this.d);
         r.profiled = p.profiled;
         r.unprofiled = p.unprofiled;
-        if (p.paused)
+        if (p.paused) {
+            r.paused ??= p.paused;
             this.d.log(`memory sleep: profiles wait for the next night (${p.paused})`);
+        }
         this.d.ltm.compact();
         // A night that stopped early (the pair cap, the fast model away) starts the next one where it stopped.
         if (this.d.sleepState) {

package/dist/core/store.d.ts

@@ -47,6 +47,8 @@ export interface MemoryItem {
     /** Last learned or refreshed. */
     ts: number;
     firstTs: number;
+    /** When it was written to long-term memory (`ts` is its source's time); absent in older logs. */
+    learnedAt?: number;
     /** The key: a content title and a ~150-word summary (what is embedded). */
     title: string;
     summary: string;
@@ -137,13 +139,16 @@ export declare class JsonlLog<T extends {
     entries(): Generator<[string, M]>;
     load(): Promise<void>;
     get(uid: string): T | null;
+    private read;
     put(recs: T[]): void;
     remove(uids: string[]): void;
     /** Every live record, in file order. */
     all(): Generator<T>;
     /** A file is written when `put` returns. */
     flush(): Promise<void>;
-    /** Rewrite the log without replaced lines once it is twice what is live. */
+    /** Rewrite the log without replaced lines once it is twice what is live. Record by record into a file beside it, made
+     *  durable and then renamed over it: a log of any size, and a crash leaves the old one whole. A record that cannot be
+     *  read is never dropped by a rewrite: it throws, and the log stays as it is. */
     compact(): void;
 }
 export type StmMeta = {
@@ -164,6 +169,7 @@ export type LtmMeta = {
     importance: number;
     cued: boolean;
     mapped: boolean;
+    learnedAt?: number;
 };
 export declare const stmMetaOf: (r: StmEntry) => StmMeta;
 export declare const ltmMetaOf: (r: MemoryItem) => LtmMeta;

package/dist/core/store.js

@@ -17,15 +17,16 @@
  * waits for it (a file write is done when `put` returns).
  */
 import { createHash } from 'node:crypto';
-import { appendFileSync, closeSync, createReadStream, existsSync, mkdirSync, openSync, readFileSync, readSync, renameSync, statSync, writeFileSync } from 'node:fs';
+import { appendFileSync, closeSync, createReadStream, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, statSync, truncateSync, writeFileSync, writeSync } from 'node:fs';
 import { dirname } from 'node:path';
-import { createInterface } from 'node:readline';
 export function fingerprint(...parts) {
     const h = createHash('sha1');
     for (const p of parts)
         h.update(typeof p === 'string' || p == null ? (p ?? '') : p.join('\u0001')).update('\u0000');
     return h.digest('hex').slice(0, 16);
 }
+/** What a compaction holds before it writes. */
+const WRITE_BYTES = 1 << 20;
 /** An append-only JSONL log, read back by byte offset. `meta` is what stays in memory for each record. */
 export class JsonlLog {
     path;
@@ -47,23 +48,51 @@ export class JsonlLog {
         this.end = 0;
         if (!existsSync(this.path))
             return;
+        // By bytes, not decoded lines: a torn multi-byte character must not shift every offset after it.
         let at = 0;
-        for await (const line of createInterface({ input: createReadStream(this.path, { encoding: 'utf8' }), crlfDelay: Infinity })) {
-            const len = Buffer.byteLength(line, 'utf8') + 1;
-            if (line) {
-                try {
-                    const rec = JSON.parse(line);
-                    if (rec.removed)
-                        this.live.delete(rec.uid);
-                    else
-                        this.live.set(rec.uid, { at, len, meta: this.project(rec) });
-                    this.lines++;
-                }
-                catch { /* a torn last line */ }
+        let pending = [];
+        const take = (line, len) => {
+            if (!line.length)
+                return true;
+            try {
+                const rec = JSON.parse(line.toString('utf8'));
+                if (rec.removed)
+                    this.live.delete(rec.uid);
+                else
+                    this.live.set(rec.uid, { at, len, meta: this.project(rec) });
+                this.lines++;
+                return true;
+            }
+            catch {
+                return false;
             }
-            at += len;
+        };
+        for await (const chunk of createReadStream(this.path)) {
+            let from = 0;
+            for (let nl = chunk.indexOf(10); nl >= 0; nl = chunk.indexOf(10, from)) {
+                const line = Buffer.concat([...pending, chunk.subarray(from, nl)]);
+                pending = [];
+                take(line, line.length + 1);
+                at += line.length + 1;
+                from = nl + 1;
+            }
+            if (from < chunk.length)
+                pending.push(chunk.subarray(from));
         }
-        this.end = at;
+        // A crash cut the last line short of its '\n': whole but for it, it is ended; torn, it goes. Either way what is
+        // added next starts on a line of its own, at the offset the index gives it.
+        const tail = Buffer.concat(pending);
+        if (tail.length) {
+            try {
+                // Its length without the '\n' it is given: readable whether or not that write lands.
+                if (take(tail, tail.length))
+                    appendFileSync(this.path, '\n');
+                else
+                    truncateSync(this.path, at);
+            }
+            catch { /* read-only: nothing is added after it either */ }
+        }
+        this.end = statSync(this.path).size;
     }
     get(uid) {
         const s = this.live.get(uid);
@@ -71,9 +100,7 @@ export class JsonlLog {
             return null;
         const fd = openSync(this.path, 'r');
         try {
-            const buf = Buffer.alloc(s.len);
-            readSync(fd, buf, 0, s.len, s.at);
-            return JSON.parse(buf.toString('utf8').trimEnd());
+            return this.read(fd, s);
         }
         catch {
             return null;
@@ -82,6 +109,11 @@ export class JsonlLog {
             closeSync(fd);
         }
     }
+    read(fd, s) {
+        const buf = Buffer.alloc(s.len);
+        readSync(fd, buf, 0, s.len, s.at);
+        return JSON.parse(buf.toString('utf8').trimEnd());
+    }
     put(recs) {
         if (!recs.length)
             return;
@@ -125,28 +157,64 @@ export class JsonlLog {
     }
     /** A file is written when `put` returns. */
     async flush() { }
-    /** Rewrite the log without replaced lines once it is twice what is live. */
+    /** Rewrite the log without replaced lines once it is twice what is live. Record by record into a file beside it, made
+     *  durable and then renamed over it: a log of any size, and a crash leaves the old one whole. A record that cannot be
+     *  read is never dropped by a rewrite: it throws, and the log stays as it is. */
     compact() {
         if (this.lines < 2 * this.live.size + 1000)
             return;
-        const recs = [...this.all()];
+        const order = [...this.live.entries()].sort((a, b) => a[1].at - b[1].at);
         const tmp = `${this.path}.compact`;
-        writeFileSync(tmp, recs.map((r) => `${JSON.stringify(r)}\n`).join(''), { mode: 0o600 });
-        renameSync(tmp, this.path);
-        // Offsets again, from the file just written.
-        this.live.clear();
-        let at = 0;
-        for (const r of recs) {
-            const len = Buffer.byteLength(`${JSON.stringify(r)}\n`, 'utf8');
-            this.live.set(r.uid, { at, len, meta: this.project(r) });
-            at += len;
+        const next = new Map();
+        const src = openSync(this.path, 'r');
+        let out = null;
+        try {
+            out = openSync(tmp, 'w', 0o600);
+            let at = 0;
+            let parts = [];
+            let size = 0;
+            const write = () => { writeSync(out, Buffer.concat(parts)); parts = []; size = 0; };
+            for (const [uid, s] of order) {
+                let rec = null;
+                try {
+                    rec = this.read(src, s);
+                }
+                catch { /* below */ }
+                if (rec?.uid !== uid)
+                    throw new Error(`${this.path}: record ${uid} cannot be read at ${s.at}; not compacted`);
+                const b = Buffer.from(`${JSON.stringify(rec)}\n`, 'utf8');
+                next.set(uid, { at, len: b.length, meta: s.meta });
+                parts.push(b);
+                size += b.length;
+                at += b.length;
+                if (size >= WRITE_BYTES)
+                    write();
+            }
+            if (size)
+                write();
+            fsyncSync(out);
+            closeSync(out);
+            out = null;
+            renameSync(tmp, this.path);
+            this.live.clear();
+            for (const [uid, s] of next)
+                this.live.set(uid, s);
+            this.end = at;
+            this.lines = next.size;
+        }
+        catch (e) {
+            if (out !== null)
+                closeSync(out);
+            rmSync(tmp, { force: true });
+            throw e;
+        }
+        finally {
+            closeSync(src);
         }
-        this.end = at;
-        this.lines = recs.length;
     }
 }
 export const stmMetaOf = (r) => ({ fp: r.fingerprint, done: r.done, ts: r.ts, session: r.session, project: r.project, synthetic: r.synthetic });
-export const ltmMetaOf = (r) => ({ fp: r.fingerprint, status: r.status, kind: r.kind, scope: r.scope, project: r.project, ts: r.ts, importance: r.importance, cued: !!r.cues?.length, mapped: !!r.concepts?.length });
+export const ltmMetaOf = (r) => ({ fp: r.fingerprint, status: r.status, kind: r.kind, scope: r.scope, project: r.project, ts: r.ts, importance: r.importance, cued: !!r.cues?.length, mapped: !!r.concepts?.length, learnedAt: r.learnedAt });
 export const stmLog = (path) => new JsonlLog(path, stmMetaOf);
 export const ltmLog = (path) => new JsonlLog(path, ltmMetaOf);
 /** How often each long-term item was shown with a turn and opened by an agent. Saved at most once a minute. */

package/dist/core/text.d.ts

@@ -12,6 +12,8 @@ export declare function fold(text: string): string;
 /** Words for the keyword channel, in any language: folded (fold()); identifiers kept whole AND split at `_ . - /`; in
  *  scripts without spaces (Chinese, Japanese, Thai…) the segmenter's words. */
 export declare function tokens(text: string): string[];
+/** Whitespace runs as one space, the ends trimmed. */
+export declare const flat: (s: string) => string;
 /** A text's first `n` words, as written (whitespace flattened). */
 export declare function firstWords(text: string, n: number): string;
 /** 32-bit FNV-1a: a term's index in the sparse vector. */
@@ -25,7 +27,9 @@ export declare const CHILD_STRIDE = 72;
 /** ~96-word windows of a body, a quarter overlapping, all of it — searched, never shown: the item is what comes back.
  *  Words as the segmenter finds them, so a body in a script without spaces is windowed too. */
 export declare function children(text: string, words?: number, stride?: number): string[];
-export declare const cut: (s: string, n: number) => string;
+/** The first `n` UTF-16 units and an ellipsis — never ending on half a surrogate pair (an emoji cut in two is not valid
+ *  UTF-8, and Qdrant refuses the whole write over it). */
+export declare function cut(s: string, n: number): string;
 /** One spelling per concept, the graph's key: folded like a word (case, accents, compatibility forms), inner spaces
- *  collapsed, stray punctuation off the ends — "Gateway.conf " and "gateway.conf" are one node. */
+ *  collapsed, stray punctuation off the ends — "Proxy.conf " and "proxy.conf" are one node. */
 export declare function conceptKey(name: string): string;

package/dist/core/text.js

@@ -1,5 +1,7 @@
 const K1 = 1.2;
 const B = 0.75;
+/** Longer than this is not a word a person would type (a hash, a base64 blob). */
+const MAX_TOKEN = 60;
 /** Scripts written without spaces between words: there the ICU word segmenter finds the words. */
 const UNSPACED = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u;
 const WORDS = new Intl.Segmenter(undefined, { granularity: 'word' });
@@ -22,12 +24,14 @@ export function tokens(text) {
                     out.push(seg.segment);
             continue;
         }
-        if (w.length < 2 || w.length > 60)
+        if (w.length < 2)
             continue;
-        out.push(w);
+        // The length cap is per word: a path too long to be one keyword still gives its parts.
+        if (w.length <= MAX_TOKEN)
+            out.push(w);
         if (/[_.\-/]/.test(w))
             for (const p of w.split(/[_.\-/]+/))
-                if (p.length >= 2 && p !== w)
+                if (p.length >= 2 && p.length <= MAX_TOKEN && p !== w)
                     out.push(p);
     }
     return out;
@@ -40,7 +44,8 @@ function wordSpans(text) {
             at.push([seg.index, seg.index + seg.segment.length]);
     return at;
 }
-const flat = (s) => s.replace(/\s+/g, ' ').trim();
+/** Whitespace runs as one space, the ends trimmed. */
+export const flat = (s) => s.replace(/\s+/g, ' ').trim();
 /** A text's first `n` words, as written (whitespace flattened). */
 export function firstWords(text, n) {
     const at = wordSpans(text);
@@ -93,9 +98,17 @@ export function children(text, words = CHILD_WORDS, stride = CHILD_STRIDE) {
         out.push(flat(text.slice(at[Math.max(0, at.length - words)][0], at[at.length - 1][1])));
     return out;
 }
-export const cut = (s, n) => (s.length > n ? `${s.slice(0, n)}…` : s);
+/** The first `n` UTF-16 units and an ellipsis — never ending on half a surrogate pair (an emoji cut in two is not valid
+ *  UTF-8, and Qdrant refuses the whole write over it). */
+export function cut(s, n) {
+    if (s.length <= n)
+        return s;
+    const high = n > 0 && s.charCodeAt(n - 1) >= 0xd800 && s.charCodeAt(n - 1) <= 0xdbff;
+    const end = high ? n - 1 : n;
+    return `${s.slice(0, end)}…`;
+}
 /** One spelling per concept, the graph's key: folded like a word (case, accents, compatibility forms), inner spaces
- *  collapsed, stray punctuation off the ends — "Gateway.conf " and "gateway.conf" are one node. */
+ *  collapsed, stray punctuation off the ends — "Proxy.conf " and "proxy.conf" are one node. */
 export function conceptKey(name) {
     return fold(name).replace(/\s+/g, ' ').replace(/^[\s.,:;"'`()[\]]+|[\s.,:;"'`()[\]]+$/g, '');
 }

package/dist/core/words.js

@@ -12,7 +12,7 @@
  * numbers with those signs, over √dims. Kept by the text's hash, so an unchanged text is never encoded twice.
  */
 import { createHash } from 'node:crypto';
-import { appendFile, mkdir, readFile } from 'node:fs/promises';
+import { appendFile, mkdir, readFile, truncate } from 'node:fs/promises';
 import { dirname } from 'node:path';
 /** Words kept of a memory (64 lost most of the gain). */
 export const WORD_LIMIT = 128;
@@ -70,7 +70,8 @@ export class WordStore {
         const b = await readFile(this.file).catch(() => null);
         if (!b)
             return;
-        for (let at = 0; at + KEY_BYTES + 4 <= b.length;) {
+        let at = 0;
+        while (at + KEY_BYTES + 4 <= b.length) {
             const key = b.subarray(at, at + KEY_BYTES).toString('hex');
             const words = b.readUInt16LE(at + KEY_BYTES);
             const dims = b.readUInt16LE(at + KEY_BYTES + 2);
@@ -81,6 +82,9 @@ export class WordStore {
             this.byKey.set(key, { words, dims, bits: new Uint8Array(b.subarray(start, start + size)) });
             at = start + size;
         }
+        // The cut-off entry goes, or every entry appended after it would be read as part of it and never load again.
+        if (at < b.length)
+            await truncate(this.file, at).catch(() => { });
     }
     async put(entries) {
         const parts = [];

package/dist/core/writer.d.ts

@@ -18,7 +18,8 @@ export declare function writeStm(d: WriterDeps, entries: StmEntry[]): Promise<vo
  *  either there are, so the two never meet. */
 export declare const cuePoint: (k: number) => number;
 /** The time a date stands for, as the first and last millisecond: a day, a month or a year (an observation's
- *  `eventAt`, or a date an agent searches by) — null for anything else. */
+ *  `eventAt`, or a date an agent searches by) — null for anything else, a month or day that does not exist too
+ *  (2024-02-30 is not 1 March). */
 export declare function dateSpan(d: string | undefined): {
     from: number;
     to: number;

package/dist/core/writer.js

@@ -36,11 +36,12 @@ export async function writeStm(d, entries) {
     const points = [];
     let at = 0;
     for (const { e, body, kids } of plan) {
-        const common = { item: stmItem(e.uid), tier: 'stm', status: 'active', kind: 'episode', scope: 'project', project: e.project, session: e.session, ts: e.ts, synthetic: e.synthetic };
+        // The record's kind (`engine`) on every point: a kinds filter must find a record by any of its chunks.
+        const common = { item: stmItem(e.uid), tier: 'stm', status: 'active', kind: 'episode', engine: e.engine, scope: 'project', project: e.project, session: e.session, ts: e.ts, synthetic: e.synthetic };
         points.push({
             id: pointId(stmItem(e.uid), 0),
             vector: { dense: Array.from(vecs[at]), bm25: docSparse(`${stmTitle(e)}\n${body}`) },
-            payload: { ...common, part: 'key', engine: e.engine, cwd: e.cwd ?? '', where: e.title, title: stmTitle(e), summary: rawKey(e), body, files: e.files, importance: 0, ...(e.record ? { record: e.record } : {}) },
+            payload: { ...common, part: 'key', cwd: e.cwd ?? '', where: e.title, title: stmTitle(e), summary: rawKey(e), body, files: e.files, importance: 0, ...(e.record ? { record: e.record } : {}) },
         });
         kids.forEach((_, k) => points.push({ id: pointId(stmItem(e.uid), k + 1), vector: { dense: Array.from(vecs[at + 1 + k]) }, payload: { ...common, part: 'child' } }));
         at += 1 + kids.length;
@@ -53,12 +54,17 @@ export async function writeStm(d, entries) {
  *  either there are, so the two never meet. */
 export const cuePoint = (k) => -(k + 1);
 /** The time a date stands for, as the first and last millisecond: a day, a month or a year (an observation's
- *  `eventAt`, or a date an agent searches by) — null for anything else. */
+ *  `eventAt`, or a date an agent searches by) — null for anything else, a month or day that does not exist too
+ *  (2024-02-30 is not 1 March). */
 export function dateSpan(d) {
     const m = /^(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?$/.exec(d ?? '');
     if (!m)
         return null;
     const [y, mo, dd] = [Number(m[1]), m[2] ? Number(m[2]) - 1 : null, m[3] ? Number(m[3]) : null];
+    if (mo !== null && (mo < 0 || mo > 11))
+        return null;
+    if (mo !== null && dd !== null && (dd < 1 || new Date(Date.UTC(y, mo, dd)).getUTCMonth() !== mo))
+        return null;
     if (mo === null)
         return { from: Date.UTC(y, 0, 1), to: Date.UTC(y + 1, 0, 1) - 1 };
     if (dd === null)