openmausbot 0.1.89 → 0.1.90

package/dist-server/server/memory-tidy.js

@@ -0,0 +1,144 @@
+// The nightly tidy-up of MEMORY.md, for a bot with Memory upkeep on. Three
+// steps, in order, each leaving a trace:
+//   1. expired  — a live entry past its `until` day moves to memory/archive.md
+//   2. duplicate — of two entries with the same fact identity (signs,
+//      symbols and case kept), the newest stays and older copies go
+//   3. contradicted — one strict-JSON model call over what is left names
+//      pairs that cannot both be true; the loser is struck through with a
+//      date, never deleted, and at most floor(live × 0.2) of them change in
+//      one pass — none at all below five entries.
+// Hand-written lines without a date are never touched. Pure here: the
+// caller reads, calls the model, writes and journals.
+import { factIdentity, isExpired, parseMemoryEntries } from "./memory-entries.js";
+export const CONTRADICTION_SHARE = 0.2;
+export const TIDY_MARKER = "You are the TIDY step of a memory system";
+const PROMPT_ENTRIES = 200;
+/** The newest-wins duplicates among live, unexpired entries. */
+function duplicatesOf(live) {
+    const byKey = new Map();
+    for (const entry of live) {
+        const key = factIdentity(entry.body);
+        if (!key)
+            continue;
+        byKey.set(key, [...(byKey.get(key) ?? []), entry]);
+    }
+    const older = [];
+    for (const group of byKey.values()) {
+        if (group.length < 2)
+            continue;
+        // newest date first; on the same day the later line wins
+        const sorted = [...group].sort((x, y) => (x.date === y.date ? y.line - x.line : x.date < y.date ? 1 : -1));
+        older.push(...sorted.slice(1));
+    }
+    return older;
+}
+/** What survives steps 1 and 2: the list the contradiction call is asked
+ * about, so two copies of a fact beside its correction never confuse it. */
+export function contradictionCandidates(text, today) {
+    const live = parseMemoryEntries(text).filter((e) => !e.struck && !isExpired(e, today));
+    const dupes = new Set(duplicatesOf(live).map((e) => e.line));
+    return live.filter((e) => !dupes.has(e.line));
+}
+/** How many contradictions one pass may act on. */
+export function contradictionBudget(liveEntries) {
+    return Math.floor(liveEntries * CONTRADICTION_SHARE);
+}
+export function planTidy(text, today, contradictions = []) {
+    const entries = parseMemoryEntries(text);
+    const expired = entries.filter((e) => isExpired(e, today));
+    const live = entries.filter((e) => !e.struck && !isExpired(e, today));
+    const duplicates = duplicatesOf(live);
+    const candidates = contradictionCandidates(text, today);
+    const budget = contradictionBudget(live.length);
+    const touched = new Set();
+    const superseded = [];
+    let deferred = 0;
+    for (const pair of contradictions) {
+        const a = candidates[pair.a];
+        const b = candidates[pair.b];
+        if (!a || !b || a.line === b.line)
+            continue;
+        const [winner, loser] = pair.keep === "a" ? [a, b] : [b, a];
+        if (touched.has(winner.line) || touched.has(loser.line))
+            continue;
+        if (superseded.length >= budget) {
+            deferred += 1;
+            continue;
+        }
+        superseded.push({ loser, winner, ...(pair.remainder ? { remainder: pair.remainder } : {}) });
+        touched.add(loser.line);
+        touched.add(winner.line);
+    }
+    return { expired, duplicates, superseded, deferred };
+}
+export function planChanges(plan) {
+    return plan.expired.length + plan.duplicates.length + plan.superseded.length;
+}
+/** MEMORY.md after the plan, and the lines the archive gains. */
+export function applyTidy(text, plan, today) {
+    const lines = text.split("\n");
+    const drop = new Set([...plan.expired, ...plan.duplicates].map((e) => e.line));
+    for (const { loser, remainder } of plan.superseded) {
+        const struck = `${loser.prefix}~~${loser.body}~~ · superseded ${today}`;
+        // the still-true part of a mixed line lives on as its own dated entry
+        lines[loser.line] = remainder ? `${struck}\n- ${today} · from tidy-up · ${remainder}` : struck;
+    }
+    const archived = plan.expired.map((e) => `${e.raw} · expired ${today}`);
+    return { text: lines.filter((_, index) => !drop.has(index)).join("\n"), archived };
+}
+export function contradictionPrompt(candidates) {
+    const listed = candidates.slice(0, PROMPT_ENTRIES).map((e, index) => `[${index}] (${e.date}) ${e.body}`).join("\n");
+    return [
+        `${TIDY_MARKER}. You have no tools. Read only the numbered notebook lines below, each with the date it was written.`,
+        "",
+        listed || "(nothing)",
+        "",
+        "Find pairs of lines that cannot both be true now: a direct contradiction about the same thing (a changed number, a new city, a reversed preference) — not a refinement, not two different topics, not two facts that can both hold.",
+        "Numbers, signs and symbols matter: \"balance is -10\" and \"balance is 10\" contradict; \"uses C\" and \"uses C++\" may both be true.",
+        "For each pair say which to keep: the later line unless the earlier one is clearly the correction.",
+        "When the line that loses also states other facts that are still true, give them as remainder, one sentence, worded as in the line; otherwise omit remainder.",
+        "Answer with one JSON object and nothing else:",
+        '{"pairs": [{"a": <index>, "b": <index>, "keep": "a"|"b", "remainder": "optional"}]}',
+        'An empty list {"pairs": []} is the normal answer. Never invent a contradiction.',
+    ].join("\n");
+}
+export function parseContradictions(text, count) {
+    const trimmed = text.trim();
+    const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(trimmed);
+    const candidate = (fenced ? fenced[1] : trimmed).trim();
+    const start = candidate.indexOf("{");
+    const end = candidate.lastIndexOf("}");
+    if (start < 0 || end <= start)
+        return [];
+    let parsed;
+    try {
+        parsed = JSON.parse(candidate.slice(start, end + 1));
+    }
+    catch {
+        return [];
+    }
+    if (!Array.isArray(parsed.pairs))
+        return [];
+    const out = [];
+    const seen = new Set();
+    for (const pair of parsed.pairs) {
+        if (!pair || typeof pair !== "object")
+            continue;
+        const { a, b, keep, remainder } = pair;
+        if (!Number.isInteger(a) || !Number.isInteger(b) || a === b)
+            continue;
+        const ia = a;
+        const ib = b;
+        if (ia < 0 || ib < 0 || ia >= count || ib >= count)
+            continue;
+        if (keep !== "a" && keep !== "b")
+            continue;
+        const key = `${Math.min(ia, ib)}:${Math.max(ia, ib)}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        const rest = typeof remainder === "string" ? remainder.replace(/\s+/g, " ").replace(/~~/g, "").trim().slice(0, 300) : "";
+        out.push({ a: ia, b: ib, keep, ...(rest ? { remainder: rest } : {}) });
+    }
+    return out;
+}

package/dist-server/server/memory-topics.js

@@ -0,0 +1,169 @@
+// The topic index: one line per memory/<topic>.md, so a bot knows what its
+// longer notes cover without a pointer in MEMORY.md. A topic may begin with
+// YAML frontmatter (`title`, `description`, `aliases`); aliases are the other
+// words someone might search with, which matters because every lookup here
+// is word matching, not meaning. Only the head of each file is read.
+import { closeSync, openSync, readSync } from "node:fs";
+/** How much of a topic file is read to find its frontmatter. */
+const HEAD_BYTES = 2_048;
+export const TOPIC_INDEX_MAX_TOPICS = 40;
+export const TOPIC_INDEX_MAX_CHARS = 2_000;
+function unquote(value) {
+    const v = value.trim();
+    if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'")))
+        return v.slice(1, -1).trim();
+    return v;
+}
+/** The frontmatter fields the index uses. Anything else, or no frontmatter,
+ * reads as an empty header — a hand-written topic is still listed by name. */
+export function parseTopicHeader(text) {
+    const header = { aliases: [] };
+    // People write a "# Title" first and the block under it (the panel's own
+    // new-topic text used to), so a heading and blank lines may come before the
+    // block; the heading is the title when the block names none.
+    const lead = /^(?:\s*\r?\n|#{1,6}[ \t]+[^\r\n]*\r?\n)*/.exec(text)?.[0] ?? "";
+    const heading = /^#{1,6}[ \t]+([^\r\n]+)/m.exec(lead)?.[1]?.trim();
+    const m = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(text.slice(lead.length));
+    if (!m) {
+        if (heading)
+            header.title = heading.slice(0, 160);
+        return header;
+    }
+    const lines = m[1].split(/\r?\n/);
+    for (let i = 0; i < lines.length; i += 1) {
+        const field = /^(title|description|aliases|tags)\s*:\s*(.*)$/.exec(lines[i]);
+        if (!field)
+            continue;
+        const [, key, value] = field;
+        if (key === "title" || key === "description") {
+            if (value.trim())
+                header[key] = unquote(value).slice(0, 160);
+            continue;
+        }
+        // aliases (and tags, which people use the same way): [a, b] or a list below
+        const items = [];
+        if (value.trim().startsWith("[")) {
+            items.push(...value.trim().replace(/^\[|\]$/g, "").split(","));
+        }
+        else if (value.trim()) {
+            items.push(...value.split(","));
+        }
+        else {
+            while (i + 1 < lines.length && /^\s*-\s+/.test(lines[i + 1]))
+                items.push(lines[(i += 1)].replace(/^\s*-\s+/, ""));
+        }
+        for (const item of items.map(unquote).filter(Boolean)) {
+            if (!header.aliases.includes(item))
+                header.aliases.push(item.slice(0, 40));
+        }
+    }
+    header.aliases = header.aliases.slice(0, 12);
+    if (!header.title && heading)
+        header.title = heading.slice(0, 160);
+    return header;
+}
+/** The words a topic answers to: its file name, title, description and
+ * aliases, lower-cased, for matching a message against without a search. */
+export function topicWords(name, header) {
+    const text = [name.replace(/\.md$/i, "").replace(/[-_.]+/g, " "), header.title ?? "", header.description ?? "", ...header.aliases].join(" ");
+    return [...new Set(text.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((word) => word.length >= 3))];
+}
+/** A topic's text without its header block, for a recalled passage. */
+export function topicBody(text) {
+    const lead = /^(?:\s*\r?\n|#{1,6}[ \t]+[^\r\n]*\r?\n)*/.exec(text)?.[0] ?? "";
+    const rest = text.slice(lead.length).replace(/^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/, "");
+    return rest.trim();
+}
+export function readTopicHead(path) {
+    let fd;
+    try {
+        fd = openSync(path, "r");
+        const buffer = Buffer.alloc(HEAD_BYTES);
+        const read = readSync(fd, buffer, 0, HEAD_BYTES, 0);
+        return buffer.subarray(0, read).toString("utf8");
+    }
+    catch {
+        return "";
+    }
+    finally {
+        if (fd !== undefined)
+            closeSync(fd);
+    }
+}
+/** One line per topic, capped by count and length. The archive the tidy-up
+ * keeps is listed last and labelled, so it is not mistaken for current notes. */
+export function renderTopicIndex(topics) {
+    if (!topics.length)
+        return "";
+    const ordered = [...topics].sort((a, b) => Number(a.name === "archive.md") - Number(b.name === "archive.md") || a.name.localeCompare(b.name));
+    const lines = [];
+    let length = 0;
+    let omitted = 0;
+    for (const { name, header } of ordered) {
+        if (lines.length >= TOPIC_INDEX_MAX_TOPICS) {
+            omitted += 1;
+            continue;
+        }
+        const about = name === "archive.md" && !header.title && !header.description
+            ? "expired and replaced notes, kept for the record"
+            : [header.title, header.description].filter(Boolean).join(" — ");
+        const also = header.aliases.length ? ` (also: ${header.aliases.join(", ")})` : "";
+        const line = `- memory/${name}${about ? ` — ${about}` : ""}${also}`.replace(/\s+/g, " ");
+        if (length + line.length + 1 > TOPIC_INDEX_MAX_CHARS) {
+            omitted += 1;
+            continue;
+        }
+        lines.push(line);
+        length += line.length + 1;
+    }
+    if (omitted)
+        lines.push(`- …and ${omitted} more in memory/`);
+    return lines.join("\n");
+}
+const LEAD = /^(?:\s*\r?\n|#{1,6}[ \t]+[^\r\n]*\r?\n)*/;
+const HEADER_BLOCK = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/;
+function aliasList(aliases) {
+    return `[${aliases.join(", ")}]`;
+}
+/** A topic file with new entry lines appended and new aliases merged into its
+ * header — created, header first, when it does not exist yet. The header is
+ * what the topic index and recall read, so a topic the bot files for itself
+ * is found by the other words it was given. */
+export function mergeTopicText(existing, input) {
+    const lines = input.lines.join("\n");
+    if (existing === null || !existing.trim()) {
+        const aliases = [...new Set(input.aliases)].slice(0, 12);
+        return `---\ntitle: ${input.title}\n${aliases.length ? `aliases: ${aliasList(aliases)}\n` : ""}---\n\n${lines}\n`;
+    }
+    const lead = LEAD.exec(existing)?.[0] ?? "";
+    const rest = existing.slice(lead.length);
+    const block = HEADER_BLOCK.exec(rest);
+    let text;
+    if (!block) {
+        const aliases = [...new Set(input.aliases)].slice(0, 12);
+        text = `---\ntitle: ${parseTopicHeader(existing).title ?? input.title}\n${aliases.length ? `aliases: ${aliasList(aliases)}\n` : ""}---\n\n${existing}`;
+    }
+    else {
+        const known = parseTopicHeader(existing).aliases;
+        const lower = new Set(known.map((alias) => alias.toLowerCase()));
+        const added = input.aliases.filter((alias) => !lower.has(alias.toLowerCase()));
+        if (!added.length)
+            text = existing;
+        else {
+            const merged = [...known, ...added].slice(0, 12);
+            const body = block[1].split(/\r?\n/);
+            const at = body.findIndex((line) => /^aliases\s*:/.test(line));
+            if (at === -1)
+                body.push(`aliases: ${aliasList(merged)}`);
+            else {
+                // a multi-line list under aliases: is replaced by the one-line form
+                let end = at + 1;
+                while (end < body.length && /^\s*-\s+/.test(body[end]))
+                    end += 1;
+                body.splice(at, end - at, `aliases: ${aliasList(merged)}`);
+            }
+            text = `${lead}---\n${body.join("\n")}\n---\n${rest.slice(block[0].length)}`;
+        }
+    }
+    return `${text}${text.endsWith("\n") ? "" : "\n"}${lines}\n`;
+}