@frockbot/plugin-memory 0.0.0 → 0.1.0

package/src/chunker.ts

@@ -0,0 +1,104 @@
+const CHUNK_SIZE_CHARS = 1600;
+const OVERLAP_CHARS = 320;
+
+export interface MemoryChunk {
+  content: string;
+  startLine: number;
+  endLine: number;
+  hash: string;
+}
+
+interface LineRange {
+  text: string;
+  startLine: number;
+  endLine: number;
+}
+
+export async function hashMemoryContent(text: string): Promise<string> {
+  const digest = await crypto.subtle.digest(
+    "SHA-256",
+    new TextEncoder().encode(text),
+  );
+  return [...new Uint8Array(digest)]
+    .map((byte) => byte.toString(16).padStart(2, "0"))
+    .join("");
+}
+
+function paragraphs(lines: string[]): LineRange[] {
+  const result: LineRange[] = [];
+  let block: string[] = [];
+  let startLine = -1;
+  const flush = (endLine: number) => {
+    if (block.length === 0) return;
+    result.push({ text: block.join("\n"), startLine, endLine });
+    block = [];
+    startLine = -1;
+  };
+  for (let index = 0; index < lines.length; index += 1) {
+    const line = lines[index] ?? "";
+    if (line.trim()) {
+      if (startLine === -1) startLine = index + 1;
+      block.push(line);
+    } else {
+      flush(index);
+    }
+  }
+  flush(lines.length);
+  return result;
+}
+
+function splitOversized(range: LineRange): LineRange[] {
+  if (range.text.length <= CHUNK_SIZE_CHARS) return [range];
+  const result: LineRange[] = [];
+  const step = CHUNK_SIZE_CHARS - OVERLAP_CHARS;
+  for (let offset = 0; offset < range.text.length; offset += step) {
+    const text = range.text.slice(offset, offset + CHUNK_SIZE_CHARS).trim();
+    if (text) result.push({ ...range, text });
+    if (offset + CHUNK_SIZE_CHARS >= range.text.length) break;
+  }
+  return result;
+}
+
+export async function chunkMarkdown(content: string): Promise<MemoryChunk[]> {
+  if (!content.trim()) return [];
+  const blocks = paragraphs(content.split("\n")).flatMap(splitOversized);
+  const chunks: MemoryChunk[] = [];
+  let index = 0;
+  while (index < blocks.length) {
+    let text = "";
+    const startLine = blocks[index]?.startLine ?? 1;
+    let endLine = blocks[index]?.endLine ?? startLine;
+    let cursor = index;
+    while (cursor < blocks.length) {
+      const block = blocks[cursor];
+      if (!block) break;
+      const addition = text ? `\n\n${block.text}` : block.text;
+      if (text && text.length + addition.length > CHUNK_SIZE_CHARS) break;
+      text += addition;
+      endLine = block.endLine;
+      cursor += 1;
+    }
+    const normalized = text.trim();
+    if (normalized) {
+      chunks.push({
+        content: normalized,
+        startLine,
+        endLine,
+        hash: await hashMemoryContent(normalized),
+      });
+    }
+    if (cursor >= blocks.length) break;
+    let overlap = 0;
+    let next = cursor;
+    for (
+      let candidate = cursor - 1;
+      candidate > index && overlap < OVERLAP_CHARS;
+      candidate -= 1
+    ) {
+      overlap += (blocks[candidate]?.text.length ?? 0) + 2;
+      next = candidate;
+    }
+    index = next <= index ? cursor : next;
+  }
+  return chunks;
+}

package/src/documents.ts

@@ -0,0 +1,97 @@
+// Memory files as documents, for the derived index.
+//
+// "Indexes, embeddings, and summaries are derived from Memory files and are
+// always rebuildable from them." That sentence is only true if there is one
+// place the index reads its inputs from, and it is the files — not a sidecar,
+// not a cache, not a second store. This module is that place: it enumerates
+// every Memory file of every tier a Bot can see, through the same
+// `WorkspaceReadsV1` the renderer uses, and hands back bytes and content
+// addresses. Nothing derived is stored here.
+import type {
+  MemoryScopeNameV1,
+  WorkspaceMemoryRootV1,
+  WorkspaceReadsV1,
+} from "@frockbot/kernel-contracts";
+import {
+  memoryFileKindV1,
+  memoryProjectIdOfRootV1,
+  memoryScopeOfRootV1,
+} from "./roots.js";
+import { MEMORY_MAX_FILES_PER_TIER, MEMORY_MAX_LIST_PAGES } from "./store.js";
+
+/** One Memory file, addressed by its content and its generation. */
+export interface MemoryDocumentV1 {
+  scope: MemoryScopeNameV1;
+  projectId: string;
+  /** Relative to its root, shard prefix included. */
+  path: string;
+  /** The Bot whose shard holds it. */
+  botId: string;
+  kind: "profile" | "log";
+  text: string;
+  contentHash: string;
+  generationId: string;
+}
+
+/** A stable key for one document across every tier. */
+export function memoryDocumentKeyV1(document: {
+  scope: MemoryScopeNameV1;
+  projectId: string;
+  path: string;
+}): string {
+  return `${document.scope}:${document.projectId}:${document.path}`;
+}
+
+/**
+ * Reads every Memory file under one root. A file that cannot be read is
+ * skipped rather than thrown: an index is derived state, and a partial rebuild
+ * that says so beats a Turn that fails because one object was briefly
+ * unreachable.
+ */
+export async function listMemoryDocumentsV1(
+  reads: WorkspaceReadsV1,
+  root: WorkspaceMemoryRootV1,
+): Promise<MemoryDocumentV1[]> {
+  const scope = memoryScopeOfRootV1(root);
+  const projectId = memoryProjectIdOfRootV1(root);
+  const documents: MemoryDocumentV1[] = [];
+  let cursor: string | undefined;
+  for (let page = 0; page < MEMORY_MAX_LIST_PAGES; page += 1) {
+    const outcome = await reads.list(
+      cursor === undefined ? { root } : { root, cursor },
+    );
+    if (outcome.status !== "ok") return documents;
+    for (const entry of outcome.entries) {
+      if (documents.length >= MEMORY_MAX_FILES_PER_TIER) return documents;
+      const classified = memoryFileKindV1(root, entry.path.path);
+      if (!classified) continue;
+      const read = await reads.read(entry.path);
+      if (read.status !== "ok") continue;
+      documents.push({
+        scope,
+        projectId,
+        path: entry.path.path,
+        botId: classified.shard,
+        kind: classified.kind,
+        text: new TextDecoder().decode(read.file.bytes),
+        contentHash: read.file.generation.contentHash,
+        generationId: read.file.generation.generationId,
+      });
+    }
+    if (!outcome.cursor) break;
+    cursor = outcome.cursor;
+  }
+  return documents;
+}
+
+/** Every Memory document of every root a Bot can see, in tier order. */
+export async function listAllMemoryDocumentsV1(
+  reads: WorkspaceReadsV1,
+  roots: WorkspaceMemoryRootV1[],
+): Promise<MemoryDocumentV1[]> {
+  const documents: MemoryDocumentV1[] = [];
+  for (const root of roots) {
+    documents.push(...(await listMemoryDocumentsV1(reads, root)));
+  }
+  return documents;
+}

package/src/embeddings.ts

@@ -0,0 +1,23 @@
+import {
+  EMBEDDING_MODEL,
+  type EmbedMemory,
+  type MemoryAiBinding,
+} from "./types.js";
+
+const MAX_BATCH_SIZE = 100;
+
+export function createMemoryEmbedder(
+  ai: MemoryAiBinding,
+  model = EMBEDDING_MODEL,
+): EmbedMemory {
+  return async (texts) => {
+    const vectors: number[][] = [];
+    for (let index = 0; index < texts.length; index += MAX_BATCH_SIZE) {
+      const response = await ai.run(model, {
+        text: texts.slice(index, index + MAX_BATCH_SIZE),
+      });
+      vectors.push(...response.data);
+    }
+    return vectors;
+  };
+}

package/src/facts.test.ts

@@ -0,0 +1,126 @@
+// The typed markers on a fact text: `[note] ` and the reserved `[episode] `.
+//
+// The claim every case here defends is that typing the markers changed no
+// bytes. A file written before markers were parsed parses to the same facts,
+// renders to the same file, and injects the same line.
+import { describe, expect, test } from "bun:test";
+import {
+  MEMORY_MARKERS_V1,
+  isMemoryRetractionV1,
+  memoryFactBodyV1,
+  parseMemoryFileV1,
+  parseMemoryMarkerV1,
+  renderInjectedFactLineV1,
+  renderMemoryFileV1,
+  renderMemoryMarkerV1,
+  retractedFactTextV1,
+} from "./facts.ts";
+
+describe("parseMemoryMarkerV1", () => {
+  test("recognises exactly `[note] ` and `[episode] `", () => {
+    expect(MEMORY_MARKERS_V1).toEqual(["note", "episode"]);
+    expect(parseMemoryMarkerV1("[note] we ship on Friday")).toEqual({
+      marker: "note",
+      body: "we ship on Friday",
+    });
+    expect(parseMemoryMarkerV1("[episode] the gym build week")).toEqual({
+      marker: "episode",
+      body: "the gym build week",
+    });
+  });
+
+  test("an unrecognised bracket prefix is fact text, not a marker", () => {
+    for (const text of [
+      "[todo] buy rubber matting",
+      "[via School] Tim lives in Wollongong.",
+      "[note]no space",
+      "[NOTE] shouting",
+      "a note about the floor",
+    ]) {
+      expect(parseMemoryMarkerV1(text)).toEqual({ body: text });
+      expect(memoryFactBodyV1(text)).toBe(text);
+    }
+  });
+
+  test("a retraction is checked first, so `[forgotten] [note] x` retracts the note", () => {
+    const retraction = "[forgotten] [note] we ship on Friday";
+    expect(isMemoryRetractionV1(retraction)).toBe(true);
+    // The retraction itself carries no marker…
+    expect(parseMemoryMarkerV1(retraction)).toEqual({ body: retraction });
+    // …and what it retracts is the note, marker and all.
+    expect(parseMemoryMarkerV1(retractedFactTextV1(retraction))).toEqual({
+      marker: "note",
+      body: "we ship on Friday",
+    });
+  });
+
+  test("render is the exact inverse of parse, byte for byte", () => {
+    for (const text of [
+      "[note] we ship on Friday",
+      "[episode] the gym build week",
+      "[todo] buy rubber matting",
+      "plain",
+      "[note]  two spaces after the marker",
+    ]) {
+      const { marker, body } = parseMemoryMarkerV1(text);
+      expect(renderMemoryMarkerV1(marker, body)).toBe(text);
+    }
+  });
+});
+
+describe("markers through the file format", () => {
+  const FILE = [
+    "- (2026-08-30) Tim prefers blunt answers.",
+    "- (2026-08-31) [note] we ship on Friday",
+    "- (2026-08-31) [episode] the gym build week",
+    "- (2026-08-31) [todo] buy rubber matting",
+    "",
+  ].join("\n");
+
+  test("a file written before this change parses identically, and round-trips", () => {
+    const facts = parseMemoryFileV1(FILE);
+    expect(facts.map((fact) => fact.text)).toEqual([
+      "Tim prefers blunt answers.",
+      "[note] we ship on Friday",
+      "[episode] the gym build week",
+      "[todo] buy rubber matting",
+    ]);
+    // `marker`/`body` are added *beside* `text`, which still holds the prefix.
+    expect(facts.map((fact) => fact.marker)).toEqual([
+      undefined,
+      "note",
+      "episode",
+      undefined,
+    ]);
+    expect(facts[1]?.body).toBe("we ship on Friday");
+    expect(facts[3]?.body).toBe("[todo] buy rubber matting");
+    // disk → parse → disk, byte-identical.
+    expect(renderMemoryFileV1(facts)).toBe(FILE);
+  });
+
+  test("the injected line is `(learned d) [via b] [note] body`", () => {
+    expect(
+      renderInjectedFactLineV1(
+        { date: "2026-08-31", text: "[note] we ship on Friday", via: "School" },
+        500,
+      ),
+    ).toBe("- (learned 2026-08-31) [via School] [note] we ship on Friday");
+    expect(
+      renderInjectedFactLineV1(
+        { date: "2026-08-31", text: "[note] we ship on Friday" },
+        500,
+      ),
+    ).toBe("- (learned 2026-08-31) [note] we ship on Friday");
+  });
+
+  test("the clamp applies to the whole fact text, marker included", () => {
+    const long = `[note] ${"x".repeat(900)}`;
+    const line = renderInjectedFactLineV1(
+      { date: "2026-08-31", text: long },
+      500,
+    );
+    expect(line.startsWith("- (learned 2026-08-31) [note] xxx")).toBe(true);
+    expect(line.endsWith("…")).toBe(true);
+    expect(line.length).toBe("- (learned 2026-08-31) ".length + 500);
+  });
+});

package/src/facts.ts

@@ -0,0 +1,258 @@
+// The Memory file format, both halves of it.
+//
+// GrokBot writes and injects a fact in two different shapes
+// (`docs/research/grokbot-computer.md` §4.1b), and matching parity means
+// matching both:
+//
+//   on disk    - (YYYY-MM-DD) <fact>
+//   injected   - (learned YYYY-MM-DD) [via <bot>] <fact>
+//
+// `[note] ` and `[episode] ` are prefixes *on the fact text*, not separate
+// files, so they survive a round trip through the disk form untouched. They
+// are also *parsed* here — `MEMORY_MARKERS_V1` is the one vocabulary the
+// writer, the renderer, the fade and `forget` all share — because a marker
+// that is only ever opaque text cannot mean anything at read time, and the
+// note tier's whole claim is that it fades.
+//
+// One shape this Package adds, which GrokBot has no equivalent for: a
+// retraction. "a forget on a shared tier writes a retraction in the Bot's own
+// shard — newest wins — never edits another shard", so a forget the Bot cannot
+// perform by deleting a line it owns is recorded as `[forgotten] <fact>` in
+// its own shard, and the reader drops the fact when the retraction is newer.
+// It is deliberately the same line grammar: a Bot or a User reading the file
+// with ordinary tools sees why the fact went away.
+
+/** The marker a retraction carries, as a prefix on the fact text. */
+export const MEMORY_FORGOTTEN_PREFIX = "[forgotten] ";
+
+/**
+ * The markers a fact text may carry, as a `[<marker>] ` prefix.
+ *
+ * `note` is written by `memory_write`'s `note` tier. `episode` is **reserved
+ * and not produced**: GrokBot exposes no episode tier either (§2.2 lists
+ * `profile|log|note`), and FrockBot has no episodic summariser, so inventing a
+ * producer would be product surface added to close a register row. It is
+ * recognised so that a fact carrying it — written by hand, or by a summariser
+ * that lands later — is typed rather than opaque, and it fades on the same
+ * rule as a note.
+ */
+export const MEMORY_MARKERS_V1 = ["note", "episode"] as const;
+
+/** One of the `[note] `/`[episode] ` markers a fact text may carry. */
+export type MemoryMarkerV1 = (typeof MEMORY_MARKERS_V1)[number];
+
+/** One fact, as it was parsed from a Memory file. */
+export interface MemoryFactV1 {
+  /** `YYYY-MM-DD`, the day the fact was recorded. */
+  date: string;
+  /** The fact text, including any `[note] `/`[episode] ` prefix. */
+  text: string;
+  /**
+   * The marker `text` carries, when it carries one.
+   *
+   * Derived, never authoritative: `text` is what is on disk, and
+   * `parseMemoryMarkerV1(text)` recovers this field at any time. A fact built
+   * by hand may omit it, which is why every reader in this Package parses
+   * `text` rather than trusting the field — one source of truth, and the bytes
+   * are it.
+   */
+  marker?: MemoryMarkerV1;
+  /** The fact text with its marker removed; equal to `text` when unmarked. */
+  body?: string;
+}
+
+/** A fact together with where it came from, for rendering and precedence. */
+export interface SourcedMemoryFactV1 extends MemoryFactV1 {
+  /** The Bot whose shard holds it. */
+  botId: string;
+  /** The display name of that Bot, or its id when no name is known. */
+  via: string;
+  /** `profile.md` or a monthly log. */
+  kind: "profile" | "log";
+  /** The generation the line was read from; ordering's tiebreak. */
+  generationId: string;
+}
+
+const FACT_LINE = /^-\s+\((\d{4}-\d{2}-\d{2})\)\s+(.*)$/;
+
+/**
+ * Splits a fact text into its marker and its body.
+ *
+ * Three rules, each chosen so the on-disk bytes are exactly what they were
+ * before markers were typed:
+ *
+ *  - Only `[note] ` and `[episode] ` — the exact prefix, one space — are
+ *    markers. An unrecognised bracket prefix (`[todo] `, a `[via …]` a User
+ *    typed by hand) is ordinary fact text: `marker` is `undefined` and `body`
+ *    is the whole text. Guessing at brackets would silently reclassify a
+ *    User's own words.
+ *  - A retraction is checked first and is never a marker, so
+ *    `[forgotten] [note] x` is a retraction whose retracted text is
+ *    `[note] x` — the note, not a fact named `[forgotten] …`.
+ *  - The body is sliced, not trimmed, so
+ *    `renderMemoryMarkerV1(marker, body) === text` byte for byte.
+ */
+export function parseMemoryMarkerV1(text: string): {
+  marker?: MemoryMarkerV1;
+  body: string;
+} {
+  if (!isMemoryRetractionV1(text)) {
+    for (const marker of MEMORY_MARKERS_V1) {
+      const prefix = `[${marker}] `;
+      if (text.startsWith(prefix)) {
+        return { marker, body: text.slice(prefix.length) };
+      }
+    }
+  }
+  return { body: text };
+}
+
+/** The fact text one marker and one body make; the inverse of the parse. */
+export function renderMemoryMarkerV1(
+  marker: MemoryMarkerV1 | undefined,
+  body: string,
+): string {
+  return marker ? `[${marker}] ${body}` : body;
+}
+
+/** The marker-stripped body of a fact text, for matching by what it says. */
+export function memoryFactBodyV1(text: string): string {
+  return parseMemoryMarkerV1(text).body;
+}
+
+/** `YYYY-MM-DD` in UTC. Memory dates are days, never instants. */
+export function memoryDayV1(at: Date): string {
+  return at.toISOString().slice(0, 10);
+}
+
+/** The on-disk line for one fact. */
+export function renderMemoryFactLineV1(fact: MemoryFactV1): string {
+  return `- (${fact.date}) ${fact.text}`;
+}
+
+/**
+ * Parses one Memory file. Lines that are not facts — a heading a User typed, a
+ * blank line, a stray paragraph — are ignored rather than refused: a Memory
+ * file is a file the User may edit by hand, and one malformed line must not
+ * cost the Bot the rest of its Memory.
+ */
+export function parseMemoryFileV1(
+  text: string,
+  options: { maxFacts?: number } = {},
+): MemoryFactV1[] {
+  const maximum = options.maxFacts ?? 5_000;
+  const facts: MemoryFactV1[] = [];
+  for (const line of text.split("\n")) {
+    if (facts.length >= maximum) break;
+    const match = FACT_LINE.exec(line.trim());
+    if (!match) continue;
+    const text = (match[2] ?? "").trim();
+    if (!text) continue;
+    // The marker stays in `text`: a file written before markers were typed
+    // parses to the same `text` it always did, and nothing downstream that
+    // reads `text` changes behaviour. `marker`/`body` are added beside it.
+    const { marker, body } = parseMemoryMarkerV1(text);
+    facts.push({
+      date: match[1] ?? "",
+      text,
+      ...(marker ? { marker } : {}),
+      body,
+    });
+  }
+  return facts;
+}
+
+/** Renders a whole Memory file from its facts, oldest first. */
+export function renderMemoryFileV1(facts: MemoryFactV1[]): string {
+  if (facts.length === 0) return "";
+  return `${facts.map(renderMemoryFactLineV1).join("\n")}\n`;
+}
+
+/** The comparison key for "the same fact": trimmed, case-folded text. */
+export function memoryFactKeyV1(text: string): string {
+  return text.trim().replace(/\s+/g, " ").toLowerCase();
+}
+
+/** True when a fact text is a retraction rather than an assertion. */
+export function isMemoryRetractionV1(text: string): boolean {
+  return text.startsWith(MEMORY_FORGOTTEN_PREFIX);
+}
+
+/** The fact a retraction retracts. */
+export function retractedFactTextV1(text: string): string {
+  return text.slice(MEMORY_FORGOTTEN_PREFIX.length).trim();
+}
+
+/** The retraction line for one fact. */
+export function memoryRetractionTextV1(text: string): string {
+  return `${MEMORY_FORGOTTEN_PREFIX}${text.trim()}`;
+}
+
+/**
+ * Newest-wins resolution within one tier.
+ *
+ * "readers merge shards, newest fact wins on conflict". Newest is the day
+ * first and the minted generation id second, because two Bots writing the same
+ * fact on the same day still have an order — the generation ledger's — and
+ * ordering by text or by shard would make the answer depend on the listing.
+ *
+ * A retraction is a fact like any other in that ordering: when the newest
+ * entry for a key is `[forgotten] …`, the fact is gone; when a later
+ * assertion follows it, the fact is back.
+ */
+export function resolveMemoryFactsV1(
+  facts: SourcedMemoryFactV1[],
+): SourcedMemoryFactV1[] {
+  const newest = new Map<string, SourcedMemoryFactV1>();
+  for (const fact of facts) {
+    const asserted = isMemoryRetractionV1(fact.text)
+      ? retractedFactTextV1(fact.text)
+      : fact.text;
+    const key = memoryFactKeyV1(asserted);
+    const current = newest.get(key);
+    if (!current || isNewerMemoryFactV1(fact, current)) newest.set(key, fact);
+  }
+  return [...newest.values()].filter(
+    (fact) => !isMemoryRetractionV1(fact.text),
+  );
+}
+
+/** Strictly newer: by day, then by the minted generation id. */
+export function isNewerMemoryFactV1(
+  candidate: SourcedMemoryFactV1,
+  current: SourcedMemoryFactV1,
+): boolean {
+  if (candidate.date !== current.date) return candidate.date > current.date;
+  return candidate.generationId > current.generationId;
+}
+
+/** Newest first: the order every rendered block lists facts in. */
+export function sortMemoryFactsV1(
+  facts: SourcedMemoryFactV1[],
+): SourcedMemoryFactV1[] {
+  return [...facts].sort((left, right) =>
+    isNewerMemoryFactV1(left, right) ? -1 : 1,
+  );
+}
+
+/**
+ * The injected line: `- (learned YYYY-MM-DD) [via <bot>] [note] <body>`.
+ *
+ * The marker sits *after* `[via …]` because §4.1b puts `[via …]` before "the
+ * fact text" and §2.2 puts the marker "on the fact text" — the order is
+ * derived from the two citations rather than chosen. It is re-emitted through
+ * `renderMemoryMarkerV1` rather than left riding along inside `text`, so the
+ * one vocabulary decides where a marker appears; the bytes are identical
+ * either way, and the clamp still applies to the whole fact text, marker
+ * included.
+ */
+export function renderInjectedFactLineV1(
+  fact: { date: string; text: string; via?: string },
+  clamp: number,
+): string {
+  const via = fact.via ? `[via ${fact.via}] ` : "";
+  const { marker, body } = parseMemoryMarkerV1(fact.text);
+  const full = renderMemoryMarkerV1(marker, body);
+  const text = full.length > clamp ? `${full.slice(0, clamp - 1)}…` : full;
+  return `- (learned ${fact.date}) ${via}${text}`;
+}

package/src/index.ts

@@ -0,0 +1,15 @@
+export * from "./agent.js";
+export * from "./chunker.js";
+export * from "./documents.js";
+export * from "./embeddings.js";
+export * from "./facts.js";
+export * from "./indexer.js";
+export { default as memoryManifest } from "./manifest.js";
+export * from "./projects.js";
+export * from "./render.js";
+export * from "./roots.js";
+export * from "./searcher.js";
+export * from "./secrets.js";
+export * from "./store.js";
+export * from "./testing.js";
+export * from "./types.js";