@frockbot/plugin-memory 0.0.0 → 0.1.1

package/src/searcher.ts

@@ -0,0 +1,159 @@
+// Searching Memory over the derived index.
+//
+// The index is derived and rebuildable, so it is never authoritative about a
+// fact: a hit names a document and a line range, and the caller reads the
+// bytes back out of the Workspace before showing them. That is the same rule
+// the rest of the Package follows — the files are the Memory, everything else
+// is a way of finding a part of them.
+//
+// Precedence here is the Memory precedence: own (`bot`) before `project`
+// before `user`, "the most specific wins", applied after scoring so a strong
+// shared hit still ranks above a weak own one within the same document.
+import type { MemoryScopeNameV1 } from "@frockbot/kernel-contracts";
+import {
+  memoryVectorNamespaceV1,
+  type MemoryIndexChunkV1,
+  type MemoryIndexV1,
+} from "./indexer.js";
+import type {
+  EmbedMemory,
+  MemorySearchResult,
+  MemoryVectorIndex,
+} from "./types.js";
+
+const SNIPPET_MAX_CHARS = 700;
+
+const SCOPE_ORDER: Record<MemoryScopeNameV1, number> = {
+  bot: 0,
+  project: 1,
+  user: 2,
+};
+
+function truncate(text: string): string {
+  if (text.length <= SNIPPET_MAX_CHARS) return text;
+  const slice = text.slice(0, SNIPPET_MAX_CHARS);
+  const boundary = slice.lastIndexOf(" ");
+  return `${boundary > 0 ? slice.slice(0, boundary) : slice}…`;
+}
+
+function resultOf(
+  chunk: MemoryIndexChunkV1,
+  score?: number,
+): MemorySearchResult {
+  return {
+    scope: chunk.scope,
+    projectId: chunk.projectId,
+    path: chunk.path,
+    startLine: chunk.startLine,
+    endLine: chunk.endLine,
+    snippet: truncate(chunk.content),
+    ...(score === undefined ? {} : { score }),
+  };
+}
+
+function lexicalScore(content: string, terms: string[]): number {
+  const haystack = content.toLowerCase();
+  let hits = 0;
+  for (const term of terms) if (haystack.includes(term)) hits += 1;
+  return terms.length === 0 ? 0 : hits / terms.length;
+}
+
+export interface SearchMemoryOptionsV1 {
+  index: MemoryIndexV1;
+  query: string;
+  maxResults: number;
+  scope?: MemoryScopeNameV1;
+  embed?: EmbedMemory;
+  vectorize?: MemoryVectorIndex;
+}
+
+/**
+ * Vector search when an embedder and a vector index are configured, lexical
+ * search otherwise, and lexical search as the fallback when the embedder
+ * fails. A Memory search must not fail a Turn because a model binding is
+ * briefly unavailable.
+ */
+export async function searchMemoryV1(
+  options: SearchMemoryOptionsV1,
+): Promise<MemorySearchResult[]> {
+  const query = options.query.trim();
+  if (!query) return [];
+  const candidates = options.index.chunks.filter(
+    (chunk) => options.scope === undefined || chunk.scope === options.scope,
+  );
+  if (candidates.length === 0) return [];
+
+  const scored = new Map<MemoryIndexChunkV1, number>();
+  if (options.embed && options.vectorize) {
+    try {
+      const [vector] = await options.embed([query]);
+      if (vector) {
+        const namespaces = new Set(candidates.map(memoryVectorNamespaceV1));
+        const byHash = new Map(
+          candidates.map((chunk) => [chunk.hash, chunk] as const),
+        );
+        for (const namespace of namespaces) {
+          const response = await options.vectorize.query(vector, {
+            topK: Math.min(options.maxResults * 3, 20),
+            namespace,
+            returnMetadata: "all",
+          });
+          for (const match of response.matches) {
+            const hash = match.metadata?.hash;
+            const chunk =
+              typeof hash === "string" ? byHash.get(hash) : undefined;
+            if (!chunk) continue;
+            scored.set(
+              chunk,
+              Math.max(scored.get(chunk) ?? 0, match.score ?? 0),
+            );
+          }
+        }
+      }
+    } catch (error) {
+      console.error(
+        "[memory] vector search failed; using lexical search",
+        error,
+      );
+    }
+  }
+
+  if (scored.size === 0) {
+    const terms = query
+      .toLowerCase()
+      .split(/\s+/)
+      .filter((term) => term.length > 1);
+    for (const chunk of candidates) {
+      const score = lexicalScore(chunk.content, terms);
+      if (score > 0) scored.set(chunk, score);
+    }
+  }
+
+  return [...scored.entries()]
+    .sort(([leftChunk, leftScore], [rightChunk, rightScore]) => {
+      if (rightScore !== leftScore) return rightScore - leftScore;
+      const order =
+        SCOPE_ORDER[leftChunk.scope] - SCOPE_ORDER[rightChunk.scope];
+      if (order !== 0) return order;
+      return leftChunk.path.localeCompare(rightChunk.path);
+    })
+    .slice(0, options.maxResults)
+    .map(([chunk, score]) => resultOf(chunk, score));
+}
+
+/** The compact rendering a tool result carries. */
+export function formatMemoryResultsV1(results: MemorySearchResult[]): string {
+  if (results.length === 0) return "No memory matches.";
+  return results
+    .map((result, index) => {
+      const where = result.projectId
+        ? `${result.scope}/${result.projectId}`
+        : result.scope;
+      const score =
+        result.score === undefined
+          ? ""
+          : ` (score: ${result.score.toFixed(3)})`;
+      return `[${index + 1}] ${where}:${result.path}:${result.startLine}-${result.endLine}${score}\n${result.snippet}`;
+    })
+    .join("\n\n---\n\n");
+}

package/src/secrets.ts

@@ -0,0 +1,45 @@
+// "Memory contains no secrets and no credential references."
+//
+// This is Package policy and it is only here. `kernel-contracts` deliberately
+// declares nothing about it, because the kernel's file contract carries bytes
+// and cannot classify them; the Memory Package owns what may be written into a
+// Memory root, so the refusal belongs at its write path.
+//
+// The *shapes* it recognises are not owned here. They live in
+// `@frockbot/secret-shapes`, because the Audit Package needs the same list to
+// redact a durable preview and two drifting copies of a redaction list is the
+// failure mode worth one small package. What is owned here is the policy: a
+// match is a refusal, in words a Bot can act on.
+//
+// The check is deliberately bounded and deliberately shallow. It refuses
+// *obvious credential shapes* — the ones a model pastes into a fact because it
+// just read them out of a config file — and it makes no claim to be a secret
+// scanner. A determined encoding gets through, and that is the honest bound:
+// the rule it enforces is "do not write the API key into Memory", not "prove
+// this string holds no entropy". A refusal is a declared outcome the tool
+// reports, never a throw and never a silent redaction, because a Bot told its
+// fact was refused can write a different one.
+import { matchSecretShapeV1 } from "@frockbot/secret-shapes";
+
+/** One refusal: the pattern that matched, in words a Bot can act on. */
+export interface MemorySecretRefusalV1 {
+  reason: string;
+}
+
+/**
+ * Answers a refusal when a fact looks like a credential, and `undefined`
+ * otherwise. Pure and total: the same text always gets the same answer, and no
+ * input throws.
+ */
+export function refuseMemorySecretV1(
+  text: string,
+): MemorySecretRefusalV1 | undefined {
+  // Bounded input: a fact is already length-capped by the tool, and the shared
+  // table caps its own input too, so the bound is the scanner's rather than a
+  // caller's promise.
+  const match = matchSecretShapeV1(text);
+  if (!match) return undefined;
+  return {
+    reason: `Memory contains no secrets and no credential references, and ${match.reason}`,
+  };
+}

package/src/store.test.ts

@@ -0,0 +1,475 @@
+// The Memory Package's writer and reader, against the production
+// object-storage store over an in-memory bucket and generation ledger.
+import { describe, expect, test } from "bun:test";
+import type {
+  WorkspaceFilesV1,
+  WorkspaceWriterV1,
+} from "@frockbot/kernel-contracts";
+import {
+  botMemoryRootV1,
+  projectMemoryRootV1,
+  userMemoryRootV1,
+} from "./roots.ts";
+import { MemoryStore, MEMORY_MAX_FILES_PER_TIER } from "./store.ts";
+import { createTestMemoryFilesV1 } from "./testing.ts";
+
+const OWNER = { userId: "user-1", botId: "bot-1" };
+const AT = new Date("2026-08-31T10:00:00.000Z");
+
+function writerFor(botId: string): WorkspaceWriterV1 {
+  return {
+    kind: "bot",
+    botId,
+    sessionId: `user-1:${botId}`,
+    turnId: "turn-1",
+    runId: "run-1",
+  };
+}
+
+function storeFor(
+  botId: string,
+  files = createTestMemoryFilesV1({ userId: "user-1" }),
+) {
+  return {
+    files,
+    store: new MemoryStore({
+      files,
+      owner: { userId: "user-1", botId },
+      botNames: { "bot-1": "General", "bot-2": "School" },
+      clock: () => AT,
+    }),
+  };
+}
+
+describe("the Memory writer", () => {
+  test("writes a Bot fact to profile.md and a log fact to log/YYYY-MM.md", async () => {
+    const { store } = storeFor("bot-1");
+    const root = botMemoryRootV1(OWNER);
+
+    const profile = await store.write({
+      root,
+      tier: "profile",
+      fact: "Tim lives in Wollongong.",
+      writer: writerFor("bot-1"),
+    });
+    const log = await store.write({
+      root,
+      tier: "log",
+      fact: "Term ends on the 12th.",
+      writer: writerFor("bot-1"),
+    });
+
+    expect(profile).toMatchObject({ status: "ok", path: "profile.md" });
+    expect(log).toMatchObject({ status: "ok", path: "log/2026-08.md" });
+
+    const tier = await store.read(root);
+    expect(tier.profile.map((fact) => fact.text)).toEqual([
+      "Tim lives in Wollongong.",
+    ]);
+    expect(tier.recent.map((fact) => fact.text)).toEqual([
+      "Term ends on the 12th.",
+    ]);
+    expect(tier.profile[0]?.date).toBe("2026-08-31");
+  });
+
+  test("writes a shared fact into the writing Bot's own shard only", async () => {
+    const files = createTestMemoryFilesV1({ userId: "user-1" });
+    const { store } = storeFor("bot-1", files);
+    const root = userMemoryRootV1(OWNER);
+
+    const written = await store.write({
+      root,
+      tier: "profile",
+      fact: "Tim prefers blunt answers.",
+      writer: writerFor("bot-1"),
+    });
+    expect(written).toMatchObject({
+      status: "ok",
+      path: "by-agent/bot-1/profile.md",
+    });
+
+    // Another Bot's shard is refused, whatever this Bot asks for.
+    const foreign = await new MemoryStore({
+      files,
+      owner: { userId: "user-1", botId: "bot-1" },
+      clock: () => AT,
+    }).writeFile({
+      path: { root, path: "by-agent/bot-2/profile.md" },
+      text: "- (2026-08-31) Forged.\n",
+      writer: writerFor("bot-1"),
+    });
+    expect(foreign.status).toBe("refused");
+  });
+
+  test("merges shards and tags every shared fact with the Bot that learned it", async () => {
+    const files = createTestMemoryFilesV1({ userId: "user-1" });
+    const root = userMemoryRootV1(OWNER);
+    const one = storeFor("bot-1", files);
+    const two = storeFor("bot-2", files);
+    await one.store.write({
+      root,
+      tier: "profile",
+      fact: "Tim lives in Wollongong.",
+      writer: writerFor("bot-1"),
+    });
+    await two.store.write({
+      root,
+      tier: "profile",
+      fact: "Tim teaches on Tuesdays.",
+      writer: writerFor("bot-2"),
+    });
+
+    const tier = await one.store.read(root);
+    expect(
+      tier.profile.map((fact) => `${fact.via}: ${fact.text}`).sort(),
+    ).toEqual([
+      "General: Tim lives in Wollongong.",
+      "School: Tim teaches on Tuesdays.",
+    ]);
+  });
+
+  test("dedupes a fact that is already recorded, without writing", async () => {
+    const { store } = storeFor("bot-1");
+    const root = botMemoryRootV1(OWNER);
+    const first = await store.write({
+      root,
+      tier: "profile",
+      fact: "Tim lives in Wollongong.",
+      writer: writerFor("bot-1"),
+    });
+    const again = await store.write({
+      root,
+      tier: "profile",
+      fact: "tim lives in wollongong.",
+      writer: writerFor("bot-1"),
+    });
+    expect(first).toMatchObject({ status: "ok", duplicate: false });
+    expect(again).toMatchObject({ status: "ok", duplicate: true });
+    expect((await store.read(root)).profile).toHaveLength(1);
+  });
+
+  test("refuses a fact that looks like a credential", async () => {
+    const { store } = storeFor("bot-1");
+    for (const fact of [
+      "The API key is sk-abcdefghijklmnopqrstuvwx.",
+      "Use Bearer eyJhbGciOiJIUzI1NiJ9.aaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbbbbbbbbb",
+      "-----BEGIN RSA PRIVATE KEY-----",
+      "password = hunter2hunter2hunter2",
+    ]) {
+      const outcome = await store.write({
+        root: botMemoryRootV1(OWNER),
+        tier: "log",
+        fact,
+        writer: writerFor("bot-1"),
+      });
+      expect(outcome.status).toBe("refused");
+      expect(outcome.status === "refused" ? outcome.reason : "").toContain(
+        "no secrets",
+      );
+    }
+    expect((await store.read(botMemoryRootV1(OWNER))).recent).toHaveLength(0);
+  });
+});
+
+describe("forgetting", () => {
+  test("removes a fact this Bot recorded from its own shard", async () => {
+    const { store } = storeFor("bot-1");
+    const root = botMemoryRootV1(OWNER);
+    await store.write({
+      root,
+      tier: "profile",
+      fact: "Tim lives in Wollongong.",
+      writer: writerFor("bot-1"),
+    });
+
+    const forgotten = await store.forget({
+      root,
+      fact: "Tim lives in Wollongong.",
+      writer: writerFor("bot-1"),
+    });
+
+    expect(forgotten.status).toBe("ok");
+    expect(forgotten.retracted).toBeUndefined();
+    expect((await store.read(root)).profile).toEqual([]);
+  });
+
+  test("retracts another Bot's shared fact in this Bot's own shard, never editing theirs", async () => {
+    const files = createTestMemoryFilesV1({ userId: "user-1" });
+    const root = userMemoryRootV1(OWNER);
+    const two = storeFor("bot-2", files);
+    await two.store.write({
+      root,
+      tier: "profile",
+      fact: "Tim teaches on Tuesdays.",
+      writer: writerFor("bot-2"),
+    });
+    const one = storeFor("bot-1", files);
+
+    const forgotten = await one.store.forget({
+      root,
+      fact: "Tim teaches on Tuesdays.",
+      writer: writerFor("bot-1"),
+    });
+
+    expect(forgotten).toMatchObject({ status: "ok", retracted: true });
+    // The other Bot's file is untouched: its bytes still hold the fact.
+    const theirs = await files.read({
+      root,
+      path: "by-agent/bot-2/profile.md",
+    });
+    expect(
+      theirs.status === "ok" ? new TextDecoder().decode(theirs.file.bytes) : "",
+    ).toContain("Tim teaches on Tuesdays.");
+    // But the merged tier no longer carries it: newest wins.
+    expect((await one.store.read(root)).profile).toEqual([]);
+    expect((await two.store.read(root)).profile).toEqual([]);
+  });
+
+  test("removes every marker variant of a fact, given the body or the marker", async () => {
+    const { store } = storeFor("bot-1");
+    const root = botMemoryRootV1(OWNER);
+    const writer = writerFor("bot-1");
+    // A note and a durable log fact making the same claim are two records:
+    // dedupe is on the full text, so the second write is not a duplicate.
+    await store.write({ root, tier: "log", fact: "we ship on Friday", writer });
+    const second = await store.write({
+      root,
+      tier: "note",
+      fact: "[note] we ship on Friday",
+      writer,
+    });
+    expect(second).toMatchObject({ status: "ok", duplicate: false });
+    expect((await store.read(root)).recent.map((f) => f.text).sort()).toEqual([
+      "[note] we ship on Friday",
+      "we ship on Friday",
+    ]);
+
+    // Forgetting the body removes both — a User can forget a note whose
+    // `[note] ` prefix they were never shown.
+    const forgotten = await store.forget({
+      root,
+      fact: "we ship on Friday",
+      writer,
+    });
+    expect(forgotten.status).toBe("ok");
+    expect((await store.read(root)).recent).toEqual([]);
+  });
+
+  test("a forget that does pass the marker matches too", async () => {
+    const { store } = storeFor("bot-1");
+    const root = botMemoryRootV1(OWNER);
+    const writer = writerFor("bot-1");
+    await store.write({
+      root,
+      tier: "note",
+      fact: "[note] we ship on Friday",
+      writer,
+    });
+    const forgotten = await store.forget({
+      root,
+      fact: "[note] we ship on Friday",
+      writer,
+    });
+    expect(forgotten.status).toBe("ok");
+    expect((await store.read(root)).recent).toEqual([]);
+  });
+
+  test("retracts another Bot's note by its body, naming the recorded text", async () => {
+    const files = createTestMemoryFilesV1({ userId: "user-1" });
+    const root = userMemoryRootV1(OWNER);
+    const two = storeFor("bot-2", files);
+    await two.store.write({
+      root,
+      tier: "note",
+      fact: "[note] Tim teaches on Tuesdays.",
+      writer: writerFor("bot-2"),
+    });
+    const one = storeFor("bot-1", files);
+
+    const forgotten = await one.store.forget({
+      root,
+      fact: "Tim teaches on Tuesdays.",
+      writer: writerFor("bot-1"),
+    });
+    expect(forgotten).toMatchObject({ status: "ok", retracted: true });
+
+    // The retraction names the *recorded* text, marker and all, because that
+    // is the text newest-wins resolves against.
+    const mine = await files.read({
+      root,
+      path: "by-agent/bot-1/log/2026-08.md",
+    });
+    expect(
+      mine.status === "ok" ? new TextDecoder().decode(mine.file.bytes) : "",
+    ).toContain("[forgotten] [note] Tim teaches on Tuesdays.");
+    expect((await one.store.read(root)).recent).toEqual([]);
+  });
+
+  test("refuses a forget of a fact nobody recorded", async () => {
+    const { store } = storeFor("bot-1");
+    const outcome = await store.forget({
+      root: botMemoryRootV1(OWNER),
+      fact: "Never said.",
+      writer: writerFor("bot-1"),
+    });
+    expect(outcome.status).toBe("refused");
+  });
+});
+
+describe("project memory", () => {
+  test("shards a Project tier per writing Bot exactly as the User tier does", async () => {
+    const { store } = storeFor("bot-1");
+    const root = projectMemoryRootV1(OWNER, "ghetto-movement");
+    const written = await store.write({
+      root,
+      tier: "log",
+      fact: "The shoot is on Friday.",
+      writer: writerFor("bot-1"),
+    });
+    expect(written).toMatchObject({
+      status: "ok",
+      path: "by-agent/bot-1/log/2026-08.md",
+    });
+    expect((await store.read(root)).recent.map((fact) => fact.via)).toEqual([
+      "General",
+    ]);
+  });
+});
+
+describe("a tier read that a declared bound cut short", () => {
+  /** One profile shard per Bot, past the per-tier file bound. */
+  async function crowdedUserTier(shards: number) {
+    const files = createTestMemoryFilesV1({ userId: "user-1" });
+    for (let index = 0; index < shards; index += 1) {
+      const botId = `bot-${String(index).padStart(3, "0")}`;
+      const store = new MemoryStore({
+        files,
+        owner: { userId: "user-1", botId },
+        clock: () => AT,
+      });
+      const written = await store.write({
+        root: userMemoryRootV1(OWNER),
+        tier: "profile",
+        fact: `Shard ${String(index).padStart(3, "0")} learned something.`,
+        writer: writerFor(botId),
+      });
+      expect(written.status).toBe("ok");
+    }
+    return files;
+  }
+
+  test("keeps the newest files, because injection is about recent facts", async () => {
+    const shards = MEMORY_MAX_FILES_PER_TIER + 2;
+    const files = await crowdedUserTier(shards);
+    const store = new MemoryStore({
+      files,
+      owner: { userId: "user-1", botId: "bot-000" },
+      clock: () => AT,
+    });
+
+    const tier = await store.read(userMemoryRootV1(OWNER));
+
+    expect(tier.sources).toHaveLength(MEMORY_MAX_FILES_PER_TIER);
+    expect(tier.omitted).toContain("the newest");
+    const shardIds = tier.sources.map((source) => source.botId).sort();
+    // The two oldest shards were dropped; the newest write is still read.
+    expect(shardIds).not.toContain("bot-000");
+    expect(shardIds).not.toContain("bot-001");
+    expect(shardIds).toContain(`bot-${String(shards - 1).padStart(3, "0")}`);
+  });
+
+  test("a forget refuses rather than reporting a fact it could not have removed", async () => {
+    const files = await crowdedUserTier(MEMORY_MAX_FILES_PER_TIER + 1);
+    const store = new MemoryStore({
+      files,
+      owner: { userId: "user-1", botId: "bot-000" },
+      clock: () => AT,
+    });
+
+    const forgotten = await store.forget({
+      root: userMemoryRootV1(OWNER),
+      fact: "Shard 000 learned something.",
+      writer: writerFor("bot-000"),
+    });
+
+    expect(forgotten.status).toBe("unavailable");
+    if (forgotten.status !== "unavailable") throw new Error("unreachable");
+    expect(forgotten.reason).toContain("read bound");
+    // And the fact is still on disk, which is what the refusal is about.
+    const own = await files.read({
+      root: userMemoryRootV1(OWNER),
+      path: "by-agent/bot-000/profile.md",
+    });
+    expect(own.status).toBe("ok");
+    if (own.status !== "ok") return;
+    expect(new TextDecoder().decode(own.file.bytes)).toContain(
+      "Shard 000 learned something.",
+    );
+  });
+
+  test("keeps every omission when two bounds bite at once", async () => {
+    // A listing that never ends, of more files than the tier bound reads:
+    // both the page bound and the file bound cut this read, and a caller that
+    // must refuse on an incomplete read needs to be told both.
+    const entries = (page: number) =>
+      Array.from({ length: 100 }, (_, index) => {
+        const botId = `bot-${String(page * 100 + index).padStart(4, "0")}`;
+        return {
+          path: {
+            root: userMemoryRootV1(OWNER),
+            path: `by-agent/${botId}/profile.md`,
+          },
+          generation: {
+            schemaVersion: 1 as const,
+            generationId: `${String(page * 100 + index).padStart(9, "0")}`,
+            contentHash: "0".repeat(64),
+            size: 12,
+            writer: writerFor(botId),
+            writtenAt: AT.toISOString(),
+          },
+        };
+      });
+    const endless: WorkspaceFilesV1 = {
+      list: (request) => {
+        const page = Number(request.cursor ?? "0");
+        return Promise.resolve({
+          status: "ok",
+          entries: entries(page),
+          cursor: String(page + 1),
+        });
+      },
+      read: (path) =>
+        Promise.resolve({
+          status: "ok",
+          file: {
+            path,
+            generation: {
+              schemaVersion: 1,
+              generationId: "000000001",
+              contentHash: "0".repeat(64),
+              size: 12,
+              writer: writerFor("bot-0000"),
+              writtenAt: AT.toISOString(),
+            },
+            bytes: new TextEncoder().encode("- 2026-08-31 A fact.\n"),
+          },
+        }),
+      stat: () =>
+        Promise.resolve({ status: "not-found", reason: "unused in this test" }),
+      write: () =>
+        Promise.resolve({ status: "refused", reason: "unused in this test" }),
+      delete: () =>
+        Promise.resolve({ status: "refused", reason: "unused in this test" }),
+    };
+    const store = new MemoryStore({
+      files: endless,
+      owner: { userId: "user-1", botId: "bot-0000" },
+      clock: () => AT,
+    });
+
+    const tier = await store.read(userMemoryRootV1(OWNER));
+
+    expect(tier.omitted).toContain("did not finish listing");
+    expect(tier.omitted).toContain("read bound were not read");
+  });
+});