@open-everywhere/core 0.1.0

package/agent/memory/graph.ts

@@ -0,0 +1,178 @@
+// A wikilink graph over a silo's memory files, derived on demand and never persisted: a `graph.json` beside the silo would land in the agent-visible memory tree and go stale when a file changes out of band.
+// No I/O lives here. `graphToJSON` exists for a caller that does want to cache it.
+
+import type { MemoryFiles } from "./store.js";
+
+export interface MemoryEdge {
+  /** Source node id — the markdown file that contains the link (relPath minus `.md`). */
+  readonly from: string;
+  /** Target node id — a resolved file/topic, or the cleaned raw link text when dangling. */
+  readonly to: string;
+  /** True when `to` matched a known file/topic; false marks a dangling link (kept: it's intent). */
+  readonly resolved: boolean;
+}
+
+export interface MemoryGraph {
+  readonly nodes: readonly string[];
+  /** Deduped edges (one per distinct from→to pair). */
+  readonly edges: readonly MemoryEdge[];
+}
+
+export interface BoostOptions {
+  /**
+   * Which edges connect a seed to a neighbour. Default `both` — recall benefits from a symmetric association.
+   * `in` matches resolved edges only, because only a resolved edge has a real node on its `to` side.
+   */
+  readonly direction?: "out" | "in" | "both";
+  /** Include dangling targets as neighbours. Default false — a dangling link points at no file, so there is nothing to resurface. */
+  readonly includeUnresolved?: boolean;
+}
+
+/** A neighbor surfaced by the boost, with its (edge-count) weight. */
+export interface NeighborBoost {
+  readonly node: string;
+  readonly weight: number;
+}
+
+const WIKILINK = /\[\[([^[\]]+?)\]\]/g;
+
+function stripMd(relPath: string): string {
+  return relPath.replace(/\.md$/i, "");
+}
+
+/**
+ * Normalise a name for MATCHING (lower-case, spaces/underscores → `-`, drop other
+ * punctuation) while preserving `/` so path-qualified links (`[[decisions/foo]]`) still
+ * resolve. Purely a lookup key — never shown to the agent.
+ */
+function slug(s: string): string {
+  return s
+    .trim()
+    .toLowerCase()
+    .replace(/\.md$/i, "")
+    .replace(/[\s_]+/g, "-")
+    .replace(/[^a-z0-9/-]+/g, "")
+    .replace(/-*\/-*/g, "/") // collapse dashes hugging a path separator (`a / b` → `a/b`)
+    .replace(/-{2,}/g, "-")
+    .replace(/^[-/]+|[-/]+$/g, "");
+}
+
+function baseOf(s: string): string {
+  const i = s.lastIndexOf("/");
+  return i === -1 ? s : s.slice(i + 1);
+}
+
+function cleanTarget(raw: string): string {
+  return raw.split("|")[0].split("#")[0].trim();
+}
+
+/**
+ * The wikilink targets in document order, with a `|alias` and a `#anchor` stripped and empty links dropped.
+ * Duplicates are kept — the caller dedups at edge level.
+ */
+export function parseWikiLinks(content: string): string[] {
+  const out: string[] = [];
+  for (const m of content.matchAll(WIKILINK)) {
+    const target = cleanTarget(m[1]);
+    if (target.length > 0) out.push(target);
+  }
+  return out;
+}
+
+// A resolver from an arbitrary link/ref to a canonical node id. Full-path slugs win over
+// bare basenames; an ambiguous basename (two files share it) resolves to neither.
+interface NodeIndex {
+  readonly byPath: Map<string, string>;
+  readonly byBase: Map<string, string | null>; // null == ambiguous (>1 file)
+}
+
+function buildIndex(nodes: readonly string[]): NodeIndex {
+  const byPath = new Map<string, string>();
+  const byBase = new Map<string, string | null>();
+  for (const id of nodes) {
+    const s = slug(id);
+    if (!byPath.has(s)) byPath.set(s, id);
+    const b = baseOf(s);
+    byBase.set(b, byBase.has(b) ? null : id);
+  }
+  return { byPath, byBase };
+}
+
+function resolveWith(index: NodeIndex, ref: string): string | null {
+  const s = slug(ref);
+  if (s.length === 0) return null;
+  const byPath = index.byPath.get(s);
+  if (byPath) return byPath;
+  const byBase = index.byBase.get(baseOf(s));
+  return byBase ?? null; // null both when absent and when ambiguous
+}
+
+export function resolveNode(graph: MemoryGraph, ref: string): string | null {
+  return resolveWith(buildIndex(graph.nodes), ref);
+}
+
+/**
+ * Only markdown files are nodes; each file's links become edges to the resolved target, or to the cleaned raw text when dangling (`resolved:false`).
+ * Self-links are kept, edges are deduped per from-to pair, and a dangling link never throws — an unresolved link is valid intent.
+ */
+export function buildGraph(files: MemoryFiles): MemoryGraph {
+  const mdPaths = Object.keys(files).filter((p) => /\.md$/i.test(p));
+  const nodes = mdPaths.map(stripMd);
+  const index = buildIndex(nodes);
+
+  const seen = new Set<string>();
+  const edges: MemoryEdge[] = [];
+  for (const relPath of mdPaths) {
+    const from = stripMd(relPath);
+    for (const target of parseWikiLinks(files[relPath])) {
+      const resolvedId = resolveWith(index, target);
+      const to = resolvedId ?? slug(target);
+      if (to.length === 0) continue;
+      const key = `${from} ${to}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      edges.push({ from, to, resolved: resolvedId !== null });
+    }
+  }
+  return { nodes, edges };
+}
+
+/**
+ * 1-hop boost: the neighbours of the seed refs, weighted by how many edges tie them to the seed set.
+ * A seed that does not resolve to a node is ignored, and a neighbour that is itself a seed is dropped.
+ */
+export function adjacencyBoost(
+  graph: MemoryGraph,
+  seeds: readonly string[],
+  opts: BoostOptions = {},
+): NeighborBoost[] {
+  const direction = opts.direction ?? "both";
+  const index = buildIndex(graph.nodes);
+  const seedSet = new Set<string>();
+  for (const s of seeds) {
+    const id = resolveWith(index, s);
+    if (id) seedSet.add(id);
+  }
+  if (seedSet.size === 0) return [];
+
+  const weights = new Map<string, number>();
+  const bump = (node: string): void => {
+    if (seedSet.has(node)) return; // don't boost a seed back to itself
+    weights.set(node, (weights.get(node) ?? 0) + 1);
+  };
+
+  for (const e of graph.edges) {
+    const wantOut = direction !== "in";
+    const wantIn = direction !== "out";
+    if (wantOut && seedSet.has(e.from) && (e.resolved || opts.includeUnresolved)) bump(e.to);
+    if (wantIn && e.resolved && seedSet.has(e.to)) bump(e.from);
+  }
+
+  return [...weights.entries()]
+    .map(([node, weight]) => ({ node, weight }))
+    .sort((a, b) => b.weight - a.weight || a.node.localeCompare(b.node));
+}
+
+export function graphToJSON(graph: MemoryGraph): string {
+  return JSON.stringify({ nodes: graph.nodes, edges: graph.edges }, null, 2);
+}

package/agent/memory/host-owned-files.ts

@@ -0,0 +1,3 @@
+// Files a sibling session-end hook writes host-direct: hooks/learn.ts enriches RECURRING.md on the host before the memory hook persists, so the sandbox's session-start snapshot must never overlay it nor count as a conflict. Host always wins for these.
+// The set lives here, below both hooks and memory, so hooks/memory.ts and remember-plan.ts's TOPIC guard read one set without an import going the wrong direction.
+export const HOST_OWNED_FILES: ReadonlySet<string> = new Set(["RECURRING.md"]);

package/agent/memory/layout.ts

@@ -0,0 +1,85 @@
+// Provider-agnostic memory layout: the git provider (store.ts) and the SQL providers both import `tierRelDir` and `seedFor`, so a tier keys and seeds identically on every backend and a migration is a straight copy.
+// The Tier/Partition imports are type-only, so there is no runtime dependency on store.ts and no import cycle.
+
+import type { Partition } from "./partition.js";
+import type { Tier } from "./store.js";
+import type { MemoryFiles } from "./store.js";
+
+// Defensive: domain names come from config but build filesystem paths / keys.
+export function safeName(name: string): string {
+  return name.trim().replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^[-.]+|[-.]+$/g, "").slice(0, 100) || "unnamed";
+}
+
+/** The canonical scope path a tier occupies: a directory under the git provider, the `tier_path` primary key under the SQL providers. */
+export function tierRelDir(p: Partition, tier: Tier): string {
+  switch (tier.kind) {
+    case "workspace":
+      return `${p.workspaceKey}/@workspace`;
+    case "silo":
+      // Keyed by SILO NAME alone (org is the store-root prefix) — so the SAME silo
+      // referenced by many domains resolves to ONE shared bucket.
+      return `@silos/${safeName(tier.name)}`;
+    case "surface":
+      return `${p.workspaceKey}/@surfaces/${p.surface}`;
+  }
+}
+
+export function seedFor(p: Partition, tier: Tier): MemoryFiles {
+  // Seeded rather than left blank because the agent writes these files freehand (OPE-389): the headings are the convention, so a new fact lands under an existing section instead of a fresh heading at EOF.
+  const body = `
+## Standing instructions
+(none yet)
+
+## People / ownership
+(none yet)
+
+## Decisions
+(none yet)
+
+## Notes
+(none yet)
+`;
+  // The provenance sentence is load-bearing, not manners: the host marks a bullet added on a turn that ingested external content (OPE-426), but only this sentence asks for the source behind prose and a heading.
+  // So is the `State each fact ONCE` pair before it (OPE-1977). A silo that keeps every statement of one fact makes the reader pick, and on the e2e S-MEM-READ leg of board #224 the reader picked a previous board's canary and called the tree inconsistent. The first sentence stops the stack forming; the second tells whoever finds one which line wins. 🔴 This seed reaches a silo only on its FIRST materialisation — `ensureTier` returns the existing map once the tier has keys — so an already-seeded silo never gains these sentences and reads them from the memory index instead. A HOST road cannot apply this rule for the writer — the repo's word-overlap kernel scores two different facts 1.00, so an auto-applied rewrite deletes live facts (see docs/capabilities/memory.md, Writing memory). The writer is the only thing that knows the fact, and this seed is ONE of two copies it reads — the memory index (`memory/listing.ts`) carries the same pair, and for an already-seeded silo the index is the only one.
+  const linkGuidance =
+    "Link related subjects like `[[name]]` — facts about the same person/system/decision " +
+    "connect up. This index keeps only the most recent few bullets per section; each " +
+    "category's full history lives in its own topic file (`people.md`, `decisions.md`, …). " +
+    "State each fact ONCE: if a line here already states the fact you are saving, EDIT that line " +
+    "to the new value instead of adding a second line about the same subject. Where you do find " +
+    "two lines stating one fact, the one with the LATER date is the current one — answer with it, " +
+    "and do not report the pair as a conflict. If the two carry the SAME date, or either is " +
+    "undated, say the files disagree rather than choosing one. " +
+    "If a fact came from outside this conversation — a fetched page, an issue or PR body, a " +
+    "connector result, an uploaded file — say so on the bullet: nothing else records where it " +
+    "came from.";
+
+  switch (tier.kind) {
+    case "workspace":
+      return {
+        "MEMORY.md": `# Workspace memory — index (${p.workspaceKey})
+
+Shared across **every surface** (channel/DM) in this workspace. Put facts here
+when they're true org-wide — deploy process, who owns what, standing decisions —
+so a new or similar surface inherits them instead of starting blank. ${linkGuidance}
+${body}`,
+      };
+    case "silo":
+      return {
+        "MEMORY.md": `# Memory silo — "${tier.name}"
+
+A shared brain. Every channel whose domain references the **${tier.name}** silo reads
+and writes here, so a sibling channel isn't context-stranded. Put facts here that are
+true for this team/area/topic. ${linkGuidance}
+${body}`,
+      };
+    case "surface":
+      return {
+        "MEMORY.md": `# Personal memory — index (${p.surfaceKey})
+
+Specific to **this DM only** — your private universe with this person. Nothing here
+leaks to shared (silo) memory. ${linkGuidance}
+${body}`,
+      };
+  }
+}

package/agent/memory/listing.ts

@@ -0,0 +1,199 @@
+// Renders the top-level memory index (memory/MEMORY.md), the table of contents the agent reads first — memory is never in the model's context automatically.
+// It carries what the filesystem cannot: each silo's purpose, a source label, and the route for reading and updating it.
+
+import type { ResolvedMemory } from "./resolve.js";
+import type { MemorySilo } from "./silo.js";
+import { isProjectedCache, isAppendOnlyBacking } from "./readonly.js";
+import { tierSandboxDir } from "./sandbox-sync.js";
+import type { MemoryRecallRoute } from "../access/window-gate-manifest.js";
+
+const code = (s: string): string => "`" + s + "`";
+
+export function sourceLabel(silo: MemorySilo): string {
+  switch (silo.provider) {
+    case "notion":
+      return "Notion";
+    case "github":
+      return silo.container ? `GitHub (${silo.container})` : "GitHub";
+    case "git":
+      return "git";
+    case "postgres":
+    case "sqlite":
+      return "database";
+    default:
+      return silo.provider;
+  }
+}
+
+/**
+ * There is no remember tool. The only write path is a FILE EDIT in the tier's sandbox directory, which hooks/memory.ts persistMemoryNow diffs and persists at the turn's end.
+ * The clause is route-aware: a delegating seat cannot write the file itself, and a none seat can neither open nor write it.
+ */
+function accessClause(silo: MemorySilo, readOnly: boolean, recall: MemoryRecallRoute, dir: string): string {
+  const delegated = recall === "agent";
+  const unreachable = recall === "none";
+  const file = code(dir + "/MEMORY.md");
+  // 🔴 THE REWRITE RULE ONLY REACHES A BACKING THAT CAN REWRITE (OPE-1977). A save supersedes the
+  // section's previous statement of the same subject, so the writer is told to change that line. An
+  // APPEND-ONLY silo (Notion) REFUSES an edit above the last line, so on that backing the same
+  // sentence would instruct a save the persist then drops — the OPE-70 shape two clauses below, with
+  // the rule pointing the wrong way. Such a silo appends, and the newest-wins rule in the recall lead
+  // is what resolves the pair it leaves.
+  const appendOnly = isAppendOnlyBacking(silo);
+  const rewriteInPlace = appendOnly ? "" : " Where a line already states the fact you are saving, CHANGE that line to the new value rather than adding a second one.";
+  const rewriteInBrief = appendOnly ? "" : "If a line there already states that fact, CHANGE that line to the new value instead of adding a second one. ";
+  // The brief is a TEMPLATE, not a description. An exact shape is what survives a cheap model, and the sync mode is what makes the seat's own reply wait for the worker's report.
+  const saveVerb = unreachable
+    ? "this seat CANNOT write these files and cannot delegate — say plainly that you can't save anything to memory from here, never that you saved it"
+    : delegated
+      ? `to save something, delegate it THIS turn — \`${recall}\` with \`mode: "sync"\` and a brief of exactly this shape: ` +
+        `"Open ${dir}/MEMORY.md with your file tools and add this fact for the team under the right heading (as a note, or as a decision if it records a choice): <the fact, verbatim>. ` +
+        rewriteInBrief +
+        `Reply with the exact line you wrote and the path you wrote it to.". ` +
+        `A fact is not a standing rule — never frame it as one. Only after the worker reports the line it WROTE, reply naming what was saved ("Saved: <the fact>") — never before, and never a bare "Done."`
+      : `save durable facts here by EDITING ${file} (or another file in this silo) with your file tools — the edit IS the save, and the host persists it when the turn ends. Nothing is saved by saying it is.${rewriteInPlace}`;
+  if (!readOnly) {
+    // An APPEND-ONLY backing (Notion) keeps the writable verb but not the plain writable promise:
+    // an edit above the last line is refused at persist, so the seat is told here rather than
+    // discovering it from a host log it never sees.
+    return appendOnly
+      ? `writable but APPEND-ONLY — ${saveVerb}. Add new lines at the END of a file; changing or removing an existing line is REFUSED and does NOT save`
+      : `writable — ${saveVerb}`;
+  }
+  if (isProjectedCache(silo)) {
+    const via = silo.skill ? `the ${code(silo.skill)} skill` : "the Notion tools";
+    return `read-only cache — edits are BLOCKED; to update, use ${via} (never edit these files)`;
+  }
+  return delegated
+    ? "read-only — edits are blocked; this channel's own memory is written the same delegated way"
+    : unreachable
+      ? "read-only — and this seat cannot open or write these files at all"
+      : "read-only — edits are blocked; edit this channel's OWN writable silo above instead";
+}
+
+/**
+ * 🔴 THE NEWEST STATEMENT OF A FACT IS THE CURRENT ONE (OPE-1977). A silo can hold two lines about one
+ * subject — an older save the writer did not rewrite, or a value the dream cycle re-enriched. Read as a
+ * conflict, that pair makes the seat pick, and on the e2e S-MEM-READ leg of board #224 it picked the
+ * older line and reported the files as inconsistent. The date stamp already orders them, so the reader
+ * needs the rule, not a repair. `memory/layout.ts` seeds the same sentence into a silo's own
+ * `MEMORY.md` on its FIRST materialisation only — `ensureTier` returns the existing map once the tier
+ * has keys, and the Notion backing never seeds — so an already-seeded silo reads the rule HERE and
+ * nowhere else.
+ *
+ * 🔴 THE RULE IS BOUNDED BY WHAT ORDERS THE PAIR. Bullets are DAY-granular, so two saves of one fact on
+ * one day — or an undated line — leave nothing to read the rule against. Unbounded, the sentence would
+ * forbid the honest answer there and send the seat back to picking, which is the board-#224 failure
+ * with its one remaining escape closed. So the clause names that case and asks for "the files disagree".
+ *
+ * It rides the `file-tools` and `agent` leads only. A `none` seat reads nothing, so a rule for resolving
+ * what it found is the dead-affordance shape OPE-771 removed.
+ */
+const NEWEST_WINS =
+  "Where two lines state one fact, the one with the LATER date is the current value — answer with it, " +
+  "and do not report the pair as a conflict. If the two carry the SAME date, or either is undated, " +
+  "nothing orders them: say the files disagree rather than choosing one.";
+
+/**
+ * The how-to-read lead for each recall route. The index is the one place a seat is told which verb it holds, and naming a verb it lacks is a dead affordance.
+ * `renderMemoryIndex` defaults to `file-tools`, so the on-disk copy at `/workspace/memory/MEMORY.md` carries that route's lead — only the in-prompt copy passes the seat's real route.
+ */
+const RECALL_GUIDANCE: Readonly<Record<MemoryRecallRoute, { readonly lead: string; readonly heading: string }>> = {
+  "file-tools": {
+    lead: "Read the relevant silo with your tools before acting. " + NEWEST_WINS,
+    heading: "## Silos — read with file tools",
+  },
+  agent: {
+    lead:
+      "This seat has NO file tools, so you cannot open these files yourself. To recall anything from " +
+      "memory, delegate a lookup: `agent` with `mode: \"sync\"` and a brief naming the paths below and " +
+      "what you need out of them (the worker has the file tools, greps the tree, and returns what it " +
+      "found this turn). Do that before answering from guesswork — and if the lookup comes back empty, " +
+      "say so rather than inventing the answer. " + NEWEST_WINS,
+    heading: "## Silos — read by delegating a lookup",
+  },
+  none: {
+    lead:
+      "This seat has NO file tools and cannot delegate, so it CANNOT read these files. If a question " +
+      "turns on what is in them, say plainly that you can't check memory from here — never answer as " +
+      "if you had read it.",
+    heading: "## Silos — on disk, but unreachable from this seat",
+  },
+};
+
+/**
+ * Kept beside RECALL_GUIDANCE so one route can never be described two ways — an honest index followed by a wake reminder naming file tools the seat lacks is what that prevents.
+ * Two tails per route: the CONTENT delta is followed by a pointer list, so it ends on a colon, and the INDEX delta is a standalone sentence.
+ */
+const RECALL_DELTA_TAILS: Readonly<Record<MemoryRecallRoute, { readonly content: string; readonly index: string }>> = {
+  "file-tools": {
+    content: "re-read a path with your file tools ONLY if it bears on the current message, otherwise ignore it:",
+    index: "re-read the relevant path before answering from memory if it bears on the current message.",
+  },
+  agent: {
+    content:
+      "this seat has no file tools — ONLY if it bears on the current message, delegate a lookup " +
+      "(`agent`, `mode: \"sync\"`) naming the path; otherwise ignore it:",
+    index:
+      "this seat has no file tools, so delegate a lookup (`agent`, `mode: \"sync\"`) naming the path " +
+      "before answering from memory, if it bears on the current message.",
+  },
+  none: {
+    content:
+      "this seat CANNOT open these files — ONLY if it bears on the current message, say you can't " +
+      "check memory from here rather than answering as if you had:",
+    index:
+      "this seat CANNOT open it — if it bears on the current message, say you can't check memory from " +
+      "here rather than answering as if you had.",
+  },
+};
+
+export function recallDeltaTail(recall: MemoryRecallRoute, shape: "content" | "index"): string {
+  return (RECALL_DELTA_TAILS[recall] ?? RECALL_DELTA_TAILS["file-tools"])[shape];
+}
+
+/**
+ * failedSilos names sources that were PLANNED but failed to materialise. The index must say so, or the agent reads missing as empty and answers from partial memory.
+ * recall defaults to file-tools: the on-disk copy is only ever read by something holding the file tools, and only the in-prompt copy passes the seat's real route.
+ */
+export function renderMemoryIndex(
+  reads: readonly ResolvedMemory[],
+  failedSilos: readonly string[] = [],
+  recall: MemoryRecallRoute = "file-tools",
+): string {
+  const guidance = RECALL_GUIDANCE[recall] ?? RECALL_GUIDANCE["file-tools"];
+  const lines: string[] = [
+    "# Memory index",
+    "",
+    "Your durable memory for this channel — files on disk, **not** in your context automatically.",
+    guidance.lead,
+    "",
+  ];
+
+  if (reads.length === 0 && failedSilos.length === 0) {
+    lines.push("(No memory is configured for this channel.)", "");
+    return lines.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
+  }
+
+  if (reads.length > 0) {
+    lines.push(guidance.heading);
+    for (const r of reads) {
+      const dir = tierSandboxDir(r.tier);
+      const desc = r.silo.description ? `${r.silo.description} — ` : "";
+      // "Start with …" tells a seat to OPEN a path, which a none seat holds no tool to do. The path is still named on every route, because a delegating seat has to put it in the brief.
+      const entry = recall === "none" ? `The index entry is ${code(dir + "/MEMORY.md")}` : `Start with ${code(dir + "/MEMORY.md")}`;
+      lines.push(`- **${r.silo.name}** (${dir + "/"}) — ${desc}source: ${sourceLabel(r.silo)}; ${accessClause(r.silo, r.readOnly, recall, dir)}. ${entry}.`);
+    }
+    lines.push("");
+  }
+
+  if (failedSilos.length > 0) {
+    lines.push(
+      "## ⚠ Unavailable this session",
+      ...failedSilos.map((name) => `- **${name}** failed to load — its knowledge is MISSING from disk right now, not empty. Say so if asked about anything it might hold; do not answer as if memory were complete.`),
+      "",
+    );
+  }
+
+  return lines.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
+}