@open-everywhere/core 0.1.0

package/agent/memory/resolve.ts

@@ -0,0 +1,181 @@
+// The memory resolver: given a session's binding and partition it returns the MemoryPlan — which silos to read into the sandbox and which to persist back. Pure, so the config is passed in.
+// It replaces the old automatic tier stack: there is no automatic workspace or surface memory, and a channel has memory only through a domain.
+
+import type { SurfaceBinding } from "../identity/binding.js";
+import { domainKeyOf, type Partition } from "./partition.js";
+import type { Domain } from "./domain.js";
+import type { MemorySilo } from "./silo.js";
+import { SURFACE, siloTier, type Tier } from "./store.js";
+import { domainForChannel, criteriaDomain, memoryRefs } from "./domain.js";
+import { dbProviderId, getProvider, providerById, type MemoryAccess, type MemoryProvider } from "./provider.js";
+import { getConfig } from "../access/config.js";
+
+export interface ResolvedMemory {
+  readonly silo: MemorySilo;
+  readonly tier: Tier;
+  readonly readOnly: boolean;
+  /**
+   * Whose credential this source is read under — the domain whose `memory` list NAMED the silo, not the session's domain: under `includeOrg` the org's silos are read under the ORG's credential.
+   * Absent on the built-in personal (DB) silo, whose credential the deployment holds.
+   */
+  readonly access?: MemoryAccess;
+}
+
+export interface MemoryPlan {
+  /** Sources materialised into the sandbox (what the agent may read). */
+  readonly reads: readonly ResolvedMemory[];
+  /** Sources persisted back at session end (subset of reads; never the read-only ones). */
+  readonly writes: readonly ResolvedMemory[];
+  /** Human-readable rationale, for logs/diagnostics. */
+  readonly reason: string;
+}
+
+/** The slice of AppConfig the resolver needs (passed explicitly so the core stays pure/testable). */
+export interface MemoryResolveConfig {
+  readonly domains: readonly Domain[];
+  readonly silos: readonly MemorySilo[];
+}
+
+const NONE = (reason: string): MemoryPlan => ({ reads: [], writes: [], reason });
+
+/**
+ * The built-in personal silo for DMs — HARD-pinned to the DB provider (sqlite/postgres),
+ * never git/Notion, so personal memory can't leak into a shared/durable repo store.
+ */
+function personalSilo(): MemorySilo {
+  return { name: "@personal", provider: dbProviderId() };
+}
+
+/**
+ * The provider a SURFACE-tier read or write must use for THIS session. A DM's SURFACE tier IS the per-user personal silo, pinned to the DB provider.
+ * A caller that writes SURFACE-tier bytes outside the resolved plan (a status line, the onboarding marker) must come through here, or the write lands on the env-selected provider and defeats the pin.
+ */
+export function surfaceProvider(binding: SurfaceBinding): MemoryProvider {
+  return binding.mode === "dm" ? providerById(dbProviderId()) : getProvider();
+}
+
+function findSilo(silos: readonly MemorySilo[], name: string, referencedBy: string): MemorySilo {
+  const silo = silos.find((s) => s.name === name);
+  if (!silo) {
+    throw new Error(
+      `memory silo "${name}" referenced by ${referencedBy} is not defined — add a ` +
+        `defineMemorySilo({ name: "${name}", … }) (memory/silo.ts / docs/capabilities/memory.md).`,
+    );
+  }
+  return silo;
+}
+
+/**
+ * Resolve a domain's memory into read sources: its own silos, plus the `org` pseudo-domain's when it sets `includeOrg`. Returns [] for a grouping-only domain.
+ * `forceReadOnly` overrides every silo; otherwise each keeps its own `readOnly`, `includeOrg: "readonly"` forces the org slice, and an explicitly-listed silo wins over the includeOrg copy.
+ */
+function resolveDomainReads(domain: Domain, cfg: MemoryResolveConfig, forceReadOnly: boolean): ResolvedMemory[] {
+  const out: ResolvedMemory[] = [];
+  const seen = new Set<string>();
+  for (const name of memoryRefs(domain)) {
+    const silo = findSilo(cfg.silos, name, `domain "${domain.name}"`);
+    if (seen.has(silo.name)) continue;
+    seen.add(silo.name);
+    out.push({ silo, tier: siloTier(silo.name), readOnly: forceReadOnly || silo.readOnly === true, access: { domain } });
+  }
+  if (domain.includeOrg) {
+    const org = criteriaDomain("org", cfg.domains);
+    // assertDomainsValid guarantees an org domain exists when includeOrg is set; guard anyway.
+    for (const name of org ? memoryRefs(org) : []) {
+      const silo = findSilo(cfg.silos, name, `org default (via includeOrg on "${domain.name}")`);
+      if (seen.has(silo.name)) continue; // an explicit memory entry wins
+      seen.add(silo.name);
+      const readOnly = forceReadOnly || domain.includeOrg === "readonly" || silo.readOnly === true;
+      // The ORG owns these pages, so the ORG's credential reads them — see ResolvedMemory.access.
+      out.push({ silo, tier: siloTier(silo.name), readOnly, access: org ? { domain: org } : undefined });
+    }
+  }
+  return out;
+}
+
+function planFromReads(reads: ResolvedMemory[], reason: string): MemoryPlan {
+  return { reads, writes: reads.filter((r) => !r.readOnly), reason };
+}
+
+/**
+ * Resolve the memory plan for a session. PURE — all config is passed in. The order of
+ * the checks IS the policy; in particular the trust gate sits BEFORE any explicit-domain
+ * lookup so an externally-shared channel can never reach an internal domain, even if it
+ * was (mis)placed in one at config time — it routes to the `public` pseudo-domain instead.
+ */
+export function resolveMemoryPlan(
+  binding: SurfaceBinding,
+  partition: Partition,
+  cfg: MemoryResolveConfig,
+): MemoryPlan {
+  if (binding.mode === "mpim") return NONE("group chat — execution-only, no memory");
+
+  // A DM always gets its per-user DB silo (writable, isolated per principal) plus a shared read layer chosen by the PARTNER's trust, forced read-only — a DM never writes into a shared brain.
+  if (binding.mode === "dm") {
+    const personal: ResolvedMemory = { silo: personalSilo(), tier: SURFACE, readOnly: false };
+    const sharedDomain = criteriaDomain(binding.trust === "internal" ? "dm" : "public", cfg.domains);
+    const shared = sharedDomain ? resolveDomainReads(sharedDomain, cfg, true) : [];
+    return {
+      reads: [personal, ...shared],
+      writes: [personal],
+      reason: shared.length
+        ? `DM — personal (DB) + ${binding.trust === "internal" ? "dm" : "public"} shared reads (${shared.length}, read-only)`
+        : "DM — personal memory (DB)",
+    };
+  }
+
+  // THE TRUST GATE: an externally-shared or guest-bearing channel never reaches an internal domain. It routes to the `public` pseudo-domain, read-only.
+  // Re-evaluated every session, and config cannot place an external channel in an internal domain — there is no override.
+  if (binding.trust === "external") {
+    const pub = criteriaDomain("public", cfg.domains);
+    const reads = pub ? resolveDomainReads(pub, cfg, true) : [];
+    return {
+      reads,
+      writes: [], // external sessions never persist into shared memory
+      reason: reads.length
+        ? `external channel → public memory (${reads.length}, read-only)`
+        : "externally-shared / guest channel — no public memory configured (trust gate)",
+    };
+  }
+
+  // DOMAIN grain, not surface grain: a Discord thread's surfaceKey is the thread's own channel id and would never match a domain listing the parent channel.
+  // `domainKeyOf` resolves the parent there, and is the surfaceKey itself everywhere else.
+  const domain = domainForChannel(domainKeyOf(partition), cfg.domains);
+  if (domain) {
+    const reads = resolveDomainReads(domain, cfg, false);
+    if (reads.length > 0) {
+      const refs = memoryRefs(domain);
+      const detail = domain.includeOrg ? `${reads.length} silos incl. org` : refs.length === 1 ? refs[0] : `${reads.length} silos`;
+      return planFromReads(reads, `domain "${domain.name}" (${detail})`);
+    }
+    // A grouping-only domain does NOT fall through to org (single hop, v1).
+    return NONE(`domain "${domain.name}" has no memory silo — grouping only`);
+  }
+
+  // Silos reached through the org fallback keep their own read/write — read-only is the recommended default for a curated baseline, not forced.
+  const org = criteriaDomain("org", cfg.domains);
+  if (org) {
+    const reads = resolveDomainReads(org, cfg, false);
+    if (reads.length > 0) return planFromReads(reads, `org default (${reads.length})`);
+  }
+
+  return NONE("unassigned channel — no domain, no org default");
+}
+
+/**
+ * Whose credential a SILO-FIRST caller reads under — the control plane, the tidy schedule, the purge worker, which hold a silo and an org but never a session.
+ * The owning domain is the one whose `memory` list names the silo, EXACTLY ONE or none: a silo named by two domains falls back to the org default rather than to an arbitrary owner, even when that reads empty.
+ */
+export function siloAccess(siloName: string, domains: readonly Domain[]): MemoryAccess {
+  const owners = domains.filter((d) => memoryRefs(d).includes(siloName));
+  return owners.length === 1 ? { domain: owners[0] } : {};
+}
+
+/** Runtime convenience: resolve from the registered app config. */
+export function planFor(binding: SurfaceBinding, partition: Partition): MemoryPlan {
+  const cfg = getConfig();
+  return resolveMemoryPlan(binding, partition, {
+    domains: cfg.loadDomains(),
+    silos: cfg.loadSilos(),
+  });
+}

package/agent/memory/root-fence.ts

@@ -0,0 +1,54 @@
+// Takes write off the `memory/` ROOT directory only, so a shell write that would CREATE a silo directory fails with EACCES instead of being accepted and dropped at flush time (OPE-999).
+// NON-recursive on purpose — `chmod -R a-w` would break the writable tiers — and a FAIL-LOUD guardrail, not a security boundary: the agent's own shell can `chmod u+w memory`.
+
+/** The memory tree's root directory, sandbox-relative — `/workspace/memory`. */
+export const MEMORY_ROOT_DIR = "memory";
+
+export function memoryRootLockCommand(): string {
+  return `chmod a-w '${MEMORY_ROOT_DIR}'`;
+}
+
+export function memoryRootUnlockCommand(): string {
+  return `chmod u+w '${MEMORY_ROOT_DIR}'`;
+}
+
+/**
+ * The minimum a sandbox session has to offer this module — `run` and nothing else.
+ * `PromiseLike`, not `Promise`: the AI SDK sandbox surface types `run` that way, so a real `SandboxSession` is assignable only if this shape says `PromiseLike` too.
+ */
+export interface RootFenceSandbox {
+  run(options: { command: string }): PromiseLike<unknown>;
+}
+
+/**
+ * Best-effort: a chmod failure degrades to an unfenced root and never breaks the session, because a session that cannot write memory at all is the worse outcome.
+ * Returns whether the lock command ran without throwing, so a caller can log the degradation.
+ */
+export async function lockMemoryRoot(sandbox: RootFenceSandbox): Promise<boolean> {
+  try {
+    await sandbox.run({ command: memoryRootLockCommand() });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Unlock the root, run `fn`, and RE-LOCK in a `finally`: a mid-write transport error must never leave the root writable for the rest of a session, because an unfenced root is silent.
+ * The unlock is a WINDOW — a `bash` call in the same model step could create a silo directory, and those bytes are then dropped at flush time as they were before the fence.
+ */
+export async function withMemoryRootWritable<T>(sandbox: RootFenceSandbox, fn: () => Promise<T>): Promise<T> {
+  let unlocked = false;
+  try {
+    await sandbox.run({ command: memoryRootUnlockCommand() });
+    unlocked = true;
+  } catch {
+    // Never materialised / already writable / transport blip — run the work anyway. A refusal to do
+    // the host's own write because a chmod failed would turn a degraded fence into a broken feature.
+  }
+  try {
+    return await fn();
+  } finally {
+    if (unlocked) await lockMemoryRoot(sandbox);
+  }
+}

package/agent/memory/sandbox-sync.ts

@@ -0,0 +1,84 @@
+// Shuttles a tier's memory files between the durable host store and the ephemeral sandbox: materialized under /workspace/memory at session start, captured back at session end.
+// The agent reads and writes them as plain files, so every path here anchors to /workspace.
+
+import type { SandboxSession } from "eve/sandbox";
+import type { MemoryFiles, Tier } from "./store.js";
+import { safeName } from "./layout.js";
+
+const BASE = "memory"; // under /workspace -> /workspace/memory
+
+export function tierSandboxDir(tier: Tier): string {
+  switch (tier.kind) {
+    case "workspace":
+      return `${BASE}/workspace`;
+    case "silo":
+      return `${BASE}/${safeName(tier.name)}`;
+    case "surface":
+      return `${BASE}/surface`;
+  }
+}
+
+export async function syncTierIntoSandbox(
+  sandbox: SandboxSession,
+  tier: Tier,
+  files: MemoryFiles,
+): Promise<void> {
+  const dir = tierSandboxDir(tier);
+  for (const [rel, content] of Object.entries(files)) {
+    await sandbox.writeTextFile({ path: `${dir}/${rel}`, content });
+  }
+}
+
+// Lists with `find` for portability, then reads each file as text. A missing dir or an unreadable file yields an empty or skipped entry, never a throw.
+export async function readTierFromSandbox(
+  sandbox: SandboxSession,
+  tier: Tier,
+): Promise<MemoryFiles> {
+  const dir = tierSandboxDir(tier);
+  const listing = await sandbox.run({
+    command: `cd ${dir} 2>/dev/null && find . -type f | sed 's|^\\./||' || true`,
+  });
+  const out: MemoryFiles = {};
+  const rels = listing.stdout
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l.length > 0 && l !== ".git" && !l.startsWith(".git/"));
+  for (const rel of rels) {
+    const content = await sandbox.readTextFile({ path: `${dir}/${rel}` });
+    if (content !== null) out[rel] = content;
+  }
+  return out;
+}
+
+/**
+ * Propagates an upstream delete into the live sandbox: without it a file deleted in the durable store stays model-visible for the session's life and is resurrected by persistMemoryNow's host-then-sandbox union.
+ * Each path is single-quoted so a rel with odd characters cannot break out of the shell word.
+ */
+export async function removeTierFiles(
+  sandbox: SandboxSession,
+  tier: Tier,
+  rels: readonly string[],
+): Promise<void> {
+  if (rels.length === 0) return;
+  const dir = tierSandboxDir(tier);
+  const quoted = rels.map((rel) => `'${`${dir}/${rel}`.replace(/'/g, "'\\''")}'`).join(" ");
+  await sandbox.run({ command: `rm -f ${quoted}` });
+}
+
+/**
+ * removeTierFiles propagates a delete within a tier the session still reads; this is for a tier that dropped out of the plan entirely (a surface un-vouched, a silo removed from the domain).
+ * The caller must unlock a read-only tier first: `rm` needs write on a directory to unlink what is inside it.
+ */
+export async function removeTierDir(sandbox: SandboxSession, tier: Tier): Promise<void> {
+  const dir = tierSandboxDir(tier);
+  await sandbox.run({ command: `rm -rf '${dir.replace(/'/g, "'\\''")}'` });
+}
+
+export async function writeTierFile(
+  sandbox: SandboxSession,
+  tier: Tier,
+  rel: string,
+  content: string,
+): Promise<void> {
+  await sandbox.writeTextFile({ path: `${tierSandboxDir(tier)}/${rel}`, content });
+}

package/agent/memory/selection.ts

@@ -0,0 +1,120 @@
+// Which nodes of a tree backing (a Notion page subtree, a git folder) a silo materialises: a default for unlisted nodes plus per-node include, recurse and exclude rules.
+// Most-specific wins — the deepest rule at or above a node governs it, and a rule strictly under a node forces a descent to reach it.
+
+import { safeName } from "./layout.js";
+import type { MemorySilo } from "./silo.js";
+
+export type SelectMode = "recurse" | "include" | "exclude";
+
+export interface SelectRule {
+  /**
+   * Node path RELATIVE to the silo root, "/"-joined (e.g. "Onboarding/Deep Dive"), OR a
+   * raw node id (a Notion page id) for a single node. Matched case-insensitively on
+   * safeName-normalised segments.
+   */
+  readonly path: string;
+  readonly mode: SelectMode;
+}
+
+export interface MemorySelection {
+  /** What nodes with no covering rule get. "recurse" = take all (prune with excludes);
+   *  "include" = take only the root + explicitly ticked nodes (manual selection). */
+  readonly default: "recurse" | "include";
+  readonly rules?: readonly SelectRule[];
+}
+
+export interface Selector {
+  /** Render/take this container's OWN content (a page's blocks, a dir's direct files)? */
+  includes(segs: readonly string[], id?: string): boolean;
+  /** Walk into this container's child containers (sub-pages / sub-dirs)? */
+  descends(segs: readonly string[], id?: string): boolean;
+  /** A leaf file named `leaf` directly in container `segs`: honoured by a file-specific
+   *  rule if one exists, else inherits the container's `includes` decision. (git only.) */
+  includesLeaf(segs: readonly string[], leaf: string): boolean;
+}
+
+const norm = (s: string): string => safeName(s).toLowerCase();
+
+function normSegs(path: string): string[] {
+  return path
+    .split("/")
+    .map((s) => s.trim())
+    .filter((s) => s.length > 0)
+    .map(norm);
+}
+
+interface NormRule {
+  readonly segs: string[]; // normalised path segments
+  readonly id?: string; // lowercase raw path when it's a single segment (id match)
+  readonly mode: SelectMode;
+}
+
+function isPrefix(prefix: readonly string[], of: readonly string[]): boolean {
+  if (prefix.length > of.length) return false;
+  for (let i = 0; i < prefix.length; i++) if (prefix[i] !== of[i]) return false;
+  return true;
+}
+
+function arrayEq(a: readonly string[], b: readonly string[]): boolean {
+  return a.length === b.length && a.every((x, i) => x === b[i]);
+}
+
+export function normalizeSelection(silo: Pick<MemorySilo, "select" | "recursive">): MemorySelection {
+  if (silo.select) return silo.select;
+  return { default: silo.recursive === false ? "include" : "recurse" };
+}
+
+export function makeSelector(sel: MemorySelection): Selector {
+  const rules: NormRule[] = (sel.rules ?? []).map((r) => {
+    const raw = r.path.trim();
+    return { segs: normSegs(raw), id: raw.includes("/") ? undefined : raw.toLowerCase(), mode: r.mode };
+  });
+  const rootMode = sel.default;
+
+  function selfRule(ns: readonly string[], id?: string): NormRule | undefined {
+    return rules.find((r) => arrayEq(r.segs, ns) || (r.id !== undefined && id !== undefined && r.id === id.toLowerCase()));
+  }
+  function ancestorRule(ns: readonly string[]): NormRule | undefined {
+    let best: NormRule | undefined;
+    for (const r of rules) {
+      if (r.segs.length < ns.length && isPrefix(r.segs, ns)) {
+        if (!best || r.segs.length > best.segs.length) best = r;
+      }
+    }
+    return best;
+  }
+  function hasRuleUnder(ns: readonly string[]): boolean {
+    return rules.some((r) => r.segs.length > ns.length && isPrefix(ns, r.segs));
+  }
+
+  function govern(ns: readonly string[], id?: string): { mode: SelectMode; atSelf: boolean } {
+    const self = selfRule(ns, id);
+    if (self) return { mode: self.mode, atSelf: true };
+    const anc = ancestorRule(ns);
+    if (anc) return { mode: anc.mode, atSelf: false };
+    return { mode: rootMode, atSelf: ns.length === 0 };
+  }
+
+  function includes(segs: readonly string[], id?: string): boolean {
+    const ns = segs.map(norm);
+    const { mode, atSelf } = govern(ns, id);
+    if (mode === "exclude") return false;
+    return atSelf ? true : mode === "recurse"; // inherited content needs an ancestor recurse
+  }
+
+  function descends(segs: readonly string[], id?: string): boolean {
+    const ns = segs.map(norm);
+    if (hasRuleUnder(ns)) return true; // a deeper tick — must walk to reach it
+    return govern(ns, id).mode === "recurse"; // else children inherit only under recurse
+  }
+
+  function includesLeaf(segs: readonly string[], leaf: string): boolean {
+    const ns = segs.map(norm);
+    const fileSegs = [...ns, norm(leaf)];
+    const self = selfRule(fileSegs);
+    if (self) return self.mode !== "exclude"; // explicit file rule (include/recurse vs exclude)
+    return includes(segs); // inherit the container's decision
+  }
+
+  return { includes, descends, includesLeaf };
+}

package/agent/memory/silo.ts

@@ -0,0 +1,71 @@
+// A named, REUSABLE durable backing for a domain's memory: vendor + container + subtree. A silo is defined ONCE and referenced by NAME from one or more domains, so many domains can share one brain.
+// The agent never sees the silo — the resolver maps a channel to its domain to this silo to a provider. See docs/capabilities/memory.md.
+
+import type { MemorySelection } from "./selection.js";
+
+// Every silo is MATERIALISED to on-disk files; there is no live, not-on-disk pointer mode. An external source the agent reads and writes live is a connector plus a bundle instruction, not a memory silo.
+export interface MemorySilo {
+  /** Stable id, referenced by Domain.memory and the org default. Also the on-disk dir name. */
+  readonly name: string;
+  /**
+   * One-line PURPOSE, surfaced to the agent in the top-level memory index (memory/listing.ts).
+   * Describe the silo's role, which is stable, not its contents, which change — so it does not drift.
+   */
+  readonly description?: string;
+  /** Resolved against the MemoryProvider registry (memory/provider.ts): sqlite, postgres, git, github or notion. Required — every silo materialises. */
+  readonly provider: string;
+  /** Provider-specific container: a repo for github, a teamspace or parent-page id for notion, unused for the plain DB providers. */
+  readonly container?: string;
+  readonly path?: string;
+  /**
+   * The resolver reads this silo INTO the sandbox but never persists back, and the hook locks the files (fail-loud).
+   * For a curated baseline, or a hot-cache of an external source that is updated through that source's own tools — named by `skill`.
+   */
+  readonly readOnly?: boolean;
+  /** The skill that teaches how to USE or UPDATE this source — surfaced in the index and in the read-only banner. */
+  readonly skill?: string;
+  /**
+   * For tree backings (Notion pages, github folders): whether to descend into the subtree. Default true; false takes just the selected node.
+   * Ignored by the flat DB backings, and mutually exclusive with `select` — use `select` for a curated subset.
+   */
+  readonly recursive?: boolean;
+  /**
+   * Curated SUBSET of a tree backing (memory/selection.ts): a `default` of recurse or include, plus per-node `rules` of recurse, include or exclude, keyed by path relative to the root or by Notion page id.
+   * Most-specific rule wins. Mutually exclusive with `recursive`, and ignored by the flat DB backings.
+   */
+  readonly select?: MemorySelection;
+}
+
+export function defineMemorySilo(silo: MemorySilo): MemorySilo {
+  return silo;
+}
+
+/**
+ * Validate the silo set at config load (fail loud): every silo has a provider and any
+ * `select` is well-formed (and not combined with the `recursive` toggle).
+ */
+export function assertSilosValid(silos: readonly MemorySilo[]): void {
+  const bad: string[] = [];
+  for (const s of silos) {
+    const hasProvider = typeof s.provider === "string" && s.provider.length > 0;
+    if (!hasProvider) bad.push(`  "${s.name ?? "(unnamed)"}" has no provider (every silo is materialized — set provider)`);
+    if (s.select && s.recursive !== undefined) {
+      bad.push(`  "${s.name}" sets BOTH select and recursive (select supersedes the simple toggle — drop recursive)`);
+    }
+    if (s.select) {
+      const d = s.select.default;
+      if (d !== "recurse" && d !== "include") bad.push(`  "${s.name}" select.default must be "recurse" or "include" (got ${JSON.stringify(d)})`);
+      for (const r of s.select.rules ?? []) {
+        if (r.mode !== "recurse" && r.mode !== "include" && r.mode !== "exclude") {
+          bad.push(`  "${s.name}" select rule "${r.path}" has invalid mode ${JSON.stringify(r.mode)}`);
+        }
+        if (typeof r.path !== "string" || r.path.trim().length === 0) bad.push(`  "${s.name}" has a select rule with an empty path`);
+      }
+    }
+  }
+  if (bad.length > 0) {
+    throw new Error(
+      "open-everywhere: invalid memory silo(s):\n" + bad.join("\n") + "\n(memory/silo.ts / docs/capabilities/memory.md)",
+    );
+  }
+}