@remnic/core 1.1.11 → 1.1.13

package/src/memory-extension/shared-instructions.ts

@@ -0,0 +1,133 @@
+/**
+ * @remnic/core — Shared Instruction Blocks
+ *
+ * Reusable markdown fragments that every host-specific publisher can
+ * compose into its instructions.md file. Keeping them here avoids
+ * per-host copy-paste drift.
+ */
+
+/**
+ * Describes the Remnic memory types a host agent may encounter.
+ */
+export const REMNIC_SEMANTIC_OVERVIEW = `\
+## Remnic Memory Types
+
+Remnic stores memories as plain Markdown files with YAML front-matter.
+Each memory has a **type** that describes its semantic role:
+
+| Type | Description |
+|------|-------------|
+| \`fact\` | An objective piece of knowledge the user confirmed or that was extracted from a session. |
+| \`preference\` | A stated or inferred user preference (e.g. coding style, tool choice). |
+| \`decision\` | An explicit decision or trade-off the user made. |
+| \`entity\` | A named thing the user cares about (project, service, person, API). |
+| \`skill\` | A reusable workflow or procedure documented for future sessions. |
+| \`correction\` | A fix or amendment to a previously stored memory. |
+| \`question\` | An open question or uncertainty flagged for future resolution. |
+| \`observation\` | A pattern noticed across sessions (e.g. "user always runs tests before commits"). |
+| \`summary\` | A condensed roll-up of recent sessions or a topic area. |
+
+When reading Remnic content, the front-matter \`type\` field tells you what
+kind of knowledge you are looking at and how much weight to give it.
+`;
+
+/**
+ * Explains the oai-mem-citation block format hosts should use when
+ * referencing Remnic-sourced content.
+ */
+export const REMNIC_CITATION_FORMAT = `\
+## Citing Remnic Memories
+
+When a piece of your output draws on a Remnic file, cite it using the
+memory citation block format so the user can trace the source:
+
+\`\`\`
+<oai-mem-citation path="<path-relative-to-remnic-memory-base>" />
+\`\`\`
+
+The path must be **relative to the Remnic memory base** (the directory
+named \`memories/\` under \`<remnic-home>\`), not absolute. Examples:
+
+- \`<oai-mem-citation path="default/MEMORY.md" />\`
+- \`<oai-mem-citation path="my-project/skills/deploy/SKILL.md" />\`
+- \`<oai-mem-citation path="shared/memory_summary.md" />\`
+
+Cite each distinct source once near the fact it supports. Do not invent
+citations for files you have not actually read.
+`;
+
+/**
+ * Table of MCP tools the Remnic daemon exposes. Hosts that can reach
+ * the MCP server should prefer these over raw file reads.
+ *
+ * Tool names use the canonical `remnic.*` prefix. Legacy `engram.*`
+ * aliases are also accepted by the server for backward compatibility.
+ */
+export const REMNIC_MCP_TOOL_INVENTORY = `\
+## Remnic MCP Tools
+
+When the Remnic MCP server is reachable, the following tools are
+available. Prefer MCP tools over direct file reads when the host
+supports MCP connections.
+
+| Tool | Purpose |
+|------|---------|
+| \`remnic.recall\` | Retrieve contextually relevant memories for the current session. |
+| \`remnic.recall_explain\` | Like recall, but includes an explanation of why each memory was selected. |
+| \`remnic.memory_store\` | Persist a new memory (fact, preference, decision, etc.). |
+| \`remnic.memory_get\` | Fetch a specific memory by ID. |
+| \`remnic.memory_search\` | Full-text + semantic search across all memories. |
+| \`remnic.memory_timeline\` | Retrieve memories in chronological order within a time range. |
+| \`remnic.observe\` | Record an observation from the current conversation turn. |
+| \`remnic.entity_get\` | Look up a named entity and its relationships. |
+| \`remnic.memory_entities_list\` | List all known entities. |
+| \`remnic.memory_profile\` | Retrieve the user profile summary. |
+| \`remnic.day_summary\` | Generate a summary of memories from a specific day. |
+| \`remnic.briefing\` | Generate a structured briefing for an upcoming session. |
+| \`remnic.memory_feedback\` | Submit feedback on a recalled memory (useful, outdated, wrong). |
+| \`remnic.memory_promote\` | Promote a memory to a higher confidence tier. |
+| \`remnic.context_checkpoint\` | Save a conversation checkpoint for continuity. |
+| \`remnic.suggestion_submit\` | Submit a suggestion for a new memory to the review queue. |
+| \`remnic.review_queue_list\` | List pending suggestions in the review queue. |
+| \`remnic.work_task\` | Create or update a work task. |
+| \`remnic.work_project\` | Create or update a work project. |
+| \`remnic.work_board\` | View the work board. |
+
+Legacy \`engram.*\` prefixed names are accepted as aliases for all tools.
+`;
+
+/**
+ * Decision rules for when a host agent should use MCP recall vs
+ * reading Remnic files directly from disk.
+ */
+export const REMNIC_RECALL_DECISION_RULES = `\
+## When to Use Recall vs Direct Read
+
+### Use \`remnic.recall\` (MCP) when:
+
+- The Remnic MCP server is reachable (the host has an active MCP
+  connection to the Remnic daemon).
+- You want contextually relevant memories ranked by the recall planner
+  (semantic search + reranking + importance scoring).
+- You need memories across multiple namespaces or topics.
+- The conversation benefits from Remnic's intent detection and
+  adaptive recall depth.
+
+### Use direct file reads when:
+
+- You are in a sandboxed environment with no network or MCP access
+  (e.g. Codex phase-2 consolidation).
+- You need a specific file you already know the path to.
+- The MCP server is unavailable or unhealthy.
+- You are operating on the raw memory files for maintenance or
+  migration purposes.
+
+### General guidance:
+
+- Prefer MCP tools when available — they provide ranked, deduplicated,
+  and context-aware results.
+- Fall back to file reads gracefully — never block on a failed MCP call
+  when the data is also on disk.
+- Never write directly to the Remnic memory directory unless you are an
+  authorized extraction or consolidation process.
+`;

package/src/memory-extension/types.ts

@@ -0,0 +1,86 @@
+/**
+ * @remnic/core — Memory Extension Publisher Contract
+ *
+ * Defines the interface that every host-specific publisher must implement.
+ * Each publisher knows how to write Remnic instruction files into the
+ * host's extension directory so the host agent can discover and use
+ * Remnic memories.
+ */
+
+import type { TokenEntry } from "../tokens.js";
+
+/**
+ * A publisher that can install (and remove) Remnic memory-extension
+ * artefacts into a specific host's extension directory.
+ */
+export interface MemoryExtensionPublisher {
+  /** Unique host identifier (e.g. "codex", "claude-code", "hermes"). */
+  readonly hostId: string;
+
+  /**
+   * Resolve the absolute path to the extension root for this host.
+   * @param env Optional environment to read HOME / host-specific vars from.
+   */
+  resolveExtensionRoot(env?: NodeJS.ProcessEnv): Promise<string>;
+
+  /** Return true when the host toolchain appears to be installed locally. */
+  isHostAvailable(): Promise<boolean>;
+
+  /** Render the full instructions markdown that will be written to disk. */
+  renderInstructions(ctx: PublishContext): Promise<string>;
+
+  /** Write extension artefacts to the host's extension directory. */
+  publish(ctx: PublishContext): Promise<PublishResult>;
+
+  /** Remove extension artefacts previously written by publish(). */
+  unpublish(): Promise<void>;
+}
+
+/**
+ * Context passed to every publisher method that needs configuration,
+ * paths, or logging.
+ */
+export interface PublishContext {
+  readonly config: {
+    memoryDir: string;
+    daemonUrl?: string;
+    daemonPort?: number;
+    namespace?: string;
+  };
+  readonly skillsRoot: string;
+  /**
+   * Token entry captured before connector installation rotated the token store.
+   * Publishers that perform post-install side effects can restore this snapshot
+   * if publish fails after the connector install has already committed.
+   */
+  readonly rollbackTokenEntry?: TokenEntry | null;
+  readonly log: {
+    info: (msg: string) => void;
+    warn: (msg: string) => void;
+    error: (msg: string) => void;
+  };
+}
+
+/** Result returned by a successful publish() call. */
+export interface PublishResult {
+  readonly hostId: string;
+  readonly extensionRoot: string;
+  readonly filesWritten: string[];
+  readonly skipped: string[];
+}
+
+/**
+ * Declarative capability flags that describe what a given publisher
+ * can produce. Useful for feature-gating UI or doctor output without
+ * instantiating the publisher.
+ */
+export interface PublisherCapabilities {
+  /** Whether the publisher writes an instructions.md file. */
+  readonly instructionsMd: boolean;
+  /** Whether the publisher populates a skills folder. */
+  readonly skillsFolder: boolean;
+  /** Whether the publisher embeds citation format guidance. */
+  readonly citationFormat: boolean;
+  /** Whether the publisher includes a read-path template for the host. */
+  readonly readPathTemplate: boolean;
+}

package/src/memory-extension-host/host-discovery.ts

@@ -0,0 +1,276 @@
+/**
+ * memory-extension-host/host-discovery.ts — Discover third-party memory extensions.
+ *
+ * Scans a root directory (typically ~/.remnic/memory_extensions/) for valid
+ * extension subdirectories. Each extension must contain an instructions.md.
+ * The discovery process is read-only and NEVER reads or executes files under
+ * any extension's scripts/ directory.
+ */
+
+import { readdir, readFile, lstat, realpath } from "node:fs/promises";
+import path from "node:path";
+import type { LoggerBackend } from "../logger.js";
+import type { PluginConfig } from "../types.js";
+import type { DiscoveredExtension, ExtensionSchema } from "./types.js";
+
+/** Total token budget for all discovered extension instructions combined. */
+export const REMNIC_EXTENSIONS_TOTAL_TOKEN_LIMIT = 5_000;
+
+/** Maximum number of example files collected per extension. */
+const MAX_EXAMPLES_PER_EXTENSION = 10;
+
+/** Slug validation: lowercase letters, digits, hyphens, 1-64 chars. */
+const VALID_SLUG_RE = /^[a-z0-9][a-z0-9-]{0,63}$/;
+
+const VALID_MEMORY_TYPES = new Set(["fact", "preference", "procedure", "reference"]);
+
+/**
+ * Discover all valid memory extensions under the given root directory.
+ *
+ * Returns extensions sorted by name. Skips entries with warnings when:
+ * - The slug is invalid (not lowercase alphanumeric + hyphens, or > 64 chars)
+ * - instructions.md is missing
+ * - schema.json is malformed (extension still returned but schema is undefined)
+ *
+ * NEVER reads files under any extension's scripts/ directory.
+ */
+export async function discoverMemoryExtensions(
+  root: string,
+  log: Pick<LoggerBackend, "warn" | "debug">,
+): Promise<DiscoveredExtension[]> {
+  // If root doesn't exist, return empty silently (not even a warning).
+  // Use lstat() for root — a symlinked extensions root could redirect
+  // discovery to an untrusted directory tree (#428 P2).  When the root
+  // IS a symlink, resolve it and verify it still lives under the parent
+  // memory directory so that an attacker-controlled symlink can't point
+  // discovery at /etc or another user's home.
+  let rootStat;
+  try {
+    rootStat = await lstat(root);
+  } catch {
+    return [];
+  }
+  if (rootStat.isSymbolicLink()) {
+    // Resolve and verify the real path is inside the expected parent.
+    let resolved: string;
+    try {
+      resolved = await realpath(root);
+    } catch {
+      return [];
+    }
+    // Normalize the parent path through realpath so that:
+    // 1. Relative roots (e.g. "memory_extensions") become absolute (#431 Finding 1)
+    // 2. Intermediate symlinks (e.g. macOS /var -> /private/var) are resolved (#431 Finding 2)
+    let expectedParent: string;
+    try {
+      expectedParent = await realpath(path.resolve(path.dirname(root)));
+    } catch {
+      // Parent directory doesn't exist or is inaccessible — reject.
+      return [];
+    }
+    if (!resolved.startsWith(expectedParent + path.sep) && resolved !== expectedParent) {
+      log.warn?.(
+        `[memory-extensions] root "${root}" is a symlink resolving outside the expected parent directory, skipping`,
+      );
+      return [];
+    }
+    // Re-check the resolved path is a directory.
+    try {
+      rootStat = await lstat(resolved);
+    } catch {
+      return [];
+    }
+  }
+  if (!rootStat.isDirectory()) {
+    return [];
+  }
+
+  let entries: string[];
+  try {
+    entries = await readdir(root);
+  } catch {
+    return [];
+  }
+
+  const extensions: DiscoveredExtension[] = [];
+
+  for (const entry of entries) {
+    const entryPath = path.join(root, entry);
+
+    // Must be a real directory (not a symlink) — lstat() blocks symlink
+    // traversal that could escape the extensions root (#382 P2).
+    let entryStat;
+    try {
+      entryStat = await lstat(entryPath);
+    } catch {
+      continue;
+    }
+    if (entryStat.isSymbolicLink()) {
+      log.warn?.(
+        `[memory-extensions] skipping "${entry}": symlinks are not followed for security`,
+      );
+      continue;
+    }
+    if (!entryStat.isDirectory()) continue;
+
+    // Validate slug
+    if (!VALID_SLUG_RE.test(entry)) {
+      log.warn?.(
+        `[memory-extensions] skipping "${entry}": invalid slug (must be lowercase alphanumeric + hyphens, 1-64 chars)`,
+      );
+      continue;
+    }
+
+    // Require instructions.md — reject symlinked files (#428 P1)
+    const instructionsPath = path.join(entryPath, "instructions.md");
+    if (await isSymlink(instructionsPath)) {
+      log.warn?.(
+        `[memory-extensions] skipping "${entry}": instructions.md is a symlink`,
+      );
+      continue;
+    }
+    let instructions: string;
+    try {
+      instructions = await readFile(instructionsPath, "utf-8");
+    } catch {
+      log.warn?.(
+        `[memory-extensions] skipping "${entry}": missing instructions.md`,
+      );
+      continue;
+    }
+
+    // Read optional schema.json — reject symlinked files (#428 P1)
+    let schema: ExtensionSchema | undefined;
+    const schemaPath = path.join(entryPath, "schema.json");
+    if (await isSymlink(schemaPath)) {
+      log.warn?.(
+        `[memory-extensions] "${entry}": schema.json is a symlink, ignoring schema`,
+      );
+    } else {
+      try {
+        const schemaRaw = await readFile(schemaPath, "utf-8");
+        const parsed = JSON.parse(schemaRaw);
+        if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) {
+          schema = validateSchema(parsed);
+        } else {
+          log.warn?.(
+            `[memory-extensions] "${entry}": schema.json is not a valid object, ignoring schema`,
+          );
+        }
+      } catch (err) {
+        // File doesn't exist → fine, no warning needed
+        if (isFileNotFoundError(err)) {
+          // schema remains undefined
+        } else {
+          log.warn?.(
+            `[memory-extensions] "${entry}": malformed schema.json, ignoring schema`,
+          );
+        }
+      }
+    }
+
+    // Collect examples/*.md (cap at MAX_EXAMPLES_PER_EXTENSION)
+    // NEVER read from scripts/ directory
+    const examplesPaths: string[] = [];
+    const examplesDir = path.join(entryPath, "examples");
+    try {
+      const exampleEntries = await readdir(examplesDir);
+      const mdFiles = exampleEntries
+        .filter((f) => f.endsWith(".md"))
+        .sort()
+        .slice(0, MAX_EXAMPLES_PER_EXTENSION);
+      for (const f of mdFiles) {
+        examplesPaths.push(path.join(examplesDir, f));
+      }
+    } catch {
+      // No examples dir — fine
+    }
+
+    extensions.push({
+      name: entry,
+      root: entryPath,
+      instructionsPath,
+      instructions,
+      schema,
+      examplesPaths,
+    });
+  }
+
+  // Sort by name for deterministic ordering
+  extensions.sort((a, b) => {
+    if (a.name < b.name) return -1;
+    if (a.name > b.name) return 1;
+    return 0;
+  });
+
+  return extensions;
+}
+
+function validateSchema(raw: Record<string, unknown>): ExtensionSchema {
+  const memoryTypes: ExtensionSchema["memoryTypes"] = (() => {
+    if (!Array.isArray(raw.memoryTypes)) return undefined;
+    const valid = raw.memoryTypes.filter(
+      (t): t is "fact" | "preference" | "procedure" | "reference" =>
+        typeof t === "string" && VALID_MEMORY_TYPES.has(t),
+    );
+    return valid.length > 0 ? valid : undefined;
+  })();
+
+  const groupingHints: ExtensionSchema["groupingHints"] = (() => {
+    if (!Array.isArray(raw.groupingHints)) return undefined;
+    const valid = raw.groupingHints.filter(
+      (h): h is string => typeof h === "string" && h.length > 0,
+    );
+    return valid.length > 0 ? valid : undefined;
+  })();
+
+  const version: ExtensionSchema["version"] =
+    typeof raw.version === "string" && raw.version.length > 0
+      ? raw.version
+      : undefined;
+
+  return {
+    ...(memoryTypes ? { memoryTypes } : {}),
+    ...(groupingHints ? { groupingHints } : {}),
+    ...(version ? { version } : {}),
+  };
+}
+
+function isFileNotFoundError(err: unknown): boolean {
+  return (
+    typeof err === "object" &&
+    err !== null &&
+    "code" in err &&
+    (err as { code: string }).code === "ENOENT"
+  );
+}
+
+/**
+ * Returns true if the path exists and is a symlink.
+ * Returns false if the path does not exist or is not a symlink.
+ */
+async function isSymlink(filePath: string): Promise<boolean> {
+  try {
+    const s = await lstat(filePath);
+    return s.isSymbolicLink();
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Resolve the memory extensions root directory from config.
+ * If memoryExtensionsRoot is empty, derive from memoryDir by going up to
+ * the Remnic home dir and appending memory_extensions.
+ *
+ * Moved here from semantic-consolidation.ts (#428 Finding 3) because this
+ * is a generic config-to-path resolver with no consolidation logic.
+ */
+export function resolveExtensionsRoot(config: PluginConfig): string {
+  if (config.memoryExtensionsRoot.length > 0) {
+    return config.memoryExtensionsRoot;
+  }
+  // Default: memoryDir is typically ~/.openclaw/workspace/memory/local
+  // Go up to the parent that owns the memory tree and append memory_extensions
+  return path.join(path.dirname(config.memoryDir), "memory_extensions");
+}

package/src/memory-extension-host/index.ts

@@ -0,0 +1,14 @@
+/**
+ * memory-extension-host/index.ts — Public API for memory extension discovery.
+ */
+
+export type { DiscoveredExtension, ExtensionSchema } from "./types.js";
+export {
+  discoverMemoryExtensions,
+  resolveExtensionsRoot,
+  REMNIC_EXTENSIONS_TOTAL_TOKEN_LIMIT,
+} from "./host-discovery.js";
+export {
+  renderExtensionsBlock,
+  renderExtensionsFooter,
+} from "./render-extensions-block.js";

package/src/memory-extension-host/render-extensions-block.ts

@@ -0,0 +1,73 @@
+/**
+ * memory-extension-host/render-extensions-block.ts — Render discovered extensions
+ * into a markdown block for injection into consolidation prompts.
+ *
+ * Respects the global token budget (REMNIC_EXTENSIONS_TOTAL_TOKEN_LIMIT) and
+ * truncates with a footer listing omitted extensions when over budget.
+ */
+
+import { REMNIC_EXTENSIONS_TOTAL_TOKEN_LIMIT } from "./host-discovery.js";
+import type { DiscoveredExtension } from "./types.js";
+
+/**
+ * Approximate token count using the 4 chars per token heuristic.
+ */
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * Render a markdown block containing extension instructions for injection
+ * into consolidation prompts.
+ *
+ * If the list is empty, returns "".
+ * Inlines extensions in name order until the token budget is exhausted.
+ * If the budget is exceeded, appends a truncation footer listing omitted extensions.
+ */
+export function renderExtensionsBlock(extensions: DiscoveredExtension[]): string {
+  if (extensions.length === 0) return "";
+
+  const header = `## Active memory extensions
+
+You are running with the following third-party memory extensions. Each
+extension's \`instructions.md\` tells you how to interpret memories that
+extension produces or curates.
+
+`;
+
+  let budget = REMNIC_EXTENSIONS_TOTAL_TOKEN_LIMIT;
+  budget -= estimateTokens(header);
+
+  const inlined: string[] = [];
+  const omitted: string[] = [];
+
+  for (const ext of extensions) {
+    const block = `### remnic-extension/${ext.name}\n\`\`\`\n${ext.instructions}\n\`\`\`\n\n`;
+    const cost = estimateTokens(block);
+    if (cost <= budget) {
+      inlined.push(block);
+      budget -= cost;
+    } else {
+      omitted.push(ext.name);
+    }
+  }
+
+  let result = header;
+  result += inlined.join("");
+
+  if (omitted.length > 0) {
+    result += `> **Note:** ${omitted.length} extension(s) omitted due to token budget: ${omitted.join(", ")}\n`;
+  }
+
+  return result;
+}
+
+/**
+ * Render a compact one-line footer listing active extension names.
+ * Used by day-summary and summary-snapshot where full instructions are not needed.
+ */
+export function renderExtensionsFooter(extensions: DiscoveredExtension[]): string {
+  if (extensions.length === 0) return "";
+  const names = extensions.map((ext) => ext.name).join(", ");
+  return `Active extensions: ${names}`;
+}

package/src/memory-extension-host/types.ts

@@ -0,0 +1,21 @@
+/**
+ * memory-extension-host/types.ts — Types for third-party memory extension discovery.
+ *
+ * Memory extensions live under ~/.remnic/memory_extensions/<slug>/ and provide
+ * instructions.md (required), schema.json (optional), and examples/*.md (optional).
+ */
+
+export interface DiscoveredExtension {
+  readonly name: string;
+  readonly root: string;
+  readonly instructionsPath: string;
+  readonly instructions: string;
+  readonly schema?: ExtensionSchema;
+  readonly examplesPaths: string[];
+}
+
+export interface ExtensionSchema {
+  readonly memoryTypes?: Array<"fact" | "preference" | "procedure" | "reference">;
+  readonly groupingHints?: string[];
+  readonly version?: string;
+}