openclaw-memory-hierarchical 0.1.0

package/README.md

@@ -0,0 +1,80 @@
+# openclaw-memory-hierarchical
+
+Hierarchical (2048-style) autobiographical memory plugin for [OpenClaw](https://github.com/openclaw/openclaw).
+
+Continuously summarizes conversations in the background, creating layers of progressively compressed first-person memories:
+
+- **L1** — Recent memory chunks (~6k tokens summarized to ~1k)
+- **L2** — Earlier context (6 L1 summaries merged into 1)
+- **L3** — Long-term memory (6 L2 summaries merged into 1)
+
+Memories are written in first-person ("I discussed...", "I learned that the user...") — autobiographical, not transcript summaries.
+
+## Install
+
+```bash
+openclaw plugins install openclaw-memory-hierarchical
+```
+
+## Configure
+
+Add to your OpenClaw config:
+
+```yaml
+plugins:
+  entries:
+    memory-hierarchical:
+      config:
+        # Optional: explicit API key (falls back to ANTHROPIC_API_KEY env var)
+        # apiKey: "sk-..."
+
+        # Optional: model for summarization (default: anthropic/claude-sonnet-4-5-20250929)
+        # model: "anthropic/claude-sonnet-4-5-20250929"
+
+        # Optional: worker interval (default: 5m)
+        # workerInterval: "5m"
+```
+
+Minimal config (uses `ANTHROPIC_API_KEY` from environment):
+
+```yaml
+plugins:
+  entries:
+    memory-hierarchical:
+      config: {}
+```
+
+## How it works
+
+1. A background worker runs every 5 minutes (configurable)
+2. It reads the current session transcript and finds messages old enough to summarize
+3. Chunks of ~6k tokens are summarized to ~1k token first-person memories (L1)
+4. When 6 L1 summaries accumulate, they merge into 1 L2 summary
+5. When 6 L2 summaries accumulate, they merge into 1 L3 summary
+6. Before each agent run, active memories are injected into the system prompt
+
+## CLI commands
+
+```bash
+openclaw memory-hierarchical status          # Show memory stats
+openclaw memory-hierarchical status --json   # JSON output
+openclaw memory-hierarchical inspect         # View summaries
+openclaw memory-hierarchical inspect --level L1 --limit 3
+```
+
+## Configuration options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `apiKey` | string | `$ANTHROPIC_API_KEY` | API key for the summarization model |
+| `model` | string | `anthropic/claude-sonnet-4-5-20250929` | Model to use for summarization |
+| `workerInterval` | string | `"5m"` | How often the background worker runs |
+| `chunkTokens` | number | `6000` | Minimum tokens before summarizing a chunk |
+| `summaryTargetTokens` | number | `1000` | Target summary length in tokens |
+| `mergeThreshold` | number | `6` | Summaries before merging to next level |
+| `pruningBoundaryTokens` | number | `30000` | Messages must be this far behind to summarize |
+| `maxLevels` | number | `3` | Maximum summary levels |
+
+## License
+
+MIT

package/config.ts

@@ -0,0 +1,126 @@
+/**
+ * Configuration resolution for hierarchical memory plugin.
+ */
+
+import { DEFAULT_HIERARCHICAL_MEMORY_CONFIG, type HierarchicalMemoryConfig } from "./types.js";
+
+/** Plugin-specific configuration (from openclaw.plugin.json schema) */
+export type PluginConfig = {
+  /** API key for the summarization model. Falls back to ANTHROPIC_API_KEY env var if not set. */
+  apiKey?: string;
+  workerInterval?: string;
+  chunkTokens?: number;
+  summaryTargetTokens?: number;
+  mergeThreshold?: number;
+  pruningBoundaryTokens?: number;
+  model?: string;
+  maxLevels?: number;
+};
+
+/** Parse duration string to milliseconds (e.g., "5m" → 300000) */
+function parseDurationMs(raw: string, opts?: { defaultUnit?: string }): number {
+  const trimmed = String(raw ?? "")
+    .trim()
+    .toLowerCase();
+  if (!trimmed) {
+    throw new Error("invalid duration (empty)");
+  }
+
+  const m = /^(\d+(?:\.\d+)?)(ms|s|m|h|d)?$/.exec(trimmed);
+  if (!m) {
+    throw new Error(`invalid duration: ${raw}`);
+  }
+
+  const value = Number(m[1]);
+  if (!Number.isFinite(value) || value < 0) {
+    throw new Error(`invalid duration: ${raw}`);
+  }
+
+  const unit = (m[2] ?? opts?.defaultUnit ?? "ms") as "ms" | "s" | "m" | "h" | "d";
+  const multiplier =
+    unit === "ms"
+      ? 1
+      : unit === "s"
+        ? 1000
+        : unit === "m"
+          ? 60_000
+          : unit === "h"
+            ? 3_600_000
+            : 86_400_000;
+  const ms = Math.round(value * multiplier);
+  if (!Number.isFinite(ms)) {
+    throw new Error(`invalid duration: ${raw}`);
+  }
+  return ms;
+}
+
+/** Parse workerInterval string or fall back to default */
+function resolveWorkerIntervalMs(raw: { workerInterval?: string }): number {
+  if (raw.workerInterval) {
+    try {
+      return parseDurationMs(raw.workerInterval, { defaultUnit: "m" });
+    } catch {
+      // Fall through to default
+    }
+  }
+  return DEFAULT_HIERARCHICAL_MEMORY_CONFIG.workerIntervalMs;
+}
+
+/** Resolve the API key: explicit config > ANTHROPIC_API_KEY > provider-specific env vars */
+function resolveApiKey(explicit?: string): string | undefined {
+  if (explicit) {
+    return explicit;
+  }
+  // Fall back to common provider env vars
+  return (
+    process.env.ANTHROPIC_API_KEY ??
+    process.env.OPENAI_API_KEY ??
+    undefined
+  );
+}
+
+/** Parse and validate plugin config */
+export function parsePluginConfig(raw: unknown): PluginConfig {
+  if (!raw || typeof raw !== "object") {
+    // Allow empty config — API key can come from env
+    return { apiKey: resolveApiKey() };
+  }
+  const cfg = raw as Record<string, unknown>;
+  const apiKey = typeof cfg.apiKey === "string" ? cfg.apiKey : undefined;
+  return {
+    apiKey: resolveApiKey(apiKey),
+    workerInterval: typeof cfg.workerInterval === "string" ? cfg.workerInterval : undefined,
+    chunkTokens: typeof cfg.chunkTokens === "number" ? cfg.chunkTokens : undefined,
+    summaryTargetTokens:
+      typeof cfg.summaryTargetTokens === "number" ? cfg.summaryTargetTokens : undefined,
+    mergeThreshold: typeof cfg.mergeThreshold === "number" ? cfg.mergeThreshold : undefined,
+    pruningBoundaryTokens:
+      typeof cfg.pruningBoundaryTokens === "number" ? cfg.pruningBoundaryTokens : undefined,
+    model: typeof cfg.model === "string" ? cfg.model : undefined,
+    maxLevels: typeof cfg.maxLevels === "number" ? cfg.maxLevels : undefined,
+  };
+}
+
+/** Resolve hierarchical memory config from plugin config */
+export function resolveHierarchicalMemoryConfig(
+  pluginConfig?: PluginConfig,
+): HierarchicalMemoryConfig {
+  if (!pluginConfig) {
+    return { ...DEFAULT_HIERARCHICAL_MEMORY_CONFIG, enabled: true };
+  }
+
+  return {
+    enabled: true, // If plugin is loaded, it's enabled
+    workerIntervalMs: resolveWorkerIntervalMs(pluginConfig),
+    chunkTokens: pluginConfig.chunkTokens ?? DEFAULT_HIERARCHICAL_MEMORY_CONFIG.chunkTokens,
+    summaryTargetTokens:
+      pluginConfig.summaryTargetTokens ?? DEFAULT_HIERARCHICAL_MEMORY_CONFIG.summaryTargetTokens,
+    mergeThreshold:
+      pluginConfig.mergeThreshold ?? DEFAULT_HIERARCHICAL_MEMORY_CONFIG.mergeThreshold,
+    pruningBoundaryTokens:
+      pluginConfig.pruningBoundaryTokens ??
+      DEFAULT_HIERARCHICAL_MEMORY_CONFIG.pruningBoundaryTokens,
+    model: pluginConfig.model,
+    maxLevels: pluginConfig.maxLevels ?? DEFAULT_HIERARCHICAL_MEMORY_CONFIG.maxLevels,
+  };
+}

package/context.ts

@@ -0,0 +1,148 @@
+/**
+ * Context injection for hierarchical memory.
+ *
+ * Loads summaries and formats them for injection into the system prompt.
+ */
+
+import { hasSummaries, loadSummaryContents, loadSummaryIndex } from "./storage.js";
+import { getAllSummariesForContext } from "./types.js";
+
+export type MemoryContext = {
+  /** Formatted memory section to inject into system prompt */
+  memorySection: string;
+  /** Number of summaries at each level */
+  counts: {
+    L1: number;
+    L2: number;
+    L3: number;
+  };
+  /** Estimated token count of the memory section */
+  tokenEstimate: number;
+};
+
+/**
+ * Load and format hierarchical memory for system prompt injection.
+ */
+export async function loadMemoryContext(agentId?: string): Promise<MemoryContext | null> {
+  // Quick check if there are any summaries
+  if (!(await hasSummaries(agentId))) {
+    return null;
+  }
+
+  const index = await loadSummaryIndex(agentId);
+  const summaryContext = getAllSummariesForContext(index);
+
+  // Load contents for each level
+  const L3Contents = await loadSummaryContents(summaryContext.L3, agentId);
+  const L2Contents = await loadSummaryContents(summaryContext.L2, agentId);
+  const L1Contents = await loadSummaryContents(summaryContext.L1, agentId);
+
+  // If no summaries at any level, return null
+  if (L3Contents.length === 0 && L2Contents.length === 0 && L1Contents.length === 0) {
+    return null;
+  }
+
+  // Format the memory section
+  const memorySection = formatMemorySection(L3Contents, L2Contents, L1Contents);
+
+  // Rough token estimate (4 chars per token)
+  const tokenEstimate = Math.ceil(memorySection.length / 4);
+
+  return {
+    memorySection,
+    counts: {
+      L1: L1Contents.length,
+      L2: L2Contents.length,
+      L3: L3Contents.length,
+    },
+    tokenEstimate,
+  };
+}
+
+/**
+ * Format summaries into a memory section for the system prompt.
+ */
+function formatMemorySection(L3: string[], L2: string[], L1: string[]): string {
+  const sections: string[] = [];
+
+  sections.push("## My memories of our conversation\n");
+  sections.push("(These are my autobiographical memories from our ongoing relationship.)\n");
+
+  if (L3.length > 0) {
+    sections.push("\n### Long-term memory\n");
+    sections.push(L3.join("\n\n---\n\n"));
+  }
+
+  if (L2.length > 0) {
+    sections.push("\n### Earlier context\n");
+    sections.push(L2.join("\n\n---\n\n"));
+  }
+
+  if (L1.length > 0) {
+    sections.push("\n### Recent memory\n");
+    sections.push(L1.join("\n\n---\n\n"));
+  }
+
+  return sections.join("\n");
+}
+
+/**
+ * Get the last summarized entry ID from the index.
+ * Used to filter recent messages (only include those after this ID).
+ */
+export async function getLastSummarizedEntryId(agentId?: string): Promise<string | null> {
+  try {
+    const index = await loadSummaryIndex(agentId);
+    return index.lastSummarizedEntryId;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Check if hierarchical memory has any data for an agent.
+ */
+export async function hasMemoryData(agentId?: string): Promise<boolean> {
+  return hasSummaries(agentId);
+}
+
+/**
+ * Get summary statistics for display.
+ */
+export async function getMemoryStats(agentId?: string): Promise<{
+  totalSummaries: number;
+  levels: { L1: number; L2: number; L3: number };
+  lastSummarizedAt: number | null;
+  lastWorkerRun: number | null;
+} | null> {
+  try {
+    const index = await loadSummaryIndex(agentId);
+
+    const L1Count = index.levels.L1.length;
+    const L2Count = index.levels.L2.length;
+    const L3Count = index.levels.L3.length;
+
+    if (L1Count === 0 && L2Count === 0 && L3Count === 0) {
+      return null;
+    }
+
+    // Find most recent summary timestamp
+    let lastSummarizedAt: number | null = null;
+    for (const level of [index.levels.L1, index.levels.L2, index.levels.L3]) {
+      for (const summary of level) {
+        if (!lastSummarizedAt || summary.createdAt > lastSummarizedAt) {
+          lastSummarizedAt = summary.createdAt;
+        }
+      }
+    }
+
+    return {
+      totalSummaries: L1Count + L2Count + L3Count,
+      levels: { L1: L1Count, L2: L2Count, L3: L3Count },
+      lastSummarizedAt,
+      lastWorkerRun: index.worker.lastRunAt,
+    };
+  } catch {
+    return null;
+  }
+}

package/index.ts

@@ -0,0 +1,241 @@
+/**
+ * Hierarchical Memory Plugin for OpenClaw
+ *
+ * A 2048-style autobiographical memory compression system.
+ * Continuously summarizes conversation chunks in the background,
+ * creating layers of progressively compressed context (L1 → L2 → L3).
+ *
+ * Integration points:
+ * - before_agent_start hook: injects memory section into system prompt
+ * - registerService: background worker for periodic summarization
+ * - registerCli: memory-hierarchical status/inspect commands
+ */
+
+import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
+import { parsePluginConfig, type PluginConfig } from "./config.js";
+import { loadMemoryContext, getMemoryStats, hasMemoryData } from "./context.js";
+import { setStateDir, hasSummaries, loadSummaryIndex, readSummary, resolveSummariesDir } from "./storage.js";
+import type { SummaryEntry, SummaryLevel } from "./types.js";
+import { startHierarchicalMemoryTimer, type HierarchicalMemoryTimerHandle } from "./timer.js";
+
+function formatAge(ms: number): string {
+  if (ms < 0) {
+    return "future";
+  }
+  if (ms < 60_000) {
+    return `${Math.floor(ms / 1000)}s ago`;
+  }
+  if (ms < 3600_000) {
+    return `${Math.floor(ms / 60_000)}m ago`;
+  }
+  if (ms < 86400_000) {
+    return `${Math.floor(ms / 3600_000)}h ago`;
+  }
+  return `${Math.floor(ms / 86400_000)}d ago`;
+}
+
+export default {
+  id: "memory-hierarchical",
+  name: "Hierarchical Memory",
+  description: "2048-style autobiographical memory compression for long-running conversations",
+
+  register(api: OpenClawPluginApi) {
+    const cfg = parsePluginConfig(api.pluginConfig);
+    let timer: HierarchicalMemoryTimerHandle | null = null;
+
+    // Inject memory into system prompt before each agent run
+    api.on("before_agent_start", async (_event, ctx) => {
+      const agentId = ctx.agentId ?? "main";
+      try {
+        const memCtx = await loadMemoryContext(agentId);
+        if (memCtx) {
+          api.logger.debug?.(
+            `loaded hierarchical memory: L1=${memCtx.counts.L1} L2=${memCtx.counts.L2} L3=${memCtx.counts.L3} (~${memCtx.tokenEstimate} tokens)`,
+          );
+          return { prependContext: memCtx.memorySection };
+        }
+      } catch (err) {
+        api.logger.warn(
+          `failed to load hierarchical memory: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+    });
+
+    // Background worker for summarization
+    api.registerService({
+      id: "memory-hierarchical-worker",
+      start: async (ctx) => {
+        // Set state directory for storage module
+        setStateDir(ctx.stateDir);
+
+        timer = startHierarchicalMemoryTimer({
+          agentId: "main",
+          pluginConfig: cfg,
+          stateDir: ctx.stateDir,
+          log: {
+            info: (msg) => ctx.logger.info(msg),
+            warn: (msg) => ctx.logger.warn(msg),
+            error: (msg) => ctx.logger.error(msg),
+          },
+        });
+      },
+      stop: async () => {
+        timer?.stop();
+      },
+    });
+
+    // CLI commands
+    api.registerCli(
+      ({ program, logger }) => {
+        const cmd = program
+          .command("memory-hierarchical")
+          .description("Hierarchical memory (autobiographical summaries)");
+
+        cmd
+          .command("status")
+          .description("Show hierarchical memory status and statistics")
+          .option("--agent <id>", "Agent id (default: main)")
+          .option("--json", "Print JSON")
+          .action(async (opts: { agent?: string; json?: boolean }) => {
+            const agentId = opts.agent ?? "main";
+            const hasData = await hasSummaries(agentId);
+
+            if (opts.json) {
+              const stats = hasData ? await getMemoryStats(agentId) : null;
+              logger.info(
+                JSON.stringify(
+                  { agentId, enabled: true, hasData, stats, summariesDir: resolveSummariesDir(agentId) },
+                  null,
+                  2,
+                ),
+              );
+              return;
+            }
+
+            logger.info("Hierarchical Memory Status");
+            logger.info("");
+            logger.info(`  Agent:   ${agentId}`);
+            logger.info(`  Enabled: yes`);
+            logger.info(`  Storage: ${resolveSummariesDir(agentId)}`);
+            logger.info("");
+
+            if (!hasData) {
+              logger.info("  No memory data yet.");
+              return;
+            }
+
+            const stats = await getMemoryStats(agentId);
+            if (!stats) {
+              logger.info("  No summaries found.");
+              return;
+            }
+
+            logger.info("  Summaries:");
+            logger.info(`    L1 (recent):     ${stats.levels.L1}`);
+            logger.info(`    L2 (earlier):    ${stats.levels.L2}`);
+            logger.info(`    L3 (long-term):  ${stats.levels.L3}`);
+            logger.info(`    ${"─".repeat(20)}`);
+            logger.info(`    Total:           ${stats.totalSummaries}`);
+            logger.info("");
+            if (stats.lastSummarizedAt) {
+              logger.info(`  Last summarized: ${formatAge(Date.now() - stats.lastSummarizedAt)}`);
+            }
+            if (stats.lastWorkerRun) {
+              logger.info(`  Last worker run: ${formatAge(Date.now() - stats.lastWorkerRun)}`);
+            }
+          });
+
+        cmd
+          .command("inspect")
+          .description("View hierarchical memory summaries")
+          .option("--agent <id>", "Agent id (default: main)")
+          .option("--level <level>", "Filter by level (L1, L2, or L3)")
+          .option("--limit <n>", "Maximum summaries per level", "5")
+          .option("--json", "Print JSON")
+          .action(
+            async (opts: { agent?: string; level?: string; limit?: string; json?: boolean }) => {
+              const agentId = opts.agent ?? "main";
+              const limit = parseInt(opts.limit ?? "5", 10);
+              const validLevels = ["L1", "L2", "L3"];
+
+              if (opts.level && !validLevels.includes(opts.level.toUpperCase())) {
+                logger.error(
+                  `Invalid level: ${opts.level}. Must be one of: ${validLevels.join(", ")}`,
+                );
+                return;
+              }
+              const filterLevel = opts.level?.toUpperCase() as SummaryLevel | undefined;
+
+              const hasData = await hasSummaries(agentId);
+              if (!hasData) {
+                if (opts.json) {
+                  logger.info(JSON.stringify({ agentId, summaries: [] }, null, 2));
+                } else {
+                  logger.info("No memory data yet.");
+                }
+                return;
+              }
+
+              const index = await loadSummaryIndex(agentId);
+              const summariesToShow: Array<{ entry: SummaryEntry; content: string }> = [];
+              const levels: SummaryLevel[] = filterLevel ? [filterLevel] : ["L3", "L2", "L1"];
+
+              for (const level of levels) {
+                const entries = index.levels[level]
+                  .filter((e) => !e.mergedInto)
+                  .toSorted((a, b) => b.createdAt - a.createdAt); // Node 22+ baseline
+
+                for (const entry of entries.slice(0, limit)) {
+                  const result = await readSummary(level, entry.id, agentId);
+                  if (result) {
+                    summariesToShow.push({ entry, content: result.content });
+                  }
+                }
+              }
+
+              if (opts.json) {
+                logger.info(
+                  JSON.stringify(
+                    {
+                      agentId,
+                      summaries: summariesToShow.map(({ entry, content }) => ({
+                        id: entry.id,
+                        level: entry.level,
+                        createdAt: entry.createdAt,
+                        tokenEstimate: entry.tokenEstimate,
+                        content,
+                      })),
+                    },
+                    null,
+                    2,
+                  ),
+                );
+                return;
+              }
+
+              logger.info("Memory Summaries");
+              logger.info(`Agent: ${agentId}`);
+              if (filterLevel) {
+                logger.info(`Level: ${filterLevel}`);
+              }
+              logger.info("");
+
+              if (summariesToShow.length === 0) {
+                logger.info("No active summaries found.");
+                return;
+              }
+
+              for (const { entry, content } of summariesToShow) {
+                const header = `[${entry.level}] ${entry.id} - ${formatAge(Date.now() - entry.createdAt)} (~${entry.tokenEstimate} tokens)`;
+                logger.info(header);
+                logger.info("─".repeat(60));
+                logger.info(content);
+                logger.info("");
+              }
+            },
+          );
+      },
+      { commands: ["memory-hierarchical"] },
+    );
+  },
+};

package/lock.ts

@@ -0,0 +1,94 @@
+/**
+ * Simple file-based locking for the hierarchical memory worker.
+ * Prevents concurrent runs from multiple processes.
+ *
+ * Atomicity is provided by `writeFile` with the `wx` flag (create-exclusive).
+ * Only one process can successfully create the lock file; all others get EEXIST.
+ */
+
+import fs from "node:fs/promises";
+import path from "node:path";
+import { resolveSummariesDir } from "./storage.js";
+
+const LOCK_FILENAME = ".worker.lock";
+const LOCK_STALE_MS = 10 * 60 * 1000; // 10 minutes
+
+export type WorkerLock = {
+  release: () => Promise<void>;
+};
+
+/** Try to create the lock file atomically. Returns null if it already exists. */
+async function tryCreateLock(lockPath: string): Promise<WorkerLock | null> {
+  const lockContent = JSON.stringify({
+    pid: process.pid,
+    acquiredAt: Date.now(),
+  });
+
+  try {
+    await fs.writeFile(lockPath, lockContent, { flag: "wx" });
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "EEXIST") {
+      return null;
+    }
+    throw err;
+  }
+
+  return {
+    release: async () => {
+      try {
+        await fs.unlink(lockPath);
+      } catch {
+        // Ignore errors on release (file may already be removed)
+      }
+    },
+  };
+}
+
+/** Acquire a lock for the summary worker. Returns null if already locked. */
+export async function acquireSummaryLock(agentId?: string): Promise<WorkerLock | null> {
+  const lockPath = path.join(resolveSummariesDir(agentId), LOCK_FILENAME);
+
+  // Ensure directory exists
+  await fs.mkdir(path.dirname(lockPath), { recursive: true });
+
+  // First attempt: try to create the lock file
+  const firstAttempt = await tryCreateLock(lockPath);
+  if (firstAttempt) {
+    return firstAttempt;
+  }
+
+  // Lock file exists — check if it's stale
+  try {
+    const stat = await fs.stat(lockPath);
+    const age = Date.now() - stat.mtimeMs;
+
+    if (age < LOCK_STALE_MS) {
+      return null; // Lock is fresh, another process holds it
+    }
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      // Lock was released between our failed create and this stat — retry
+      return tryCreateLock(lockPath);
+    }
+    throw err;
+  }
+
+  // Lock is stale — remove and retry. If another process also removes the stale
+  // lock, unlink may fail (harmless). The subsequent tryCreateLock is atomic:
+  // only one process wins the `wx` create.
+  await fs.unlink(lockPath).catch(() => {});
+  return tryCreateLock(lockPath);
+}
+
+/** Check if a lock is currently held (without acquiring) */
+export async function isLockHeld(agentId?: string): Promise<boolean> {
+  const lockPath = path.join(resolveSummariesDir(agentId), LOCK_FILENAME);
+
+  try {
+    const stat = await fs.stat(lockPath);
+    const age = Date.now() - stat.mtimeMs;
+    return age < LOCK_STALE_MS;
+  } catch {
+    return false;
+  }
+}