@pennyclaw/auto-compact 0.1.1

package/README.md

@@ -0,0 +1,34 @@
+# auto-compact
+
+Idle + threshold based transcript compaction with optional aggressive tool pruning.
+
+## Requires
+- `gateway.http.endpoints.chatCompletions.enabled = true`
+
+## Config
+
+```json5
+plugins: {
+  entries: {
+    "auto-compact": {
+      enabled: true,
+      config: {
+        idleMinutes: 15,
+        contextTokensThreshold: 100000,
+        triggerMode: "or",       // "or" | "and"
+        keepTurns: 5,
+        aggressive: false,        // true = prune tool_result before summary
+        modelOverride: null       // null = default model
+      }
+    }
+  }
+}
+```
+
+## Behavior
+- Keeps the last N turns intact (user + assistant + tool calls).
+- Summarizes everything before that into a single assistant message prefixed with `[context_summary]`.
+- Uses rolling summary (previous summary is used as base).
+- Chunking + fallback to avoid overflow.
+- When `aggressive=true`, prunes all tool_result content before summary.
+- Credentials are **never included**; only `/secrets` file paths are referenced.

package/dist/index.js

@@ -0,0 +1,478 @@
+// index.ts
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+function estimateTokens(msg) {
+  const content = normalizeContentToText(msg.content);
+  return Math.ceil(content.length / 4);
+}
+var SAFETY_MARGIN = 1.2;
+var MAX_CHUNK_TOKENS = 6e3;
+var MAX_SUMMARY_TOKENS = 800;
+var SUMMARY_PREFIX = "[context_summary]";
+var inMemoryLocks = /* @__PURE__ */ new Map();
+var lastRunAt = /* @__PURE__ */ new Map();
+function looksLikeGroupId(from) {
+  const lower = from.toLowerCase();
+  if (lower.includes(":group:")) return "group";
+  if (lower.includes(":channel:")) return "channel";
+  if (lower.endsWith("@g.us")) return "group";
+  return "dm";
+}
+function stripChannelPrefix(value, channelId) {
+  const prefix = `${channelId}:`;
+  return value.startsWith(prefix) ? value.slice(prefix.length) : value;
+}
+async function withFileLock(key, fn) {
+  if (inMemoryLocks.get(key)) return;
+  inMemoryLocks.set(key, true);
+  try {
+    return await fn();
+  } finally {
+    inMemoryLocks.delete(key);
+  }
+}
+function isValidSessionFile(sessionFile, sessionsDir) {
+  if (!sessionFile || !sessionsDir) return false;
+  if (!sessionFile.endsWith(".jsonl")) return false;
+  const resolved = path.resolve(sessionFile);
+  const base = path.resolve(sessionsDir) + path.sep;
+  return resolved.startsWith(base);
+}
+function parseEntryTimestamp(entry) {
+  const ts = entry?.timestamp;
+  if (typeof ts === "number" && Number.isFinite(ts)) return ts < 1e12 ? ts * 1e3 : ts;
+  if (typeof ts === "string") {
+    const parsed = Date.parse(ts);
+    if (!Number.isNaN(parsed)) return parsed;
+  }
+  return void 0;
+}
+async function getLastMessageTimestampBefore(sessionFile, cutoffMs) {
+  if (!fs.existsSync(sessionFile)) return void 0;
+  const stat = await fs.promises.stat(sessionFile);
+  const fileSize = stat.size;
+  const maxBytes = 512 * 1024;
+  const chunkSize = 64 * 1024;
+  let bytesRead = 0;
+  let position = fileSize;
+  let buffer = "";
+  const handle = await fs.promises.open(sessionFile, "r");
+  try {
+    while (position > 0 && bytesRead < maxBytes) {
+      const readSize = Math.min(chunkSize, position);
+      position -= readSize;
+      const buf = Buffer.alloc(readSize);
+      await handle.read(buf, 0, readSize, position);
+      bytesRead += readSize;
+      buffer = buf.toString("utf-8") + buffer;
+      const lines = buffer.split("\n");
+      for (let i = lines.length - 1; i >= 0; i--) {
+        const line = lines[i];
+        if (!line || !line.trim()) continue;
+        try {
+          const entry = JSON.parse(line);
+          if (entry?.type !== "message") continue;
+          const ts = parseEntryTimestamp(entry);
+          if (typeof ts !== "number") continue;
+          if (ts < cutoffMs) return ts;
+        } catch {
+          continue;
+        }
+      }
+      buffer = lines[0] || "";
+    }
+  } finally {
+    await handle.close();
+  }
+  return void 0;
+}
+function normalizeContentToText(content) {
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) {
+    return content.map((c) => typeof c?.text === "string" ? c.text : typeof c === "string" ? c : "").filter(Boolean).join("\n");
+  }
+  if (content && typeof content === "object") {
+    if (typeof content.text === "string") return content.text;
+    try {
+      return JSON.stringify(content);
+    } catch {
+      return String(content);
+    }
+  }
+  return "";
+}
+function pruneToolResultMessage(entry, placeholder) {
+  if (!entry || entry.type !== "message") return false;
+  const msg = entry.message;
+  if (!msg || msg.role !== "tool" && msg.role !== "toolResult") return false;
+  const content = msg.content;
+  if (typeof content === "string") {
+    if (content === placeholder) return false;
+    msg.content = placeholder;
+    return true;
+  }
+  if (Array.isArray(content)) {
+    if (content.length === 1 && content[0]?.type === "text" && content[0]?.text === placeholder) {
+      return false;
+    }
+    msg.content = [{ type: "text", text: placeholder }];
+    return true;
+  }
+  if (content && typeof content === "object" && "text" in content) {
+    if (content.text === placeholder) return false;
+    msg.content = { ...content, text: placeholder };
+    return true;
+  }
+  msg.content = [{ type: "text", text: placeholder }];
+  return true;
+}
+function estimateMessagesTokens(messages) {
+  return messages.reduce((sum, message) => sum + estimateTokens(message), 0);
+}
+function chunkMessagesByMaxTokens(messages, maxTokens) {
+  if (messages.length === 0) return [];
+  const chunks = [];
+  let current = [];
+  let currentTokens = 0;
+  for (const message of messages) {
+    const messageTokens = estimateTokens(message) * SAFETY_MARGIN;
+    if (current.length > 0 && currentTokens + messageTokens > maxTokens) {
+      chunks.push(current);
+      current = [];
+      currentTokens = 0;
+    }
+    current.push(message);
+    currentTokens += messageTokens;
+    if (messageTokens > maxTokens) {
+      chunks.push(current);
+      current = [];
+      currentTokens = 0;
+    }
+  }
+  if (current.length > 0) chunks.push(current);
+  return chunks;
+}
+function isOversizedForSummary(msg, contextWindow) {
+  const tokens = estimateTokens(msg) * SAFETY_MARGIN;
+  return tokens > contextWindow * 0.5;
+}
+async function callChatCompletions(params) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), params.timeoutMs ?? 3e4);
+  try {
+    const body = {
+      messages: [
+        { role: "system", content: params.systemPrompt },
+        { role: "user", content: params.userPrompt }
+      ],
+      max_tokens: MAX_SUMMARY_TOKENS,
+      temperature: 0.2
+    };
+    if (params.modelOverride) body.model = params.modelOverride;
+    const res = await fetch(`${params.baseUrl}/v1/chat/completions`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        ...params.token ? { Authorization: `Bearer ${params.token}`, "X-OpenClaw-Token": params.token } : {}
+      },
+      body: JSON.stringify(body),
+      signal: controller.signal
+    });
+    if (!res.ok) {
+      const txt = await res.text().catch(() => "");
+      throw new Error(`chatCompletions ${res.status}: ${txt}`);
+    }
+    const json = await res.json();
+    const content = json?.choices?.[0]?.message?.content ?? json?.output_text;
+    if (!content || typeof content !== "string") {
+      throw new Error("chatCompletions: empty response");
+    }
+    return content.trim();
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+function buildSummaryPrompt(params) {
+  const lines = [];
+  if (params.previousSummary) {
+    lines.push("Previous summary (for continuity):");
+    lines.push(params.previousSummary);
+    lines.push("");
+  }
+  lines.push("Conversation chunk:");
+  for (const msg of params.chunk) {
+    const role = msg.role ?? "message";
+    const content = normalizeContentToText(msg.content);
+    if (!content) continue;
+    lines.push(`${role.toUpperCase()}: ${content}`);
+  }
+  if (params.secretsList.length > 0) {
+    lines.push("");
+    lines.push("Known credential file paths (DO NOT include secrets, only paths):");
+    for (const p of params.secretsList) lines.push(`- ${p}`);
+  }
+  return lines.join("\n");
+}
+async function summarizeWithFallback(params) {
+  const { messages } = params;
+  if (messages.length === 0) return params.previousSummary ?? "No prior history.";
+  const chunks = chunkMessagesByMaxTokens(messages, MAX_CHUNK_TOKENS);
+  let summary = params.previousSummary;
+  try {
+    for (const chunk of chunks) {
+      summary = await callChatCompletions({
+        baseUrl: params.baseUrl,
+        token: params.token,
+        modelOverride: params.modelOverride,
+        systemPrompt: params.systemPrompt,
+        userPrompt: buildSummaryPrompt({
+          chunk,
+          previousSummary: summary,
+          secretsList: params.secretsList
+        })
+      });
+    }
+    return summary ?? "No prior history.";
+  } catch (fullError) {
+  }
+  const smallMessages = [];
+  const oversizedNotes = [];
+  for (const msg of messages) {
+    if (isOversizedForSummary(msg, params.contextWindow)) {
+      const role = msg.role ?? "message";
+      const tokens = estimateTokens(msg);
+      oversizedNotes.push(`[Large ${role} (~${Math.round(tokens / 1e3)}K tokens) omitted from summary]`);
+    } else {
+      smallMessages.push(msg);
+    }
+  }
+  if (smallMessages.length > 0) {
+    try {
+      const chunks2 = chunkMessagesByMaxTokens(smallMessages, MAX_CHUNK_TOKENS);
+      let summary2 = params.previousSummary;
+      for (const chunk of chunks2) {
+        summary2 = await callChatCompletions({
+          baseUrl: params.baseUrl,
+          token: params.token,
+          modelOverride: params.modelOverride,
+          systemPrompt: params.systemPrompt,
+          userPrompt: buildSummaryPrompt({
+            chunk,
+            previousSummary: summary2,
+            secretsList: params.secretsList
+          })
+        });
+      }
+      const notes = oversizedNotes.length > 0 ? `
+
+${oversizedNotes.join("\n")}` : "";
+      return (summary2 ?? "No prior history.") + notes;
+    } catch {
+    }
+  }
+  return `Context contained ${messages.length} messages (${oversizedNotes.length} oversized). Summary unavailable due to size limits.`;
+}
+function extractSecretsList() {
+  const candidates = ["/home/node/.openclaw/workspace/secrets", "/secrets"];
+  for (const dir of candidates) {
+    try {
+      if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) continue;
+      const files = fs.readdirSync(dir).map((f) => path.join(dir, f));
+      if (files.length > 0) return files;
+    } catch {
+    }
+  }
+  return [];
+}
+function findPreviousSummary(messages) {
+  let previousSummary;
+  const filtered = [];
+  for (const msg of messages) {
+    if (msg.role === "assistant") {
+      const content = normalizeContentToText(msg.content);
+      if (content.startsWith(SUMMARY_PREFIX)) {
+        previousSummary = content.replace(SUMMARY_PREFIX, "").trim();
+        continue;
+      }
+    }
+    filtered.push(msg);
+  }
+  return { previousSummary, filtered };
+}
+function generateId() {
+  return crypto.randomUUID().replace(/-/g, "").slice(0, 8);
+}
+function register(api) {
+  api.logger.info("auto-compact: loaded v0.0.2");
+  api.on("message_received", async (event, ctx) => {
+    const pluginCfg = api.pluginConfig ?? {};
+    const idleMinutes = Number.isFinite(pluginCfg.idleMinutes) ? Number(pluginCfg.idleMinutes) : 15;
+    const contextTokensThreshold = Number.isFinite(pluginCfg.contextTokensThreshold) ? Number(pluginCfg.contextTokensThreshold) : 1e5;
+    const triggerMode = pluginCfg.triggerMode === "and" ? "and" : "or";
+    const keepTurns = Number.isFinite(pluginCfg.keepTurns) ? Math.max(1, Number(pluginCfg.keepTurns)) : 5;
+    const aggressive = Boolean(pluginCfg.aggressive);
+    const modelOverride = typeof pluginCfg.modelOverride === "string" && pluginCfg.modelOverride.trim() ? pluginCfg.modelOverride.trim() : null;
+    if (!idleMinutes && !contextTokensThreshold) return;
+    const from = typeof event.from === "string" ? event.from.trim() : "";
+    if (!from) return;
+    const channelId = typeof ctx.channelId === "string" ? ctx.channelId : "";
+    const conversationId = typeof ctx.conversationId === "string" && ctx.conversationId.trim() ? ctx.conversationId.trim() : typeof event?.metadata?.to === "string" ? event.metadata.to : from;
+    const peerIdRaw = conversationId || from;
+    const peerId = channelId ? stripChannelPrefix(peerIdRaw, channelId) : peerIdRaw;
+    const peerKind = looksLikeGroupId(peerIdRaw);
+    const route = api.runtime.channel.routing.resolveAgentRoute({
+      cfg: api.config,
+      channel: channelId || "telegram",
+      accountId: ctx.accountId ?? void 0,
+      peer: { kind: peerKind, id: peerId }
+    });
+    const storePath = api.runtime.channel.session.resolveStorePath(
+      api.config.session?.store,
+      { agentId: route.agentId }
+    );
+    if (!fs.existsSync(storePath)) return;
+    let store = {};
+    try {
+      store = JSON.parse(await fs.promises.readFile(storePath, "utf-8"));
+    } catch {
+      return;
+    }
+    let sessionKey = route.sessionKey;
+    if (!store[sessionKey]) {
+      const keys = Object.keys(store);
+      const match = keys.find((key) => {
+        const e = store[key] || {};
+        const candidates = [e.origin?.from, e.origin?.to, e.lastTo, e.deliveryContext?.to].filter(Boolean);
+        return candidates.includes(conversationId) || candidates.includes(peerIdRaw) || candidates.includes(peerId);
+      });
+      if (match) sessionKey = match;
+    }
+    const entry = store[sessionKey];
+    if (!entry) return;
+    const sessionsDir = path.dirname(storePath);
+    const sessionFile = typeof entry.sessionFile === "string" && entry.sessionFile.trim() || path.join(sessionsDir, `${entry.sessionId}.jsonl`);
+    if (!isValidSessionFile(sessionFile, sessionsDir)) return;
+    const now = Date.now();
+    const recentRunAt = lastRunAt.get(sessionFile) ?? 0;
+    if (now - recentRunAt < 6e4) return;
+    let cutoff = typeof event?.timestamp === "number" ? event.timestamp : typeof event?.timestamp === "string" ? Date.parse(event.timestamp) : now;
+    if (!Number.isFinite(cutoff)) cutoff = now;
+    if (cutoff < 1e12) cutoff *= 1e3;
+    const lastTs = await getLastMessageTimestampBefore(sessionFile, cutoff);
+    const idleOk = idleMinutes > 0 && typeof lastTs === "number" && now - lastTs >= idleMinutes * 60 * 1e3;
+    const contextTokens = typeof entry.contextTokens === "number" ? entry.contextTokens : 0;
+    const thresholdOk = contextTokensThreshold > 0 && contextTokens >= contextTokensThreshold;
+    let shouldRun = false;
+    if (idleMinutes > 0 && contextTokensThreshold > 0) {
+      shouldRun = triggerMode === "and" ? idleOk && thresholdOk : idleOk || thresholdOk;
+    } else if (idleMinutes > 0) {
+      shouldRun = idleOk;
+    } else if (contextTokensThreshold > 0) {
+      shouldRun = thresholdOk;
+    }
+    api.logger.info(
+      `auto-compact: idleOk=${idleOk} thresholdOk=${thresholdOk} contextTokens=${contextTokens} idleMinutes=${idleMinutes} threshold=${contextTokensThreshold} triggerMode=${triggerMode}`
+    );
+    if (!shouldRun) return;
+    const chatCompletionsEnabled = api.config?.gateway?.http?.endpoints?.chatCompletions?.enabled === true;
+    if (!chatCompletionsEnabled) {
+      api.logger.warn("auto-compact: chatCompletions endpoint disabled; skipping summary");
+      return;
+    }
+    const token = api.config?.gateway?.auth?.token;
+    const port = api.config?.gateway?.port ?? 18789;
+    const baseUrl = `http://127.0.0.1:${port}`;
+    const systemPrompt = "You are summarizing chat history to reduce token usage. Return a concise, factual summary. Always include: decisions, configuration/parameters, credentials (paths only, never secrets), current task status, and important file/paths. Do not include secrets or raw credentials.";
+    await withFileLock(sessionFile, async () => {
+      const raw = await fs.promises.readFile(sessionFile, "utf-8");
+      const lines = raw.split("\n").filter((l) => l.trim().length > 0);
+      if (lines.length === 0) return;
+      const entries = lines.map((line) => {
+        try {
+          return JSON.parse(line);
+        } catch {
+          return null;
+        }
+      }).filter(Boolean);
+      const sessionHeaderIndex = entries.findIndex((e) => e?.type === "session");
+      const header = sessionHeaderIndex >= 0 ? entries[sessionHeaderIndex] : null;
+      const userMessageIndices = [];
+      for (let i = 0; i < entries.length; i++) {
+        const e = entries[i];
+        if (e?.type === "message" && e?.message?.role === "user") userMessageIndices.push(i);
+      }
+      api.logger.info(`auto-compact: userMessages=${userMessageIndices.length} keepTurns=${keepTurns}`);
+      if (userMessageIndices.length < keepTurns) return;
+      const keepStartIndex = userMessageIndices[userMessageIndices.length - keepTurns];
+      const keptEntries = entries.slice(keepStartIndex);
+      const summarySourceEntries = entries.slice(sessionHeaderIndex >= 0 ? sessionHeaderIndex + 1 : 0, keepStartIndex);
+      const summaryMessages = summarySourceEntries.filter((e) => e?.type === "message" && e?.message).map((e) => ({
+        role: e.message.role,
+        content: e.message.content,
+        timestamp: e.message.timestamp ?? e.timestamp ?? Date.now()
+      }));
+      const { previousSummary, filtered } = findPreviousSummary(summaryMessages);
+      const secretsList = extractSecretsList();
+      let summaryText;
+      try {
+        summaryText = await summarizeWithFallback({
+          messages: filtered,
+          baseUrl,
+          token,
+          modelOverride,
+          previousSummary,
+          secretsList,
+          contextWindow: 16e3,
+          systemPrompt
+        });
+      } catch (err) {
+        api.logger.error(`auto-compact: summary failed ${err instanceof Error ? err.message : String(err)}`);
+        return;
+      }
+      if (secretsList.length > 0) {
+        summaryText += `
+
+Credentials (paths only):
+${secretsList.map((p) => `- ${p}`).join("\n")}`;
+      }
+      const compactionEntry = {
+        type: "compaction",
+        id: generateId(),
+        parentId: header?.id ?? keptEntries[0]?.parentId,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        summary: `${SUMMARY_PREFIX}
+${summaryText}`,
+        firstKeptEntryId: keptEntries[0]?.id,
+        tokensBefore: Math.round(estimateMessagesTokens(summaryMessages) * SAFETY_MARGIN)
+      };
+      const tmp = `${sessionFile}.auto-compact.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}.tmp`;
+      const out = [];
+      for (const line of lines) {
+        if (!line.trim()) continue;
+        let entry2;
+        try {
+          entry2 = JSON.parse(line);
+        } catch {
+          out.push(line);
+          continue;
+        }
+        if (entry2?.id && entry2.id === compactionEntry.firstKeptEntryId) {
+          entry2.parentId = compactionEntry.id;
+        }
+        if (aggressive) {
+          pruneToolResultMessage(entry2, "[pruned due to compact]");
+        }
+        out.push(JSON.stringify(entry2));
+      }
+      out.push(JSON.stringify(compactionEntry));
+      await fs.promises.writeFile(tmp, out.join("\n"), "utf-8");
+      await fs.promises.rename(tmp, sessionFile);
+      lastRunAt.set(sessionFile, Date.now());
+      api.logger.info(`auto-compact: wrote compaction entry in ${sessionFile}`);
+    });
+  });
+}
+export {
+  register as default
+};

package/openclaw.plugin.json

@@ -0,0 +1,27 @@
+{
+  "id": "auto-compact",
+  "name": "Auto Compact",
+  "version": "0.0.1",
+  "description": "Idle + threshold based transcript compaction with optional aggressive tool pruning.",
+  "main": "dist/index.js",
+  "configSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+      "idleMinutes": { "type": "number", "default": 15, "minimum": 1 },
+      "contextTokensThreshold": { "type": "number", "default": 100000, "minimum": 1 },
+      "triggerMode": { "type": "string", "enum": ["or", "and"], "default": "or" },
+      "keepTurns": { "type": "number", "default": 5, "minimum": 1 },
+      "aggressive": { "type": "boolean", "default": false },
+      "modelOverride": { "type": ["string", "null"], "default": null }
+    }
+  },
+  "uiHints": {
+    "idleMinutes": { "label": "Idle minutes" },
+    "contextTokensThreshold": { "label": "Context tokens threshold" },
+    "triggerMode": { "label": "Trigger mode (or/and)" },
+    "keepTurns": { "label": "Keep turns" },
+    "aggressive": { "label": "Aggressive (prune tool_result first)" },
+    "modelOverride": { "label": "Model override" }
+  }
+}

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@pennyclaw/auto-compact",
+  "version": "0.1.1",
+  "description": "Idle + threshold based transcript compaction with optional aggressive tool pruning.",
+  "type": "module",
+  "main": "dist/index.js",
+  "files": [
+    "dist/index.js",
+    "openclaw.plugin.json",
+    "README.md"
+  ],
+  "keywords": [
+    "openclaw",
+    "plugin",
+    "compact",
+    "summary"
+  ],
+  "license": "MIT",
+  "scripts": {
+    "build": "esbuild index.ts --bundle --platform=node --format=esm --outfile=dist/index.js --external:@mariozechner/pi-coding-agent"
+  },
+  "devDependencies": {
+    "esbuild": "^0.27.3"
+  }
+}