@nowledge/openclaw-nowledge-mem 0.7.0 → 0.7.1

package/CHANGELOG.md

@@ -2,6 +2,21 @@
 
 All notable changes to the Nowledge Mem OpenClaw plugin will be documented in this file.
 
+## [Unreleased]
+
+### Fixed
+
+- **Thread sync no longer depends on argv-sized CLI payloads.** OpenClaw conversation capture now creates and appends threads through the Mem HTTP API instead of passing whole message arrays through `nmem ... -m '<json>'`. This removes the transport limit that caused repeated append failures on long or repetitive sessions.
+- **Session capture now syncs only the unsynced tail.** The plugin preserves the real transcript, asks Mem how many messages are already stored, and appends only the new tail instead of replaying the whole session on every hook or Context Engine turn.
+- **Remote config is still unified after the transport change.** The same resolved `apiUrl` and `apiKey` from OpenClaw settings / `~/.nowledge-mem/config.json` now drive both CLI-backed memory tools and API-backed thread sync.
+- **Removed lossy repetitive-session collapse.** The temporary content-based dedup workaround for cron-style sessions has been removed so conversation structure is preserved faithfully.
+
+## [0.7.1] - 2026-03-23
+
+### Fixed
+
+- **Heartbeat sessions no longer trigger thread capture.** Sessions with `ctx.trigger === "heartbeat"` are now skipped in hook handlers. For cron-triggered heartbeat sessions (which use `trigger: "cron"`), a content-based dedup detects repetitive patterns: when >50% of messages in a session are duplicates, only unique messages are kept. This collapses 20 repetitive heartbeat messages down to 2, eliminating the CLI timeout caused by oversized payloads.
+
 ## [0.7.0] - 2026-03-23
 
 ### Added

package/README.md

@@ -70,7 +70,7 @@ Connect to a Nowledge Mem server running elsewhere (a VPS, a home server, or a s
 
 Enable the plugin the same way as local mode, then set `apiUrl` and `apiKey` in the OpenClaw plugin settings.
 
-Or use a config file at `~/.nowledge-mem/openclaw.json`:
+Or use the shared config file at `~/.nowledge-mem/config.json` so OpenClaw, `nmem`, Bub, Claude Code, and other integrations all point at the same remote Mem:
 
 ```json
 {
@@ -79,7 +79,9 @@ Or use a config file at `~/.nowledge-mem/openclaw.json`:
 }
 ```
 
-The `apiKey` is injected as `NMEM_API_KEY` into the nmem CLI process. Never passed as a CLI argument, never logged.
+Legacy `~/.nowledge-mem/openclaw.json` is still honored first for backward compatibility.
+
+The resolved `apiUrl` and `apiKey` are reused across the plugin: CLI-backed memory tools and API-backed thread sync both talk to the same backend. The `apiKey` is never passed as a CLI argument and is never logged.
 
 ### Configure via WebUI of OpenClaw
 
@@ -154,6 +156,7 @@ flowchart TD
 
 **Key points:**
 - Thread capture is unconditional: every conversation is saved and searchable via `nowledge_mem_thread_search`
+- Thread sync is incremental: the plugin preserves the real transcript but appends only the unsynced tail instead of replaying the whole session
 - LLM distillation only runs at `agent_end`, not during compaction/reset checkpoints
 - Distilled memories carry `sourceThreadId`, linking them back to the source conversation
 - Cooldown (`digestMinInterval`, default 300s) prevents burst distillation

package/openclaw.plugin.json

@@ -1,6 +1,6 @@
 {
 	"id": "openclaw-nowledge-mem",
-	"version": "0.7.0",
+	"version": "0.7.1",
 	"kind": "memory",
 	"skills": ["skills/memory-guide"],
 	"uiHints": {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nowledge/openclaw-nowledge-mem",
-	"version": "0.7.0",
+	"version": "0.7.1",
 	"type": "module",
 	"description": "Nowledge Mem memory plugin for OpenClaw, local-first personal knowledge base",
 	"author": {

package/src/client.js

@@ -54,15 +54,11 @@ function patchWmSection(currentContent, heading, { content, append } = {}) {
 }
 
 /**
- * Nowledge Mem client. Wraps the nmem CLI for local-first and remote operations.
+ * Nowledge Mem client. Wraps the nmem CLI plus direct API calls.
  *
- * All operations go through the CLI first. This means:
- * - Local mode: CLI uses http://127.0.0.1:14242 automatically
- * - Remote mode: configure via apiUrl + apiKey (plugin config or env vars)
- *   (see: https://docs.nowledge.co/docs/remote-access)
- *
- * Falls back to direct API calls when a CLI command is too new for the installed
- * version. The fallback path uses the same apiUrl / apiKey.
+ * Most interactive reads/writes stay CLI-first for local-first ergonomics.
+ * Thread create/append are API-first because large message batches should
+ * travel in an HTTP body, not as a single argv-sized JSON argument.
  *
  * Credential rules:
  * - apiUrl: passed to CLI via --api-url flag (not a secret)
@@ -559,42 +555,12 @@ export class NowledgeMemClient {
 			throw new Error("createThread requires at least one message");
 		}
 
-		let data;
-		try {
-			const args = [
-				"--json",
-				"t",
-				"create",
-				"-t",
-				normalizedTitle,
-				"-m",
-				JSON.stringify(messages),
-				"-s",
-				String(source),
-			];
-			if (threadId) {
-				args.push("--id", String(threadId));
-			}
-			data = await this.execJson(args);
-		} catch (err) {
-			const message = err instanceof Error ? err.message : String(err);
-			const needsApiFallback =
-				Boolean(threadId) &&
-				(message.includes("unrecognized arguments: --id") ||
-					message.includes("invalid choice"));
-			if (!needsApiFallback) {
-				throw err;
-			}
-			this.logger.warn(
-				"createThread: CLI missing --id support, falling back to API",
-			);
-			data = await this.apiJson("POST", "/threads", {
-				thread_id: String(threadId),
-				title: normalizedTitle,
-				source: String(source),
-				messages,
-			});
-		}
+		const data = await this.apiJson("POST", "/threads", {
+			...(threadId ? { thread_id: String(threadId) } : {}),
+			title: normalizedTitle,
+			source: String(source),
+			messages,
+		});
 
 		return String(
 			data.id ?? data.thread?.thread_id ?? data.thread_id ?? "created",
@@ -615,50 +581,46 @@ export class NowledgeMemClient {
 			return { messagesAdded: 0, totalMessages: 0 };
 		}
 
+		const data = await this.apiJson(
+			"POST",
+			`/threads/${encodeURIComponent(normalizedThreadId)}/append`,
+			{
+				messages,
+				deduplicate,
+				...(idempotencyKey
+					? { idempotency_key: String(idempotencyKey) }
+					: {}),
+			},
+		);
+		return {
+			messagesAdded: Number(data.messages_added ?? 0),
+			totalMessages: Number(data.total_messages ?? 0),
+		};
+	}
+
+	async getThreadMessageCount(threadId) {
+		const normalizedThreadId = String(threadId || "").trim();
+		if (!normalizedThreadId) {
+			throw new Error("getThreadMessageCount requires threadId");
+		}
+
 		try {
-			const args = [
-				"--json",
-				"t",
-				"append",
-				normalizedThreadId,
-				"-m",
-				JSON.stringify(messages),
-				...(deduplicate ? [] : ["--no-deduplicate"]),
-			];
-			if (idempotencyKey) {
-				args.push("--idempotency-key", String(idempotencyKey));
-			}
-			const data = await this.execJson(args);
-			return {
-				messagesAdded: Number(data.messages_added ?? 0),
-				totalMessages: Number(data.total_messages ?? 0),
-			};
-		} catch (err) {
-			const message = err instanceof Error ? err.message : String(err);
-			const needsApiFallback =
-				message.includes("invalid choice") ||
-				message.includes("unrecognized arguments");
-			if (!needsApiFallback) {
-				throw err;
-			}
-			this.logger.warn(
-				"appendThread: CLI missing append support, falling back to API",
-			);
 			const data = await this.apiJson(
-				"POST",
-				`/threads/${encodeURIComponent(normalizedThreadId)}/append`,
-				{
-					messages,
-					deduplicate,
-					...(idempotencyKey
-						? { idempotency_key: String(idempotencyKey) }
-						: {}),
-				},
+				"GET",
+				`/threads/${encodeURIComponent(normalizedThreadId)}?limit=1`,
+				undefined,
+				15_000,
 			);
-			return {
-				messagesAdded: Number(data.messages_added ?? 0),
-				totalMessages: Number(data.total_messages ?? 0),
-			};
+			return Number(
+				data.message_count ??
+					data.total_messages ??
+					(Array.isArray(data.messages) ? data.messages.length : 0),
+			);
+		} catch (err) {
+			if (this.isThreadNotFoundError(err)) {
+				return null;
+			}
+			throw err;
 		}
 	}
 

package/src/hooks/capture.js

@@ -12,6 +12,8 @@ export const MIN_MESSAGES_FOR_DISTILL = 4;
 // Evicted opportunistically when new entries are set (see _setLastCapture).
 const _lastCaptureAt = new Map();
 const _MAX_COOLDOWN_ENTRIES = 200;
+const _syncedMessageCounts = new Map();
+const _MAX_SYNC_CURSOR_ENTRIES = 500;
 
 function _setLastCapture(threadId, now) {
 	_lastCaptureAt.set(threadId, now);
@@ -24,6 +26,20 @@ function _setLastCapture(threadId, now) {
 	}
 }
 
+function _setSyncedMessageCount(threadId, count) {
+	if (!threadId || !Number.isFinite(count) || count < 0) return;
+	_syncedMessageCounts.set(threadId, Math.trunc(count));
+	if (_syncedMessageCounts.size > _MAX_SYNC_CURSOR_ENTRIES) {
+		const excess = _syncedMessageCounts.size - _MAX_SYNC_CURSOR_ENTRIES;
+		let removed = 0;
+		for (const key of _syncedMessageCounts.keys()) {
+			_syncedMessageCounts.delete(key);
+			removed += 1;
+			if (removed >= excess) break;
+		}
+	}
+}
+
 export function truncate(text, max = DEFAULT_MAX_MESSAGE_CHARS) {
 	const str = String(text || "").trim();
 	if (!str) return "";
@@ -194,7 +210,7 @@ export async function appendOrCreateThread({
 		.filter(Boolean);
 	if (normalized.length === 0) return;
 
-	const messages = normalized.map((message, index) => ({
+	const allMessages = normalized.map((message, index) => ({
 		role: message.role,
 		content: message.content,
 		timestamp: message.timestamp,
@@ -210,16 +226,46 @@ export async function appendOrCreateThread({
 			session_id: sessionId || undefined,
 		},
 	}));
-	const idempotencyKey = buildAppendIdempotencyKey(threadId, reason, messages);
+
+	let syncedCount = _syncedMessageCounts.get(threadId);
+	if (syncedCount === undefined) {
+		syncedCount = await client.getThreadMessageCount(threadId);
+		if (syncedCount !== null) {
+			_setSyncedMessageCount(threadId, syncedCount);
+		}
+	}
+
+	if (typeof syncedCount === "number" && syncedCount > allMessages.length) {
+		// OpenClaw compaction can shrink the active transcript after we already
+		// stored the pre-compaction history. Reset the local cursor to the new
+		// compacted length so future turns append only the post-compaction tail.
+		_setSyncedMessageCount(threadId, allMessages.length);
+		return { threadId, normalized, messagesAdded: 0 };
+	}
+
+	const appendStart =
+		typeof syncedCount === "number" && syncedCount > 0
+			? Math.min(syncedCount, allMessages.length)
+			: 0;
+	const newMessages = allMessages.slice(appendStart);
+	if (newMessages.length === 0) {
+		return { threadId, normalized, messagesAdded: 0 };
+	}
+	const idempotencyKey = buildAppendIdempotencyKey(
+		threadId,
+		reason,
+		newMessages,
+	);
 
 	try {
 		const appended = await client.appendThread({
 			threadId,
-			messages,
+			messages: newMessages,
 			deduplicate: true,
 			idempotencyKey,
 		});
 		const added = appended.messagesAdded ?? 0;
+		_setSyncedMessageCount(threadId, allMessages.length);
 		logger.info(
 			`capture: appended ${added} messages to ${threadId} (${reason || "event"})`,
 		);
@@ -236,13 +282,14 @@ export async function appendOrCreateThread({
 		const createdId = await client.createThread({
 			threadId,
 			title,
-			messages,
+			messages: allMessages,
 			source: "openclaw",
 		});
+		_setSyncedMessageCount(threadId, allMessages.length);
 		logger.info(
-			`capture: created thread ${createdId} with ${messages.length} messages (${reason || "event"})`,
+			`capture: created thread ${createdId} with ${allMessages.length} messages (${reason || "event"})`,
 		);
-		return { threadId, normalized, messagesAdded: messages.length };
+		return { threadId, normalized, messagesAdded: allMessages.length };
 	} catch (err) {
 		const message = err instanceof Error ? err.message : String(err);
 		logger.warn(`capture: thread create failed for ${threadId}: ${message}`);
@@ -354,11 +401,17 @@ export async function triageAndDistill({
  *
  * When the context engine is active, this hook is a no-op — afterTurn
  * handles capture and distillation through the CE lifecycle.
+ *
+ * Heartbeat sessions (ctx.trigger === "heartbeat") are skipped — they
+ * produce repetitive status pings that aren't worth preserving.
+ * Other sessions use incremental tail sync: we preserve the real
+ * transcript, but only append messages that are not already stored.
  */
 export function buildAgentEndCaptureHandler(client, cfg, logger) {
 	return async (event, ctx) => {
 		if (ceState.active) return;
 		if (!event?.success) return;
+		if (ctx?.trigger === "heartbeat") return;
 
 		const captureResult = await appendOrCreateThread({
 			client,
@@ -379,10 +432,13 @@ export function buildAgentEndCaptureHandler(client, cfg, logger) {
  *
  * When the context engine is active, this hook is a no-op — afterTurn
  * handles capture through the CE lifecycle.
+ *
+ * Heartbeat sessions are skipped (same rationale as agent_end).
  */
 export function buildBeforeResetCaptureHandler(client, _cfg, logger) {
 	return async (event, ctx) => {
 		if (ceState.active) return;
+		if (ctx?.trigger === "heartbeat") return;
 		const reason = typeof event?.reason === "string" ? event.reason : undefined;
 		await appendOrCreateThread({
 			client,