@hyperspell/openclaw-hyperspell 0.14.1 → 0.15.0

package/dist/client.js

@@ -270,6 +270,57 @@ export class HyperspellClient {
         });
         return { resourceId: result.resource_id, status: result.status };
     }
+    /**
+     * POST /messages — the real-time hot buffer. Rows are full-text searchable
+     * the instant they're inserted (a Postgres GENERATED tsvector, no embedding
+     * wait) and auto-consolidated server-side into vault Resources within ~60s.
+     * The existing search path already unions this buffer with vector results,
+     * so once we write here our turns become searchable immediately.
+     *
+     * Auth: api key + `X-As-User` (REQUIRED — the endpoint 422s without it).
+     * Upsert key is (app_id, user_id, resource_id, message_id), so re-posting an
+     * identical message_id updates `content` only — safe to retry, no duplicates.
+     *
+     * Server-side limits (return 422): per-message content 1..512,000 chars;
+     * batch 1..1,000 messages. Callers should pre-enforce these.
+     */
+    async sendMessages(messages, options) {
+        const userId = options?.userId ?? this.config.userId;
+        if (!userId) {
+            // X-As-User is mandatory for /messages. Fail loud rather than firing a
+            // request we know will 422.
+            throw new Error("sendMessages requires a userId (X-As-User) — none configured");
+        }
+        if (messages.length === 0)
+            return { count: 0 };
+        const source = (options?.source ?? "vault").toLowerCase();
+        const body = {
+            source,
+            messages: messages.map((m) => ({
+                resource_id: m.resourceId,
+                message_id: m.messageId,
+                content: m.content,
+            })),
+        };
+        log.debugRequest("messages.create", {
+            source,
+            count: messages.length,
+            userId,
+        });
+        const res = await fetch(`${API_BASE_URL}/messages`, {
+            method: "POST",
+            headers: { ...this.rawHeaders(), "X-As-User": userId },
+            body: JSON.stringify(body),
+        });
+        if (!res.ok) {
+            const text = await res.text().catch(() => "");
+            throw new Error(`POST /messages failed (${res.status}): ${text}`);
+        }
+        const data = await res.json();
+        const count = data?.count ?? messages.length;
+        log.debugResponse("messages.create", { count });
+        return { count };
+    }
     async listConnections(options) {
         log.debugRequest("connections.list", { userId: options?.userId });
         const response = await this.client.connections.list(this.requestOptions(options?.userId));

package/dist/commands/setup.js

@@ -269,6 +269,12 @@ async function runSetup() {
                 userId,
                 autoContext: true,
                 autoTrace: { enabled: false, extract: ["procedure"] },
+                hotBuffer: {
+                    enabled: false,
+                    source: "vault",
+                    writeUser: true,
+                    writeAssistant: true,
+                },
                 emotionalContext: false,
                 syncMemories: true,
                 syncMemoriesConfig: {
@@ -277,6 +283,7 @@ async function runSetup() {
                     watchPaths: [],
                     debounceMs: 2000,
                     maxAgeDays: 30,
+                    ignorePaths: ["dreaming"],
                 },
                 sources: [],
                 maxResults: 10,

package/dist/config.js

@@ -15,6 +15,7 @@ const ALLOWED_KEYS = [
     "userId",
     "autoContext",
     "autoTrace",
+    "hotBuffer",
     "emotionalContext",
     "relationshipId",
     "startupOrientation",
@@ -235,6 +236,11 @@ export function parseConfig(raw) {
     const kgRaw = (cfg.knowledgeGraph ?? {});
     const atRaw = (cfg.autoTrace ?? {});
     const soRaw = (cfg.startupOrientation ?? {});
+    const hbRaw = (cfg.hotBuffer ?? {});
+    if (cfg.hotBuffer && typeof cfg.hotBuffer === "object" && !Array.isArray(cfg.hotBuffer)) {
+        assertAllowedKeys(hbRaw, ["enabled", "source", "writeUser", "writeAssistant"], "hyperspell.hotBuffer");
+    }
+    const hbSource = parseSources(hbRaw.source)[0] ?? "vault";
     // syncMemories can be a boolean (legacy) or an object (new)
     const smRaw = cfg.syncMemories;
     const syncMemoriesEnabled = typeof smRaw === "boolean"
@@ -249,7 +255,14 @@ export function parseConfig(raw) {
     // object form must be too, or a typo (sectionise/debounceMS) is silently
     // ignored and the user gets default behavior they didn't ask for.
     if (typeof smRaw === "object" && smRaw !== null && !Array.isArray(smRaw)) {
-        assertAllowedKeys(smObj, ["enabled", "sectionize", "watchPaths", "debounceMs", "maxAgeDays"], "hyperspell.syncMemories");
+        assertAllowedKeys(smObj, [
+            "enabled",
+            "sectionize",
+            "watchPaths",
+            "debounceMs",
+            "maxAgeDays",
+            "ignorePaths",
+        ], "hyperspell.syncMemories");
     }
     return {
         apiKey,
@@ -262,6 +275,14 @@ export function parseConfig(raw) {
             ],
             metadata: atRaw.metadata,
         },
+        hotBuffer: {
+            // Default OFF so shipping the plugin never changes existing installs'
+            // behavior; opt in per-install via plugin config.
+            enabled: hbRaw.enabled ?? false,
+            source: hbSource,
+            writeUser: hbRaw.writeUser ?? true,
+            writeAssistant: hbRaw.writeAssistant ?? true,
+        },
         emotionalContext: cfg.emotionalContext ?? false,
         relationshipId: cfg.relationshipId,
         startupOrientation: {
@@ -281,6 +302,9 @@ export function parseConfig(raw) {
                 : [],
             debounceMs: smObj.debounceMs ?? 2000,
             maxAgeDays: smObj.maxAgeDays ?? 30,
+            ignorePaths: Array.isArray(smObj.ignorePaths)
+                ? smObj.ignorePaths
+                : ["dreaming"],
         },
         sources: parseSources(cfg.sources),
         maxResults: cfg.maxResults ?? 10,

package/dist/hooks/auto-context.js

@@ -1,21 +1,6 @@
 import { buildScopeFilter, getCanReadScopes, resolveUser, } from "../lib/sender.js";
+import { excludeFilterFor, mergeWithExclude } from "../lib/filters.js";
 import { log } from "../logger.js";
-/**
- * Memories produced by session-end hooks (auto-trace, emotional-state) are
- * tagged `source: "openclaw_agent_end"`. The emotional-state hook already has
- * a dedicated fetch path (`getEmotionalState` + `<hyperspell-emotional-context>`
- * injection), so those memories should NOT also surface via generic retrieval —
- * double-injection dilutes results and replays the conversation verbatim.
- * Exclude them here at the search filter.
- */
-const EXCLUDE_SESSION_END_FILTER = {
-    source: { $ne: "openclaw_agent_end" },
-};
-function mergeWithExclude(base) {
-    if (!base)
-        return EXCLUDE_SESSION_END_FILTER;
-    return { $and: [base, EXCLUDE_SESSION_END_FILTER] };
-}
 function formatRelativeTime(isoTimestamp) {
     try {
         const dt = new Date(isoTimestamp);
@@ -64,8 +49,8 @@ function formatHighlightBullets(results, maxResults, threshold) {
         return null;
     return sections.join("\n\n");
 }
-const INTRO = "The following is context from the user's connected sources. Reference it only when relevant to the conversation.";
-const DISCLAIMER = "Use this context naturally when relevant — including indirect connections — but don't force it into every response or make assumptions beyond what's stated.";
+const INTRO = "The following is surfaced from the user's memory and connected sources, including past conversations. Reference it as recalled context, only when relevant to the conversation.";
+const DISCLAIMER = "Draw on it when relevant — including indirect connections — but don't force it into every response or make assumptions beyond what's stated.";
 function wrapSingle(body) {
     return `<hyperspell-context>\n${INTRO}\n\n${body}\n\n${DISCLAIMER}\n</hyperspell-context>`;
 }
@@ -84,7 +69,7 @@ export function buildAutoContextHandler(client, cfg) {
         try {
             const results = await client.search(prompt, {
                 limit: cfg.maxResults,
-                filter: EXCLUDE_SESSION_END_FILTER,
+                filter: excludeFilterFor(cfg),
             });
             const formatted = formatHighlightBullets(results, cfg.maxResults, cfg.relevanceThreshold);
             if (!formatted) {
@@ -119,7 +104,7 @@ async function multiUserSearch(client, cfg, prompt, resolved) {
         ? client.search(prompt, {
             limit: cfg.maxResults,
             userId: resolved.userId,
-            filter: EXCLUDE_SESSION_END_FILTER,
+            filter: excludeFilterFor(cfg),
         })
         : null;
     // Always search shared for unknown senders, even if includeSharedInSearch is false
@@ -130,7 +115,7 @@ async function multiUserSearch(client, cfg, prompt, resolved) {
         ? client.search(prompt, {
             limit: sharedLimit,
             userId: multiUser.sharedUserId,
-            filter: mergeWithExclude(scopeFilter),
+            filter: mergeWithExclude(scopeFilter, cfg),
         })
         : null;
     const searches = [personalSearch, sharedSearch].filter(Boolean);

package/dist/hooks/memory-sync.js

@@ -19,8 +19,22 @@ export function buildFileSyncHandler(client, cfg) {
     const sectionize = cfg.syncMemoriesConfig.sectionize;
     const watchPaths = cfg.syncMemoriesConfig.watchPaths;
     const debounceMs = cfg.syncMemoriesConfig.debounceMs;
+    const ignoreDirs = new Set(cfg.syncMemoriesConfig.ignorePaths);
     // Resolve additional watch paths to absolute paths for matching
     const resolvedWatchPaths = (watchPaths ?? []).map((wp) => wp.startsWith("/") ? wp : path.join(workspaceDir, wp));
+    /**
+     * Mirror the bulk walk's exclusions on the live path: a file under a
+     * dot-directory (e.g. .dreams) or an ignored directory (e.g. dreaming) must
+     * not live-sync either, or an edit would re-ingest exactly what the walk
+     * skips.
+     */
+    function isIgnoredPath(filePath) {
+        const rel = path.relative(memoryDir, filePath);
+        if (rel.startsWith("..") || path.isAbsolute(rel))
+            return false;
+        const segments = rel.split(path.sep).slice(0, -1); // directory segments only
+        return segments.some((seg) => seg.startsWith(".") || ignoreDirs.has(seg));
+    }
     // Debounce map: filePath -> timeout handle
     const debounceTimers = new Map();
     /**
@@ -29,6 +43,8 @@ export function buildFileSyncHandler(client, cfg) {
     function isSyncable(filePath) {
         if (!filePath.endsWith(".md"))
             return false;
+        if (isIgnoredPath(filePath))
+            return false;
         // Always sync memory/ files
         if (filePath.startsWith(memoryDir + path.sep))
             return true;
@@ -107,6 +123,7 @@ export async function syncMemoriesOnStartup(client, workspaceDir, options) {
             userId: options.userId,
             watchPaths: options.watchPaths,
             maxAgeDays: options.maxAgeDays,
+            ignorePaths: options.ignorePaths,
         });
         log.info(`Section sync complete: ${result.synced} synced, ${result.skipped} unchanged, ${result.agedOut} aged-out, ${result.removed} removed, ${result.failed} failed`);
         if (result.failed > 0) {

package/dist/index.js

@@ -6,6 +6,7 @@ import { buildAutoContextHandler } from "./hooks/auto-context.js";
 import { buildAutoTraceHandler } from "./hooks/auto-trace.js";
 import { buildEmotionalStateCompactionHandler, buildEmotionalStateFetchHandler, buildEmotionalStateSessionCleanupHandler, buildEmotionalStateStoreHandler, } from "./hooks/emotional-state.js";
 import { buildFileSyncHandler, syncMemoriesOnStartup } from "./hooks/memory-sync.js";
+import { buildHotBufferHandler, buildHotBufferSessionCleanupHandler, } from "./hooks/hot-buffer.js";
 import { buildStartupOrientationCompactionHandler, buildStartupOrientationHandler, buildStartupOrientationSessionCleanupHandler, } from "./hooks/startup-orientation.js";
 import { initLogger } from "./logger.js";
 import { createRememberToolFactory } from "./tools/remember.js";
@@ -100,6 +101,12 @@ export default {
         if (cfg.autoTrace.enabled) {
             api.on("agent_end", buildAutoTraceHandler(client, cfg));
         }
+        // Register hot-buffer hook: write each turn to POST /messages so it's
+        // instantly full-text searchable (vs. the slow /memories embedding path).
+        if (cfg.hotBuffer.enabled) {
+            api.on("agent_end", buildHotBufferHandler(client, cfg));
+            api.on("session_end", buildHotBufferSessionCleanupHandler());
+        }
         // Register memory sync hook
         if (cfg.syncMemories) {
             const fileSyncHandler = buildFileSyncHandler(client, cfg);
@@ -131,6 +138,7 @@ export default {
                         sectionize: cfg.syncMemoriesConfig.sectionize,
                         watchPaths: cfg.syncMemoriesConfig.watchPaths,
                         maxAgeDays: cfg.syncMemoriesConfig.maxAgeDays,
+                        ignorePaths: cfg.syncMemoriesConfig.ignorePaths,
                     }).catch((err) => {
                         api.logger.error("hyperspell: background memory sync failed", err);
                     });

package/dist/lib/filters.js

@@ -0,0 +1,61 @@
+/**
+ * Shared Hyperspell `options.filter` clauses, used by every retrieval path
+ * (auto-context hook + the hyperspell_search tool) so filtering stays
+ * consistent across them.
+ *
+ * Filters match against memory METADATA keys by their bare name — the same
+ * convention `buildScopeFilter` uses (`openclaw_scope`, `openclaw_user`).
+ */
+/**
+ * Memories produced by the auto-trace session-end hook are tagged in metadata
+ * as `openclaw_source: "agent_end"` (see `sendTrace` in client.ts). Those should
+ * NOT surface via generic retrieval — replaying whole sanitized transcripts back
+ * into context creates a self-amplifying pollution loop. Exclude them here.
+ *
+ * THE #40 TENSION: hot-buffer rows written via `POST /messages` carry NO
+ * `openclaw_source`, and the backend evaluates absent-field metadata predicates
+ * in SQL three-valued logic — `metadata->>'openclaw_source'` is NULL for a
+ * missing key, and `NULL != 'agent_end'` is NULL (not TRUE) — so this filter
+ * also drops every untagged hot-buffer row. We could not work around that at the
+ * filter layer: `docs/filter-dialect-test.mjs` against the live backend showed
+ * that NO `openclaw_source` predicate returns untagged rows ($exists/$or/$nin/
+ * $not all fail), AND that `POST /messages` silently ignores a `metadata` field,
+ * so the rows can't be positively tagged either. See `excludeFilterFor` for the
+ * fix we ship (gate on auto-trace), and issue #40 for the backend follow-up
+ * (make `/messages` accept metadata, or make the filter NULL-tolerant).
+ *
+ * NOTE: an earlier version checked the top-level `source` field for
+ * "openclaw_agent_end" — wrong on BOTH counts (the tag lives in metadata under
+ * `openclaw_source`, value `"agent_end"`), so it silently matched nothing.
+ */
+export const EXCLUDE_SESSION_END_FILTER = {
+    openclaw_source: { $ne: "agent_end" },
+};
+/**
+ * The exclude clause to apply for a given config — or `undefined` to skip
+ * filtering entirely (issue #40, Option 4 — the only viable plugin-side fix).
+ * `openclaw_source: "agent_end"` rows are written ONLY by the auto-trace hook;
+ * when auto-trace is disabled there are none to hide, so we skip the filter
+ * entirely — which is also the ONLY way to keep untagged hot-buffer rows
+ * visible, since (per the dialect test) no filter and no write-tag can do it.
+ *
+ * LIMITATION: when auto-trace IS enabled, this still applies `$ne agent_end`,
+ * which drops untagged hot-buffer rows along with the traces. There is no
+ * plugin-side fix for that combination today; it needs the backend change
+ * tracked in #40. (The common single-feature install has auto-trace off.)
+ */
+export function excludeFilterFor(cfg) {
+    return cfg.autoTrace.enabled ? EXCLUDE_SESSION_END_FILTER : undefined;
+}
+/**
+ * Combine a caller-supplied filter with the session-end exclude via `$and`.
+ * Returns `undefined` when neither a base filter nor an exclude applies.
+ */
+export function mergeWithExclude(base, cfg) {
+    const exclude = excludeFilterFor(cfg);
+    if (!exclude)
+        return base;
+    if (!base)
+        return exclude;
+    return { $and: [base, exclude] };
+}

package/dist/sync/markdown.js

@@ -223,15 +223,38 @@ export function withSyncLock(fn) {
 // ---------------------------------------------------------------------------
 // Public API — file-level sync (original, kept for backward compat)
 // ---------------------------------------------------------------------------
-export function getMemoryFiles(workspaceDir) {
+/**
+ * Directory names skipped by default when walking memory/. The dreaming engine
+ * writes first-person dream journals under memory/dreaming/ that are not user
+ * memories; ingesting them pollutes retrieval (they score highly on emotional/
+ * associative queries and crowd out real memories). Configurable via
+ * `syncMemories.ignorePaths`.
+ */
+export const DEFAULT_IGNORE_DIRS = ["dreaming"];
+/**
+ * Skip a directory entry during the walk when it is a dot-directory/file
+ * (e.g. `.dreams`, engine scaffolding) or a directory named in `ignore`.
+ * Dot-entries are always skipped regardless of the configured list.
+ */
+function isIgnoredEntry(entry, ignore) {
+    if (entry.name.startsWith("."))
+        return true;
+    if (entry.isDirectory() && ignore.has(entry.name))
+        return true;
+    return false;
+}
+export function getMemoryFiles(workspaceDir, ignorePaths) {
     const memoryDir = path.join(workspaceDir, "memory");
     if (!fs.existsSync(memoryDir)) {
         return [];
     }
+    const ignore = new Set(ignorePaths ?? DEFAULT_IGNORE_DIRS);
     const results = [];
     function walk(dir) {
         try {
             for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+                if (isIgnoredEntry(entry, ignore))
+                    continue;
                 const fullPath = path.join(dir, entry.name);
                 if (entry.isDirectory()) {
                     walk(fullPath);
@@ -252,8 +275,9 @@ export function getMemoryFiles(workspaceDir) {
  * Collect all syncable files based on configuration.
  * Includes memory/*.md by default, plus any additional watchPaths.
  */
-export function getSyncableFiles(workspaceDir, watchPaths) {
-    const files = getMemoryFiles(workspaceDir);
+export function getSyncableFiles(workspaceDir, watchPaths, ignorePaths) {
+    const ignore = new Set(ignorePaths ?? DEFAULT_IGNORE_DIRS);
+    const files = getMemoryFiles(workspaceDir, ignorePaths);
     if (watchPaths) {
         for (const wp of watchPaths) {
             const resolved = wp.startsWith("/") ? wp : path.join(workspaceDir, wp);
@@ -270,6 +294,8 @@ export function getSyncableFiles(workspaceDir, watchPaths) {
                 const walk = (dir) => {
                     try {
                         for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+                            if (isIgnoredEntry(entry, ignore))
+                                continue;
                             const fullPath = path.join(dir, entry.name);
                             if (entry.isDirectory()) {
                                 walk(fullPath);
@@ -497,7 +523,7 @@ export async function syncAllMemoryFiles(client, workspaceDir, options) {
  * Unchanged sections (by content hash) are skipped.
  */
 export async function syncAllFilesSectionized(client, workspaceDir, options) {
-    const files = getSyncableFiles(workspaceDir, options?.watchPaths);
+    const files = getSyncableFiles(workspaceDir, options?.watchPaths, options?.ignorePaths);
     // Age pre-filter: a file untouched for longer than maxAgeDays that is
     // already recorded in the manifest has been ingested at least once and is
     // not changing — re-parsing/hashing/diffing it every startup is pure churn.

package/dist/tools/search.js

@@ -1,4 +1,5 @@
 import { Type } from "@sinclair/typebox";
+import { mergeWithExclude } from "../lib/filters.js";
 import { buildScopeFilter, getCanReadScopes, resolveUser } from "../lib/sender.js";
 import { log } from "../logger.js";
 export function createSearchToolFactory(client, cfg) {
@@ -10,7 +11,7 @@ export function createSearchToolFactory(client, cfg) {
     return (ctx) => ({
         name: "hyperspell_search",
         label: "Memory Search",
-        description: "Search through the user's connected sources (Notion, Slack, Gmail, Google Drive, etc.) for relevant information.",
+        description: "Search the user's long-term memory and connected sources for anything not already in the current conversation. Covers: saved memories and notes; past conversations — including ones from earlier or parallel sessions you have no transcript for; and connected sources (Notion, Slack, Gmail, Google Drive, etc.). Reach for this whenever the user refers to something from before, asks what was said / decided / remembered, or when relevant context likely exists but isn't in front of you — search before concluding you don't know.",
         parameters: Type.Object({
             query: Type.String({ description: "Search query" }),
             limit: Type.Optional(Type.Number({ description: "Max results (default: 5)" })),
@@ -38,6 +39,9 @@ export function createSearchToolFactory(client, cfg) {
                     : canRead;
                 filter = buildScopeFilter(allowed, resolved?.userId ?? "");
             }
+            // Keep session-end trace memories out of agent-facing search, matching
+            // the auto-context hook so both retrieval paths filter identically.
+            filter = mergeWithExclude(filter, cfg);
             log.debug(`search tool: query="${params.query}" limit=${limit} after=${params.after ?? "none"} before=${params.before ?? "none"} userId=${userId} scope=${params.scope ?? "any"}`);
             try {
                 const response = await client.searchRaw(params.query, {

package/openclaw.plugin.json

@@ -36,6 +36,11 @@
 			"help": "Automatically send conversation traces to Hyperspell for memory extraction (procedural, mood) at the end of each session",
 			"advanced": true
 		},
+		"hotBuffer": {
+			"label": "Hot Buffer",
+			"help": "Write each turn to the realtime message buffer (POST /messages) so it is instantly full-text searchable, then auto-consolidated into vault Resources. Requires userId. { enabled, source, writeUser, writeAssistant }",
+			"advanced": true
+		},
 		"emotionalContext": {
 			"label": "Emotional Context",
 			"help": "Maintain emotional continuity across sessions — remembers not just what happened, but how it felt",
@@ -116,6 +121,15 @@
 					"metadata": { "type": "object" }
 				}
 			},
+			"hotBuffer": {
+				"type": "object",
+				"properties": {
+					"enabled": { "type": "boolean" },
+					"source": { "type": "string" },
+					"writeUser": { "type": "boolean" },
+					"writeAssistant": { "type": "boolean" }
+				}
+			},
 			"emotionalContext": {
 				"type": "boolean"
 			},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperspell/openclaw-hyperspell",
-  "version": "0.14.1",
+  "version": "0.15.0",
   "type": "module",
   "description": "OpenClaw Hyperspell memory plugin",
   "license": "MIT",
@@ -38,7 +38,7 @@
     "check-types": "tsc --noEmit",
     "lint": "bunx @biomejs/biome ci .",
     "lint:fix": "bunx @biomejs/biome check --write .",
-    "test": "node --test --experimental-strip-types lib/sender.test.ts config.test.ts sync/markdown.test.ts hooks/auto-trace.test.ts hooks/emotional-state.test.ts hooks/startup-orientation.test.ts"
+    "test": "node --test --experimental-strip-types lib/sender.test.ts lib/filters.test.ts config.test.ts sync/markdown.test.ts hooks/auto-trace.test.ts hooks/hot-buffer.test.ts hooks/emotional-state.test.ts hooks/startup-orientation.test.ts"
   },
   "openclaw": {
     "extensions": [