@hyperspell/openclaw-hyperspell 0.14.0 → 0.14.2

package/README.md

@@ -151,6 +151,22 @@ When `syncMemories: true`, the plugin syncs markdown files from your agent's wor
 - The returned `resource_id` is stored in the file's YAML frontmatter as `hyperspell_id`
 - On subsequent syncs, files with an existing `hyperspell_id` are updated rather than duplicated
 - Files are synced automatically on startup and when they change
+- Startup sync runs in the **background** — the agent boots immediately and sync proceeds without blocking it
+- A per-section content hash is tracked in `<workspace>/.hyperspell-sync-hashes.json`, so unchanged sections are skipped on subsequent syncs (no re-ingestion)
+
+**Tuning sync (object form):**
+
+```jsonc
+"syncMemories": {
+  "enabled": true,
+  "sectionize": true,     // split files on ## headings into separate memories
+  "watchPaths": [],        // extra files/dirs to sync beyond memory/
+  "debounceMs": 2000,      // wait for writes to settle before syncing
+  "maxAgeDays": 30          // startup skips already-synced files older than this
+}
+```
+
+`maxAgeDays` bounds steady-state load: on startup, files whose mtime is older than the cutoff **and** already recorded in the sync manifest are skipped without re-reading. Files not yet in the manifest are always synced once regardless of age, and editing an old file bumps its mtime back into the window. Set to `0` to disable the cutoff.
 
 **Example frontmatter after sync:**
 

package/dist/commands/setup.js

@@ -276,6 +276,8 @@ async function runSetup() {
                     sectionize: true,
                     watchPaths: [],
                     debounceMs: 2000,
+                    maxAgeDays: 30,
+                    ignorePaths: ["dreaming"],
                 },
                 sources: [],
                 maxResults: 10,

package/dist/config.js

@@ -249,7 +249,14 @@ export function parseConfig(raw) {
     // object form must be too, or a typo (sectionise/debounceMS) is silently
     // ignored and the user gets default behavior they didn't ask for.
     if (typeof smRaw === "object" && smRaw !== null && !Array.isArray(smRaw)) {
-        assertAllowedKeys(smObj, ["enabled", "sectionize", "watchPaths", "debounceMs"], "hyperspell.syncMemories");
+        assertAllowedKeys(smObj, [
+            "enabled",
+            "sectionize",
+            "watchPaths",
+            "debounceMs",
+            "maxAgeDays",
+            "ignorePaths",
+        ], "hyperspell.syncMemories");
     }
     return {
         apiKey,
@@ -280,6 +287,10 @@ export function parseConfig(raw) {
                 ? smObj.watchPaths
                 : [],
             debounceMs: smObj.debounceMs ?? 2000,
+            maxAgeDays: smObj.maxAgeDays ?? 30,
+            ignorePaths: Array.isArray(smObj.ignorePaths)
+                ? smObj.ignorePaths
+                : ["dreaming"],
         },
         sources: parseSources(cfg.sources),
         maxResults: cfg.maxResults ?? 10,

package/dist/hooks/auto-context.js

@@ -1,21 +1,6 @@
 import { buildScopeFilter, getCanReadScopes, resolveUser, } from "../lib/sender.js";
+import { EXCLUDE_SESSION_END_FILTER, mergeWithExclude } from "../lib/filters.js";
 import { log } from "../logger.js";
-/**
- * Memories produced by session-end hooks (auto-trace, emotional-state) are
- * tagged `source: "openclaw_agent_end"`. The emotional-state hook already has
- * a dedicated fetch path (`getEmotionalState` + `<hyperspell-emotional-context>`
- * injection), so those memories should NOT also surface via generic retrieval —
- * double-injection dilutes results and replays the conversation verbatim.
- * Exclude them here at the search filter.
- */
-const EXCLUDE_SESSION_END_FILTER = {
-    source: { $ne: "openclaw_agent_end" },
-};
-function mergeWithExclude(base) {
-    if (!base)
-        return EXCLUDE_SESSION_END_FILTER;
-    return { $and: [base, EXCLUDE_SESSION_END_FILTER] };
-}
 function formatRelativeTime(isoTimestamp) {
     try {
         const dt = new Date(isoTimestamp);

package/dist/hooks/memory-sync.js

@@ -19,8 +19,22 @@ export function buildFileSyncHandler(client, cfg) {
     const sectionize = cfg.syncMemoriesConfig.sectionize;
     const watchPaths = cfg.syncMemoriesConfig.watchPaths;
     const debounceMs = cfg.syncMemoriesConfig.debounceMs;
+    const ignoreDirs = new Set(cfg.syncMemoriesConfig.ignorePaths);
     // Resolve additional watch paths to absolute paths for matching
     const resolvedWatchPaths = (watchPaths ?? []).map((wp) => wp.startsWith("/") ? wp : path.join(workspaceDir, wp));
+    /**
+     * Mirror the bulk walk's exclusions on the live path: a file under a
+     * dot-directory (e.g. .dreams) or an ignored directory (e.g. dreaming) must
+     * not live-sync either, or an edit would re-ingest exactly what the walk
+     * skips.
+     */
+    function isIgnoredPath(filePath) {
+        const rel = path.relative(memoryDir, filePath);
+        if (rel.startsWith("..") || path.isAbsolute(rel))
+            return false;
+        const segments = rel.split(path.sep).slice(0, -1); // directory segments only
+        return segments.some((seg) => seg.startsWith(".") || ignoreDirs.has(seg));
+    }
     // Debounce map: filePath -> timeout handle
     const debounceTimers = new Map();
     /**
@@ -29,6 +43,8 @@ export function buildFileSyncHandler(client, cfg) {
     function isSyncable(filePath) {
         if (!filePath.endsWith(".md"))
             return false;
+        if (isIgnoredPath(filePath))
+            return false;
         // Always sync memory/ files
         if (filePath.startsWith(memoryDir + path.sep))
             return true;
@@ -106,8 +122,10 @@ export async function syncMemoriesOnStartup(client, workspaceDir, options) {
         const result = await syncAllFilesSectionized(client, workspaceDir, {
             userId: options.userId,
             watchPaths: options.watchPaths,
+            maxAgeDays: options.maxAgeDays,
+            ignorePaths: options.ignorePaths,
         });
-        log.info(`Section sync complete: ${result.synced} synced, ${result.skipped} unchanged, ${result.removed} removed, ${result.failed} failed`);
+        log.info(`Section sync complete: ${result.synced} synced, ${result.skipped} unchanged, ${result.agedOut} aged-out, ${result.removed} removed, ${result.failed} failed`);
         if (result.failed > 0) {
             for (const error of result.errors) {
                 log.error(`  - ${error}`);

package/dist/index.js

@@ -117,13 +117,23 @@ export default {
             id: "openclaw-hyperspell",
             start: async () => {
                 api.logger.info("hyperspell: connected");
-                // Sync memories on startup if enabled
+                // Sync memories on startup if enabled.
+                //
+                // Deliberately NOT awaited: the bulk sync is sequential and
+                // network-bound (one request per changed section), so awaiting
+                // it here stalls the agent's startup until the entire corpus
+                // has been processed. Run it in the background and let the
+                // agent come up immediately; failures are logged, not fatal.
                 if (cfg.syncMemories) {
                     const workspaceDir = getWorkspaceDir();
-                    await syncMemoriesOnStartup(client, workspaceDir, {
+                    void syncMemoriesOnStartup(client, workspaceDir, {
                         userId: cfg.multiUser?.sharedUserId,
                         sectionize: cfg.syncMemoriesConfig.sectionize,
                         watchPaths: cfg.syncMemoriesConfig.watchPaths,
+                        maxAgeDays: cfg.syncMemoriesConfig.maxAgeDays,
+                        ignorePaths: cfg.syncMemoriesConfig.ignorePaths,
+                    }).catch((err) => {
+                        api.logger.error("hyperspell: background memory sync failed", err);
                     });
                 }
             },

package/dist/lib/filters.js

@@ -0,0 +1,30 @@
+/**
+ * Shared Hyperspell `options.filter` clauses, used by every retrieval path
+ * (auto-context hook + the hyperspell_search tool) so filtering stays
+ * consistent across them.
+ *
+ * Filters match against memory METADATA keys by their bare name — the same
+ * convention `buildScopeFilter` uses (`openclaw_scope`, `openclaw_user`).
+ */
+/**
+ * Memories produced by session-end hooks (auto-trace, emotional-state) are
+ * tagged in metadata as `openclaw_source: "agent_end"` (see `sendTrace` in
+ * client.ts). Those should NOT surface via generic retrieval: the
+ * emotional-state hook injects them through its own dedicated path, and
+ * replaying whole sanitized transcripts back into context creates a
+ * self-amplifying pollution loop. Exclude them at the search filter.
+ *
+ * NOTE: an earlier version of this filter checked the top-level `source` field
+ * for the value `"openclaw_agent_end"` — wrong on BOTH counts (the tag lives in
+ * metadata under `openclaw_source`, and its value is `"agent_end"`), so it
+ * silently matched nothing and excluded no traces.
+ */
+export const EXCLUDE_SESSION_END_FILTER = {
+    openclaw_source: { $ne: "agent_end" },
+};
+/** Combine a caller-supplied filter with the session-end exclude via `$and`. */
+export function mergeWithExclude(base) {
+    if (!base)
+        return EXCLUDE_SESSION_END_FILTER;
+    return { $and: [base, EXCLUDE_SESSION_END_FILTER] };
+}

package/dist/sync/markdown.js

@@ -168,10 +168,14 @@ export function loadManifest(workspaceDir) {
     try {
         if (fs.existsSync(p)) {
             const parsed = JSON.parse(fs.readFileSync(p, "utf-8"));
-            // Self-healing: an unrecognized/legacy shape is discarded. This feature
-            // has never shipped a released on-disk format, so nothing to migrate —
-            // a mismatch just triggers one re-sync, which is idempotent.
-            if (parsed && parsed.version === MANIFEST_VERSION && parsed.files) {
+            // Forward-migrate rather than discard. Throwing the manifest away on a
+            // version bump means every section re-uploads on the next run — the
+            // exact full-re-ingest load this guard exists to prevent. As long as the
+            // `files` map is structurally a record we keep it: the per-section hash
+            // check still gates every upload, so a stale-but-readable manifest is
+            // strictly better than an empty one. Only a missing/corrupt `files`
+            // (truly unusable) falls back to empty.
+            if (parsed && typeof parsed.files === "object" && parsed.files !== null) {
                 return { version: MANIFEST_VERSION, files: parsed.files };
             }
         }
@@ -183,11 +187,24 @@ export function loadManifest(workspaceDir) {
 }
 export function saveManifest(workspaceDir, manifest) {
     const p = manifestPath(workspaceDir);
+    // Atomic write: a process killed mid-write (deploy/restart) would otherwise
+    // leave a truncated JSON file, which loadManifest can only treat as corrupt
+    // -> empty -> full re-ingest. Write to a temp sibling, then rename (atomic
+    // on the same filesystem) so readers only ever see a complete manifest.
+    const tmp = `${p}.${process.pid}.tmp`;
     try {
-        fs.writeFileSync(p, JSON.stringify(manifest, null, 2));
+        fs.writeFileSync(tmp, JSON.stringify(manifest, null, 2));
+        fs.renameSync(tmp, p);
     }
     catch (err) {
         log.error("Failed to write sync manifest", err);
+        try {
+            if (fs.existsSync(tmp))
+                fs.unlinkSync(tmp);
+        }
+        catch {
+            /* best-effort cleanup */
+        }
     }
 }
 /**
@@ -206,15 +223,38 @@ export function withSyncLock(fn) {
 // ---------------------------------------------------------------------------
 // Public API — file-level sync (original, kept for backward compat)
 // ---------------------------------------------------------------------------
-export function getMemoryFiles(workspaceDir) {
+/**
+ * Directory names skipped by default when walking memory/. The dreaming engine
+ * writes first-person dream journals under memory/dreaming/ that are not user
+ * memories; ingesting them pollutes retrieval (they score highly on emotional/
+ * associative queries and crowd out real memories). Configurable via
+ * `syncMemories.ignorePaths`.
+ */
+export const DEFAULT_IGNORE_DIRS = ["dreaming"];
+/**
+ * Skip a directory entry during the walk when it is a dot-directory/file
+ * (e.g. `.dreams`, engine scaffolding) or a directory named in `ignore`.
+ * Dot-entries are always skipped regardless of the configured list.
+ */
+function isIgnoredEntry(entry, ignore) {
+    if (entry.name.startsWith("."))
+        return true;
+    if (entry.isDirectory() && ignore.has(entry.name))
+        return true;
+    return false;
+}
+export function getMemoryFiles(workspaceDir, ignorePaths) {
     const memoryDir = path.join(workspaceDir, "memory");
     if (!fs.existsSync(memoryDir)) {
         return [];
     }
+    const ignore = new Set(ignorePaths ?? DEFAULT_IGNORE_DIRS);
     const results = [];
     function walk(dir) {
         try {
             for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+                if (isIgnoredEntry(entry, ignore))
+                    continue;
                 const fullPath = path.join(dir, entry.name);
                 if (entry.isDirectory()) {
                     walk(fullPath);
@@ -235,8 +275,9 @@ export function getMemoryFiles(workspaceDir) {
  * Collect all syncable files based on configuration.
  * Includes memory/*.md by default, plus any additional watchPaths.
  */
-export function getSyncableFiles(workspaceDir, watchPaths) {
-    const files = getMemoryFiles(workspaceDir);
+export function getSyncableFiles(workspaceDir, watchPaths, ignorePaths) {
+    const ignore = new Set(ignorePaths ?? DEFAULT_IGNORE_DIRS);
+    const files = getMemoryFiles(workspaceDir, ignorePaths);
     if (watchPaths) {
         for (const wp of watchPaths) {
             const resolved = wp.startsWith("/") ? wp : path.join(workspaceDir, wp);
@@ -253,6 +294,8 @@ export function getSyncableFiles(workspaceDir, watchPaths) {
                 const walk = (dir) => {
                     try {
                         for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+                            if (isIgnoredEntry(entry, ignore))
+                                continue;
                             const fullPath = path.join(dir, entry.name);
                             if (entry.isDirectory()) {
                                 walk(fullPath);
@@ -480,13 +523,45 @@ export async function syncAllMemoryFiles(client, workspaceDir, options) {
  * Unchanged sections (by content hash) are skipped.
  */
 export async function syncAllFilesSectionized(client, workspaceDir, options) {
-    const files = getSyncableFiles(workspaceDir, options?.watchPaths);
+    const files = getSyncableFiles(workspaceDir, options?.watchPaths, options?.ignorePaths);
+    // Age pre-filter: a file untouched for longer than maxAgeDays that is
+    // already recorded in the manifest has been ingested at least once and is
+    // not changing — re-parsing/hashing/diffing it every startup is pure churn.
+    // Files NOT yet in the manifest are always processed regardless of age, so
+    // a genuinely cold start (or a never-synced old file) still ingests once.
+    // The live file_changed handler is unaffected: an edit bumps mtime, so
+    // edited-but-old files re-enter the window naturally.
+    const maxAgeDays = options?.maxAgeDays ?? 0;
+    let candidates = files;
+    let agedOut = 0;
+    if (maxAgeDays > 0) {
+        const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
+        const manifest = loadManifest(workspaceDir);
+        candidates = files.filter((filePath) => {
+            const key = fileKey(workspaceDir, filePath);
+            const known = manifest.files[key] !== undefined;
+            if (!known)
+                return true;
+            try {
+                if (fs.statSync(filePath).mtimeMs >= cutoff)
+                    return true;
+            }
+            catch {
+                return true; // stat failed — be safe, process it
+            }
+            agedOut++;
+            return false;
+        });
+        if (agedOut > 0) {
+            log.info(`Skipping ${agedOut} unchanged file(s) older than ${maxAgeDays}d (already synced)`);
+        }
+    }
     let totalSynced = 0;
     let totalSkipped = 0;
     let totalFailed = 0;
     let totalRemoved = 0;
     const allErrors = [];
-    for (const filePath of files) {
+    for (const filePath of candidates) {
         const result = await syncMarkdownFileSectionized(client, filePath, workspaceDir, {
             userId: options?.userId,
         });
@@ -501,6 +576,7 @@ export async function syncAllFilesSectionized(client, workspaceDir, options) {
         skipped: totalSkipped,
         failed: totalFailed,
         removed: totalRemoved,
+        agedOut,
         errors: allErrors,
     };
 }

package/dist/tools/search.js

@@ -1,4 +1,5 @@
 import { Type } from "@sinclair/typebox";
+import { mergeWithExclude } from "../lib/filters.js";
 import { buildScopeFilter, getCanReadScopes, resolveUser } from "../lib/sender.js";
 import { log } from "../logger.js";
 export function createSearchToolFactory(client, cfg) {
@@ -38,6 +39,9 @@ export function createSearchToolFactory(client, cfg) {
                     : canRead;
                 filter = buildScopeFilter(allowed, resolved?.userId ?? "");
             }
+            // Keep session-end trace memories out of agent-facing search, matching
+            // the auto-context hook so both retrieval paths filter identically.
+            filter = mergeWithExclude(filter);
             log.debug(`search tool: query="${params.query}" limit=${limit} after=${params.after ?? "none"} before=${params.before ?? "none"} userId=${userId} scope=${params.scope ?? "any"}`);
             try {
                 const response = await client.searchRaw(params.query, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperspell/openclaw-hyperspell",
-  "version": "0.14.0",
+  "version": "0.14.2",
   "type": "module",
   "description": "OpenClaw Hyperspell memory plugin",
   "license": "MIT",
@@ -38,7 +38,7 @@
     "check-types": "tsc --noEmit",
     "lint": "bunx @biomejs/biome ci .",
     "lint:fix": "bunx @biomejs/biome check --write .",
-    "test": "node --test --experimental-strip-types lib/sender.test.ts config.test.ts sync/markdown.test.ts hooks/auto-trace.test.ts hooks/emotional-state.test.ts hooks/startup-orientation.test.ts"
+    "test": "node --test --experimental-strip-types lib/sender.test.ts lib/filters.test.ts config.test.ts sync/markdown.test.ts hooks/auto-trace.test.ts hooks/emotional-state.test.ts hooks/startup-orientation.test.ts"
   },
   "openclaw": {
     "extensions": [