@hyperspell/openclaw-hyperspell 0.19.0 → 0.21.0

package/dist/lib/ranking.js

@@ -1,3 +1,9 @@
+export const DEFAULT_ELBOW = {
+    enabled: false,
+    minResults: 3,
+    gapRatio: 2.5,
+    minGap: 0.05,
+};
 export const DEFAULT_RANKING = {
     enabled: true,
     curationBoost: 0.2,
@@ -6,8 +12,46 @@ export const DEFAULT_RANKING = {
     storyTerms: [],
     candidateMultiplier: 3,
     chatterQuota: 2,
+    recencyHalfLifeDays: 90,
+    recencyMaxPenalty: 0.1,
+    recencyCuratedFactor: 0.5,
+    sourceWeights: {},
+    dedupThreshold: 0.8,
+    elbow: DEFAULT_ELBOW,
 };
 const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+// Compiled story-term matchers, cached per storyTerms array instance — the
+// config array is built once in parseConfig and stable for the process
+// lifetime, so this avoids per-result regex compilation on the hot path.
+const matcherCache = new WeakMap();
+/**
+ * One regex per term, anchored at word boundaries so short terms can't
+ * false-positive inside longer words ("ada" no longer matches "adaptation";
+ * "mira" no longer matches "admiral"). Boundaries are word↔non-word
+ * transitions, so possessives and punctuation still match: "mira" hits
+ * "Mira's" and "mira-class". `\b` only anchors against word characters, so
+ * terms whose edges are punctuation get no anchor there and still match.
+ */
+function storyMatchers(storyTerms) {
+    let ms = matcherCache.get(storyTerms);
+    if (!ms) {
+        ms = storyTerms
+            .map((t) => t.trim().toLowerCase())
+            .filter((t) => t.length > 0)
+            .map((term) => {
+            const lead = /^\w/.test(term) ? "\\b" : "";
+            const tail = /\w$/.test(term) ? "\\b" : "";
+            return new RegExp(`${lead}${escapeRe(term)}${tail}`);
+        });
+        matcherCache.set(storyTerms, ms);
+    }
+    return ms;
+}
+/** Count ranked results by kind — the debug-tally shape auto-context logs. */
+export function kindTally(results) {
+    return results.reduce((acc, r) => ((acc[r._kind] = (acc[r._kind] ?? 0) + 1), acc), {});
+}
 /** Highest available relevance for a result (doc score or its best highlight). */
 export function baseScore(r) {
     const topHighlight = r.highlights.reduce((m, h) => Math.max(m, h.score ?? 0), 0);
@@ -25,8 +69,11 @@ export function baseScore(r) {
 export function classifyResult(r, storyTerms) {
     const title = (r.title ?? "").trim();
     if (storyTerms.length > 0) {
-        const hay = `${title} ${r.highlights.map((h) => h.text).join(" ")}`.toLowerCase();
-        if (storyTerms.some((t) => t && hay.includes(t.toLowerCase())))
+        // \n-joined (not space-joined) so a multi-word phrase term can never
+        // spuriously match across the seam of two unrelated highlights (terms are
+        // escaped literals, so a phrase's inner space cannot match the \n).
+        const hay = `${title}\n${r.highlights.map((h) => h.text).join("\n")}`.toLowerCase();
+        if (storyMatchers(storyTerms).some((re) => re.test(hay)))
             return "story";
     }
     const untitled = title === "" || /^unnamed conversation$/i.test(title);
@@ -36,24 +83,63 @@ export function classifyResult(r, storyTerms) {
         return "curated";
     return "other";
 }
-/** Composite score + classification for one result. */
-export function scoreResult(r, w) {
+/**
+ * Age-based additive penalty: exponential decay with a configurable half-life,
+ * capped at recencyMaxPenalty so recency stays a tiebreaker with teeth, never
+ * a dominant signal — an infinitely old result loses at most the cap, so it
+ * can reorder near-ties but never bury a result that out-relevances a rival
+ * by more than the cap. Additive (not a relevance multiplier) to stay on the
+ * same tuned scale as the boost/penalty algebra above.
+ */
+function recencyPenalty(createdAt, kind, w, now) {
+    if (w.recencyHalfLifeDays <= 0 || w.recencyMaxPenalty <= 0)
+        return 0;
+    if (!createdAt)
+        return 0; // unknown age — never punish missing data
+    const ts = Date.parse(createdAt);
+    if (Number.isNaN(ts))
+        return 0; // unparseable — same fail-open rule
+    // max(0, …) clamps future timestamps (clock skew) to zero age, never a boost.
+    const ageDays = Math.max(0, (now - ts) / 86_400_000);
+    const decay = 0.5 ** (ageDays / w.recencyHalfLifeDays);
+    // Kept memory (curated/story) is deliberately durable — it ages at the
+    // reduced curated factor so old truths keep their edge over old chatter.
+    const kept = kind === "curated" || kind === "story";
+    const factor = kept ? w.recencyCuratedFactor : 1;
+    return w.recencyMaxPenalty * (1 - decay) * factor;
+}
+/** Weight for a source; anything unlisted or malformed is neutral, never zero.
+ * The lookup-time guard is the safety floor: an unrecognized or unweighted
+ * source degrades to 1.0 — it never crashes and never zeroes a result out. */
+export function sourceWeight(w, source) {
+    const v = w.sourceWeights[source];
+    return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : 1;
+}
+/** Composite score + classification for one result. `now` is injectable so
+ * tests and the eval harness stay deterministic; runtime callers omit it. */
+export function scoreResult(r, w, now = Date.now()) {
     const kind = classifyResult(r, w.storyTerms);
     const base = baseScore(r);
-    let composite = base;
+    // The weight multiplies BASE only — kind boosts/penalties stay in the same
+    // additive currency regardless of source, so tuning sourceWeights can never
+    // silently retune chatterPenalty/curationBoost (proposal 11 §3.1). _base
+    // stays unweighted for debuggability; the weight shows only in _composite.
+    let composite = base * sourceWeight(w, r.source);
     if (kind === "story")
         composite += w.storyBoost + w.curationBoost; // the story is kept memory too
     else if (kind === "curated")
         composite += w.curationBoost;
     else if (kind === "chatter")
         composite -= w.chatterPenalty;
+    composite -= recencyPenalty(r.createdAt, kind, w, now);
     return { kind, base, composite };
 }
-/** Re-rank results by composite score (descending). Pure; stable enough. */
-export function rerank(results, w) {
+/** Re-rank results by composite score (descending). Pure; stable enough.
+ * One `now` per call so every candidate scores against a consistent clock. */
+export function rerank(results, w, now = Date.now()) {
     return results
         .map((r) => {
-        const s = scoreResult(r, w);
+        const s = scoreResult(r, w, now);
         return Object.assign({}, r, {
             _kind: s.kind,
             _base: s.base,
@@ -62,6 +148,59 @@ export function rerank(results, w) {
     })
         .sort((a, b) => b._composite - a._composite);
 }
+/** The text a result would lead its injection with: its highest-scored
+ * highlight, falling back to the title. Near-duplicate KEYS ≈ near-duplicate
+ * injected content, which is exactly what the diversity check must catch. */
+export function dedupKey(r) {
+    let best = "";
+    let bestScore = -1;
+    for (const h of r.highlights) {
+        const s = h.score ?? 0;
+        if (s > bestScore) {
+            bestScore = s;
+            best = h.text;
+        }
+    }
+    return best !== "" ? best : (r.title ?? "");
+}
+function tokens(s) {
+    return new Set(s
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s]/gu, " ")
+        .split(/\s+/)
+        .filter((t) => t.length > 0));
+}
+/**
+ * Token-set OVERLAP COEFFICIENT (|A∩B| / min|A|,|B|), not Jaccard: duplicated
+ * memories here typically differ mainly by length (a note vs the doc it
+ * quotes), and containment must read as duplication — 10 tokens fully inside
+ * 40 scores 1.0 here but only 0.25 under Jaccard. Dependency-free and linear;
+ * this runs synchronously in the ranking hot path.
+ */
+export function nearDuplicate(a, b, threshold) {
+    if (threshold <= 0)
+        return false;
+    const ta = tokens(a);
+    const tb = tokens(b);
+    const min = Math.min(ta.size, tb.size);
+    if (min === 0)
+        return false;
+    // Tiny keys (short titles, 2-3 common words) make overlap-coefficient
+    // trigger-happy; require exact token-set equality below 5 tokens.
+    if (min < 5) {
+        if (ta.size !== tb.size)
+            return false;
+        for (const t of ta)
+            if (!tb.has(t))
+                return false;
+        return true;
+    }
+    let inter = 0;
+    for (const t of ta)
+        if (tb.has(t))
+            inter++;
+    return inter / min >= threshold;
+}
 /**
  * One pass over ranked results, annotating each with its selection outcome.
  * Same policy as selectRanked — which is now derived from this, so the two
@@ -73,10 +212,22 @@ export function rerank(results, w) {
  * "max-results", not "chatter-quota" — it would have been cut regardless, so
  * the quota was not the binding constraint (proposal 03 §3.2 semantics).
  */
-export function explainSelection(ranked, maxResults, threshold, chatterQuota) {
+export function explainSelection(ranked, maxResults, threshold, chatterQuota, dedupThreshold = 0, elbow) {
     const out = [];
+    // Dedup state is exactly "what was accepted": only SELECTED results record
+    // keys (and only they consume quota), so a skipped candidate — whatever cut
+    // it — can never shadow later different-kind content sharing its text, and
+    // a diversity-skipped chatter echo never burns a quota slot (proposal 09's
+    // double-count invariant).
+    const keys = [];
     let chatter = 0;
     let kept = 0;
+    // Elbow bookkeeping: gaps between consecutive ACCEPTED results only —
+    // threshold/quota/dup-skipped rows widen the observed gap on purpose (the
+    // drop that matters is between results that could actually be injected).
+    let gapSum = 0;
+    let lastAccepted = 0;
+    let elbowFired = false;
     for (const r of ranked) {
         if (r._composite < threshold) {
             out.push({ result: r, selected: false, cut: "threshold" });
@@ -86,13 +237,39 @@ export function explainSelection(ranked, maxResults, threshold, chatterQuota) {
             out.push({ result: r, selected: false, cut: "max-results" });
             continue;
         }
+        // Cliff = outlier vs the decline so far AND material in absolute terms;
+        // the two-part test keeps flat lists (tiny meanGap) and steady declines
+        // (every gap "big") from tripping it. Gated on the minResults floor, so
+        // a huge gap right after result #1 is never even examined — the elbow
+        // can only ever cut EARLIER than maxResults, never below the floor.
+        if (!elbowFired && elbow?.enabled && kept >= Math.max(2, elbow.minResults)) {
+            const gap = lastAccepted - r._composite;
+            if (gap >= elbow.minGap && gap >= elbow.gapRatio * (gapSum / (kept - 1)))
+                elbowFired = true;
+        }
+        if (elbowFired) {
+            out.push({ result: r, selected: false, cut: "elbow" });
+            continue;
+        }
+        // After max-results (a dup past a full cap was cut regardless — the cap
+        // was binding), before chatter-quota (a dup echo injects nothing, so the
+        // quota was not the binding constraint and must not be charged).
+        const key = dedupKey(r);
+        if (keys.some((k) => nearDuplicate(k, key, dedupThreshold))) {
+            out.push({ result: r, selected: false, cut: "near-duplicate" });
+            continue;
+        }
         if (r._kind === "chatter" && chatter >= chatterQuota) {
             out.push({ result: r, selected: false, cut: "chatter-quota" });
             continue;
         }
         if (r._kind === "chatter")
             chatter++;
+        if (kept > 0)
+            gapSum += lastAccepted - r._composite;
+        lastAccepted = r._composite;
         kept++;
+        keys.push(key);
         out.push({ result: r, selected: true, cut: null });
     }
     return out;
@@ -109,8 +286,8 @@ export function explainSelection(ranked, maxResults, threshold, chatterQuota) {
  * It is a thin projection of explainSelection so selection policy lives in
  * exactly one place.
  */
-export function selectRanked(ranked, maxResults, threshold, chatterQuota) {
-    return explainSelection(ranked, maxResults, threshold, chatterQuota)
+export function selectRanked(ranked, maxResults, threshold, chatterQuota, dedupThreshold = 0, elbow) {
+    return explainSelection(ranked, maxResults, threshold, chatterQuota, dedupThreshold, elbow)
         .filter((e) => e.selected)
         .map((e) => e.result);
 }

package/dist/logger.js

@@ -19,6 +19,22 @@ export const log = {
             _logger.debug(`hyperspell: ${message}`, ...args);
         }
     },
+    /**
+     * Operator-meaningful diagnostics (one line per event: injection summaries,
+     * cut attribution, ranked tallies). Emits via the host's INFO channel when the
+     * plugin's own `debug: true` is set, and stays silent otherwise.
+     *
+     * Why not `debug`: the host drops plugin debug-channel output from gateway.log
+     * even at `logging.level: "debug"` (issue #118 — 2,026 info lines vs zero debug
+     * lines, live). The plugin's `debug` flag is explicit operator intent, so these
+     * diagnostics must not depend on host debug-channel plumbing. Truly-verbose
+     * output (per-request dumps, per-candidate lines) stays on `debug`.
+     */
+    diag: (message, ...args) => {
+        if (_debug) {
+            _logger.info(`hyperspell: ${message}`, ...args);
+        }
+    },
     debugRequest: (method, params) => {
         if (_debug) {
             _logger.debug(`hyperspell: [${method}] request`, params);

package/dist/sync/markdown.js

@@ -40,6 +40,7 @@ function readMarkdownFile(filePath) {
             title,
             content: body.trim(),
             hyperspellId: frontmatter.hyperspell_id || null,
+            graphEntity: frontmatter.graph_entity === "true",
         };
     }
     catch (err) {
@@ -271,6 +272,39 @@ export function getMemoryFiles(workspaceDir, ignorePaths) {
     walk(memoryDir);
     return results;
 }
+/** Resolve a watchPath (workspace-relative or absolute) to an absolute path. */
+export function resolveWatchPath(workspaceDir, p) {
+    return path.isAbsolute(p) ? p : path.join(workspaceDir, p);
+}
+/** Slug for unlabeled watchPaths: "notes/brainstem" -> "notes_brainstem". */
+function watchPathSlug(workspaceDir, resolved) {
+    const rel = path.relative(workspaceDir, resolved);
+    const base = rel.startsWith("..") ? path.basename(resolved) : rel;
+    return base.split(path.sep).join("_").replace(/[^a-zA-Z0-9_]/g, "_");
+}
+/**
+ * Provenance for a syncable file: the longest-prefix-matching watchPath's
+ * label (explicit `source` or derived slug), else "memory" for memory/ files.
+ * Longest prefix wins so a watchPath nested inside another (or inside memory/)
+ * keeps its more specific label.
+ */
+export function resolveSyncSource(filePath, workspaceDir, watchPaths) {
+    let best;
+    for (const wp of watchPaths) {
+        const resolved = resolveWatchPath(workspaceDir, wp.path);
+        if (filePath === resolved || filePath.startsWith(resolved + path.sep)) {
+            if (!best || resolved.length > best.len) {
+                best = { len: resolved.length, label: wp.source ?? watchPathSlug(workspaceDir, resolved) };
+            }
+        }
+    }
+    if (best)
+        return best.label;
+    const memoryDir = path.join(workspaceDir, "memory");
+    if (filePath.startsWith(memoryDir + path.sep))
+        return "memory";
+    return undefined;
+}
 /**
  * Collect all syncable files based on configuration.
  * Includes memory/*.md by default, plus any additional watchPaths.
@@ -280,7 +314,7 @@ export function getSyncableFiles(workspaceDir, watchPaths, ignorePaths) {
     const files = getMemoryFiles(workspaceDir, ignorePaths);
     if (watchPaths) {
         for (const wp of watchPaths) {
-            const resolved = wp.startsWith("/") ? wp : path.join(workspaceDir, wp);
+            const resolved = resolveWatchPath(workspaceDir, wp.path);
             if (!fs.existsSync(resolved))
                 continue;
             const stat = fs.statSync(resolved);
@@ -333,6 +367,13 @@ export async function syncMarkdownFile(client, filePath, options) {
             metadata: {
                 openclaw_source: "memory_sync",
                 file_path: filePath,
+                // Content origin (memory/ vs a labeled watchPath). Additive key —
+                // openclaw_source stays the pipeline discriminator that retrieval
+                // filters and startup-orientation dedupe depend on.
+                ...(options?.syncSource ? { openclaw_sync_source: options.syncSource } : {}),
+                // Honor the extraction prompt's contract: entity files synced back
+                // to Hyperspell must be skippable by the Memory Network scan.
+                ...(file.graphEntity ? { graph_entity: "true" } : {}),
             },
             userId: options?.userId,
         });
@@ -426,6 +467,14 @@ export async function syncMarkdownFileSectionized(client, filePath, workspaceDir
                             file_name: fileName,
                             section_title: section.title,
                             content_hash: section.contentHash,
+                            // Content origin (memory/ vs a labeled watchPath). Additive key —
+                            // openclaw_source stays the pipeline discriminator that retrieval
+                            // filters and startup-orientation dedupe depend on.
+                            ...(options?.syncSource ? { openclaw_sync_source: options.syncSource } : {}),
+                            // Honor the extraction prompt's contract: entity files synced
+                            // back to Hyperspell must be skippable by the Memory Network
+                            // scan, or the extractor is re-fed its own output every cycle.
+                            ...(file.graphEntity ? { graph_entity: "true" } : {}),
                         },
                         userId: options?.userId,
                     });
@@ -503,7 +552,12 @@ export async function syncAllMemoryFiles(client, workspaceDir, options) {
     let failed = 0;
     const errors = [];
     for (const filePath of files) {
-        const result = await syncMarkdownFile(client, filePath, { userId: options?.userId });
+        // Legacy bulk mode only walks memory/, so everything it uploads is
+        // memory-origin by construction.
+        const result = await syncMarkdownFile(client, filePath, {
+            userId: options?.userId,
+            syncSource: "memory",
+        });
         if (result.success) {
             synced++;
             log.info(`Synced: ${path.basename(filePath)} -> ${result.resourceId}`);
@@ -564,6 +618,7 @@ export async function syncAllFilesSectionized(client, workspaceDir, options) {
     for (const filePath of candidates) {
         const result = await syncMarkdownFileSectionized(client, filePath, workspaceDir, {
             userId: options?.userId,
+            syncSource: resolveSyncSource(filePath, workspaceDir, options?.watchPaths ?? []),
         });
         totalSynced += result.synced;
         totalSkipped += result.skipped;

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
 	"id": "openclaw-hyperspell",
 	"name": "Hyperspell",
 	"description": "Context and memory for your AI agents — search across Notion, Slack, Google Drive, and more",
-	"version": "0.19.0",
+	"version": "0.21.0",
 	"kind": "memory",
 	"contracts": {
 		"tools": [
@@ -75,6 +75,16 @@
 			"help": "Maximum memories injected into context per turn",
 			"advanced": true
 		},
+		"ranking": {
+			"label": "Composite Ranking",
+			"help": "Rank memories by more than raw relevance: boost curated memory and the active story, penalize auto-saved conversation fragments. Set storyTerms to 3-15 distinctive names/terms of the active thread (characters, codenames, invented words; matched case-insensitively at word boundaries) so it outranks chatter; update the list when the active story changes. Old results accrue a small bounded recency penalty (half-life recencyHalfLifeDays, cap recencyMaxPenalty; kept memory ages at recencyCuratedFactor); recencyHalfLifeDays 0 disables it. { enabled, storyTerms, curationBoost, storyBoost, chatterPenalty, candidateMultiplier, chatterQuota, recencyHalfLifeDays, recencyMaxPenalty, recencyCuratedFactor, sourceWeights, dedupThreshold, elbow }",
+			"advanced": true
+		},
+		"coverageLog": {
+			"label": "Coverage Log",
+			"help": "Append a local-only JSONL event when auto-context finds no relevant memories (zero results or all below threshold), so capture gaps can be told apart from ranking misses. Events include prompt text (truncated), so this is off by default. Never sent to Hyperspell.",
+			"advanced": true
+		},
 		"debug": {
 			"label": "Debug Logging",
 			"help": "Enable verbose debug logs for API calls and responses",
@@ -180,9 +190,32 @@
 					"storyBoost": { "type": "number", "minimum": 0, "maximum": 1 },
 					"storyTerms": { "type": "array", "items": { "type": "string" } },
 					"candidateMultiplier": { "type": "number", "minimum": 1, "maximum": 10 },
-					"chatterQuota": { "type": "number", "minimum": 0, "maximum": 20 }
+					"chatterQuota": { "type": "number", "minimum": 0, "maximum": 20 },
+					"recencyHalfLifeDays": { "type": "number", "minimum": 0, "maximum": 3650 },
+					"recencyMaxPenalty": { "type": "number", "minimum": 0, "maximum": 1 },
+					"recencyCuratedFactor": { "type": "number", "minimum": 0, "maximum": 1 },
+					"dedupThreshold": { "type": "number", "minimum": 0, "maximum": 1 },
+					"elbow": {
+						"type": "object",
+						"additionalProperties": false,
+						"description": "Stop injecting early at a natural score cliff instead of always filling maxResults. Conservative: only ever cuts earlier, never later; falls back to plain maxResults/threshold when no clear cliff exists. minGap is calibrated to the current composite scale (boosts of 0.15-0.2).",
+						"properties": {
+							"enabled": { "type": "boolean" },
+							"minResults": { "type": "number", "minimum": 2, "maximum": 20 },
+							"gapRatio": { "type": "number", "minimum": 1, "maximum": 10 },
+							"minGap": { "type": "number", "minimum": 0, "maximum": 1 }
+						}
+					},
+					"sourceWeights": {
+						"type": "object",
+						"additionalProperties": { "type": "number", "exclusiveMinimum": 0, "maximum": 5 },
+						"description": "Per-source multiplier on base relevance before ranking boosts/penalties (e.g. {\"notion\": 1.15, \"slack\": 0.85}). Sources not listed default to 1.0. To exclude a source entirely use the top-level sources filter, not a weight."
+					}
 				}
 			},
+			"coverageLog": {
+				"type": "boolean"
+			},
 			"debug": {
 				"type": "boolean"
 			},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperspell/openclaw-hyperspell",
-  "version": "0.19.0",
+  "version": "0.21.0",
   "type": "module",
   "description": "OpenClaw Hyperspell memory plugin",
   "license": "MIT",
@@ -38,7 +38,7 @@
     "check-types": "tsc --noEmit",
     "lint": "bunx @biomejs/biome ci .",
     "lint:fix": "bunx @biomejs/biome check --write .",
-    "test": "node --test --experimental-strip-types client.test.ts lib/sender.test.ts lib/filters.test.ts lib/search-error.test.ts lib/session.test.ts lib/exclude-channels.test.ts lib/ranking.test.ts lib/eval-matchers.test.ts tools/search.test.ts tools/emotional-arc.test.ts commands/preview.test.ts commands/purge-channel.test.ts config.test.ts sync/markdown.test.ts hooks/auto-trace.test.ts hooks/auto-context.test.ts hooks/hot-buffer.test.ts hooks/emotional-state.test.ts hooks/startup-orientation.test.ts hooks/mood-weather.test.ts eval/eval.test.ts"
+    "test": "node --test --experimental-strip-types client.test.ts logger.test.ts lib/sender.test.ts lib/filters.test.ts lib/search-error.test.ts lib/coverage-log.test.ts lib/session.test.ts lib/exclude-channels.test.ts lib/ranking.test.ts lib/eval-matchers.test.ts lib/loops-audit.test.ts lib/mood-skew-audit.test.ts tools/search.test.ts tools/emotional-arc.test.ts commands/preview.test.ts commands/purge-channel.test.ts config.test.ts sync/markdown.test.ts graph/ops.test.ts hooks/auto-trace.test.ts hooks/auto-context.test.ts hooks/hot-buffer.test.ts hooks/emotional-state.test.ts hooks/startup-orientation.test.ts hooks/mood-weather.test.ts eval/eval.test.ts"
   },
   "openclaw": {
     "extensions": [