@hyperspell/openclaw-hyperspell 0.20.0 → 0.21.0

package/dist/lib/ranking.js

@@ -1,3 +1,9 @@
+export const DEFAULT_ELBOW = {
+    enabled: false,
+    minResults: 3,
+    gapRatio: 2.5,
+    minGap: 0.05,
+};
 export const DEFAULT_RANKING = {
     enabled: true,
     curationBoost: 0.2,
@@ -6,6 +12,12 @@ export const DEFAULT_RANKING = {
     storyTerms: [],
     candidateMultiplier: 3,
     chatterQuota: 2,
+    recencyHalfLifeDays: 90,
+    recencyMaxPenalty: 0.1,
+    recencyCuratedFactor: 0.5,
+    sourceWeights: {},
+    dedupThreshold: 0.8,
+    elbow: DEFAULT_ELBOW,
 };
 const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
@@ -71,24 +83,63 @@ export function classifyResult(r, storyTerms) {
         return "curated";
     return "other";
 }
-/** Composite score + classification for one result. */
-export function scoreResult(r, w) {
+/**
+ * Age-based additive penalty: exponential decay with a configurable half-life,
+ * capped at recencyMaxPenalty so recency stays a tiebreaker with teeth, never
+ * a dominant signal — an infinitely old result loses at most the cap, so it
+ * can reorder near-ties but never bury a result that out-relevances a rival
+ * by more than the cap. Additive (not a relevance multiplier) to stay on the
+ * same tuned scale as the boost/penalty algebra above.
+ */
+function recencyPenalty(createdAt, kind, w, now) {
+    if (w.recencyHalfLifeDays <= 0 || w.recencyMaxPenalty <= 0)
+        return 0;
+    if (!createdAt)
+        return 0; // unknown age — never punish missing data
+    const ts = Date.parse(createdAt);
+    if (Number.isNaN(ts))
+        return 0; // unparseable — same fail-open rule
+    // max(0, …) clamps future timestamps (clock skew) to zero age, never a boost.
+    const ageDays = Math.max(0, (now - ts) / 86_400_000);
+    const decay = 0.5 ** (ageDays / w.recencyHalfLifeDays);
+    // Kept memory (curated/story) is deliberately durable — it ages at the
+    // reduced curated factor so old truths keep their edge over old chatter.
+    const kept = kind === "curated" || kind === "story";
+    const factor = kept ? w.recencyCuratedFactor : 1;
+    return w.recencyMaxPenalty * (1 - decay) * factor;
+}
+/** Weight for a source; anything unlisted or malformed is neutral, never zero.
+ * The lookup-time guard is the safety floor: an unrecognized or unweighted
+ * source degrades to 1.0 — it never crashes and never zeroes a result out. */
+export function sourceWeight(w, source) {
+    const v = w.sourceWeights[source];
+    return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : 1;
+}
+/** Composite score + classification for one result. `now` is injectable so
+ * tests and the eval harness stay deterministic; runtime callers omit it. */
+export function scoreResult(r, w, now = Date.now()) {
     const kind = classifyResult(r, w.storyTerms);
     const base = baseScore(r);
-    let composite = base;
+    // The weight multiplies BASE only — kind boosts/penalties stay in the same
+    // additive currency regardless of source, so tuning sourceWeights can never
+    // silently retune chatterPenalty/curationBoost (proposal 11 §3.1). _base
+    // stays unweighted for debuggability; the weight shows only in _composite.
+    let composite = base * sourceWeight(w, r.source);
     if (kind === "story")
         composite += w.storyBoost + w.curationBoost; // the story is kept memory too
     else if (kind === "curated")
         composite += w.curationBoost;
     else if (kind === "chatter")
         composite -= w.chatterPenalty;
+    composite -= recencyPenalty(r.createdAt, kind, w, now);
     return { kind, base, composite };
 }
-/** Re-rank results by composite score (descending). Pure; stable enough. */
-export function rerank(results, w) {
+/** Re-rank results by composite score (descending). Pure; stable enough.
+ * One `now` per call so every candidate scores against a consistent clock. */
+export function rerank(results, w, now = Date.now()) {
     return results
         .map((r) => {
-        const s = scoreResult(r, w);
+        const s = scoreResult(r, w, now);
         return Object.assign({}, r, {
             _kind: s.kind,
             _base: s.base,
@@ -97,6 +148,59 @@ export function rerank(results, w) {
     })
         .sort((a, b) => b._composite - a._composite);
 }
+/** The text a result would lead its injection with: its highest-scored
+ * highlight, falling back to the title. Near-duplicate KEYS ≈ near-duplicate
+ * injected content, which is exactly what the diversity check must catch. */
+export function dedupKey(r) {
+    let best = "";
+    let bestScore = -1;
+    for (const h of r.highlights) {
+        const s = h.score ?? 0;
+        if (s > bestScore) {
+            bestScore = s;
+            best = h.text;
+        }
+    }
+    return best !== "" ? best : (r.title ?? "");
+}
+function tokens(s) {
+    return new Set(s
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s]/gu, " ")
+        .split(/\s+/)
+        .filter((t) => t.length > 0));
+}
+/**
+ * Token-set OVERLAP COEFFICIENT (|A∩B| / min|A|,|B|), not Jaccard: duplicated
+ * memories here typically differ mainly by length (a note vs the doc it
+ * quotes), and containment must read as duplication — 10 tokens fully inside
+ * 40 scores 1.0 here but only 0.25 under Jaccard. Dependency-free and linear;
+ * this runs synchronously in the ranking hot path.
+ */
+export function nearDuplicate(a, b, threshold) {
+    if (threshold <= 0)
+        return false;
+    const ta = tokens(a);
+    const tb = tokens(b);
+    const min = Math.min(ta.size, tb.size);
+    if (min === 0)
+        return false;
+    // Tiny keys (short titles, 2-3 common words) make overlap-coefficient
+    // trigger-happy; require exact token-set equality below 5 tokens.
+    if (min < 5) {
+        if (ta.size !== tb.size)
+            return false;
+        for (const t of ta)
+            if (!tb.has(t))
+                return false;
+        return true;
+    }
+    let inter = 0;
+    for (const t of ta)
+        if (tb.has(t))
+            inter++;
+    return inter / min >= threshold;
+}
 /**
  * One pass over ranked results, annotating each with its selection outcome.
  * Same policy as selectRanked — which is now derived from this, so the two
@@ -108,10 +212,22 @@ export function rerank(results, w) {
  * "max-results", not "chatter-quota" — it would have been cut regardless, so
  * the quota was not the binding constraint (proposal 03 §3.2 semantics).
  */
-export function explainSelection(ranked, maxResults, threshold, chatterQuota) {
+export function explainSelection(ranked, maxResults, threshold, chatterQuota, dedupThreshold = 0, elbow) {
     const out = [];
+    // Dedup state is exactly "what was accepted": only SELECTED results record
+    // keys (and only they consume quota), so a skipped candidate — whatever cut
+    // it — can never shadow later different-kind content sharing its text, and
+    // a diversity-skipped chatter echo never burns a quota slot (proposal 09's
+    // double-count invariant).
+    const keys = [];
     let chatter = 0;
     let kept = 0;
+    // Elbow bookkeeping: gaps between consecutive ACCEPTED results only —
+    // threshold/quota/dup-skipped rows widen the observed gap on purpose (the
+    // drop that matters is between results that could actually be injected).
+    let gapSum = 0;
+    let lastAccepted = 0;
+    let elbowFired = false;
     for (const r of ranked) {
         if (r._composite < threshold) {
             out.push({ result: r, selected: false, cut: "threshold" });
@@ -121,13 +237,39 @@ export function explainSelection(ranked, maxResults, threshold, chatterQuota) {
             out.push({ result: r, selected: false, cut: "max-results" });
             continue;
         }
+        // Cliff = outlier vs the decline so far AND material in absolute terms;
+        // the two-part test keeps flat lists (tiny meanGap) and steady declines
+        // (every gap "big") from tripping it. Gated on the minResults floor, so
+        // a huge gap right after result #1 is never even examined — the elbow
+        // can only ever cut EARLIER than maxResults, never below the floor.
+        if (!elbowFired && elbow?.enabled && kept >= Math.max(2, elbow.minResults)) {
+            const gap = lastAccepted - r._composite;
+            if (gap >= elbow.minGap && gap >= elbow.gapRatio * (gapSum / (kept - 1)))
+                elbowFired = true;
+        }
+        if (elbowFired) {
+            out.push({ result: r, selected: false, cut: "elbow" });
+            continue;
+        }
+        // After max-results (a dup past a full cap was cut regardless — the cap
+        // was binding), before chatter-quota (a dup echo injects nothing, so the
+        // quota was not the binding constraint and must not be charged).
+        const key = dedupKey(r);
+        if (keys.some((k) => nearDuplicate(k, key, dedupThreshold))) {
+            out.push({ result: r, selected: false, cut: "near-duplicate" });
+            continue;
+        }
         if (r._kind === "chatter" && chatter >= chatterQuota) {
             out.push({ result: r, selected: false, cut: "chatter-quota" });
             continue;
         }
         if (r._kind === "chatter")
             chatter++;
+        if (kept > 0)
+            gapSum += lastAccepted - r._composite;
+        lastAccepted = r._composite;
         kept++;
+        keys.push(key);
         out.push({ result: r, selected: true, cut: null });
     }
     return out;
@@ -144,8 +286,8 @@ export function explainSelection(ranked, maxResults, threshold, chatterQuota) {
  * It is a thin projection of explainSelection so selection policy lives in
  * exactly one place.
  */
-export function selectRanked(ranked, maxResults, threshold, chatterQuota) {
-    return explainSelection(ranked, maxResults, threshold, chatterQuota)
+export function selectRanked(ranked, maxResults, threshold, chatterQuota, dedupThreshold = 0, elbow) {
+    return explainSelection(ranked, maxResults, threshold, chatterQuota, dedupThreshold, elbow)
         .filter((e) => e.selected)
         .map((e) => e.result);
 }

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
 	"id": "openclaw-hyperspell",
 	"name": "Hyperspell",
 	"description": "Context and memory for your AI agents — search across Notion, Slack, Google Drive, and more",
-	"version": "0.20.0",
+	"version": "0.21.0",
 	"kind": "memory",
 	"contracts": {
 		"tools": [
@@ -77,7 +77,12 @@
 		},
 		"ranking": {
 			"label": "Composite Ranking",
-			"help": "Rank memories by more than raw relevance: boost curated memory and the active story, penalize auto-saved conversation fragments. Set storyTerms to 3-15 distinctive names/terms of the active thread (characters, codenames, invented words; matched case-insensitively at word boundaries) so it outranks chatter; update the list when the active story changes. { enabled, storyTerms, curationBoost, storyBoost, chatterPenalty, candidateMultiplier, chatterQuota }",
+			"help": "Rank memories by more than raw relevance: boost curated memory and the active story, penalize auto-saved conversation fragments. Set storyTerms to 3-15 distinctive names/terms of the active thread (characters, codenames, invented words; matched case-insensitively at word boundaries) so it outranks chatter; update the list when the active story changes. Old results accrue a small bounded recency penalty (half-life recencyHalfLifeDays, cap recencyMaxPenalty; kept memory ages at recencyCuratedFactor); recencyHalfLifeDays 0 disables it. { enabled, storyTerms, curationBoost, storyBoost, chatterPenalty, candidateMultiplier, chatterQuota, recencyHalfLifeDays, recencyMaxPenalty, recencyCuratedFactor, sourceWeights, dedupThreshold, elbow }",
+			"advanced": true
+		},
+		"coverageLog": {
+			"label": "Coverage Log",
+			"help": "Append a local-only JSONL event when auto-context finds no relevant memories (zero results or all below threshold), so capture gaps can be told apart from ranking misses. Events include prompt text (truncated), so this is off by default. Never sent to Hyperspell.",
 			"advanced": true
 		},
 		"debug": {
@@ -185,9 +190,32 @@
 					"storyBoost": { "type": "number", "minimum": 0, "maximum": 1 },
 					"storyTerms": { "type": "array", "items": { "type": "string" } },
 					"candidateMultiplier": { "type": "number", "minimum": 1, "maximum": 10 },
-					"chatterQuota": { "type": "number", "minimum": 0, "maximum": 20 }
+					"chatterQuota": { "type": "number", "minimum": 0, "maximum": 20 },
+					"recencyHalfLifeDays": { "type": "number", "minimum": 0, "maximum": 3650 },
+					"recencyMaxPenalty": { "type": "number", "minimum": 0, "maximum": 1 },
+					"recencyCuratedFactor": { "type": "number", "minimum": 0, "maximum": 1 },
+					"dedupThreshold": { "type": "number", "minimum": 0, "maximum": 1 },
+					"elbow": {
+						"type": "object",
+						"additionalProperties": false,
+						"description": "Stop injecting early at a natural score cliff instead of always filling maxResults. Conservative: only ever cuts earlier, never later; falls back to plain maxResults/threshold when no clear cliff exists. minGap is calibrated to the current composite scale (boosts of 0.15-0.2).",
+						"properties": {
+							"enabled": { "type": "boolean" },
+							"minResults": { "type": "number", "minimum": 2, "maximum": 20 },
+							"gapRatio": { "type": "number", "minimum": 1, "maximum": 10 },
+							"minGap": { "type": "number", "minimum": 0, "maximum": 1 }
+						}
+					},
+					"sourceWeights": {
+						"type": "object",
+						"additionalProperties": { "type": "number", "exclusiveMinimum": 0, "maximum": 5 },
+						"description": "Per-source multiplier on base relevance before ranking boosts/penalties (e.g. {\"notion\": 1.15, \"slack\": 0.85}). Sources not listed default to 1.0. To exclude a source entirely use the top-level sources filter, not a weight."
+					}
 				}
 			},
+			"coverageLog": {
+				"type": "boolean"
+			},
 			"debug": {
 				"type": "boolean"
 			},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperspell/openclaw-hyperspell",
-  "version": "0.20.0",
+  "version": "0.21.0",
   "type": "module",
   "description": "OpenClaw Hyperspell memory plugin",
   "license": "MIT",
@@ -38,7 +38,7 @@
     "check-types": "tsc --noEmit",
     "lint": "bunx @biomejs/biome ci .",
     "lint:fix": "bunx @biomejs/biome check --write .",
-    "test": "node --test --experimental-strip-types client.test.ts logger.test.ts lib/sender.test.ts lib/filters.test.ts lib/search-error.test.ts lib/session.test.ts lib/exclude-channels.test.ts lib/ranking.test.ts lib/eval-matchers.test.ts lib/loops-audit.test.ts tools/search.test.ts tools/emotional-arc.test.ts commands/preview.test.ts commands/purge-channel.test.ts config.test.ts sync/markdown.test.ts graph/ops.test.ts hooks/auto-trace.test.ts hooks/auto-context.test.ts hooks/hot-buffer.test.ts hooks/emotional-state.test.ts hooks/startup-orientation.test.ts hooks/mood-weather.test.ts eval/eval.test.ts"
+    "test": "node --test --experimental-strip-types client.test.ts logger.test.ts lib/sender.test.ts lib/filters.test.ts lib/search-error.test.ts lib/coverage-log.test.ts lib/session.test.ts lib/exclude-channels.test.ts lib/ranking.test.ts lib/eval-matchers.test.ts lib/loops-audit.test.ts lib/mood-skew-audit.test.ts tools/search.test.ts tools/emotional-arc.test.ts commands/preview.test.ts commands/purge-channel.test.ts config.test.ts sync/markdown.test.ts graph/ops.test.ts hooks/auto-trace.test.ts hooks/auto-context.test.ts hooks/hot-buffer.test.ts hooks/emotional-state.test.ts hooks/startup-orientation.test.ts hooks/mood-weather.test.ts eval/eval.test.ts"
   },
   "openclaw": {
     "extensions": [