@mnemoai/core 1.1.0 → 1.1.2

package/dist/src/retriever.d.ts

@@ -0,0 +1,228 @@
+/**
+ * Hybrid Retrieval System
+ * Combines vector search + BM25 full-text search with RRF fusion
+ */
+import type { MemoryStore, MemorySearchResult } from "./store.js";
+import type { Embedder } from "./embedder.js";
+import type { DecayEngine } from "./decay-engine.js";
+import type { TierManager } from "./tier-manager.js";
+type AccessTracker = {
+    bumpAccess: (id: string) => Promise<void>;
+};
+export interface RetrievalConfig {
+    mode: "hybrid" | "vector";
+    vectorWeight: number;
+    bm25Weight: number;
+    minScore: number;
+    rerank: "cross-encoder" | "lightweight" | "none";
+    candidatePoolSize: number;
+    /** Recency boost half-life in days (default: 14). Set 0 to disable. */
+    recencyHalfLifeDays: number;
+    /** Max recency boost factor (default: 0.10) */
+    recencyWeight: number;
+    /** Filter noise from results (default: true) */
+    filterNoise: boolean;
+    /** Reranker API key (enables cross-encoder reranking) */
+    rerankApiKey?: string;
+    /** Reranker model (default: jina-reranker-v3) */
+    rerankModel?: string;
+    /** Reranker API endpoint (default: https://api.jina.ai/v1/rerank). */
+    rerankEndpoint?: string;
+    /** Reranker provider format. Determines request/response shape and auth header.
+     *  - "jina" (default): Authorization: Bearer, string[] documents, results[].relevance_score
+     *  - "siliconflow": same format as jina (alias, for clarity)
+     *  - "voyage": Authorization: Bearer, string[] documents, data[].relevance_score
+     *  - "pinecone": Api-Key header, {text}[] documents, data[].score */
+    rerankProvider?: "jina" | "siliconflow" | "voyage" | "pinecone" | "ollama";
+    /**
+     * Length normalization: penalize long entries that dominate via sheer keyword
+     * density. Formula: score *= 1 / (1 + log2(charLen / anchor)).
+     * anchor = reference length (default: 500 chars). Entries shorter than anchor
+     * get a slight boost; longer entries get penalized progressively.
+     * Set 0 to disable. (default: 300)
+     */
+    lengthNormAnchor: number;
+    /**
+     * Hard cutoff after rerank: discard results below this score.
+     * Applied after all scoring stages (rerank, recency, importance, length norm).
+     * Higher = fewer but more relevant results. (default: 0.35)
+     */
+    hardMinScore: number;
+    /**
+     * Time decay half-life in days. Entries older than this lose score.
+     * Different from recencyBoost (additive bonus for new entries):
+     * this is a multiplicative penalty for old entries.
+     * Formula: score *= 0.5 + 0.5 * exp(-ageDays / halfLife)
+     * At halfLife days: ~0.68x. At 2*halfLife: ~0.59x. At 4*halfLife: ~0.52x.
+     * Set 0 to disable. (default: 60)
+     */
+    timeDecayHalfLifeDays: number;
+    /** Access reinforcement factor for time decay half-life extension.
+     *  Higher = stronger reinforcement. 0 to disable. (default: 0.5) */
+    reinforcementFactor: number;
+    /** Maximum half-life multiplier from access reinforcement.
+     *  Prevents frequently accessed memories from becoming immortal. (default: 3) */
+    maxHalfLifeMultiplier: number;
+    /** Enable multi-hop query routing: detected multi-hop queries skip Graphiti
+     *  spread and rely on LanceDB secondary retrieval instead. (default: true) */
+    multiHopRouting: boolean;
+}
+export interface RetrievalContext {
+    query: string;
+    limit: number;
+    scopeFilter?: string[];
+    category?: string;
+    /** Retrieval source: "manual" for user-triggered, "auto-recall" for system-initiated, "cli" for CLI commands. */
+    source?: "manual" | "auto-recall" | "cli";
+}
+export interface RetrievalResult extends MemorySearchResult {
+    sources: {
+        vector?: {
+            score: number;
+            rank: number;
+        };
+        bm25?: {
+            score: number;
+            rank: number;
+        };
+        graphiti?: {
+            score: number;
+            rank: number;
+        };
+        fused?: {
+            score: number;
+        };
+        reranked?: {
+            score: number;
+        };
+    };
+}
+export declare const DEFAULT_RETRIEVAL_CONFIG: RetrievalConfig;
+export declare class MemoryRetriever {
+    private store;
+    private embedder;
+    private config;
+    private decayEngine;
+    private accessTracker;
+    private tierManager;
+    constructor(store: MemoryStore, embedder: Embedder, config?: RetrievalConfig, decayEngine?: DecayEngine | null);
+    setAccessTracker(tracker: AccessTracker): void;
+    /**
+     * Resonance check: fast vector probe to see if the query "resonates"
+     * with any high-salience memories. Returns true if at least one memory
+     * has cosine similarity above threshold. This mimics human associative
+     * memory — most inputs don't trigger recall, only resonant ones do.
+     */
+    private resonanceCheck;
+    /**
+     * Strip metadata blocks from auto-recall queries.
+     * OpenClaw core may pass the full message including Conversation info,
+     * Replied message, and Sender metadata JSON blocks. We only want the
+     * actual user text for semantic search.
+     */
+    private cleanAutoRecallQuery;
+    /**
+     * Expand date expressions in query to multiple formats for BM25 matching.
+     * "3月17日" → "3月17日 3/17 03-17 2026-03-17"
+     * "2026年3月17日" → "2026年3月17日 3/17 2026-03-17 03-17"
+     * "昨天/前天/上周" → resolved to absolute dates
+     */
+    private expandDateFormats;
+    retrieve(context: RetrievalContext): Promise<RetrievalResult[]>;
+    private vectorOnlyRetrieval;
+    private hybridRetrieval;
+    private runVectorSearch;
+    private runBM25Search;
+    private fuseResults;
+    /**
+     * Rerank results using cross-encoder API (Jina, Pinecone, or compatible).
+     * Falls back to cosine similarity if API is unavailable or fails.
+     */
+    private rerankResults;
+    private getRerankPreservationFloor;
+    /**
+     * Apply recency boost: newer memories get a small score bonus.
+     * This ensures corrections/updates naturally outrank older entries
+     * when semantic similarity is close.
+     * Formula: boost = exp(-ageDays / halfLife) * weight
+     */
+    private applyRecencyBoost;
+    /**
+     * Apply importance weighting: memories with higher importance get a score boost.
+     * This ensures critical memories (importance=1.0) outrank casual ones (importance=0.5)
+     * when semantic similarity is close.
+     * Formula: score *= (baseWeight + (1 - baseWeight) * importance)
+     * With baseWeight=0.7: importance=1.0 → ×1.0, importance=0.5 → ×0.85, importance=0.0 → ×0.7
+     */
+    private applyImportanceWeight;
+    private applyDecayBoost;
+    /**
+     * Detect multi-hop queries that involve multiple entities or relationships.
+     * Multi-hop queries benefit from LanceDB secondary retrieval rather than
+     * Graphiti spread (which follows single-entity neighborhoods).
+     */
+    private isMultiHopQuery;
+    /**
+     * via sheer keyword density and broad semantic coverage.
+     * Short, focused entries (< anchor) get a slight boost.
+     * Long, sprawling entries (> anchor) get penalized.
+     * Formula: score *= 1 / (1 + log2(charLen / anchor))
+     */
+    private applyLengthNormalization;
+    /**
+     * Time decay: multiplicative penalty for old entries.
+     * Unlike recencyBoost (additive bonus for new entries), this actively
+     * penalizes stale information so recent knowledge wins ties.
+     * Formula: score *= 0.5 + 0.5 * exp(-ageDays / halfLife)
+     * At 0 days: 1.0x (no penalty)
+     * At halfLife: ~0.68x
+     * At 2*halfLife: ~0.59x
+     * Floor at 0.5x (never penalize more than half)
+     */
+    private applyTimeDecay;
+    /**
+     * Apply lifecycle-aware score adjustment (decay + tier floors).
+     *
+     * This is intentionally lightweight:
+     * - reads tier/access metadata (if any)
+     * - multiplies scores by max(tierFloor, decayComposite)
+     */
+    private applyLifecycleBoost;
+    /**
+     * Record access stats (access_count, last_accessed_at) and apply tier
+     * promotion/demotion for a small number of top results.
+     *
+     * Note: this writes back to LanceDB via delete+readd; keep it bounded.
+     */
+    private recordAccessAndMaybeTransition;
+    /**
+     * MMR-inspired diversity filter: greedily select results that are both
+     * relevant (high score) and diverse (low similarity to already-selected).
+     *
+     * Uses cosine similarity between memory vectors. If two memories have
+     * cosine similarity > threshold (default 0.92), the lower-scored one
+     * is demoted to the end rather than removed entirely.
+     *
+     * This prevents top-k from being filled with near-identical entries
+     * (e.g. 3 similar "SVG style" memories) while keeping them available
+     * if the pool is small.
+     */
+    private applyMMRDiversity;
+    updateConfig(newConfig: Partial<RetrievalConfig>): void;
+    getConfig(): RetrievalConfig;
+    test(query?: string): Promise<{
+        success: boolean;
+        mode: string;
+        hasFtsSupport: boolean;
+        error?: string;
+    }>;
+}
+export interface RetrieverLifecycleOptions {
+    decayEngine?: DecayEngine;
+    tierManager?: TierManager;
+}
+export declare function createRetriever(store: MemoryStore, embedder: Embedder, config?: Partial<RetrievalConfig>, options?: {
+    decayEngine?: DecayEngine | null;
+}): MemoryRetriever;
+export {};
+//# sourceMappingURL=retriever.d.ts.map

package/dist/src/retriever.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retriever.d.ts","sourceRoot":"","sources":["../../src/retriever.ts"],"names":[],"mappings":"AACA;;;GAGG;AAEH,OAAO,KAAK,EAAe,WAAW,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,WAAW,EAAmB,MAAM,mBAAmB,CAAC;AACtE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAOrD,KAAK,aAAa,GAAG;IAAE,UAAU,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAAC;AAqInE,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,eAAe,GAAG,aAAa,GAAG,MAAM,CAAC;IACjD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,uEAAuE;IACvE,mBAAmB,EAAE,MAAM,CAAC;IAC5B,+CAA+C;IAC/C,aAAa,EAAE,MAAM,CAAC;IACtB,gDAAgD;IAChD,WAAW,EAAE,OAAO,CAAC;IACrB,yDAAyD;IACzD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;yEAIqE;IACrE,cAAc,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,QAAQ,GAAG,UAAU,GAAG,QAAQ,CAAC;IAC3E;;;;;;OAMG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;;;OAOG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;wEACoE;IACpE,mBAAmB,EAAE,MAAM,CAAC;IAC5B;qFACiF;IACjF,qBAAqB,EAAE,MAAM,CAAC;IAC9B;kFAC8E;IAC9E,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iHAAiH;IACjH,MAAM,CAAC,EAAE,QAAQ,GAAG,aAAa,GAAG,KAAK,CAAC;CAC3C;AAED,MAAM,WAAW,eAAgB,SAAQ,kBAAkB;IACzD,OAAO,EAAE;QACP,MAAM,CAAC,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC;QACzC,IAAI,CAAC,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC;QACvC,QAAQ,CAAC,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3C,KAAK,CAAC,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC;QAC1B,QAAQ,CAAC,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC;KAC9B,CAAC;CACH;AAMD,eAAO,MAAM,wBAAwB,EAAE,eAkBtC,CAAC;AAsMF,qBAAa,eAAe;IAKxB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;IAPrB,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,WAAW,CAA4B;gBAGrC,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,QAAQ,EAClB,MAAM,GAAE,eAA0C,EAClD,WAAW,GAAE,WAAW,GAAG,IAAW;IAGhD,gBAAgB,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI;IAI9C;;;;;OAKG;YACW,cAAc;IAqC5B;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IAgB5B;;;;;OAKG;IACH,OAAO,CAAC,iBAAiB;IAuCnB,QAAQ,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;YAoLvD,mBAAmB;YA6CnB,eAAe;YAyEf,eAAe;YAwBf,aAAa;YAqBb,WAAW;IA8FzB;;;OAGG;YACW,aAAa;IAkI3B,OAAO,CAAC,0BAA0B;IAclC;;;;;OAKG;IACH,OAAO,CAAC,iBAAiB;IAqBzB;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;IAa7B,OAAO,CAAC,eAAe;IAkBvB;;;;OAIG;IACH,OAAO,CAAC,eAAe;IA6BvB;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;IAyBhC;;;;;;;;;OASG;IACH,OAAO,CAAC,cAAc;IAiCtB;;;;;;OAMG;IACH,OAAO,CAAC,mBAAmB;IAgB3B;;;;;OAKG;YACW,8BAA8B;IAgD5C;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,iBAAiB;IAmCzB,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,IAAI;IAKvD,SAAS,IAAI,eAAe;IAKtB,IAAI,CAAC,KAAK,SAAe,GAAG,OAAO,CAAC;QACxC,OAAO,EAAE,OAAO,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,EAAE,OAAO,CAAC;QACvB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CAqBH;AAMD,MAAM,WAAW,yBAAyB;IACxC,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAED,wBAAgB,eAAe,CAC7B,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,QAAQ,EAClB,MAAM,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,EACjC,OAAO,CAAC,EAAE;IAAE,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAA;CAAE,GAC7C,eAAe,CAGjB"}