@betterdb/semantic-cache 0.5.1 → 0.6.0

package/dist/SemanticCache.d.ts

@@ -8,6 +8,7 @@ export declare class SemanticCache {
     private readonly entryPrefix;
     private readonly statsKey;
     private readonly similarityWindowKey;
+    private readonly missPendingKey;
     private readonly configKey;
     private defaultThreshold;
     private readonly defaultTtl;
@@ -159,6 +160,19 @@ export declare class SemanticCache {
     private recordStat;
     /** Append to the rolling similarity window sorted set and trim to 10,000 entries or 7 days. */
     private recordSimilarityWindow;
+    /**
+     * Track a miss so a subsequent store() can backfill its cost into the
+     * similarity-window record. Bounded by a 5-minute TTL on the bookkeeping
+     * zset — entries beyond that are pruned on every record and backfill.
+     */
+    private recordMissPending;
+    /**
+     * After a successful store(), find the oldest pending miss for the same
+     * query and update its similarity-window record with the now-known cost.
+     * Best-effort — silently no-op if no pending miss exists or the bookkeeping
+     * entry has already been pruned.
+     */
+    private applyCostToPendingMiss;
     private assertInitialized;
     private assertDimension;
     private isIndexNotFoundError;

package/dist/SemanticCache.js

@@ -16,6 +16,19 @@ const PACKAGE_VERSION = require('../package.json').version;
 function errMsg(err) {
     return err instanceof Error ? err.message : String(err);
 }
+function parseHitCostMicros(raw) {
+    if (raw === undefined || raw === null) {
+        return null;
+    }
+    const n = parseInt(raw, 10);
+    if (!Number.isFinite(n) || n < 0) {
+        return null;
+    }
+    return n;
+}
+function correlationIdFor(prompt) {
+    return (0, node_crypto_2.createHash)('sha256').update(prompt).digest('hex').slice(0, 16);
+}
 class SemanticCache {
     client;
     embedFn;
@@ -24,6 +37,7 @@ class SemanticCache {
     entryPrefix;
     statsKey;
     similarityWindowKey;
+    missPendingKey;
     configKey;
     defaultThreshold;
     defaultTtl;
@@ -68,6 +82,7 @@ class SemanticCache {
         this.entryPrefix = `${this.name}:entry:`;
         this.statsKey = `${this.name}:__stats`;
         this.similarityWindowKey = `${this.name}:__similarity_window`;
+        this.missPendingKey = `${this.name}:__miss_pending`;
         this.configKey = `${this.name}:__config`;
         this.embedKeyPrefix = `${this.name}:embed:`;
         this.defaultThreshold = options.defaultThreshold ?? 0.1;
@@ -141,10 +156,7 @@ class SemanticCache {
             }
         }
         // Cluster-aware SCAN for entry keys and embed cache keys
-        const patterns = [
-            `${this.name}:entry:*`,
-            `${this.name}:embed:*`,
-        ];
+        const patterns = [`${this.name}:entry:*`, `${this.name}:embed:*`];
         for (const pattern of patterns) {
             await (0, cluster_1.clusterScan)(this.client, pattern, async (keys, nodeClient) => {
                 await nodeClient.del(keys);
@@ -152,6 +164,7 @@ class SemanticCache {
         }
         await this.client.del(this.statsKey);
         await this.client.del(this.similarityWindowKey);
+        await this.client.del(this.missPendingKey);
         this.analytics.capture('cache_flush');
     }
     /**
@@ -212,9 +225,9 @@ class SemanticCache {
             const userFilter = options?.filter;
             // AND semantics: each ref must be present — chain separate TAG clauses.
             const binaryFilter = binaryRefs.length > 0 && this._hasBinaryRefs
-                ? (binaryRefs.length === 1
+                ? binaryRefs.length === 1
                     ? `@binary_refs:{${(0, utils_1.escapeTag)(binaryRefs[0])}}`
-                    : binaryRefs.map((r) => `@binary_refs:{${(0, utils_1.escapeTag)(r)}}`).join(' '))
+                    : binaryRefs.map((r) => `@binary_refs:{${(0, utils_1.escapeTag)(r)}}`).join(' ')
                 : null;
             const combinedFilter = [userFilter, binaryFilter].filter(Boolean).join(' ');
             const filterExpr = combinedFilter ? `(${combinedFilter})` : '*';
@@ -230,15 +243,18 @@ class SemanticCache {
             const searchMs = performance.now() - searchStart;
             const parsed = (0, utils_1.parseFtSearchResponse)(rawResult);
             const categoryLabel = category || 'none';
-            const timingAttrs = { 'embedding_latency_ms': embedSec * 1000, 'search_latency_ms': searchMs };
+            const timingAttrs = { embedding_latency_ms: embedSec * 1000, search_latency_ms: searchMs };
             // No candidates at all
             if (parsed.length === 0) {
                 await this.recordStat('misses');
                 this.telemetry.metrics.requestsTotal
-                    .labels({ cache_name: this.name, result: 'miss', category: categoryLabel }).inc();
+                    .labels({ cache_name: this.name, result: 'miss', category: categoryLabel })
+                    .inc();
                 span.setAttributes({
-                    'cache.hit': false, 'cache.name': this.name,
-                    'cache.category': categoryLabel, ...timingAttrs,
+                    'cache.hit': false,
+                    'cache.name': this.name,
+                    'cache.category': categoryLabel,
+                    ...timingAttrs,
                 });
                 return { hit: false, confidence: 'miss' };
             }
@@ -246,19 +262,24 @@ class SemanticCache {
             const score = scoreStr !== undefined ? parseFloat(scoreStr) : NaN;
             if (!isNaN(score)) {
                 this.telemetry.metrics.similarityScore
-                    .labels({ cache_name: this.name, category: categoryLabel }).observe(score);
+                    .labels({ cache_name: this.name, category: categoryLabel })
+                    .observe(score);
             }
             // Miss (no usable score, or score exceeds threshold)
             if (isNaN(score) || score > threshold) {
                 if (!isNaN(score)) {
-                    await this.recordSimilarityWindow(score, 'miss', category);
+                    const missMember = await this.recordSimilarityWindow(score, 'miss', category, null);
+                    await this.recordMissPending(promptText, missMember);
                 }
                 await this.recordStat('misses');
                 this.telemetry.metrics.requestsTotal
-                    .labels({ cache_name: this.name, result: 'miss', category: categoryLabel }).inc();
+                    .labels({ cache_name: this.name, result: 'miss', category: categoryLabel })
+                    .inc();
                 span.setAttributes({
-                    'cache.hit': false, 'cache.name': this.name,
-                    'cache.category': categoryLabel, ...timingAttrs,
+                    'cache.hit': false,
+                    'cache.name': this.name,
+                    'cache.category': categoryLabel,
+                    ...timingAttrs,
                     ...(isNaN(score) ? {} : { 'cache.similarity': score, 'cache.threshold': threshold }),
                 });
                 const result = { hit: false, confidence: 'miss' };
@@ -278,17 +299,23 @@ class SemanticCache {
                     .filter(({ s }) => !isNaN(s))
                     .map(({ i, s }) => ({
                     origIdx: i,
-                    candidate: { response: parsed[i].fields['response'] ?? '', similarity: s },
+                    candidate: { response: parsed[i].fields['response'] ?? '', similarity: s, prompt: parsed[i].fields['prompt'] ?? '' },
                 }));
                 const picked = await rerankOpts.rerankFn(promptText, indexedCandidates.map((x) => x.candidate));
                 // Explicit bounds check: -1 means "reject all"; out-of-range is a caller bug
                 // treated as a miss rather than silently falling back to the top candidate.
                 if (picked === -1 || picked < 0 || picked >= indexedCandidates.length) {
-                    await this.recordSimilarityWindow(score, 'miss', category);
+                    const missMember = await this.recordSimilarityWindow(score, 'miss', category, null);
+                    await this.recordMissPending(promptText, missMember);
                     await this.recordStat('misses');
                     this.telemetry.metrics.requestsTotal
-                        .labels({ cache_name: this.name, result: 'miss', category: categoryLabel }).inc();
-                    span.setAttributes({ 'cache.hit': false, 'cache.name': this.name, 'cache.reranked': true });
+                        .labels({ cache_name: this.name, result: 'miss', category: categoryLabel })
+                        .inc();
+                    span.setAttributes({
+                        'cache.hit': false,
+                        'cache.name': this.name,
+                        'cache.reranked': true,
+                    });
                     return { hit: false, confidence: 'miss' };
                 }
                 // Map back to the original parsed[] index (not the candidates[] index)
@@ -304,12 +331,16 @@ class SemanticCache {
                     try {
                         await this.client.del(winner.key);
                     }
-                    catch { /* best effort */ }
-                    await this.recordSimilarityWindow(winnerScore, 'miss', category);
+                    catch {
+                        /* best effort */
+                    }
+                    const missMember = await this.recordSimilarityWindow(winnerScore, 'miss', category, null);
+                    await this.recordMissPending(promptText, missMember);
                     this.telemetry.metrics.staleModelEvictions.labels({ cache_name: this.name }).inc();
                     await this.recordStat('misses');
                     this.telemetry.metrics.requestsTotal
-                        .labels({ cache_name: this.name, result: 'miss', category: categoryLabel }).inc();
+                        .labels({ cache_name: this.name, result: 'miss', category: categoryLabel })
+                        .inc();
                     span.setAttributes({ 'cache.hit': false, 'cache.stale_evicted': true });
                     return { hit: false, confidence: 'miss' };
                 }
@@ -330,6 +361,8 @@ class SemanticCache {
                         similarity: winnerScore,
                         threshold,
                         category: category || undefined,
+                        // Reserved for consumer judge functions; not consumed by the built-in judge path.
+                        cachedPrompt: winner.fields['prompt'] ?? '',
                     }), timeoutMs);
                     decision = accepted ? 'accept' : 'reject';
                 }
@@ -363,7 +396,8 @@ class SemanticCache {
                 }
                 else {
                     // reject / error_reject / timeout_reject → treat as miss
-                    await this.recordSimilarityWindow(winnerScore, 'miss', category);
+                    const missMember = await this.recordSimilarityWindow(winnerScore, 'miss', category, null);
+                    await this.recordMissPending(promptText, missMember);
                     await this.recordStat('misses');
                     this.telemetry.metrics.requestsTotal
                         .labels({ cache_name: this.name, result: 'miss', category: categoryLabel })
@@ -387,27 +421,26 @@ class SemanticCache {
                 }
             }
             // --- End judge ---
+            const hitCostMicros = parseHitCostMicros(winner.fields['cost_micros']);
             // Record as genuine hit (moved here from before the judge block)
-            await this.recordSimilarityWindow(winnerScore, 'hit', category);
+            await this.recordSimilarityWindow(winnerScore, 'hit', category, hitCostMicros);
             await this.recordStat('hits');
             const metricResult = confidence === 'uncertain' ? 'uncertain_hit' : 'hit';
             this.telemetry.metrics.requestsTotal
-                .labels({ cache_name: this.name, result: metricResult, category: categoryLabel }).inc();
+                .labels({ cache_name: this.name, result: metricResult, category: categoryLabel })
+                .inc();
             if (this.defaultTtl !== undefined && matchedKey) {
                 await this.client.expire(matchedKey, this.defaultTtl);
             }
             // Cost saved
             let costSaved;
-            const costMicrosStr = winner.fields['cost_micros'];
-            if (costMicrosStr) {
-                const costMicros = parseInt(costMicrosStr, 10);
-                if (!isNaN(costMicros) && costMicros > 0) {
-                    costSaved = costMicros / 1_000_000;
-                    // Atomically increment cost_saved_micros in stats
-                    await this.client.hincrby(this.statsKey, 'cost_saved_micros', costMicros);
-                    this.telemetry.metrics.costSavedTotal
-                        .labels({ cache_name: this.name, category: categoryLabel }).inc(costSaved);
-                }
+            if (hitCostMicros !== null) {
+                costSaved = hitCostMicros / 1_000_000;
+                // Atomically increment cost_saved_micros in stats
+                await this.client.hincrby(this.statsKey, 'cost_saved_micros', hitCostMicros);
+                this.telemetry.metrics.costSavedTotal
+                    .labels({ cache_name: this.name, category: categoryLabel })
+                    .inc(costSaved);
             }
             // Content blocks
             let contentBlocks;
@@ -416,16 +449,25 @@ class SemanticCache {
                 try {
                     contentBlocks = JSON.parse(contentBlocksStr);
                 }
-                catch { /* ignore parse errors */ }
+                catch {
+                    /* ignore parse errors */
+                }
             }
             span.setAttributes({
-                'cache.hit': true, 'cache.similarity': winnerScore, 'cache.threshold': threshold,
-                'cache.confidence': confidence, 'cache.matched_key': matchedKey,
-                'cache.category': categoryLabel, ...timingAttrs,
+                'cache.hit': true,
+                'cache.similarity': winnerScore,
+                'cache.threshold': threshold,
+                'cache.confidence': confidence,
+                'cache.matched_key': matchedKey,
+                'cache.category': categoryLabel,
+                ...timingAttrs,
             });
             const result = {
-                hit: true, response: winner.fields['response'],
-                similarity: winnerScore, confidence, matchedKey,
+                hit: true,
+                response: winner.fields['response'],
+                similarity: winnerScore,
+                confidence,
+                matchedKey,
             };
             if (costSaved !== undefined)
                 result.costSaved = costSaved;
@@ -451,8 +493,9 @@ class SemanticCache {
                 this.costTable) {
                 const pricing = this.costTable[options.model];
                 if (pricing) {
-                    costMicros = Math.round((options.inputTokens * pricing.inputPer1k / 1000 +
-                        options.outputTokens * pricing.outputPer1k / 1000) * 1_000_000);
+                    costMicros = Math.round(((options.inputTokens * pricing.inputPer1k) / 1000 +
+                        (options.outputTokens * pricing.outputPer1k) / 1000) *
+                        1_000_000);
                 }
             }
             const hashFields = {
@@ -489,10 +532,16 @@ class SemanticCache {
             if (ttl !== undefined)
                 await this.client.expire(entryKey, ttl);
             span.setAttributes({
-                'cache.name': this.name, 'cache.key': entryKey, 'cache.ttl': ttl ?? -1,
-                'cache.category': category || 'none', 'cache.model': model || 'none',
-                'embedding_latency_ms': embedSec * 1000,
+                'cache.name': this.name,
+                'cache.key': entryKey,
+                'cache.ttl': ttl ?? -1,
+                'cache.category': category || 'none',
+                'cache.model': model || 'none',
+                embedding_latency_ms: embedSec * 1000,
             });
+            if (costMicros !== undefined && costMicros >= 0) {
+                await this.applyCostToPendingMiss(promptText, costMicros);
+            }
             return entryKey;
         });
     }
@@ -512,11 +561,15 @@ class SemanticCache {
             const category = options?.category ?? '';
             const model = options?.model ?? '';
             let costMicros;
-            if (options?.model && options?.inputTokens !== undefined && options?.outputTokens !== undefined && this.costTable) {
+            if (options?.model &&
+                options?.inputTokens !== undefined &&
+                options?.outputTokens !== undefined &&
+                this.costTable) {
                 const pricing = this.costTable[options.model];
                 if (pricing) {
-                    costMicros = Math.round((options.inputTokens * pricing.inputPer1k / 1000 +
-                        options.outputTokens * pricing.outputPer1k / 1000) * 1_000_000);
+                    costMicros = Math.round(((options.inputTokens * pricing.inputPer1k) / 1000 +
+                        (options.outputTokens * pricing.outputPer1k) / 1000) *
+                        1_000_000);
                 }
             }
             const hashFields = {
@@ -535,8 +588,9 @@ class SemanticCache {
             if (costMicros !== undefined && costMicros > 0) {
                 hashFields['cost_micros'] = String(costMicros);
             }
-            if (options?.temperature !== undefined)
+            if (options?.temperature !== undefined) {
                 hashFields['temperature'] = String(options.temperature);
+            }
             if (options?.topP !== undefined)
                 hashFields['top_p'] = String(options.topP);
             if (options?.seed !== undefined)
@@ -551,10 +605,16 @@ class SemanticCache {
             if (ttl !== undefined)
                 await this.client.expire(entryKey, ttl);
             span.setAttributes({
-                'cache.name': this.name, 'cache.key': entryKey, 'cache.ttl': ttl ?? -1,
-                'cache.category': category || 'none', 'cache.model': model || 'none',
-                'embedding_latency_ms': embedSec * 1000,
+                'cache.name': this.name,
+                'cache.key': entryKey,
+                'cache.ttl': ttl ?? -1,
+                'cache.category': category || 'none',
+                'cache.model': model || 'none',
+                embedding_latency_ms: embedSec * 1000,
             });
+            if (costMicros !== undefined && costMicros >= 0) {
+                await this.applyCostToPendingMiss(promptText, costMicros);
+            }
             return entryKey;
         });
     }
@@ -592,9 +652,9 @@ class SemanticCache {
                 const { binaryRefs } = resolved[i];
                 const { vector: embedding } = embeddings[i];
                 const binaryFilter = binaryRefs.length > 0 && this._hasBinaryRefs
-                    ? (binaryRefs.length === 1
+                    ? binaryRefs.length === 1
                         ? `@binary_refs:{${(0, utils_1.escapeTag)(binaryRefs[0])}}`
-                        : binaryRefs.map((r) => `@binary_refs:{${(0, utils_1.escapeTag)(r)}}`).join(' '))
+                        : binaryRefs.map((r) => `@binary_refs:{${(0, utils_1.escapeTag)(r)}}`).join(' ')
                     : null;
                 const combinedFilter = [userFilter, binaryFilter].filter(Boolean).join(' ');
                 const filterExpr = combinedFilter ? `(${combinedFilter})` : '*';
@@ -612,7 +672,8 @@ class SemanticCache {
                 if (err) {
                     await this.recordStat('misses');
                     this.telemetry.metrics.requestsTotal
-                        .labels({ cache_name: this.name, result: 'miss', category: categoryLabel }).inc();
+                        .labels({ cache_name: this.name, result: 'miss', category: categoryLabel })
+                        .inc();
                     results.push({ hit: false, confidence: 'miss' });
                     continue;
                 }
@@ -620,7 +681,8 @@ class SemanticCache {
                 if (parsed.length === 0) {
                     await this.recordStat('misses');
                     this.telemetry.metrics.requestsTotal
-                        .labels({ cache_name: this.name, result: 'miss', category: categoryLabel }).inc();
+                        .labels({ cache_name: this.name, result: 'miss', category: categoryLabel })
+                        .inc();
                     results.push({ hit: false, confidence: 'miss' });
                     continue;
                 }
@@ -628,11 +690,13 @@ class SemanticCache {
                 const score = scoreStr !== undefined ? parseFloat(scoreStr) : NaN;
                 if (isNaN(score) || score > threshold) {
                     if (!isNaN(score)) {
-                        await this.recordSimilarityWindow(score, 'miss', category);
+                        const missMember = await this.recordSimilarityWindow(score, 'miss', category, null);
+                        await this.recordMissPending(resolved[i].text, missMember);
                     }
                     await this.recordStat('misses');
                     this.telemetry.metrics.requestsTotal
-                        .labels({ cache_name: this.name, result: 'miss', category: categoryLabel }).inc();
+                        .labels({ cache_name: this.name, result: 'miss', category: categoryLabel })
+                        .inc();
                     const result = { hit: false, confidence: 'miss' };
                     if (!isNaN(score)) {
                         result.similarity = score;
@@ -641,26 +705,25 @@ class SemanticCache {
                     results.push(result);
                     continue;
                 }
-                await this.recordSimilarityWindow(score, 'hit', category);
+                const hitCostMicros = parseHitCostMicros(parsed[0].fields['cost_micros']);
+                await this.recordSimilarityWindow(score, 'hit', category, hitCostMicros);
                 const confidence = score >= threshold - this.uncertaintyBand ? 'uncertain' : 'high';
                 await this.recordStat('hits');
                 const metricResult = confidence === 'uncertain' ? 'uncertain_hit' : 'hit';
                 this.telemetry.metrics.requestsTotal
-                    .labels({ cache_name: this.name, result: metricResult, category: categoryLabel }).inc();
+                    .labels({ cache_name: this.name, result: metricResult, category: categoryLabel })
+                    .inc();
                 const matchedKey = parsed[0].key;
                 if (this.defaultTtl !== undefined && matchedKey) {
                     await this.client.expire(matchedKey, this.defaultTtl);
                 }
                 let costSaved;
-                const costMicrosStr = parsed[0].fields['cost_micros'];
-                if (costMicrosStr) {
-                    const costMicros = parseInt(costMicrosStr, 10);
-                    if (!isNaN(costMicros) && costMicros > 0) {
-                        costSaved = costMicros / 1_000_000;
-                        await this.client.hincrby(this.statsKey, 'cost_saved_micros', costMicros);
-                        this.telemetry.metrics.costSavedTotal
-                            .labels({ cache_name: this.name, category: categoryLabel }).inc(costSaved);
-                    }
+                if (hitCostMicros !== null) {
+                    costSaved = hitCostMicros / 1_000_000;
+                    await this.client.hincrby(this.statsKey, 'cost_saved_micros', hitCostMicros);
+                    this.telemetry.metrics.costSavedTotal
+                        .labels({ cache_name: this.name, category: categoryLabel })
+                        .inc(costSaved);
                 }
                 let contentBlocks;
                 const contentBlocksStr = parsed[0].fields['content_blocks'];
@@ -668,11 +731,16 @@ class SemanticCache {
                     try {
                         contentBlocks = JSON.parse(contentBlocksStr);
                     }
-                    catch { /* ignore */ }
+                    catch {
+                        /* ignore */
+                    }
                 }
                 const result = {
-                    hit: true, response: parsed[0].fields['response'],
-                    similarity: score, confidence, matchedKey,
+                    hit: true,
+                    response: parsed[0].fields['response'],
+                    similarity: score,
+                    confidence,
+                    matchedKey,
                 };
                 if (costSaved !== undefined)
                     result.costSaved = costSaved;
@@ -703,8 +771,10 @@ class SemanticCache {
             const parsed = (0, utils_1.parseFtSearchResponse)(rawResult);
             if (parsed.length === 0) {
                 span.setAttributes({
-                    'cache.name': this.name, 'cache.filter': filter,
-                    'cache.deleted_count': 0, 'cache.truncated': false,
+                    'cache.name': this.name,
+                    'cache.filter': filter,
+                    'cache.deleted_count': 0,
+                    'cache.truncated': false,
                 });
                 return { deleted: 0, truncated: false };
             }
@@ -717,8 +787,10 @@ class SemanticCache {
                 throw new errors_1.ValkeyCommandError('DEL', err);
             }
             span.setAttributes({
-                'cache.name': this.name, 'cache.filter': filter,
-                'cache.deleted_count': keys.length, 'cache.truncated': truncated,
+                'cache.name': this.name,
+                'cache.filter': filter,
+                'cache.deleted_count': keys.length,
+                'cache.truncated': truncated,
             });
             return { deleted: keys.length, truncated };
         });
@@ -803,7 +875,9 @@ class SemanticCache {
                     }
                 }
             }
-            catch { /* skip corrupt entries */ }
+            catch {
+                /* skip corrupt entries */
+            }
         }
         const sampleCount = entries.length;
         const categoryLabel = category ?? 'all';
@@ -890,11 +964,15 @@ class SemanticCache {
                 if (entry.category)
                     categories.add(entry.category);
             }
-            catch { /* skip */ }
+            catch {
+                /* skip */
+            }
         }
         const results = await Promise.all([
             this.thresholdEffectiveness({ minSamples: options?.minSamples }),
-            ...[...categories].filter(Boolean).map((cat) => this.thresholdEffectiveness({ category: cat, minSamples: options?.minSamples })),
+            ...[...categories]
+                .filter(Boolean)
+                .map((cat) => this.thresholdEffectiveness({ category: cat, minSamples: options?.minSamples })),
         ]);
         return results;
     }
@@ -947,7 +1025,9 @@ class SemanticCache {
     }
     // -- Internal helpers exposed to package adapters --
     /** @internal Default similarity threshold. */
-    get _defaultThreshold() { return this.defaultThreshold; }
+    get _defaultThreshold() {
+        return this.defaultThreshold;
+    }
     /** @internal Test-only getter. */
     get _categoryThresholds() {
         return this.categoryThresholds;
@@ -980,15 +1060,11 @@ class SemanticCache {
             this.refreshConfig()
                 .then((ok) => {
                 if (!ok) {
-                    this.telemetry.metrics.configRefreshFailed
-                        .labels({ cache_name: this.name })
-                        .inc();
+                    this.telemetry.metrics.configRefreshFailed.labels({ cache_name: this.name }).inc();
                 }
             })
                 .catch(() => {
-                this.telemetry.metrics.configRefreshFailed
-                    .labels({ cache_name: this.name })
-                    .inc();
+                this.telemetry.metrics.configRefreshFailed.labels({ cache_name: this.name }).inc();
             });
         };
         // Synchronous first refresh: process started immediately after a proposal
@@ -1044,9 +1120,7 @@ class SemanticCache {
             metadata,
             heartbeatIntervalMs: this.discoveryOptions.heartbeatIntervalMs,
             onWriteFailed: () => {
-                this.telemetry.metrics.discoveryWriteFailed
-                    .labels({ cache_name: this.name })
-                    .inc();
+                this.telemetry.metrics.discoveryWriteFailed.labels({ cache_name: this.name }).inc();
             },
         });
         await manager.register();
@@ -1164,7 +1238,8 @@ class SemanticCache {
                 const cached = await this.client.getBuffer(embedKey);
                 if (cached) {
                     this.telemetry.metrics.embeddingCacheTotal
-                        .labels({ cache_name: this.name, result: 'hit' }).inc();
+                        .labels({ cache_name: this.name, result: 'hit' })
+                        .inc();
                     // Decode Float32 buffer
                     const vector = [];
                     for (let i = 0; i < cached.length; i += 4) {
@@ -1173,9 +1248,12 @@ class SemanticCache {
                     return { vector, durationSec: 0 };
                 }
             }
-            catch { /* ignore cache read errors */ }
+            catch {
+                /* ignore cache read errors */
+            }
             this.telemetry.metrics.embeddingCacheTotal
-                .labels({ cache_name: this.name, result: 'miss' }).inc();
+                .labels({ cache_name: this.name, result: 'miss' })
+                .inc();
         }
         const start = performance.now();
         let vector;
@@ -1186,9 +1264,7 @@ class SemanticCache {
             throw new errors_1.EmbeddingError(`embedFn failed: ${errMsg(err)}`, err);
         }
         const durationSec = (performance.now() - start) / 1000;
-        this.telemetry.metrics.embeddingDuration
-            .labels({ cache_name: this.name })
-            .observe(durationSec);
+        this.telemetry.metrics.embeddingDuration.labels({ cache_name: this.name }).observe(durationSec);
         // Store in embedding cache
         if (this.embeddingCacheEnabled && text) {
             const hash = (0, node_crypto_2.createHash)('sha256').update(text).digest('hex');
@@ -1197,7 +1273,9 @@ class SemanticCache {
                 const buf = (0, utils_1.encodeFloat32)(vector);
                 await this.client.set(embedKey, buf, 'EX', this.embeddingCacheTtl);
             }
-            catch { /* ignore cache write errors */ }
+            catch {
+                /* ignore cache write errors */
+            }
         }
         return { vector, durationSec };
     }
@@ -1235,22 +1313,99 @@ class SemanticCache {
         await pipeline.exec();
     }
     /** Append to the rolling similarity window sorted set and trim to 10,000 entries or 7 days. */
-    async recordSimilarityWindow(score, result, category) {
+    async recordSimilarityWindow(score, result, category, costSavedMicros) {
         const now = Date.now();
-        // Include a unique nonce so identical (score, result, category) tuples are
-        // each recorded as distinct ZADD members instead of overwriting each other.
-        const member = JSON.stringify({ score, result, category, _n: Math.random() });
+        const member = JSON.stringify({
+            score,
+            result,
+            category,
+            _n: Math.random(),
+            cost_saved_micros: costSavedMicros,
+        });
         const sevenDaysAgo = now - 7 * 24 * 60 * 60 * 1000;
         try {
             const pipeline = this.client.pipeline();
             pipeline.zadd(this.similarityWindowKey, now, member);
-            // Trim by time: remove entries older than 7 days
             pipeline.zremrangebyscore(this.similarityWindowKey, '-inf', sevenDaysAgo);
-            // Trim by count: keep at most 10,000 most recent
             pipeline.zremrangebyrank(this.similarityWindowKey, 0, -10001);
             await pipeline.exec();
         }
-        catch { /* best effort - never fail on window writes */ }
+        catch {
+            /* best effort - never fail on window writes */
+        }
+        return member;
+    }
+    /**
+     * Track a miss so a subsequent store() can backfill its cost into the
+     * similarity-window record. Bounded by a 5-minute TTL on the bookkeeping
+     * zset — entries beyond that are pruned on every record and backfill.
+     */
+    async recordMissPending(prompt, similarityMember) {
+        const correlationId = correlationIdFor(prompt);
+        const now = Date.now();
+        const fiveMinutesAgo = now - 5 * 60 * 1000;
+        const entry = JSON.stringify({ correlationId, similarityMember });
+        try {
+            await this.client.zadd(this.missPendingKey, now, entry);
+            await this.client.zremrangebyscore(this.missPendingKey, '-inf', `(${fiveMinutesAgo}`);
+        }
+        catch {
+            /* best effort */
+        }
+    }
+    /**
+     * After a successful store(), find the oldest pending miss for the same
+     * query and update its similarity-window record with the now-known cost.
+     * Best-effort — silently no-op if no pending miss exists or the bookkeeping
+     * entry has already been pruned.
+     */
+    async applyCostToPendingMiss(prompt, costMicros) {
+        const correlationId = correlationIdFor(prompt);
+        const fiveMinutesAgo = Date.now() - 5 * 60 * 1000;
+        try {
+            await this.client.zremrangebyscore(this.missPendingKey, '-inf', `(${fiveMinutesAgo}`);
+            const raw = (await this.client.zrange(this.missPendingKey, '0', '-1', 'WITHSCORES'));
+            let matchedEntry = null;
+            let matchedSimilarityMember = null;
+            for (let i = 0; i < raw.length; i += 2) {
+                const entryStr = raw[i];
+                try {
+                    const parsed = JSON.parse(entryStr);
+                    if (parsed.correlationId === correlationId) {
+                        matchedEntry = entryStr;
+                        matchedSimilarityMember = parsed.similarityMember;
+                        break;
+                    }
+                }
+                catch {
+                    /* skip malformed */
+                }
+            }
+            if (matchedEntry === null || matchedSimilarityMember === null) {
+                return;
+            }
+            const rawScore = await this.client.zscore(this.similarityWindowKey, matchedSimilarityMember);
+            if (rawScore === null) {
+                await this.client.zrem(this.missPendingKey, matchedEntry);
+                return;
+            }
+            const similarityScore = Number(rawScore);
+            if (!Number.isFinite(similarityScore)) {
+                await this.client.zrem(this.missPendingKey, matchedEntry);
+                return;
+            }
+            const parsedMember = JSON.parse(matchedSimilarityMember);
+            parsedMember.cost_saved_micros = costMicros;
+            const updatedMember = JSON.stringify(parsedMember);
+            const updatePipeline = this.client.pipeline();
+            updatePipeline.zrem(this.similarityWindowKey, matchedSimilarityMember);
+            updatePipeline.zadd(this.similarityWindowKey, similarityScore, updatedMember);
+            updatePipeline.zrem(this.missPendingKey, matchedEntry);
+            await updatePipeline.exec();
+        }
+        catch {
+            /* never fail store() because of bookkeeping */
+        }
     }
     assertInitialized(method) {
         if (!this._initialized) {

package/dist/index.d.ts

@@ -2,6 +2,7 @@ export { SemanticCache } from './SemanticCache';
 export type { ThresholdEffectivenessResult } from './SemanticCache';
 export { DEFAULT_COST_TABLE } from './defaultCostTable';
 export type { SemanticCacheOptions, CacheCheckOptions, CacheStoreOptions, CacheCheckResult, CacheStats, IndexInfo, InvalidateResult, CacheConfidence, EmbedFn, ModelCost, RerankOptions, JudgeOptions, ConfigRefreshOptions, } from './types';
+export { createKeywordOverlapRerank } from './rerank';
 export { SemanticCacheUsageError, EmbeddingError, ValkeyCommandError, } from './errors';
 export type { ContentBlock, TextBlock, BinaryBlock, ToolCallBlock, ToolResultBlock, ReasoningBlock, BlockHints, } from './utils';
 export { escapeTag } from './utils';

package/dist/index.js

@@ -1,10 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.defaultNormalizer = exports.composeNormalizer = exports.passthrough = exports.fetchAndHash = exports.hashUrl = exports.hashBytes = exports.hashBase64 = exports.escapeTag = exports.ValkeyCommandError = exports.EmbeddingError = exports.SemanticCacheUsageError = exports.DEFAULT_COST_TABLE = exports.SemanticCache = void 0;
+exports.defaultNormalizer = exports.composeNormalizer = exports.passthrough = exports.fetchAndHash = exports.hashUrl = exports.hashBytes = exports.hashBase64 = exports.escapeTag = exports.ValkeyCommandError = exports.EmbeddingError = exports.SemanticCacheUsageError = exports.createKeywordOverlapRerank = exports.DEFAULT_COST_TABLE = exports.SemanticCache = void 0;
 var SemanticCache_1 = require("./SemanticCache");
 Object.defineProperty(exports, "SemanticCache", { enumerable: true, get: function () { return SemanticCache_1.SemanticCache; } });
 var defaultCostTable_1 = require("./defaultCostTable");
 Object.defineProperty(exports, "DEFAULT_COST_TABLE", { enumerable: true, get: function () { return defaultCostTable_1.DEFAULT_COST_TABLE; } });
+var rerank_1 = require("./rerank");
+Object.defineProperty(exports, "createKeywordOverlapRerank", { enumerable: true, get: function () { return rerank_1.createKeywordOverlapRerank; } });
 var errors_1 = require("./errors");
 Object.defineProperty(exports, "SemanticCacheUsageError", { enumerable: true, get: function () { return errors_1.SemanticCacheUsageError; } });
 Object.defineProperty(exports, "EmbeddingError", { enumerable: true, get: function () { return errors_1.EmbeddingError; } });

package/dist/rerank.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Built-in rerank factories for @betterdb/semantic-cache.
+ */
+/**
+ * Built-in keyword-overlap reranker.
+ *
+ * Blends cosine similarity with word overlap and returns the index of the
+ * best candidate.
+ *
+ * @param compare
+ *   `"prompt"`  – overlap of the incoming query against each candidate's stored
+ *                 prompt. Equivalence signal. Catches entity mismatches
+ *                 (e.g. "weather in Paris" vs "weather in Berlin"). Default.
+ *   `"response"` – overlap of the incoming query against each candidate's cached
+ *                 response. Relevance signal.
+ *
+ * @param cosineWeight
+ *   Weight on cosine similarity in [0, 1]. Overlap weight is `1 - cosineWeight`.
+ *   Default: 0.7 (overlap 0.3).
+ *
+ * Candidate objects carry: `similarity` (cosine distance, lower = more similar),
+ * `response` (string), and `prompt` (string, stored prompt).
+ */
+export declare function createKeywordOverlapRerank(options?: {
+    compare?: 'prompt' | 'response';
+    cosineWeight?: number;
+}): (query: string, candidates: Array<{
+    response: string;
+    similarity: number;
+    prompt: string;
+}>) => Promise<number>;

package/dist/rerank.js

@@ -0,0 +1,73 @@
+"use strict";
+/**
+ * Built-in rerank factories for @betterdb/semantic-cache.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createKeywordOverlapRerank = createKeywordOverlapRerank;
+/**
+ * Tokenize: lowercase, split on whitespace, strip surrounding punctuation.
+ * Deterministic and dependency-free.
+ * IDF weighting would attach here at the token-weighting step.
+ */
+function tokenize(text) {
+    const out = new Set();
+    for (const raw of text.toLowerCase().split(/\s+/)) {
+        const tok = raw.replace(/^[.,!?;:"'()\[\]{}<>]+|[.,!?;:"'()\[\]{}<>]+$/g, '');
+        if (tok)
+            out.add(tok);
+    }
+    return out;
+}
+/**
+ * Built-in keyword-overlap reranker.
+ *
+ * Blends cosine similarity with word overlap and returns the index of the
+ * best candidate.
+ *
+ * @param compare
+ *   `"prompt"`  – overlap of the incoming query against each candidate's stored
+ *                 prompt. Equivalence signal. Catches entity mismatches
+ *                 (e.g. "weather in Paris" vs "weather in Berlin"). Default.
+ *   `"response"` – overlap of the incoming query against each candidate's cached
+ *                 response. Relevance signal.
+ *
+ * @param cosineWeight
+ *   Weight on cosine similarity in [0, 1]. Overlap weight is `1 - cosineWeight`.
+ *   Default: 0.7 (overlap 0.3).
+ *
+ * Candidate objects carry: `similarity` (cosine distance, lower = more similar),
+ * `response` (string), and `prompt` (string, stored prompt).
+ */
+function createKeywordOverlapRerank(options) {
+    const compare = options?.compare ?? 'prompt';
+    const cosineWeight = options?.cosineWeight ?? 0.7;
+    if (cosineWeight < 0 || cosineWeight > 1) {
+        throw new Error('cosineWeight must be in [0, 1]');
+    }
+    const overlapWeight = 1.0 - cosineWeight;
+    return async (query, candidates) => {
+        const queryTokens = tokenize(query);
+        let bestIdx = 0;
+        let bestScore = -Infinity;
+        for (let i = 0; i < candidates.length; i++) {
+            const text = candidates[i][compare] ?? '';
+            const candTokens = tokenize(text);
+            let overlap = 0;
+            if (queryTokens.size > 0) {
+                let intersection = 0;
+                for (const t of queryTokens) {
+                    if (candTokens.has(t))
+                        intersection++;
+                }
+                overlap = intersection / queryTokens.size;
+            }
+            const cosineSim = 1.0 - candidates[i].similarity;
+            const score = cosineWeight * cosineSim + overlapWeight * overlap;
+            if (score > bestScore) {
+                bestScore = score;
+                bestIdx = i;
+            }
+        }
+        return bestIdx;
+    };
+}

package/dist/types.d.ts

@@ -127,6 +127,7 @@ export interface RerankOptions {
     rerankFn: (query: string, candidates: Array<{
         response: string;
         similarity: number;
+        prompt: string;
     }>) => Promise<number>;
 }
 /**
@@ -162,6 +163,8 @@ export interface JudgeOptions {
         similarity: number;
         threshold: number;
         category: string | undefined;
+        /** The stored prompt text for the matched entry. */
+        cachedPrompt: string;
     }) => Promise<boolean>;
     /**
      * Behavior when judgeFn throws or exceeds timeoutMs.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@betterdb/semantic-cache",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "Valkey-native semantic cache for LLM applications with built-in OpenTelemetry and Prometheus instrumentation",
   "keywords": [
     "valkey",