@betterdb/semantic-cache 0.1.0 → 0.4.0

package/dist/index.js

@@ -1,9 +1,21 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ValkeyCommandError = exports.EmbeddingError = exports.SemanticCacheUsageError = exports.SemanticCache = void 0;
+exports.defaultNormalizer = exports.composeNormalizer = exports.passthrough = exports.fetchAndHash = exports.hashUrl = exports.hashBytes = exports.hashBase64 = exports.escapeTag = exports.ValkeyCommandError = exports.EmbeddingError = exports.SemanticCacheUsageError = exports.DEFAULT_COST_TABLE = exports.SemanticCache = void 0;
 var SemanticCache_1 = require("./SemanticCache");
 Object.defineProperty(exports, "SemanticCache", { enumerable: true, get: function () { return SemanticCache_1.SemanticCache; } });
+var defaultCostTable_1 = require("./defaultCostTable");
+Object.defineProperty(exports, "DEFAULT_COST_TABLE", { enumerable: true, get: function () { return defaultCostTable_1.DEFAULT_COST_TABLE; } });
 var errors_1 = require("./errors");
 Object.defineProperty(exports, "SemanticCacheUsageError", { enumerable: true, get: function () { return errors_1.SemanticCacheUsageError; } });
 Object.defineProperty(exports, "EmbeddingError", { enumerable: true, get: function () { return errors_1.EmbeddingError; } });
 Object.defineProperty(exports, "ValkeyCommandError", { enumerable: true, get: function () { return errors_1.ValkeyCommandError; } });
+var utils_1 = require("./utils");
+Object.defineProperty(exports, "escapeTag", { enumerable: true, get: function () { return utils_1.escapeTag; } });
+var normalizer_1 = require("./normalizer");
+Object.defineProperty(exports, "hashBase64", { enumerable: true, get: function () { return normalizer_1.hashBase64; } });
+Object.defineProperty(exports, "hashBytes", { enumerable: true, get: function () { return normalizer_1.hashBytes; } });
+Object.defineProperty(exports, "hashUrl", { enumerable: true, get: function () { return normalizer_1.hashUrl; } });
+Object.defineProperty(exports, "fetchAndHash", { enumerable: true, get: function () { return normalizer_1.fetchAndHash; } });
+Object.defineProperty(exports, "passthrough", { enumerable: true, get: function () { return normalizer_1.passthrough; } });
+Object.defineProperty(exports, "composeNormalizer", { enumerable: true, get: function () { return normalizer_1.composeNormalizer; } });
+Object.defineProperty(exports, "defaultNormalizer", { enumerable: true, get: function () { return normalizer_1.defaultNormalizer; } });

package/dist/normalizer.d.ts

@@ -0,0 +1,68 @@
+export interface BinaryRef {
+    kind: "image" | "audio" | "document";
+    source: {
+        type: "base64";
+        data: string;
+        mediaType?: string;
+    } | {
+        type: "url";
+        url: string;
+    } | {
+        type: "fileId";
+        fileId: string;
+        provider: string;
+    } | {
+        type: "bytes";
+        data: Uint8Array | Buffer;
+    };
+    context?: Record<string, unknown>;
+}
+export type BinaryNormalizer = (ref: BinaryRef) => Promise<string>;
+export interface NormalizerConfig {
+    base64?: (data: string) => string | Promise<string>;
+    url?: (urlStr: string) => string | Promise<string>;
+    fileId?: (fileId: string, provider: string) => string | Promise<string>;
+    bytes?: (data: Uint8Array | Buffer) => string | Promise<string>;
+    byKind?: {
+        image?: BinaryNormalizer;
+        audio?: BinaryNormalizer;
+        document?: BinaryNormalizer;
+    };
+}
+/**
+ * Strip any "data:<mime>;base64," prefix, decode the base64 bytes,
+ * and return "sha256:<hex>" of the decoded bytes.
+ */
+export declare function hashBase64(data: string): string;
+/**
+ * Return "sha256:<hex>" of the raw bytes.
+ */
+export declare function hashBytes(data: Uint8Array | Buffer): string;
+/**
+ * Normalize a URL: lowercase scheme+host, drop default ports (80/443),
+ * sort query params, return "url:<normalized>".
+ */
+export declare function hashUrl(urlStr: string): string;
+/**
+ * Fetch a URL, throw if response is not ok, and return "sha256:<hex>"
+ * of the response body bytes.
+ */
+export declare function fetchAndHash(url: string): Promise<string>;
+/**
+ * Return a scheme-prefixed reference without any normalization:
+ * - base64 source  -> "base64:<data>"
+ * - url source     -> "url:<url>"
+ * - fileId source  -> "fileid:<provider>:<fileId>"
+ * - bytes source   -> "sha256:<hex>" (hashes the bytes)
+ */
+export declare function passthrough(ref: BinaryRef): string;
+/**
+ * Build a BinaryNormalizer from a config.
+ *
+ * Dispatch order:
+ * 1. If cfg.byKind[ref.kind] is defined, call it with the full BinaryRef.
+ * 2. Otherwise dispatch on ref.source.type using the per-source handlers.
+ * 3. Fall back to passthrough for any unhandled source types.
+ */
+export declare function composeNormalizer(cfg?: NormalizerConfig): BinaryNormalizer;
+export declare const defaultNormalizer: BinaryNormalizer;

package/dist/normalizer.js

@@ -0,0 +1,102 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultNormalizer = void 0;
+exports.hashBase64 = hashBase64;
+exports.hashBytes = hashBytes;
+exports.hashUrl = hashUrl;
+exports.fetchAndHash = fetchAndHash;
+exports.passthrough = passthrough;
+exports.composeNormalizer = composeNormalizer;
+const node_crypto_1 = require("node:crypto");
+// --- Normalizer functions ---
+/**
+ * Strip any "data:<mime>;base64," prefix, decode the base64 bytes,
+ * and return "sha256:<hex>" of the decoded bytes.
+ */
+function hashBase64(data) {
+    const raw = data.includes(";base64,") ? data.split(";base64,")[1] : data;
+    const bytes = Buffer.from(raw, "base64");
+    return "sha256:" + (0, node_crypto_1.createHash)("sha256").update(bytes).digest("hex");
+}
+/**
+ * Return "sha256:<hex>" of the raw bytes.
+ */
+function hashBytes(data) {
+    return "sha256:" + (0, node_crypto_1.createHash)("sha256").update(data).digest("hex");
+}
+/**
+ * Normalize a URL: lowercase scheme+host, drop default ports (80/443),
+ * sort query params, return "url:<normalized>".
+ */
+function hashUrl(urlStr) {
+    const url = new URL(urlStr);
+    // URL constructor lowercases scheme and hostname; also drops default ports
+    url.searchParams.sort();
+    return "url:" + url.toString();
+}
+/**
+ * Fetch a URL, throw if response is not ok, and return "sha256:<hex>"
+ * of the response body bytes.
+ */
+async function fetchAndHash(url) {
+    const res = await fetch(url);
+    if (!res.ok) {
+        throw new Error(`fetchAndHash: HTTP ${res.status} for ${url}`);
+    }
+    const buf = await res.arrayBuffer();
+    return "sha256:" + (0, node_crypto_1.createHash)("sha256").update(Buffer.from(buf)).digest("hex");
+}
+/**
+ * Return a scheme-prefixed reference without any normalization:
+ * - base64 source  -> "base64:<data>"
+ * - url source     -> "url:<url>"
+ * - fileId source  -> "fileid:<provider>:<fileId>"
+ * - bytes source   -> "sha256:<hex>" (hashes the bytes)
+ */
+function passthrough(ref) {
+    const { source } = ref;
+    switch (source.type) {
+        case "base64":
+            return "base64:" + source.data;
+        case "url":
+            return "url:" + source.url;
+        case "fileId":
+            return "fileid:" + source.provider + ":" + source.fileId;
+        case "bytes":
+            return hashBytes(source.data);
+    }
+}
+// --- Factory ---
+/**
+ * Build a BinaryNormalizer from a config.
+ *
+ * Dispatch order:
+ * 1. If cfg.byKind[ref.kind] is defined, call it with the full BinaryRef.
+ * 2. Otherwise dispatch on ref.source.type using the per-source handlers.
+ * 3. Fall back to passthrough for any unhandled source types.
+ */
+function composeNormalizer(cfg = {}) {
+    return async (ref) => {
+        // byKind takes priority
+        const kindFn = cfg.byKind?.[ref.kind];
+        if (kindFn)
+            return kindFn(ref);
+        const { source } = ref;
+        switch (source.type) {
+            case "base64":
+                return cfg.base64 ? cfg.base64(source.data) : passthrough(ref);
+            case "url":
+                return cfg.url ? cfg.url(source.url) : passthrough(ref);
+            case "fileId":
+                return cfg.fileId
+                    ? cfg.fileId(source.fileId, source.provider)
+                    : passthrough(ref);
+            case "bytes":
+                return cfg.bytes ? cfg.bytes(source.data) : passthrough(ref);
+        }
+    };
+}
+exports.defaultNormalizer = composeNormalizer({
+    base64: hashBase64,
+    bytes: hashBytes,
+});

package/dist/telemetry.d.ts

@@ -10,6 +10,11 @@ interface CacheMetrics {
     similarityScore: Histogram;
     operationDuration: Histogram;
     embeddingDuration: Histogram;
+    costSavedTotal: Counter;
+    embeddingCacheTotal: Counter;
+    staleModelEvictions: Counter;
+    discoveryWriteFailed: Counter;
+    configRefreshFailed: Counter;
 }
 export interface Telemetry {
     tracer: Tracer;

package/dist/telemetry.js

@@ -42,6 +42,31 @@ function createTelemetry(opts) {
         labelNames: ['cache_name'],
         buckets: operationBuckets,
     });
+    const costSavedTotal = getOrCreateCounter(registry, {
+        name: `${opts.prefix}_cost_saved_total`,
+        help: 'Estimated cost saved in dollars from semantic cache hits',
+        labelNames: ['cache_name', 'category'],
+    });
+    const embeddingCacheTotal = getOrCreateCounter(registry, {
+        name: `${opts.prefix}_embedding_cache_total`,
+        help: 'Total embedding cache lookups (hit or miss)',
+        labelNames: ['cache_name', 'result'],
+    });
+    const staleModelEvictions = getOrCreateCounter(registry, {
+        name: `${opts.prefix}_stale_model_evictions_total`,
+        help: 'Entries evicted due to staleAfterModelChange detection',
+        labelNames: ['cache_name'],
+    });
+    const discoveryWriteFailed = getOrCreateCounter(registry, {
+        name: `${opts.prefix}_discovery_write_failed_total`,
+        help: 'Count of failed discovery-marker writes (best-effort HGET/HSET/SET operations against __betterdb:* keys)',
+        labelNames: ['cache_name'],
+    });
+    const configRefreshFailed = getOrCreateCounter(registry, {
+        name: `${opts.prefix}_config_refresh_failed_total`,
+        help: 'Count of failed periodic config refreshes (HGETALL on __config).',
+        labelNames: ['cache_name'],
+    });
     return {
         tracer,
         metrics: {
@@ -49,6 +74,11 @@ function createTelemetry(opts) {
             similarityScore,
             operationDuration,
             embeddingDuration,
+            costSavedTotal,
+            embeddingCacheTotal,
+            staleModelEvictions,
+            discoveryWriteFailed,
+            configRefreshFailed,
         },
     };
 }

package/dist/types.d.ts

@@ -1,7 +1,18 @@
 import type Valkey from 'iovalkey';
 import type { Registry } from 'prom-client';
+import type { DiscoveryOptions } from './discovery';
 export type { Valkey };
+export interface ConfigRefreshOptions {
+    /** Enable periodic config refresh from Valkey. Default: true. */
+    enabled?: boolean;
+    /** Refresh interval in milliseconds. Default: 30000. Minimum: 1000. */
+    intervalMs?: number;
+}
 export type EmbedFn = (text: string) => Promise<number[]>;
+export interface ModelCost {
+    inputPer1k: number;
+    outputPer1k: number;
+}
 export interface SemanticCacheOptions {
     /** Index name prefix used for Valkey keys. Default: 'betterdb_scache'. */
     name?: string;
@@ -9,6 +20,16 @@ export interface SemanticCacheOptions {
     client: Valkey;
     /** Async function that returns a float embedding vector for a text string. Required. */
     embedFn: EmbedFn;
+    /**
+     * Model pricing for cost savings tracking. Optional.
+     * Keys are model names (e.g. 'gpt-4o'), values are per-1k-token costs.
+     */
+    costTable?: Record<string, ModelCost>;
+    /**
+     * Use bundled default cost table from LiteLLM. User costTable entries override defaults.
+     * Default: true.
+     */
+    useDefaultCostTable?: boolean;
     /**
      * Default similarity threshold as cosine DISTANCE (0–2 scale, lower = more similar).
      * A lookup is a hit when score <= threshold. Default: 0.1.
@@ -39,6 +60,22 @@ export interface SemanticCacheOptions {
      * Default: 0.05. Set to 0 to disable uncertainty flagging (all hits are 'high').
      */
     uncertaintyBand?: number;
+    /**
+     * Pluggable binary content normalizer for stable hashing of images, audio, and documents.
+     * Default: passthrough (uses the ref string as-is).
+     * Pass this to adapter prepareSemanticParams() calls to share the same normalization strategy.
+     */
+    normalizer?: import('./normalizer').BinaryNormalizer;
+    /**
+     * Embedding cache configuration. When enabled, computed embeddings are stored in Valkey
+     * so that repeated check() calls on the same text skip the embedFn call.
+     */
+    embeddingCache?: {
+        /** Enable embedding caching. Default: true. */
+        enabled?: boolean;
+        /** TTL for cached embeddings in seconds. Default: 86400 (24 hours). */
+        ttl?: number;
+    };
     telemetry?: {
         /** OTel tracer name. Default: '@betterdb/semantic-cache'. */
         tracerName?: string;
@@ -52,11 +89,50 @@ export interface SemanticCacheOptions {
          */
         registry?: Registry;
     };
+    analytics?: {
+        /** PostHog API key. Overrides the build-time baked key if set. */
+        apiKey?: string;
+        /** PostHog host. Overrides the build-time baked host if set. */
+        host?: string;
+        /** Disable analytics. Also controlled by BETTERDB_TELEMETRY env var. */
+        disabled?: boolean;
+        /** Interval in ms for periodic stats snapshots. Default: 300_000 (5 min). 0 to disable. */
+        statsIntervalMs?: number;
+    };
+    /**
+     * Discovery-marker protocol controls. See
+     * docs/plans/specs/spec-semantic-cache-discovery-markers.md.
+     * Defaults: enabled=true, heartbeatIntervalMs=30000, includeCategories=true.
+     */
+    discovery?: DiscoveryOptions;
+    /**
+     * Periodic refresh of in-memory threshold config from Valkey.
+     * When enabled, the cache re-reads `{name}:__config` on the configured
+     * interval. Field `threshold` updates `defaultThreshold`; fields named
+     * `threshold:{category}` update `categoryThresholds[category]`.
+     * Defaults: enabled=true, intervalMs=30000.
+     */
+    configRefresh?: ConfigRefreshOptions;
+}
+export interface RerankOptions {
+    /**
+     * Number of top-k candidates to retrieve before reranking.
+     * A higher k gives the rerankFn more candidates to choose from.
+     */
+    k: number;
+    /**
+     * Function that receives the query text and ranked candidates, and returns
+     * the index of the best candidate. Return -1 to reject all candidates (miss).
+     */
+    rerankFn: (query: string, candidates: Array<{
+        response: string;
+        similarity: number;
+    }>) => Promise<number>;
 }
 export interface CacheCheckOptions {
-    /** Per-request threshold override (cosine distance 0–2). Highest priority. */
+    /** Per-request threshold override (cosine distance 0-2). Highest priority. */
     threshold?: number;
-    /** Category tag — used for per-category threshold lookup and metric labels. */
+    /** Category tag - used for per-category threshold lookup and metric labels. */
     category?: string;
     /**
      * Additional FT.SEARCH pre-filter expression.
@@ -64,16 +140,33 @@ export interface CacheCheckOptions {
      * Applied as: "({filter})=>[KNN {k} @embedding $vec AS __score]"
      *
      * **Security note:** this string is interpolated directly into the FT.SEARCH
-     * query. Only pass trusted, programmatically-constructed expressions — never
+     * query. Only pass trusted, programmatically-constructed expressions - never
      * unsanitised user input.
      */
     filter?: string;
     /**
      * Number of nearest neighbours to fetch via KNN. Default: 1.
-     * Currently only the closest result is evaluated for hit/miss.
-     * Values > 1 are reserved for future multi-candidate support.
+     * Ignored when rerank is set (rerank.k takes precedence).
      */
     k?: number;
+    /**
+     * When true, a cache hit whose stored model differs from currentModel is
+     * treated as a miss and the stale entry is deleted. Useful for automatically
+     * evicting cache entries when you upgrade the model you use for a given prompt.
+     * Requires currentModel to be set.
+     * Default: false.
+     */
+    staleAfterModelChange?: boolean;
+    /** The model name to compare against stored entries when staleAfterModelChange is true. */
+    currentModel?: string;
+    /**
+     * Optional rerank hook. When set, FT.SEARCH retrieves rerank.k candidates
+     * and passes them to rerank.rerankFn. The function returns the index of the
+     * best candidate, or -1 to treat all as a miss.
+     * The threshold is NOT applied to the reranked pick unless you filter candidates
+     * in rerankFn yourself.
+     */
+    rerank?: RerankOptions;
 }
 export interface CacheStoreOptions {
     /** Per-entry TTL in seconds. Overrides SemanticCacheOptions.defaultTtl. */
@@ -84,16 +177,33 @@ export interface CacheStoreOptions {
     model?: string;
     /**
      * Arbitrary metadata stored as JSON alongside the entry.
-     * Stored for external consumption (e.g. BetterDB Monitor) — not returned by check().
+     * Stored for external consumption (e.g. BetterDB Monitor) - not returned by check().
      */
     metadata?: Record<string, string | number>;
+    /**
+     * Number of input tokens used to generate the cached response.
+     * When provided along with outputTokens and model, the cost is computed and stored.
+     * On future cache hits, the stored cost is reported as costSaved in CacheCheckResult.
+     */
+    inputTokens?: number;
+    /**
+     * Number of output tokens in the cached response.
+     * See inputTokens for full description.
+     */
+    outputTokens?: number;
+    /** LLM sampling temperature stored as a NUMERIC field for opt-in filtering. */
+    temperature?: number;
+    /** Top-p nucleus sampling parameter stored as a NUMERIC field for opt-in filtering. */
+    topP?: number;
+    /** Random seed stored as a NUMERIC field for opt-in filtering. */
+    seed?: number;
 }
 export type CacheConfidence = 'high' | 'uncertain' | 'miss';
 export interface CacheCheckResult {
     hit: boolean;
     response?: string;
     /**
-     * Cosine distance score (0–2). Present when a nearest neighbour was found,
+     * Cosine distance score (0-2). Present when a nearest neighbour was found,
      * regardless of whether it was a hit or miss.
      */
     similarity?: number;
@@ -118,6 +228,15 @@ export interface CacheCheckResult {
         similarity: number;
         deltaToThreshold: number;
     };
+    /**
+     * Estimated cost saved (in dollars) by returning this cached result instead of calling the LLM.
+     * Present on hit when the original store() call included inputTokens/outputTokens and model.
+     */
+    costSaved?: number;
+    /**
+     * Structured response content blocks. Present on hit when the entry was stored via storeMultipart().
+     */
+    contentBlocks?: import('./utils').ContentBlock[];
 }
 export interface InvalidateResult {
     /** Number of entries deleted in this call. */
@@ -133,6 +252,8 @@ export interface CacheStats {
     misses: number;
     total: number;
     hitRate: number;
+    /** Accumulated cost saved in microdollars (divide by 1_000_000 for dollars). */
+    costSavedMicros: number;
 }
 export interface IndexInfo {
     name: string;

package/dist/utils.d.ts

@@ -1,5 +1,63 @@
 /** SHA-256 hex digest of a string. */
 export declare function sha256(text: string): string;
+/** Escape a string for safe use as a Valkey Search TAG filter value.
+ * Spaces are included because Valkey Search treats unescaped spaces as term
+ * separators (OR semantics), which would broaden the filter unintentionally.
+ */
+export declare function escapeTag(value: string): string;
+export type ContentBlock = TextBlock | BinaryBlock | ToolCallBlock | ToolResultBlock | ReasoningBlock;
+export interface TextBlock {
+    type: 'text';
+    text: string;
+    hints?: BlockHints;
+}
+export interface BinaryBlock {
+    type: 'binary';
+    kind: 'image' | 'audio' | 'document';
+    mediaType: string;
+    ref: string;
+    detail?: 'auto' | 'low' | 'high' | 'original';
+    filename?: string;
+    hints?: BlockHints;
+}
+export interface ToolCallBlock {
+    type: 'tool_call';
+    id: string;
+    name: string;
+    args: unknown;
+    hints?: BlockHints;
+}
+export interface ToolResultBlock {
+    type: 'tool_result';
+    toolCallId: string;
+    content: Array<TextBlock | BinaryBlock>;
+    isError?: boolean;
+    hints?: BlockHints;
+}
+export interface ReasoningBlock {
+    type: 'reasoning';
+    text: string;
+    opaqueSignature?: string;
+    redacted?: boolean;
+    hints?: BlockHints;
+}
+export interface BlockHints {
+    anthropicCacheControl?: {
+        type: 'ephemeral';
+        ttl?: '5m' | '1h';
+    };
+    [k: string]: unknown;
+}
+/**
+ * Extract all text from a ContentBlock array, joining TextBlock.text values with a space.
+ * Used to derive the embedding text from a multi-modal prompt.
+ */
+export declare function extractText(blocks: ContentBlock[]): string;
+/**
+ * Extract all binary refs from a ContentBlock array, sorted for stability.
+ * Used for the binary_refs TAG field on cache entries.
+ */
+export declare function extractBinaryRefs(blocks: ContentBlock[]): string[];
 /**
  * Encode number[] as a little-endian Float32 Buffer.
  * Used to store embeddings as binary HSET field values.

package/dist/utils.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.sha256 = sha256;
+exports.escapeTag = escapeTag;
+exports.extractText = extractText;
+exports.extractBinaryRefs = extractBinaryRefs;
 exports.encodeFloat32 = encodeFloat32;
 exports.parseFtSearchResponse = parseFtSearchResponse;
 const node_crypto_1 = require("node:crypto");
@@ -8,6 +11,33 @@ const node_crypto_1 = require("node:crypto");
 function sha256(text) {
     return (0, node_crypto_1.createHash)('sha256').update(text).digest('hex');
 }
+/** Escape a string for safe use as a Valkey Search TAG filter value.
+ * Spaces are included because Valkey Search treats unescaped spaces as term
+ * separators (OR semantics), which would broaden the filter unintentionally.
+ */
+function escapeTag(value) {
+    return value.replace(/[,.<>{}[\]"':;!@#$%^&*()\-+=~|/\\ ]/g, '\\$&');
+}
+/**
+ * Extract all text from a ContentBlock array, joining TextBlock.text values with a space.
+ * Used to derive the embedding text from a multi-modal prompt.
+ */
+function extractText(blocks) {
+    return blocks
+        .filter((b) => b.type === 'text')
+        .map((b) => b.text)
+        .join(' ');
+}
+/**
+ * Extract all binary refs from a ContentBlock array, sorted for stability.
+ * Used for the binary_refs TAG field on cache entries.
+ */
+function extractBinaryRefs(blocks) {
+    return blocks
+        .filter((b) => b.type === 'binary')
+        .map((b) => b.ref)
+        .sort();
+}
 /**
  * Encode number[] as a little-endian Float32 Buffer.
  * Used to store embeddings as binary HSET field values.