@betterdb/semantic-cache 0.1.0

package/dist/errors.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ValkeyCommandError = exports.EmbeddingError = exports.SemanticCacheUsageError = void 0;
+/**
+ * Thrown when the caller does something wrong — e.g. calling check()
+ * before initialize(), or providing an embedding with the wrong dimension.
+ * The message is always actionable: it tells the caller what to fix.
+ */
+class SemanticCacheUsageError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'SemanticCacheUsageError';
+    }
+}
+exports.SemanticCacheUsageError = SemanticCacheUsageError;
+/**
+ * Thrown when the embedding function fails.
+ * Check the underlying cause for the original error from the embedding provider.
+ */
+class EmbeddingError extends Error {
+    cause;
+    constructor(message, cause) {
+        super(message);
+        this.cause = cause;
+        this.name = 'EmbeddingError';
+    }
+}
+exports.EmbeddingError = EmbeddingError;
+/**
+ * Thrown when a Valkey command fails unexpectedly.
+ * Includes the command name and the underlying error.
+ */
+class ValkeyCommandError extends Error {
+    command;
+    cause;
+    constructor(command, cause) {
+        super(`Valkey command '${command}' failed: ${cause instanceof Error ? cause.message : String(cause)}`);
+        this.command = command;
+        this.cause = cause;
+        this.name = 'ValkeyCommandError';
+    }
+}
+exports.ValkeyCommandError = ValkeyCommandError;

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { SemanticCache } from './SemanticCache';
+export type { SemanticCacheOptions, CacheCheckOptions, CacheStoreOptions, CacheCheckResult, CacheStats, IndexInfo, InvalidateResult, CacheConfidence, EmbedFn, } from './types';
+export { SemanticCacheUsageError, EmbeddingError, ValkeyCommandError, } from './errors';

package/dist/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ValkeyCommandError = exports.EmbeddingError = exports.SemanticCacheUsageError = exports.SemanticCache = void 0;
+var SemanticCache_1 = require("./SemanticCache");
+Object.defineProperty(exports, "SemanticCache", { enumerable: true, get: function () { return SemanticCache_1.SemanticCache; } });
+var errors_1 = require("./errors");
+Object.defineProperty(exports, "SemanticCacheUsageError", { enumerable: true, get: function () { return errors_1.SemanticCacheUsageError; } });
+Object.defineProperty(exports, "EmbeddingError", { enumerable: true, get: function () { return errors_1.EmbeddingError; } });
+Object.defineProperty(exports, "ValkeyCommandError", { enumerable: true, get: function () { return errors_1.ValkeyCommandError; } });

package/dist/telemetry.d.ts

@@ -0,0 +1,19 @@
+import { type Tracer } from '@opentelemetry/api';
+import { Counter, Histogram, Registry } from 'prom-client';
+interface TelemetryFactoryOptions {
+    prefix: string;
+    tracerName: string;
+    registry?: Registry;
+}
+interface CacheMetrics {
+    requestsTotal: Counter;
+    similarityScore: Histogram;
+    operationDuration: Histogram;
+    embeddingDuration: Histogram;
+}
+export interface Telemetry {
+    tracer: Tracer;
+    metrics: CacheMetrics;
+}
+export declare function createTelemetry(opts: TelemetryFactoryOptions): Telemetry;
+export {};

package/dist/telemetry.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTelemetry = createTelemetry;
+const api_1 = require("@opentelemetry/api");
+const prom_client_1 = require("prom-client");
+function getOrCreateCounter(registry, config) {
+    const existing = registry.getSingleMetric(config.name);
+    if (existing)
+        return existing;
+    return new prom_client_1.Counter({ ...config, registers: [registry] });
+}
+function getOrCreateHistogram(registry, config) {
+    const existing = registry.getSingleMetric(config.name);
+    if (existing)
+        return existing;
+    return new prom_client_1.Histogram({ ...config, registers: [registry] });
+}
+function createTelemetry(opts) {
+    const registry = opts.registry ?? prom_client_1.register;
+    const tracer = api_1.trace.getTracer(opts.tracerName);
+    const operationBuckets = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0];
+    const requestsTotal = getOrCreateCounter(registry, {
+        name: `${opts.prefix}_requests_total`,
+        help: 'Total number of semantic cache requests',
+        labelNames: ['cache_name', 'result', 'category'],
+    });
+    const similarityScore = getOrCreateHistogram(registry, {
+        name: `${opts.prefix}_similarity_score`,
+        help: 'Cosine distance similarity scores for cache lookups',
+        labelNames: ['cache_name', 'category'],
+        buckets: [0.02, 0.05, 0.08, 0.1, 0.12, 0.15, 0.2, 0.3, 0.5, 1.0, 2.0],
+    });
+    const operationDuration = getOrCreateHistogram(registry, {
+        name: `${opts.prefix}_operation_duration_seconds`,
+        help: 'Duration of semantic cache operations in seconds',
+        labelNames: ['cache_name', 'operation'],
+        buckets: operationBuckets,
+    });
+    const embeddingDuration = getOrCreateHistogram(registry, {
+        name: `${opts.prefix}_embedding_duration_seconds`,
+        help: 'Duration of embedding function calls in seconds',
+        labelNames: ['cache_name'],
+        buckets: operationBuckets,
+    });
+    return {
+        tracer,
+        metrics: {
+            requestsTotal,
+            similarityScore,
+            operationDuration,
+            embeddingDuration,
+        },
+    };
+}

package/dist/types.d.ts

@@ -0,0 +1,142 @@
+import type Valkey from 'iovalkey';
+import type { Registry } from 'prom-client';
+export type { Valkey };
+export type EmbedFn = (text: string) => Promise<number[]>;
+export interface SemanticCacheOptions {
+    /** Index name prefix used for Valkey keys. Default: 'betterdb_scache'. */
+    name?: string;
+    /** iovalkey client instance. Required. */
+    client: Valkey;
+    /** Async function that returns a float embedding vector for a text string. Required. */
+    embedFn: EmbedFn;
+    /**
+     * Default similarity threshold as cosine DISTANCE (0–2 scale, lower = more similar).
+     * A lookup is a hit when score <= threshold. Default: 0.1.
+     * NOTE: this is cosine DISTANCE not cosine SIMILARITY.
+     * Distance 0 = identical, distance 2 = opposite.
+     */
+    defaultThreshold?: number;
+    /** Default TTL in seconds for stored entries. undefined = no expiry. */
+    defaultTtl?: number;
+    /**
+     * Per-category threshold overrides (cosine distance, 0–2).
+     * Applied when CacheCheckOptions.category matches a key here.
+     * Example: { faq: 0.08, search: 0.15 }
+     */
+    categoryThresholds?: Record<string, number>;
+    /**
+     * Width of the "uncertainty band" below the threshold.
+     * A hit whose cosine distance falls within [threshold - band, threshold]
+     * is returned with confidence 'uncertain' instead of 'high'.
+     *
+     * What to do with an uncertain hit:
+     * - Use the cached response but flag it for downstream review
+     * - Fall back to the LLM and optionally update the cache entry
+     * - Collect uncertain hits via Prometheus/OTel and review them to tune
+     *   your threshold — a high rate of uncertain hits suggests your threshold
+     *   is too loose
+     *
+     * Default: 0.05. Set to 0 to disable uncertainty flagging (all hits are 'high').
+     */
+    uncertaintyBand?: number;
+    telemetry?: {
+        /** OTel tracer name. Default: '@betterdb/semantic-cache'. */
+        tracerName?: string;
+        /** Prefix for Prometheus metric names. Default: 'semantic_cache'. */
+        metricsPrefix?: string;
+        /**
+         * prom-client Registry to register metrics on.
+         * If omitted, uses the prom-client default registry.
+         * Pass a custom Registry in library/multi-tenant contexts to avoid
+         * polluting the host application's default registry.
+         */
+        registry?: Registry;
+    };
+}
+export interface CacheCheckOptions {
+    /** Per-request threshold override (cosine distance 0–2). Highest priority. */
+    threshold?: number;
+    /** Category tag — used for per-category threshold lookup and metric labels. */
+    category?: string;
+    /**
+     * Additional FT.SEARCH pre-filter expression.
+     * Example: '@model:{gpt-4o}'
+     * Applied as: "({filter})=>[KNN {k} @embedding $vec AS __score]"
+     *
+     * **Security note:** this string is interpolated directly into the FT.SEARCH
+     * query. Only pass trusted, programmatically-constructed expressions — never
+     * unsanitised user input.
+     */
+    filter?: string;
+    /**
+     * Number of nearest neighbours to fetch via KNN. Default: 1.
+     * Currently only the closest result is evaluated for hit/miss.
+     * Values > 1 are reserved for future multi-candidate support.
+     */
+    k?: number;
+}
+export interface CacheStoreOptions {
+    /** Per-entry TTL in seconds. Overrides SemanticCacheOptions.defaultTtl. */
+    ttl?: number;
+    /** Category tag stored with the entry. */
+    category?: string;
+    /** Model name stored with the entry (e.g. 'gpt-4o'). Enables invalidation by model. */
+    model?: string;
+    /**
+     * Arbitrary metadata stored as JSON alongside the entry.
+     * Stored for external consumption (e.g. BetterDB Monitor) — not returned by check().
+     */
+    metadata?: Record<string, string | number>;
+}
+export type CacheConfidence = 'high' | 'uncertain' | 'miss';
+export interface CacheCheckResult {
+    hit: boolean;
+    response?: string;
+    /**
+     * Cosine distance score (0–2). Present when a nearest neighbour was found,
+     * regardless of whether it was a hit or miss.
+     */
+    similarity?: number;
+    /**
+     * Confidence classification for the result.
+     *
+     * - 'high': similarity score is comfortably below the threshold (distance <= threshold - uncertaintyBand).
+     *   Safe to return directly.
+     * - 'uncertain': similarity score is close to the threshold boundary
+     *   (threshold - uncertaintyBand < distance <= threshold).
+     *   Consider falling back to the LLM or flagging for review.
+     * - 'miss': no hit. response is undefined.
+     */
+    confidence: CacheConfidence;
+    /** Valkey key of the matched entry. Present on hit only. */
+    matchedKey?: string;
+    /**
+     * On a miss where a candidate existed but didn't clear the threshold,
+     * describes how close it was. Useful for threshold tuning.
+     */
+    nearestMiss?: {
+        similarity: number;
+        deltaToThreshold: number;
+    };
+}
+export interface InvalidateResult {
+    /** Number of entries deleted in this call. */
+    deleted: number;
+    /**
+     * True if the result set was truncated at 1000 entries.
+     * If true, call invalidate() again with the same filter until truncated is false.
+     */
+    truncated: boolean;
+}
+export interface CacheStats {
+    hits: number;
+    misses: number;
+    total: number;
+    hitRate: number;
+}
+export interface IndexInfo {
+    name: string;
+    numDocs: number;
+    dimension: number;
+    indexingState: string;
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils.d.ts

@@ -0,0 +1,25 @@
+/** SHA-256 hex digest of a string. */
+export declare function sha256(text: string): string;
+/**
+ * Encode number[] as a little-endian Float32 Buffer.
+ * Used to store embeddings as binary HSET field values.
+ */
+export declare function encodeFloat32(vec: number[]): Buffer;
+/**
+ * Parse a raw FT.SEARCH response from iovalkey's client.call().
+ *
+ * iovalkey returns FT.SEARCH results in the following shape:
+ *   [totalCount, key1, [field1, val1, field2, val2, ...], key2, [...], ...]
+ *
+ * - totalCount is a string (e.g. "2")
+ * - Each key is a string
+ * - Each field list is a flat string array: [fieldName, value, fieldName, value, ...]
+ *
+ * Returns an array of { key: string, fields: Record<string, string> }.
+ * Returns [] if totalCount is "0" or the response is empty/malformed.
+ * Never throws — on any parse error, returns [].
+ */
+export declare function parseFtSearchResponse(raw: unknown): Array<{
+    key: string;
+    fields: Record<string, string>;
+}>;

package/dist/utils.js

@@ -0,0 +1,77 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sha256 = sha256;
+exports.encodeFloat32 = encodeFloat32;
+exports.parseFtSearchResponse = parseFtSearchResponse;
+const node_crypto_1 = require("node:crypto");
+/** SHA-256 hex digest of a string. */
+function sha256(text) {
+    return (0, node_crypto_1.createHash)('sha256').update(text).digest('hex');
+}
+/**
+ * Encode number[] as a little-endian Float32 Buffer.
+ * Used to store embeddings as binary HSET field values.
+ */
+function encodeFloat32(vec) {
+    const buf = Buffer.alloc(vec.length * 4);
+    for (let i = 0; i < vec.length; i++) {
+        buf.writeFloatLE(vec[i], i * 4);
+    }
+    return buf;
+}
+/**
+ * Parse a raw FT.SEARCH response from iovalkey's client.call().
+ *
+ * iovalkey returns FT.SEARCH results in the following shape:
+ *   [totalCount, key1, [field1, val1, field2, val2, ...], key2, [...], ...]
+ *
+ * - totalCount is a string (e.g. "2")
+ * - Each key is a string
+ * - Each field list is a flat string array: [fieldName, value, fieldName, value, ...]
+ *
+ * Returns an array of { key: string, fields: Record<string, string> }.
+ * Returns [] if totalCount is "0" or the response is empty/malformed.
+ * Never throws — on any parse error, returns [].
+ */
+function parseFtSearchResponse(raw) {
+    try {
+        if (!Array.isArray(raw) || raw.length < 1) {
+            return [];
+        }
+        const totalCount = typeof raw[0] === 'string' ? parseInt(raw[0], 10) : Number(raw[0]);
+        if (!totalCount || totalCount <= 0) {
+            return [];
+        }
+        const results = [];
+        let i = 1;
+        while (i < raw.length) {
+            const key = raw[i];
+            if (typeof key !== 'string') {
+                i++;
+                continue;
+            }
+            const fieldList = raw[i + 1];
+            const fields = {};
+            if (Array.isArray(fieldList)) {
+                const len = fieldList.length - (fieldList.length % 2);
+                for (let j = 0; j < len; j += 2) {
+                    const fieldName = String(fieldList[j]);
+                    const fieldValue = String(fieldList[j + 1]);
+                    fields[fieldName] = fieldValue;
+                }
+                i += 2;
+            }
+            else {
+                // No field list follows the key (e.g. RETURN 0 mode)
+                results.push({ key, fields });
+                i++;
+                continue;
+            }
+            results.push({ key, fields });
+        }
+        return results;
+    }
+    catch {
+        return [];
+    }
+}

package/package.json

@@ -0,0 +1,69 @@
+{
+  "name": "@betterdb/semantic-cache",
+  "version": "0.1.0",
+  "description": "Valkey-native semantic cache for LLM applications with built-in OpenTelemetry and Prometheus instrumentation",
+  "keywords": [
+    "valkey",
+    "redis",
+    "semantic-cache",
+    "llm",
+    "opentelemetry",
+    "prometheus"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/BetterDB-inc/monitor",
+    "directory": "packages/semantic-cache"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./langchain": {
+      "import": "./dist/adapters/langchain.js",
+      "require": "./dist/adapters/langchain.js",
+      "types": "./dist/adapters/langchain.d.ts"
+    },
+    "./ai": {
+      "import": "./dist/adapters/ai.js",
+      "require": "./dist/adapters/ai.js",
+      "types": "./dist/adapters/ai.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "clean": "rm -rf dist"
+  },
+  "dependencies": {
+    "@opentelemetry/api": "^1.9.0",
+    "prom-client": "^15.1.3"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "peerDependencies": {
+    "iovalkey": ">=0.3.0",
+    "@langchain/core": ">=0.3.0",
+    "ai": ">=4.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@langchain/core": {
+      "optional": true
+    },
+    "ai": {
+      "optional": true
+    }
+  }
+}