@viveka10/cap-ai-token-optimizer 0.0.0-stage → 0.1.0

package/SECURITY.md

@@ -0,0 +1,39 @@
+# Security Policy
+
+## Supported versions
+
+Security fixes are made for the latest minor release. Before 1.0.0, that is the latest
+0.x release.
+
+| Version | Supported |
+|---|---|
+| latest 0.x | ✅ |
+| older | ❌ |
+
+## Reporting a vulnerability
+
+Please do **not** open a public issue for security problems.
+
+Report them privately through GitHub:
+[Security → Report a vulnerability](https://github.com/viveka10/npm_token_optimizer/security/advisories/new).
+Include the package version, a description, and steps to reproduce if possible.
+
+You can expect an acknowledgement within 7 days. Fixes are released as patch versions and
+noted in the [CHANGELOG](CHANGELOG.md).
+
+## How this package handles data
+
+- **Prompt and completion text** are never logged, exported or stored by default. Events
+  contain a SHA-256 hash and the length. Raw text only appears in events with
+  `captureContent: true`, and is never written to the database.
+- **User ids** are recorded as a salted hash (`metrics.userHashSalt`; set it from the
+  environment, not in committed files).
+- **Credentials** are never read or stored by this package. AI Core credentials are handled
+  by the SAP Cloud SDK for AI; destination credentials by the SAP Cloud SDK. Don't put
+  secrets in `metrics.http.headers` in committed files.
+- **The response cache** is in memory per app instance. By default it is scoped per tenant,
+  so cached answers are never shared across tenants (`cache.scope`).
+- **The admin service** is read-only and requires a role (`admin.role`).
+- **Published package contents** are checked before every publish (`npm run check:pack`):
+  no tests, `.env`, `.cdsrc-private.json`, service keys, certificates or benchmark results.
+  Releases are published from GitHub Actions with npm provenance.

package/cds-plugin.js

@@ -0,0 +1,50 @@
+'use strict'
+/**
+ * CAP plugin, loaded automatically by @sap/cds when this package is a dependency.
+ * Reads cds.requires.tokenOptimizer, starts the exporters and flushes them on shutdown.
+ */
+const cds = globalThis.cds ?? require('@sap/cds')
+const { optimizer, countText } = require('./index')
+const log = require('./lib/util/log')
+const path = require('node:path')
+
+try {
+  optimizer.configure(cds.env.requires?.tokenOptimizer)
+  log.debug('configured, mode =', optimizer.config.mode, 'exporters =', optimizer.config.metrics.exporters)
+  setupModels(optimizer.config)
+} catch (e) {
+  log.error('could not configure token optimizer:', e?.message)
+}
+
+/**
+ * Add our CDS models to the app's model when the features using them are enabled:
+ * the TokenUsageLog entity for the db exporter, the read-only admin service, and the
+ * UsageLog projection when both are on. Plugins load before the model, so adding a `model`
+ * to our cds.requires entry is enough.
+ */
+function setupModels (cfg) {
+  const exporters = cfg.metrics.exporters ?? []
+  const dbLog = exporters.includes('db') || exporters.includes('hana')
+  const admin = cfg.admin?.enabled
+  const models = []
+  if (dbLog) models.push('db/token-usage-log')
+  if (admin) models.push('srv/admin-service')
+  if (admin && dbLog) models.push('srv/admin-usage-log')
+  if (!models.length) return
+
+  const req = cds.env.requires.tokenOptimizer ??= {}
+  req.model = [...new Set([].concat(req.model ?? [], models.map(m => path.join(__dirname, m))))]
+  if (!admin) return
+  const role = cfg.admin.role || 'TokenOptimizerAdmin'
+  cds.on('loaded', (csn) => {
+    const srv = csn?.definitions?.TokenOptimizerService
+    if (srv) srv['@requires'] = role
+  })
+}
+
+// Load the tokenizer tables once the server is up, so the first AI call doesn't pay for it.
+cds.once('served', () => setImmediate(() => {
+  try { countText('warm up', 'gpt-4o') } catch { /* estimation falls back to the heuristic */ }
+}))
+
+cds.on('shutdown', () => optimizer.shutdown())

package/db/token-usage-log.cds

@@ -0,0 +1,44 @@
+namespace cap.ai.token_optimizer;
+
+/**
+ * One row per AI Core call, written by the `db` metrics exporter (alias `hana`).
+ * Contains metrics only: never prompt or completion text. In multitenant apps each tenant's
+ * calls are written to that tenant's database container.
+ */
+entity TokenUsageLog {
+  key ID                    : UUID;
+      timestamp             : Timestamp;
+      tenant                : String(255);
+      userHash              : String(16);
+      cdsService            : String(255);
+      useCase               : String(255);
+      mode                  : String(10);
+      model                 : String(255);
+      deploymentId          : String(255);
+      resourceGroup         : String(255);
+      sdk                   : String(30);
+      operation             : String(10);
+      requestId             : String(255);
+      finishReason          : String(50);
+      promptEstimatedBefore : Integer;
+      promptEstimatedAfter  : Integer;
+      promptTokens          : Integer;
+      completionTokens      : Integer;
+      totalTokens           : Integer;
+      cachedTokens          : Integer;
+      savedTokensEstimate   : Integer;
+      /** JSON object: estimated tokens saved per step */
+      savingsByStep         : LargeString;
+      cacheHit              : Boolean;
+      promptPrefixChanged   : Boolean;
+      costEstimate          : Decimal(20, 6);
+      latencyMs             : Integer;
+      retries               : Integer;
+      status                : String(10);
+      errorCategory         : String(30);
+      errorHttpStatus       : Integer;
+      errorCode             : String(100);
+      errorMessage          : String(300);
+      promptHash            : String(64);
+      promptLength          : Integer;
+}

package/docs/integration-guide.md

@@ -0,0 +1,140 @@
+# Add @viveka10/cap-ai-token-optimizer to an existing CAP app in 5 minutes
+
+This guide adds token measurement to a CAP Node.js app that already calls SAP AI Core
+through the SAP Cloud SDK for AI. The first steps change no code and no prompts. You start
+in shadow mode, look at the numbers, and switch to optimization when you're ready.
+
+## 1. Check the prerequisites (30 seconds)
+
+```sh
+node --version                                  # v22.12 or later
+npm ls @sap/cds @sap-ai-sdk/orchestration @sap-ai-sdk/foundation-models
+```
+
+You need `@sap/cds` 9 or 10, and `@sap-ai-sdk/orchestration` and/or
+`@sap-ai-sdk/foundation-models` 2.8 or later. Each `@sap-ai-sdk` package should appear once in
+the tree (see [Limits](#limits-of-automatic-instrumentation)).
+
+## 2. Install (1 minute)
+
+```sh
+npm add @viveka10/cap-ai-token-optimizer
+```
+
+The package is a CAP plugin, so there's nothing to register.
+
+## 3. Turn on measurement without code changes (1 minute)
+
+Add to `package.json` (or `.cdsrc.json`):
+
+```json
+{
+  "cds": {
+    "requires": {
+      "tokenOptimizer": {
+        "mode": "shadow",
+        "autoInstrument": true
+      }
+    }
+  }
+}
+```
+
+- `autoInstrument` hooks the SDK's `OrchestrationClient` and `AzureOpenAiChatClient`, so every
+  call your app already makes is measured.
+- `shadow` runs the optimizer but sends your original prompts: you see what would be saved,
+  and nothing about your AI calls changes.
+
+## 4. Run and look (1 minute)
+
+```sh
+cds watch
+```
+
+Use the feature of your app that calls AI Core. Each call prints one line like:
+
+```json
+{"type":"ai-core-call","useCase":"default","mode":"shadow","model":"gpt-4.1",
+ "tokens":{"promptEstimatedBefore":2015,"promptEstimatedAfter":805,"prompt":2010,"completion":41,...},
+ "savings":{"estimatedTokens":1210,"byStep":{"history":774,"dedupe":340,"normalize":96,...},...},
+ "status":"success","latencyMs":640,...}
+```
+
+`savings.estimatedTokens` is what active mode would have saved on this call;
+`savings.byStep` shows which step saved it.
+
+For a summary, enable the admin service in the same configuration block
+(`"admin": { "enabled": true }`), give your user the `TokenOptimizerAdmin` role, and open
+`/odata/v4/token-optimizer/summary()`.
+
+## 5. Name your use cases (1 minute, optional)
+
+Calls found by `autoInstrument` are grouped under the use case `default`. Name them where
+your handlers call AI Core, without touching the client code:
+
+```js
+const { optimizer } = require('@viveka10/cap-ai-token-optimizer')
+
+this.on('summarizeTicket', req =>
+  optimizer.withUseCase('ticket-summary', () => summarize(req.data.ID)))
+```
+
+For LangChain models from `@sap-ai-sdk/langchain`, wrap the model once:
+
+```js
+const { OrchestrationClient } = require('@sap-ai-sdk/langchain')
+const model = optimizer.wrapLangChain(new OrchestrationClient(config), { useCase: 'support-agent' })
+```
+
+For clients you create yourself, you can also use `optimizer.wrap(client, { useCase })`.
+
+## 6. Switch on optimization (30 seconds)
+
+When the savings and the answers look right, switch the mode, for one use case or for all:
+
+```json
+"tokenOptimizer": {
+  "mode": "shadow",
+  "autoInstrument": true,
+  "useCases": {
+    "ticket-summary": { "mode": "active" }
+  }
+}
+```
+
+In `active` mode the optimized prompt is sent and identical low-temperature calls are
+answered from the cache. Compare answer quality before and after with
+`npm run bench -- --live` in the package, or by comparing `shadow` and `active` events
+for the same requests.
+
+## Before going to production
+
+- **`pricing`:** set prices per model from your contract to get `costEstimate`.
+- **`metrics.userHashSalt`:** set it from the environment
+  (`cds_requires_tokenOptimizer_metrics_userHashSalt`), not in committed files.
+- **Reporting:** add `otel` (with `@cap-js/telemetry`, for SAP Cloud ALM or Dynatrace), `db`
+  (usage log table) or `http` to `metrics.exporters`.
+- **`captureContent`:** keep it off.
+- **Admin role:** add the `TokenOptimizerAdmin` role to `xs-security.json` if you enabled the
+  admin service.
+
+## Limits of automatic instrumentation
+
+- **Clients:** only `OrchestrationClient` (`chatCompletion`, `stream`) and
+  `AzureOpenAiChatClient` (`run`, `stream`) from `@sap-ai-sdk` 2.x are hooked. This includes the
+  ones that `@sap-ai-sdk/langchain` creates internally. Embedding clients and direct HTTP calls
+  to AI Core are not measured.
+- **SDK copy:** the hooks are installed on the copy of each SDK package resolved from your
+  app's root. If `npm ls @sap-ai-sdk/orchestration` shows a second, nested copy, calls through
+  that copy are not measured; run `npm dedupe` or align the versions.
+- **Scope:** the hooks apply to every caller in the process, including libraries. If you want
+  to choose exactly which calls are optimized, leave `autoInstrument` off and use `wrap()`,
+  `withUseCase()` and `wrapLangChain()` instead.
+- **Streaming:** `stream()` calls are measured but not optimized, and have no token usage.
+- **Use cases:** without `withUseCase()`, all hooked calls share the use case set in
+  `autoInstrument.useCase` (default `default`), so per-use-case settings apply to all of them.
+- **Start-up:** the hooks are installed when the CAP plugin loads. Outside CAP, call
+  `optimizer.configure({ autoInstrument: true })`.
+- **Turning it off:** `"autoInstrument": false` or
+  `cds_requires_tokenOptimizer_autoInstrument=false`. With it off, the hooks call the SDK's
+  original methods unchanged.

package/index.d.ts

@@ -0,0 +1,291 @@
+// Type definitions for @viveka10/cap-ai-token-optimizer
+
+export type Mode = 'active' | 'shadow' | 'off'
+
+export type FailureCategory =
+  | 'RATE_LIMIT' | 'CONTENT_FILTER' | 'CONTEXT_LENGTH' | 'TIMEOUT' | 'AUTH'
+  | 'DEPLOYMENT_NOT_FOUND' | 'SERVER' | 'NETWORK' | 'UNKNOWN'
+
+export interface ModelPrice {
+  inputPer1k?: number
+  outputPer1k?: number
+  /** Price for cached prompt tokens; defaults to inputPer1k. */
+  cachedInputPer1k?: number
+}
+
+type Message = Record<string, unknown>
+type Tool = Record<string, unknown>
+/** A step is off (false), on with defaults (true), or on with options ({ enabled?: false } turns it off). */
+type Step<T> = boolean | (T & { enabled?: boolean })
+
+export interface StepsConfig {
+  normalize?: Step<{ minifyJson?: boolean, minifyJsonFences?: boolean, collapseSpaces?: boolean }>
+  dedupe?: Step<{ messages?: boolean, paragraphs?: boolean, minParagraphLength?: number }>
+  history?: Step<{
+    maxTurns?: number
+    /** 'sliding-window' (default) or a function returning the messages to keep. */
+    strategy?: 'sliding-window' | ((input: { messages: Message[], maxTurns: number, model: string | null }) => Message[])
+    /** Called with the removed messages; the returned message is inserted after the system messages. */
+    summarizer?: ((input: { dropped: Message[], kept: Message[], model: string | null }) => Promise<Message | null> | Message | null) | null
+  }>
+  tools?: Step<{
+    maxTools?: number
+    maxDescriptionLength?: number
+    /** Tool names that are never removed. */
+    pinned?: string[]
+    selector?: ((input: { tools: Tool[], messages: Message[], maxTools: number, keep: Set<string> }) => Tool[]) | null
+  }>
+  outputCap?: Step<{ default?: number, reasoningModels?: boolean }>
+}
+
+/** Response cache store. Methods may be sync or async. Values are plain JSON. */
+export interface CacheStore {
+  get(key: string): unknown | Promise<unknown>
+  set(key: string, value: unknown, ttlMs: number): void | Promise<void>
+  delete?(key: string): void | Promise<void>
+  clear?(): void | Promise<void>
+}
+
+export declare class MemoryCacheStore implements CacheStore {
+  constructor(opts?: { maxEntries?: number, now?: () => number })
+  get(key: string): unknown
+  set(key: string, value: unknown, ttlMs: number): void
+  delete(key: string): void
+  clear(): void
+  readonly size: number
+}
+
+export interface TokenOptimizerConfig {
+  mode?: Mode
+  steps?: StepsConfig
+  cache?: {
+    enabled?: boolean
+    ttlSeconds?: number
+    maxEntries?: number
+    /** Only calls with temperature <= this are cached. null caches regardless of temperature. Default 0.2. */
+    maxTemperature?: number | null
+    /** Who may share cached answers. Default 'tenant'. */
+    scope?: 'tenant' | 'user' | 'global'
+    /** Custom store (e.g. Redis), set in code. Default: in-memory LRU. */
+    store?: CacheStore
+  }
+  metrics?: {
+    /** Built-in exporter names ('console') or custom exporter objects. */
+    exporters?: Array<string | Exporter> | string
+    flushIntervalMs?: number
+    /** Salt for the user-id hash. Set it from the environment, not in committed files. */
+    userHashSalt?: string
+    http?: {
+      /** Endpoint URL. Use either url or destination. */
+      url?: string
+      /** BTP destination name (Internet destinations), resolved with @sap-cloud-sdk/connectivity. */
+      destination?: string
+      headers?: Record<string, string>
+      batchSize?: number
+      /** When full, the oldest events are dropped (and counted). */
+      maxQueueSize?: number
+      maxRetries?: number
+      baseDelayMs?: number
+      maxDelayMs?: number
+      timeoutMs?: number
+      destinationCacheMs?: number
+    }
+    /** TokenUsageLog rows in the app database (exporter `db`, alias `hana`). */
+    db?: { batchSize?: number, maxQueueSize?: number, /** 0 keeps all rows. Default 30. */ retentionDays?: number }
+    otel?: {
+      /** Shortcut for adding 'otel' to exporters. */
+      enabled?: boolean
+      /** Add sap.tenancy.tenant_id to metrics and spans. Default true. */
+      tenantAttribute?: boolean
+    }
+  }
+  /** Price per 1k tokens by model name. */
+  pricing?: Record<string, ModelPrice>
+  /** Per-use-case overrides, merged over the global config. */
+  useCases?: Record<string, Omit<TokenOptimizerConfig, 'useCases'>>
+  /** Default request timeout (ms) when the caller sets none. null = SDK default. */
+  timeoutMs?: number | null
+  retry?: {
+    enabled?: boolean
+    maxRetries?: number
+    baseDelayMs?: number
+    maxDelayMs?: number
+    on?: FailureCategory[]
+  }
+  /**
+   * Hook the @sap-ai-sdk client classes so existing code needs no changes.
+   * `useCase` labels calls made outside wrap(), withUseCase() and wrapLangChain(). Default off.
+   */
+  autoInstrument?: boolean | { enabled?: boolean, useCase?: string }
+  /** promptCache: flag calls whose system prompt or tools changed for the same use case and model. Default true. */
+  advisories?: { promptCache?: boolean }
+  /** Read-only OData service at /odata/v4/token-optimizer. */
+  admin?: { enabled?: boolean, role?: string }
+  /** Include raw prompt and completion text in events. Off by default. */
+  captureContent?: boolean
+}
+
+export interface CallEvent {
+  schemaVersion: number
+  id: string
+  timestamp: string
+  tenant: string | null
+  /** Hashed user id. */
+  user: string | null
+  cdsService: string | null
+  useCase: string
+  mode: Mode
+  model: string | null
+  deploymentId: string | null
+  resourceGroup: string | null
+  sdk: 'orchestration' | 'foundation-models' | null
+  operation: 'chat' | 'stream'
+  requestId: string | null
+  finishReason: string | null
+  estimateMethod: 'tiktoken' | 'heuristic' | null
+  tokens: {
+    promptEstimatedBefore: number | null
+    promptEstimatedAfter: number | null
+    prompt: number | null
+    completion: number | null
+    total: number | null
+    cached: number | null
+  }
+  /** In shadow mode all savings are what active mode would have saved. */
+  savings: {
+    estimatedTokens: number
+    /** Estimated prompt tokens saved per step; `cache` holds the tokens of a cached call. */
+    byStep: Record<string, number>
+    cacheHit: boolean
+    /** Short step notes (counts only, never content). */
+    notes: string[]
+    /** The pipeline failed and the original request was sent. */
+    failed: boolean
+  }
+  costEstimate: number | null
+  latencyMs: number | null
+  retries: number
+  status: 'success' | 'error'
+  error: { category: FailureCategory, httpStatus: number | null, code: string | null, message: string } | null
+  promptHash: string | null
+  promptLength: number | null
+  /** Prompt-caching advisory; null when not checked (disabled, cache hit). */
+  advisory: { promptPrefixChanged: boolean, changed: Array<'system' | 'tools'>, causedByOptimizer: boolean } | null
+  /** Only present when captureContent is true. */
+  content?: { messages: unknown[], completion: string | null }
+}
+
+export interface Advisory {
+  type: 'PROMPT_PREFIX_CHANGED'
+  useCase: string
+  model: string | null
+  promptPrefixChanged: true
+  changed: Array<'system' | 'tools'>
+  /** The caller sent the same prefix as before; an optimizer step (e.g. tools) changed it. */
+  causedByOptimizer: boolean
+}
+
+export interface Exporter {
+  name?: string
+  export(event: CallEvent): void | Promise<void>
+  flush?(): void | Promise<void>
+  shutdown?(): void | Promise<void>
+  stats?(): Record<string, number>
+}
+
+export interface Totals {
+  calls: number
+  success: number
+  errors: number
+  cacheHits: number
+  tokens: { prompt: number, completion: number, total: number, cached: number }
+  savedTokensEstimate: number
+  costEstimate: number
+}
+
+export interface Stats extends Totals {
+  since: string
+  errorsByCategory: Partial<Record<FailureCategory, number>>
+  savingsByStep: Record<string, number>
+  latencyMs: { count: number, avg: number | null, p50: number | null, p95: number | null, max: number | null }
+  byUseCase: Record<string, Totals>
+  byModel: Record<string, Totals>
+  /** Calls whose system prompt or tools changed, per use case (prompt-caching advisory). */
+  promptPrefixChanges: Record<string, number>
+  listenerErrors: number
+  exporterErrors: Record<string, number>
+  /** From exporters with a stats() method, e.g. http: { queued, sent, dropped, failedAttempts, rejectedBatches }. */
+  exporters: Record<string, Record<string, number>>
+}
+
+export interface WrapOptions {
+  useCase?: string
+  /** CDS service name; defaults to the request's base path from cds.context. */
+  cdsService?: string
+}
+
+export interface ChatOptions extends WrapOptions {
+  /** Default 'orchestration'. */
+  sdk?: 'orchestration' | 'foundation-models'
+  model?: string
+  messages: Message[]
+  tools?: Tool[]
+  maxTokens?: number
+  temperature?: number
+  /** Other model parameters. */
+  params?: Record<string, unknown>
+  deploymentId?: string
+  resourceGroup?: string
+  /** Passed to the SDK as requestConfig (e.g. { timeout }). */
+  requestConfig?: Record<string, unknown>
+}
+
+export interface ChatResult {
+  content: string | null
+  toolCalls: unknown[] | null
+  finishReason: string | null
+  usage: Record<string, unknown> | null
+  /** The original SDK response object. */
+  response: unknown
+}
+
+export declare class TokenOptimizer {
+  constructor(config?: TokenOptimizerConfig | boolean, deps?: Record<string, unknown>)
+  readonly config: Required<TokenOptimizerConfig>
+  readonly collector: { readonly exporters: Exporter[] }
+  configure(config?: TokenOptimizerConfig | boolean): this
+  wrap<T extends object>(client: T, opts?: WrapOptions): T
+  chat(opts: ChatOptions): Promise<ChatResult>
+  /** Run fn so that @sap-ai-sdk calls inside it (any client, also ones created by libraries) are optimized and measured with useCase. */
+  withUseCase<R>(useCase: string, fn: () => R, opts?: { cdsService?: string }): R
+  /** Instrument a LangChain chat model from @sap-ai-sdk/langchain. Returns the same model. */
+  wrapLangChain<T extends object>(model: T, opts?: WrapOptions): T
+  stats(): Stats
+  resetStats(): void
+  on(type: 'call', listener: (event: CallEvent) => void): this
+  on(type: 'advisory', listener: (advisory: Advisory) => void): this
+  off(type: 'call' | 'advisory', listener: (arg: any) => void): this
+  once(type: 'call', listener: (event: CallEvent) => void): this
+  once(type: 'advisory', listener: (advisory: Advisory) => void): this
+  flush(): Promise<void>
+  shutdown(): Promise<void>
+}
+
+export interface ClassifiedError {
+  category: FailureCategory
+  httpStatus: number | null
+  code: string | null
+  message: string
+  retryAfterMs: number | null
+}
+
+/** Process-wide optimizer, configured from cds.requires.tokenOptimizer. */
+export declare const optimizer: TokenOptimizer
+export declare function createOptimizer(config?: TokenOptimizerConfig, deps?: Record<string, unknown>): TokenOptimizer
+export declare function resolveConfig(config?: TokenOptimizerConfig | boolean, log?: { warn?: (msg: string) => void }): Required<TokenOptimizerConfig>
+export declare const DEFAULTS: Readonly<Required<TokenOptimizerConfig>>
+export declare function classifyError(err: unknown, opts?: { now?: number }): ClassifiedError
+export declare const CATEGORIES: readonly FailureCategory[]
+export declare function estimateMessages(messages: unknown[], opts?: { model?: string, tools?: unknown[] }): { tokens: number, method: 'tiktoken' | 'heuristic' }
+export declare function countText(text: string, model?: string): { tokens: number, method: 'tiktoken' | 'heuristic' }
+export declare function createConsoleExporter(opts?: { stream?: { write(s: string): unknown } }): Exporter

package/index.js

@@ -0,0 +1,36 @@
+'use strict'
+const { TokenOptimizer } = require('./lib/optimizer')
+const { resolveConfig, DEFAULTS } = require('./lib/config')
+const { classifyError, CATEGORIES } = require('./lib/metrics/classifier')
+const { estimateMessages, countText } = require('./lib/tokens/estimator')
+const { createConsoleExporter } = require('./lib/exporters/console')
+const { MemoryCacheStore } = require('./lib/cache')
+
+// One optimizer per process, even if two copies of this package get loaded
+// (e.g. a `file:` install next to a registry install).
+const KEY = Symbol.for('cap-ai-token-optimizer.singleton')
+globalThis[KEY] ??= new TokenOptimizer()
+
+/** @type {TokenOptimizer} */
+const optimizer = globalThis[KEY]
+
+/**
+ * Create a separate optimizer instance with its own config, stats and exporters.
+ * @param {object} [config]
+ * @param {object} [deps]
+ */
+const createOptimizer = (config = {}, deps) => new TokenOptimizer(config, deps)
+
+module.exports = {
+  optimizer,
+  createOptimizer,
+  TokenOptimizer,
+  resolveConfig,
+  DEFAULTS,
+  classifyError,
+  CATEGORIES,
+  estimateMessages,
+  countText,
+  createConsoleExporter,
+  MemoryCacheStore
+}

package/lib/cache/index.js

@@ -0,0 +1,67 @@
+'use strict'
+/**
+ * Exact-match response cache.
+ *
+ * CacheStore interface (sync or async; plug in Redis etc. via `cache.store` in code):
+ *   get(key) → value | undefined
+ *   set(key, value, ttlMs)
+ *   delete(key), clear()
+ * Values are plain JSON ({ sdk, data, headers }), so any store that can hold JSON works.
+ */
+const { sha256, stableStringify } = require('../util/hash')
+
+class MemoryCacheStore {
+  /** @param {{ maxEntries?: number, now?: () => number }} [opts] */
+  constructor ({ maxEntries = 1000, now = Date.now } = {}) {
+    this.maxEntries = maxEntries
+    this._now = now
+    this._map = new Map()
+  }
+
+  get (key) {
+    const entry = this._map.get(key)
+    if (!entry) return undefined
+    if (entry.expires <= this._now()) { this._map.delete(key); return undefined }
+    // refresh LRU position
+    this._map.delete(key)
+    this._map.set(key, entry)
+    return entry.value
+  }
+
+  set (key, value, ttlMs) {
+    if (!(this.maxEntries > 0) || !(ttlMs > 0)) return
+    this._map.delete(key)
+    this._map.set(key, { value, expires: this._now() + ttlMs })
+    while (this._map.size > this.maxEntries) this._map.delete(this._map.keys().next().value)
+  }
+
+  delete (key) { this._map.delete(key) }
+  clear () { this._map.clear() }
+  get size () { return this._map.size }
+}
+
+/**
+ * Cache key: hash of everything that determines the response.
+ * @param {object} material what the adapter will send (config + request, or deployment + request)
+ * @param {{ scope?: 'tenant'|'user'|'global', tenant?: string|null, userId?: string|null }} [opts]
+ */
+function cacheKey (material, { scope = 'tenant', tenant = null, userId = null } = {}) {
+  const scopeKey = scope === 'global' ? null : scope === 'user' ? { tenant, userId } : { tenant }
+  return sha256(stableStringify({ v: 1, scope: scopeKey, material }))
+}
+
+/**
+ * Whether a call may use the cache: enabled, non-streaming, and temperature at most
+ * `maxTemperature`. An unset temperature means the provider default (usually 1), so it is
+ * not cached unless `maxTemperature` is null.
+ * @param {object} cacheCfg
+ * @param {object} params effective model params
+ */
+function isCacheable (cacheCfg, params) {
+  if (!cacheCfg?.enabled) return false
+  if (cacheCfg.maxTemperature == null) return true
+  const t = params?.temperature
+  return typeof t === 'number' && t <= cacheCfg.maxTemperature
+}
+
+module.exports = { MemoryCacheStore, cacheKey, isCacheable }