@viveka10/cap-ai-token-optimizer 0.0.0-stage → 0.1.0

package/lib/client/adapters.js

@@ -0,0 +1,263 @@
+'use strict'
+/**
+ * SDK adapters: read a call's messages, tools and params from each supported
+ * @sap-ai-sdk client method, and build the (optimized) call to send. Verified against
+ * @sap-ai-sdk/* 2.16.0:
+ *
+ *   OrchestrationClient.chatCompletion(request?, requestConfig?)
+ *   OrchestrationClient.stream(request?, signal?, options?, requestConfig?)
+ *   AzureOpenAiChatClient.run(request, requestConfig?)
+ *   AzureOpenAiChatClient.stream(request, signal?, requestConfig?)
+ *
+ * OrchestrationClient fixes model, params and tools at construction time and keeps them in
+ * its (TypeScript-private, runtime-visible) `config`, `deploymentConfig` and `destination`
+ * fields. When the pipeline changes tools or params, we construct a sibling client with a
+ * modified copy of the config, so the caller's client is never mutated and concurrent calls
+ * stay independent. The parsed config is accepted again by the constructor.
+ */
+const { ORIGIN, tag, untag } = require('../pipeline/messages')
+const { stableStringify, sha256 } = require('../util/hash')
+const { loadPeer } = require('../util/peer')
+
+/**
+ * @typedef {object} CallInfo
+ * @property {string|null} model
+ * @property {string|null} deploymentId
+ * @property {string|null} resourceGroup
+ * @property {Array<object>} messages all messages that reach the model, in order, tagged with their origin
+ * @property {Array<object>|undefined} tools
+ * @property {object} params model parameters (max_tokens, temperature, …)
+ * @property {{ tools: boolean, params: boolean }} editable whether tools / params can be changed per call
+ */
+
+const DERIVED_CLIENTS_PER_CLIENT = 32
+const derivedClients = new WeakMap()
+
+const sameJson = (a, b) => stableStringify(a) === stableStringify(b)
+const byOrigin = (messages, ...origins) => messages.filter(m => origins.includes(m[ORIGIN])).map(untag)
+
+const orchestration = {
+  sdk: 'orchestration',
+  /** method name → { operation, requestConfigIndex } */
+  methods: {
+    chatCompletion: { operation: 'chat', requestConfigIndex: 1 },
+    stream: { operation: 'stream', requestConfigIndex: 3 }
+  },
+  matches: (client) => typeof client?.chatCompletion === 'function',
+
+  /** @returns {CallInfo} */
+  extract (client, args) {
+    const request = args[0] ?? {}
+    const config = client.config
+    const module = firstModuleConfig(config)
+    const tpl = module?.promptTemplating
+    const template = Array.isArray(tpl?.prompt?.template) ? tpl.prompt.template : []
+    const dc = client.deploymentConfig ?? {}
+    // only a single inline module config of a real client class can be modified per call
+    const single = isInlineConfig(config) && canDerive(client)
+    return {
+      model: tpl?.model?.name ?? null,
+      deploymentId: dc.deploymentId ?? null,
+      resourceGroup: dc.resourceGroup ?? null,
+      // orchestration sends messages_history first, then template + request messages
+      messages: [
+        ...(request.messagesHistory ?? []).map(m => tag(m, 'history')),
+        ...template.map(m => tag(m, 'template')),
+        ...(request.messages ?? []).map(m => tag(m, 'request'))
+      ],
+      tools: tpl?.prompt?.tools,
+      params: tpl?.model?.params ?? {},
+      editable: { tools: single, params: single }
+    }
+  },
+
+  /**
+   * Build the call that sends `result` (the pipeline output).
+   * @returns {{ client: object, args: any[], keyMaterial: object }}
+   */
+  apply (client, args, info, result) {
+    const request = args[0] ?? {}
+    const nextRequest = { ...request }
+    const history = byOrigin(result.messages, 'history')
+    const messages = result.messages.filter(m => m[ORIGIN] !== 'history' && m[ORIGIN] !== 'template').map(untag)
+    if (history.length) nextRequest.messagesHistory = history
+    else delete nextRequest.messagesHistory
+    if (messages.length || request.messages) nextRequest.messages = messages
+
+    let target = client
+    let config = client.config
+    const toolsChanged = !sameJson(result.tools, info.tools)
+    const paramsChanged = !sameJson(result.params, info.params)
+    if ((toolsChanged || paramsChanged) && isInlineConfig(config) && canDerive(client)) {
+      const pt = config.promptTemplating
+      const prompt = { ...(pt.prompt ?? {}) }
+      if (result.tools?.length) prompt.tools = result.tools
+      else delete prompt.tools
+      config = { ...config, promptTemplating: { ...pt, model: { ...pt.model, params: result.params }, prompt } }
+      target = derivedClient(client, config)
+    }
+    return {
+      client: target,
+      args: [nextRequest, ...args.slice(1)],
+      keyMaterial: { sdk: 'orchestration', deployment: client.deploymentConfig ?? null, destination: destinationId(client.destination), config, request: nextRequest }
+    }
+  },
+
+  responseClass: () => loadPeer('@sap-ai-sdk/orchestration', 'the response cache').OrchestrationResponse
+}
+
+const foundationModels = {
+  sdk: 'foundation-models',
+  methods: {
+    run: { operation: 'chat', requestConfigIndex: 1 },
+    stream: { operation: 'stream', requestConfigIndex: 2 }
+  },
+  matches: (client) => typeof client?.run === 'function' && 'modelDeployment' in client,
+
+  /** @returns {CallInfo} */
+  extract (client, args) {
+    const request = args[0] ?? {}
+    const md = client.modelDeployment
+    const { messages, tools, ...params } = request
+    return {
+      model: typeof md === 'string' ? md : (md?.modelName ?? null),
+      deploymentId: typeof md === 'object' ? (md?.deploymentId ?? null) : null,
+      resourceGroup: typeof md === 'object' ? (md?.resourceGroup ?? null) : null,
+      messages: (messages ?? []).map(m => tag(m, 'request')),
+      tools,
+      params,
+      editable: { tools: true, params: true }
+    }
+  },
+
+  apply (client, args, info, result) {
+    const request = args[0] ?? {}
+    const { messages: _m, tools: _t, ...rest } = request
+    const nextRequest = { ...rest, ...result.params, messages: result.messages.map(untag) }
+    if (result.tools?.length || 'tools' in request) nextRequest.tools = result.tools
+    if (nextRequest.tools === undefined) delete nextRequest.tools
+    return {
+      client,
+      args: [nextRequest, ...args.slice(1)],
+      keyMaterial: { sdk: 'foundation-models', deployment: client.modelDeployment ?? null, destination: destinationId(client.destination), request: nextRequest }
+    }
+  },
+
+  responseClass: () => loadPeer('@sap-ai-sdk/foundation-models', 'the response cache').AzureOpenAiChatCompletionResponse
+}
+
+const ADAPTERS = [orchestration, foundationModels]
+
+/** @param {object} client */
+function detectAdapter (client) {
+  return ADAPTERS.find(a => a.matches(client)) ?? null
+}
+
+/**
+ * A sibling of `client` built with `config`. Reused per (client, config) so repeated calls
+ * with the same optimized config don't construct new clients.
+ */
+function derivedClient (client, config) {
+  let byConfig = derivedClients.get(client)
+  if (!byConfig) { byConfig = new Map(); derivedClients.set(client, byConfig) }
+  const key = sha256(stableStringify(config))
+  let derived = byConfig.get(key)
+  if (!derived) {
+    const Ctor = client.constructor
+    derived = new Ctor(config, client.deploymentConfig, client.destination)
+    byConfig.set(key, derived)
+    if (byConfig.size > DERIVED_CLIENTS_PER_CLIENT) byConfig.delete(byConfig.keys().next().value)
+  }
+  return derived
+}
+
+/** Identify a destination for the cache key without including credentials. */
+function destinationId (destination) {
+  if (!destination || typeof destination !== 'object') return null
+  return destination.destinationName ?? destination.name ?? destination.url ?? null
+}
+
+const canDerive = (client) => typeof client?.constructor === 'function' && client.constructor !== Object
+
+const isInlineConfig = (config) => !!config && typeof config === 'object' && !Array.isArray(config) && !!config.promptTemplating
+
+/**
+ * Return a copy of `args` whose requestConfig carries `timeout`, unless the caller set one.
+ * @param {any[]} args
+ * @param {number} index position of requestConfig
+ * @param {number|null} timeoutMs
+ */
+function withTimeout (args, index, timeoutMs) {
+  if (!(timeoutMs > 0)) return args
+  const current = args[index]
+  if (current?.timeout != null) return args
+  const next = [...args]
+  while (next.length < index) next.push(undefined)
+  next[index] = { ...(current ?? {}), timeout: timeoutMs }
+  return next
+}
+
+/**
+ * The orchestration config may be an object, a module-fallback list, a JSON string or a
+ * stored-config reference. Return the first inline module config, if any.
+ */
+function firstModuleConfig (config) {
+  if (Array.isArray(config)) return config[0]
+  if (config && typeof config === 'object' && config.promptTemplating) return config
+  if (typeof config === 'string') {
+    try {
+      const parsed = JSON.parse(config)
+      const modules = parsed?.config?.modules ?? parsed?.modules
+      const first = Array.isArray(modules) ? modules[0] : modules
+      const pt = first?.prompt_templating
+      return pt ? { promptTemplating: { model: pt.model, prompt: pt.prompt } } : null
+    } catch { return null }
+  }
+  return null
+}
+
+const CACHED_HEADERS = ['x-aicore-request-id']
+
+/**
+ * Plain-JSON form of an SDK response for the cache, or null if it has no data.
+ * @param {string} sdk
+ * @param {any} response
+ */
+function serializeResponse (sdk, response) {
+  const data = response?._data ?? response?.rawResponse?.data
+  if (!data || typeof data !== 'object') return null
+  const headers = {}
+  const raw = response.rawResponse?.headers ?? {}
+  for (const name of CACHED_HEADERS) if (raw[name] != null) headers[name] = String(raw[name])
+  return structuredClone({ sdk, data, headers })
+}
+
+const responseClasses = new Map()
+
+/** Remember the response class of a live response, so cached entries can be rebuilt. */
+function rememberResponseClass (sdk, response) {
+  if (response && typeof response.constructor === 'function' && response.constructor !== Object) {
+    responseClasses.set(sdk, response.constructor)
+  }
+}
+
+/**
+ * Rebuild an SDK response object from a cache entry. Each call gets its own copy.
+ * @param {{ sdk: string, data: object, headers: object }} entry
+ */
+function rehydrateResponse (entry) {
+  const adapter = ADAPTERS.find(a => a.sdk === entry.sdk)
+  const Cls = responseClasses.get(entry.sdk) ?? adapter.responseClass()
+  return new Cls({ data: structuredClone(entry.data), headers: { ...entry.headers }, status: 200 })
+}
+
+module.exports = {
+  orchestration,
+  foundationModels,
+  detectAdapter,
+  withTimeout,
+  firstModuleConfig,
+  serializeResponse,
+  rememberResponseClass,
+  rehydrateResponse
+}

package/lib/client/instrument.js

@@ -0,0 +1,124 @@
+'use strict'
+/**
+ * Hooks on the @sap-ai-sdk client classes, for automatic instrumentation and for clients
+ * the app does not create itself (e.g. inside @sap-ai-sdk/langchain).
+ *
+ * The prototype methods of OrchestrationClient (chatCompletion, stream) and
+ * AzureOpenAiChatClient (run, stream) are replaced by a hook that:
+ *  - routes the call through an optimizer when one is active for it: the optimizer of the
+ *    current scope (wrapLangChain(), withUseCase()), else the one with autoInstrument enabled;
+ *  - otherwise calls the original method unchanged.
+ * The original is kept on the hook (ORIGINAL), so the optimizer and explicitly wrapped
+ * clients call it directly and nothing is measured twice.
+ *
+ * Safety: hooks are only installed when needed, once per class, only for SDK major version 2,
+ * and failures to install are logged, never thrown. uninstall() restores the originals.
+ * Only the SDK copy resolved from the app root is hooked.
+ */
+const { AsyncLocalStorage } = require('node:async_hooks')
+const { orchestration, foundationModels } = require('./adapters')
+const { loadPeer, peerVersion } = require('../util/peer')
+const log = require('../util/log')
+
+const ORIGINAL = Symbol.for('cap-ai-token-optimizer.original')
+const scopes = new AsyncLocalStorage()
+const installed = [] // { proto, methodName, original }
+let autoTarget = null // { optimizer, opts }
+
+const TARGETS = [
+  { pkg: '@sap-ai-sdk/orchestration', className: 'OrchestrationClient', adapter: orchestration },
+  { pkg: '@sap-ai-sdk/foundation-models', className: 'AzureOpenAiChatClient', adapter: foundationModels }
+]
+
+function hook (adapter, methodName, method, original) {
+  const hooked = function (...args) {
+    const scope = scopes.getStore()
+    const target = scope ?? autoTarget
+    if (!target) return original.apply(this, args)
+    return target.optimizer._call({ adapter, method, methodName, client: this, args, opts: target.opts ?? {} })
+  }
+  hooked[ORIGINAL] = original
+  return hooked
+}
+
+/**
+ * Install the hooks (idempotent). Returns the names of the hooked classes.
+ * @param {{ load?: (pkg: string) => object, version?: (pkg: string) => string|null }} [deps]
+ */
+function install (deps = {}) {
+  const load = deps.load ?? ((pkg) => loadPeer(pkg))
+  const version = deps.version ?? peerVersion
+  const hooked = []
+  for (const t of TARGETS) {
+    let Cls
+    try {
+      Cls = load(t.pkg)?.[t.className]
+    } catch (e) {
+      if (e?.code !== 'PEER_DEPENDENCY_MISSING') log.warn(`could not load ${t.pkg} for instrumentation:`, e?.message)
+      continue
+    }
+    if (typeof Cls !== 'function') continue
+    const v = version(t.pkg)
+    if (v && !/^2\./.test(v)) {
+      log.warn(`${t.pkg}@${v} is not supported for instrumentation (supported: 2.x); its calls are not instrumented`)
+      continue
+    }
+    for (const [methodName, method] of Object.entries(t.adapter.methods)) {
+      const proto = Cls.prototype
+      const current = proto[methodName]
+      if (typeof current !== 'function' || current[ORIGINAL]) continue
+      try {
+        proto[methodName] = hook(t.adapter, methodName, method, current)
+        installed.push({ proto, methodName, original: current })
+      } catch (e) {
+        log.warn(`could not instrument ${t.className}.${methodName}:`, e?.message)
+      }
+    }
+    if (Cls.prototype[Object.keys(t.adapter.methods)[0]]?.[ORIGINAL]) hooked.push(t.className)
+  }
+  return hooked
+}
+
+/** Restore the original methods (mainly for tests). */
+function uninstall () {
+  for (const { proto, methodName, original } of installed.splice(0)) proto[methodName] = original
+  autoTarget = null
+}
+
+/**
+ * Route all SDK calls without a scope through `optimizer`.
+ * @param {object} optimizer
+ * @param {{ useCase?: string }} [opts]
+ * @param {object} [deps] see install()
+ */
+function enableAuto (optimizer, opts = {}, deps) {
+  if (autoTarget && autoTarget.optimizer !== optimizer) log.warn('autoInstrument moved to another optimizer instance')
+  autoTarget = { optimizer, opts }
+  const hooked = install(deps)
+  if (hooked.length) log.info(`autoInstrument: ${hooked.join(', ')} calls are optimized and measured`)
+  else log.warn('autoInstrument: no supported @sap-ai-sdk client found to instrument')
+  return hooked
+}
+
+/** Stop automatic routing for `optimizer` (hooks stay installed as pass-through). */
+function disableAuto (optimizer) {
+  if (autoTarget?.optimizer === optimizer) autoTarget = null
+}
+
+/**
+ * Run `fn` with SDK calls routed through `optimizer` with `opts`.
+ * @template T
+ * @param {object} optimizer
+ * @param {object} opts
+ * @param {() => T} fn
+ * @returns {T}
+ */
+function runInScope (optimizer, opts, fn) {
+  install()
+  return scopes.run({ optimizer, opts }, fn)
+}
+
+/** The original (unhooked) implementation of a possibly hooked method. */
+const originalOf = (fn) => fn?.[ORIGINAL] ?? fn
+
+module.exports = { install, uninstall, enableAuto, disableAuto, runInScope, originalOf, scopes, ORIGINAL }

package/lib/config.js

@@ -0,0 +1,128 @@
+'use strict'
+/**
+ * Configuration defaults and merge logic.
+ * Users configure the optimizer via `cds.requires.tokenOptimizer` (or programmatically).
+ */
+
+const MODES = ['active', 'shadow', 'off']
+
+/** @type {import('../index').TokenOptimizerConfig} */
+const DEFAULTS = Object.freeze({
+  mode: 'active',
+  // Each step: false (off), true (defaults) or an options object. See lib/pipeline/steps/*.
+  steps: {
+    normalize: true,
+    dedupe: true,
+    history: { maxTurns: 10 },
+    tools: { maxTools: 15, maxDescriptionLength: 600, pinned: [] },
+    outputCap: { default: 1024 }
+  },
+  // Exact-match response cache. Only temperature <= maxTemperature is cached (null: any).
+  // scope: 'tenant' (default), 'user' or 'global'. `store` may be set in code (CacheStore).
+  cache: { enabled: true, ttlSeconds: 3600, maxEntries: 1000, maxTemperature: 0.2, scope: 'tenant' },
+  metrics: {
+    exporters: ['console'],
+    flushIntervalMs: 15000,
+    // Salt for the user-id hash in events. Set it from the environment, not in committed files.
+    userHashSalt: '',
+    // POST batches of events to a URL or a BTP destination (see lib/exporters/http.js for all options)
+    http: { url: '', destination: '', headers: {}, batchSize: 100, maxQueueSize: 10000, maxRetries: 3, timeoutMs: 10000 },
+    // OpenTelemetry metrics and spans through the providers registered by @cap-js/telemetry
+    otel: { enabled: false, tenantAttribute: true },
+    // TokenUsageLog rows in the app database (exporter `db`, alias `hana`)
+    db: { batchSize: 50, maxQueueSize: 10000, retentionDays: 30 }
+  },
+  // Price per 1k tokens by model name. Empty by default — prices vary per contract.
+  pricing: {},
+  // Per-use-case overrides, e.g. { "invoice-agent": { steps: { outputCap: { default: 256 } } } }
+  useCases: {},
+  // Request timeout applied when the caller does not set one (ms). null = SDK default.
+  timeoutMs: null,
+  // Opt-in retries performed by the wrapper (the SDK itself does not retry).
+  retry: {
+    enabled: false,
+    maxRetries: 2,
+    baseDelayMs: 500,
+    maxDelayMs: 10000,
+    on: ['RATE_LIMIT', 'SERVER', 'NETWORK', 'TIMEOUT']
+  },
+  // promptCache: warn when the system prompt or tools sent for a use case and model change
+  // between calls, which defeats the provider's prompt caching
+  advisories: { promptCache: true },
+  // Hook the @sap-ai-sdk client classes so existing code needs no changes. `true` or
+  // { enabled, useCase }; useCase labels calls made outside withUseCase()/wrap().
+  autoInstrument: { enabled: false, useCase: 'default' },
+  // Read-only admin service at /odata/v4/token-optimizer, for users with `role`
+  admin: { enabled: false, role: 'TokenOptimizerAdmin' },
+  captureContent: false
+})
+
+// Only object literals merge; class instances (e.g. a custom cache store) are kept as they are.
+const isPlainObject = (v) => v !== null && typeof v === 'object' && [Object.prototype, null].includes(Object.getPrototypeOf(v))
+
+/**
+ * Deep merge. Objects merge recursively; arrays and scalars from `override` replace.
+ * `undefined` in override is ignored. Neither input is mutated.
+ * @template T
+ * @param {T} base
+ * @param {any} override
+ * @returns {T}
+ */
+function deepMerge (base, override) {
+  if (!isPlainObject(override)) return override === undefined ? clone(base) : clone(override)
+  const out = isPlainObject(base) ? clone(base) : {}
+  for (const [k, v] of Object.entries(override)) {
+    if (v === undefined) continue
+    out[k] = isPlainObject(v) && isPlainObject(out[k]) ? deepMerge(out[k], v) : clone(v)
+  }
+  return out
+}
+
+function clone (v) {
+  if (Array.isArray(v)) return v.map(clone)
+  if (isPlainObject(v)) return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, clone(x)]))
+  return v
+}
+
+/**
+ * Builds the effective configuration from user input on top of the defaults.
+ * Accepts `true` (use defaults) or an object. Invalid values fall back to defaults with a warning.
+ * @param {object|boolean|undefined} userConfig
+ * @param {{ warn?: (msg: string) => void }} [log]
+ */
+function resolveConfig (userConfig, log = {}) {
+  const input = isPlainObject(userConfig) ? stripCdsKeys(userConfig) : {}
+  const cfg = deepMerge(DEFAULTS, input)
+  if (!MODES.includes(cfg.mode)) {
+    log.warn?.(`Invalid tokenOptimizer.mode "${cfg.mode}", using "active". Allowed: ${MODES.join(', ')}`)
+    cfg.mode = 'active'
+  }
+  for (const [name, uc] of Object.entries(cfg.useCases ?? {})) {
+    if (uc?.mode !== undefined && !MODES.includes(uc.mode)) {
+      log.warn?.(`Invalid tokenOptimizer.useCases.${name}.mode "${uc.mode}", using "${cfg.mode}"`)
+      delete uc.mode
+    }
+  }
+  if (typeof cfg.autoInstrument === 'boolean') cfg.autoInstrument = { ...DEFAULTS.autoInstrument, enabled: cfg.autoInstrument }
+  if (typeof cfg.metrics.exporters === 'string') cfg.metrics.exporters = [cfg.metrics.exporters]
+  if (!Array.isArray(cfg.metrics.exporters)) cfg.metrics.exporters = [...DEFAULTS.metrics.exporters]
+  return cfg
+}
+
+/** Remove CAP-internal keys (kind, impl, model, …) that cds adds to `requires` entries. */
+function stripCdsKeys (obj) {
+  const { kind, impl, model, ...rest } = obj
+  return rest
+}
+
+/**
+ * Effective config for a use case: global config with `useCases[useCase]` merged on top.
+ * @param {object} cfg resolved config
+ * @param {string} [useCase]
+ */
+function forUseCase (cfg, useCase) {
+  const override = useCase && cfg.useCases?.[useCase]
+  return override ? deepMerge(cfg, override) : cfg
+}
+
+module.exports = { DEFAULTS, MODES, resolveConfig, deepMerge, forUseCase }

package/lib/exporters/console.js

@@ -0,0 +1,18 @@
+'use strict'
+/**
+ * Console exporter: writes one JSON line per event to stdout (or a given stream).
+ * @param {{ stream?: { write: (s: string) => any } }} [opts]
+ * @returns {import('../../index').Exporter}
+ */
+function createConsoleExporter ({ stream = process.stdout } = {}) {
+  return {
+    name: 'console',
+    export (event) {
+      stream.write(`${JSON.stringify({ type: 'ai-core-call', ...event })}\n`)
+    },
+    flush () {},
+    shutdown () {}
+  }
+}
+
+module.exports = { createConsoleExporter }