@viveka10/cap-ai-token-optimizer 0.0.0-stage → 0.1.0

package/lib/tokens/estimator.js

@@ -0,0 +1,163 @@
+'use strict'
+/**
+ * Pre-call token estimation.
+ * Uses js-tiktoken for OpenAI model families and a chars/4 heuristic for everything else
+ * (Anthropic, Gemini, Mistral, … tokenizers are not public / not bundled).
+ * Estimates are approximate; post-call metrics prefer real usage from the response.
+ */
+
+// Overhead per message / per reply, following OpenAI's published counting recipe.
+const TOKENS_PER_MESSAGE = 3
+const REPLY_PRIMING = 3
+// Rough flat cost for a non-text content part (e.g. an image) when estimating.
+const NON_TEXT_PART_TOKENS = 85
+
+const FAMILIES = [
+  // order matters: more specific prefixes first
+  { re: /^(gpt-4o|gpt-4\.1|gpt-4\.5|gpt-5|gpt-oss|o1|o3|o4|chatgpt-4o|gpt-realtime)/, encoding: 'o200k_base' },
+  { re: /^(gpt-4|gpt-3\.5|text-embedding-3|text-embedding-ada)/, encoding: 'cl100k_base' }
+]
+
+const encoders = new Map()
+let tiktoken // lazy: the rank tables are large
+
+/**
+ * Strip vendor prefixes used by AI Core model names, e.g. "azure-openai--gpt-4.1" or "openai/gpt-4.1".
+ * @param {string} model
+ */
+const baseModelName = (model) => String(model ?? '').toLowerCase().replace(/^.*(--|\/)/, '')
+
+/**
+ * @param {string} [model]
+ * @returns {string|null} tiktoken encoding name, or null when the heuristic should be used
+ */
+function encodingFor (model) {
+  const name = baseModelName(model)
+  return FAMILIES.find(f => f.re.test(name))?.encoding ?? null
+}
+
+function getEncoder (encoding) {
+  if (!encoders.has(encoding)) {
+    tiktoken ??= require('js-tiktoken')
+    encoders.set(encoding, tiktoken.getEncoding(encoding))
+  }
+  return encoders.get(encoding)
+}
+
+/** @param {string} text */
+const heuristic = (text) => Math.ceil(text.length / 4)
+
+/**
+ * Count tokens of a string.
+ * @param {string} text
+ * @param {string} [model]
+ * @returns {{ tokens: number, method: 'tiktoken'|'heuristic' }}
+ */
+function countText (text, model) {
+  const str = typeof text === 'string' ? text : String(text ?? '')
+  if (!str) return { tokens: 0, method: encodingFor(model) ? 'tiktoken' : 'heuristic' }
+  const encoding = encodingFor(model)
+  if (encoding) {
+    try {
+      return { tokens: getEncoder(encoding).encode(str).length, method: 'tiktoken' }
+    } catch { /* fall through to heuristic */ }
+  }
+  return { tokens: heuristic(str), method: 'heuristic' }
+}
+
+/**
+ * Flatten message content into text and count non-text parts.
+ * @param {any} content
+ * @returns {{ text: string, extraTokens: number }}
+ */
+function contentText (content) {
+  if (content == null) return { text: '', extraTokens: 0 }
+  if (typeof content === 'string') return { text: content, extraTokens: 0 }
+  if (Array.isArray(content)) {
+    let text = ''
+    let extraTokens = 0
+    for (const part of content) {
+      if (typeof part === 'string') text += part
+      else if (typeof part?.text === 'string') text += part.text
+      else extraTokens += NON_TEXT_PART_TOKENS
+    }
+    return { text, extraTokens }
+  }
+  return { text: JSON.stringify(content), extraTokens: 0 }
+}
+
+/**
+ * Tokens of one message, including the per-message overhead.
+ * @param {object} m
+ * @param {string} [model]
+ */
+function messageTokens (m, model) {
+  if (!m) return 0
+  const { text, extraTokens } = contentText(m.content)
+  const parts = [m.role ?? '', text]
+  if (m.name) parts.push(m.name)
+  if (m.tool_calls) parts.push(JSON.stringify(m.tool_calls))
+  if (m.tool_call_id) parts.push(m.tool_call_id)
+  return TOKENS_PER_MESSAGE + countText(parts.join('\n'), model).tokens + extraTokens
+}
+
+/**
+ * Tokens of one tool definition (serialised JSON; providers use a similar but not
+ * identical internal format).
+ * @param {object} tool
+ * @param {string} [model]
+ */
+const toolTokens = (tool, model) => countText(JSON.stringify(tool ?? null), model).tokens
+
+/**
+ * Estimate the prompt tokens of a chat request.
+ * @param {Array<object>} messages
+ * @param {{ model?: string, tools?: Array<object> }} [opts]
+ * @returns {{ tokens: number, method: 'tiktoken'|'heuristic' }}
+ */
+function estimateMessages (messages, { model, tools } = {}) {
+  return createMemoEstimator(model).estimate(messages, tools)
+}
+
+/**
+ * Estimate tokens consumed by tool definitions.
+ * @param {Array<object>} tools
+ * @param {string} [model]
+ */
+function estimateTools (tools, model) {
+  const method = encodingFor(model) ? 'tiktoken' : 'heuristic'
+  if (!Array.isArray(tools) || !tools.length) return { tokens: 0, method }
+  return { tokens: tools.reduce((n, t) => n + toolTokens(t, model), 0), method }
+}
+
+/**
+ * Estimator that remembers the count of every message and tool object it has seen.
+ * Pipeline steps keep unchanged messages by identity, so re-estimating after each
+ * step only counts what the step changed.
+ * @param {string} [model]
+ */
+function createMemoEstimator (model) {
+  const method = encodingFor(model) ? 'tiktoken' : 'heuristic'
+  const memo = new WeakMap()
+  const cached = (obj, fn) => {
+    if (obj === null || typeof obj !== 'object') return fn(obj, model)
+    if (!memo.has(obj)) memo.set(obj, fn(obj, model))
+    return memo.get(obj)
+  }
+  return {
+    /**
+     * @param {Array<object>} messages
+     * @param {Array<object>} [tools]
+     * @returns {{ tokens: number, method: 'tiktoken'|'heuristic' }}
+     */
+    estimate (messages, tools) {
+      let tokens = 0
+      for (const m of messages ?? []) tokens += cached(m, messageTokens)
+      if (tokens > 0) tokens += REPLY_PRIMING
+      if (Array.isArray(tools)) for (const t of tools) tokens += cached(t, toolTokens)
+      return { tokens, method }
+    }
+  }
+}
+
+module.exports = { countText, estimateMessages, estimateTools, createMemoEstimator, encodingFor, baseModelName }

package/lib/util/cds.js

@@ -0,0 +1,13 @@
+'use strict'
+/**
+ * Access the application's @sap/cds instance.
+ * @sap/cds sets `global.cds`; preferring it avoids picking up a second copy from this
+ * package's own node_modules (common with `file:` or `npm link` installs).
+ * Returns null when CAP is not loaded — the optimizer also works outside CAP.
+ * @returns {any|null}
+ */
+function getCds () {
+  return globalThis.cds ?? null
+}
+
+module.exports = { getCds }

package/lib/util/hash.js

@@ -0,0 +1,40 @@
+'use strict'
+const { createHash } = require('node:crypto')
+
+/** @param {string} text */
+const sha256 = (text) => createHash('sha256').update(text).digest('hex')
+
+/**
+ * Deterministic JSON serialisation (sorted object keys) so equal values hash equally.
+ * @param {any} value
+ * @returns {string}
+ */
+function stableStringify (value) {
+  if (value === undefined) return 'null'
+  if (value === null || typeof value !== 'object') return JSON.stringify(value)
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`
+  const keys = Object.keys(value).filter(k => value[k] !== undefined).sort()
+  return `{${keys.map(k => `${JSON.stringify(k)}:${stableStringify(value[k])}`).join(',')}}`
+}
+
+/**
+ * Pseudonymise a user id. Returns null for empty input.
+ * @param {string|undefined|null} userId
+ * @param {string} [salt]
+ */
+function hashUser (userId, salt = '') {
+  if (!userId) return null
+  return sha256(`${salt}:${userId}`).slice(0, 16)
+}
+
+/**
+ * Hash + length fingerprint of a prompt — what gets logged instead of the raw text.
+ * @param {Array<object>} messages
+ * @returns {{ promptHash: string, promptLength: number }}
+ */
+function fingerprint (messages) {
+  const text = stableStringify(messages ?? [])
+  return { promptHash: sha256(text), promptLength: text.length }
+}
+
+module.exports = { sha256, stableStringify, hashUser, fingerprint }

package/lib/util/log.js

@@ -0,0 +1,37 @@
+'use strict'
+/**
+ * Logger: uses cds.log('token-optimizer') when CAP is loaded, else console.
+ * Resolved lazily on first use. Never throws.
+ */
+const { getCds } = require('./cds')
+
+let _log
+
+function getLogger () {
+  if (_log) return _log
+  const cds = getCds()
+  if (cds?.log) _log = cds.log('token-optimizer')
+  // the console fallback is not cached, so cds.log is picked up once CAP has loaded
+  return _log ?? fallback
+}
+
+const fallback = {
+  debug: () => {},
+  info: (...a) => console.info('[token-optimizer]', ...a),
+  warn: (...a) => console.warn('[token-optimizer]', ...a),
+  error: (...a) => console.error('[token-optimizer]', ...a)
+}
+
+/** @param {'debug'|'info'|'warn'|'error'} level */
+const safe = (level) => (...args) => {
+  try { getLogger()[level](...args) } catch { /* logging must never break the caller */ }
+}
+
+module.exports = {
+  debug: safe('debug'),
+  info: safe('info'),
+  warn: safe('warn'),
+  error: safe('error'),
+  /** @internal test hook */
+  _set (logger) { _log = logger }
+}

package/lib/util/peer.js

@@ -0,0 +1,55 @@
+'use strict'
+const { getCds } = require('./cds')
+
+/**
+ * Load an optional peer dependency (e.g. @sap-ai-sdk/orchestration).
+ * Resolves from the application root first so the app's copy is used even when this package
+ * is installed via a symlink (`file:` / `npm link`), then falls back to normal resolution.
+ * @param {string} name
+ * @param {string} [purpose] shown in the error message
+ */
+function loadPeer (name, purpose) {
+  const roots = [getCds()?.root, process.cwd()].filter(Boolean)
+  for (const root of roots) {
+    try {
+      return require(require.resolve(name, { paths: [root] }))
+    } catch (e) {
+      if (e?.code !== 'MODULE_NOT_FOUND') throw e
+    }
+  }
+  try {
+    return require(name)
+  } catch (e) {
+    if (e?.code !== 'MODULE_NOT_FOUND') throw e
+    const err = new Error(`${purpose ? `${purpose} requires` : 'Missing'} peer dependency "${name}". Install it with: npm i ${name}`)
+    err.code = 'PEER_DEPENDENCY_MISSING'
+    throw err
+  }
+}
+
+/**
+ * Version of an installed peer, read from its package.json next to the resolved entry
+ * (works when package.json is not in the package's "exports"). Null if unknown.
+ * @param {string} name
+ */
+function peerVersion (name) {
+  const fs = require('node:fs')
+  const path = require('node:path')
+  const roots = [getCds()?.root, process.cwd()].filter(Boolean)
+  let entry = null
+  for (const root of roots) {
+    try { entry = require.resolve(name, { paths: [root] }); break } catch { /* next */ }
+  }
+  try { entry ??= require.resolve(name) } catch { return null }
+  for (let dir = path.dirname(entry); dir !== path.dirname(dir); dir = path.dirname(dir)) {
+    const file = path.join(dir, 'package.json')
+    if (!fs.existsSync(file)) continue
+    try {
+      const pkg = JSON.parse(fs.readFileSync(file, 'utf8'))
+      if (pkg.name === name) return pkg.version ?? null
+    } catch { return null }
+  }
+  return null
+}
+
+module.exports = { loadPeer, peerVersion }

package/package.json

@@ -1,6 +1,133 @@
 {
   "name": "@viveka10/cap-ai-token-optimizer",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Reduce, measure and report SAP AI Core token usage in SAP CAP Node.js apps: prompt optimization, response cache, metrics for SAP Cloud ALM / OpenTelemetry, HTTP and HANA",
+  "keywords": [
+    "sap",
+    "cap",
+    "cds",
+    "cds-plugin",
+    "sap-ai-core",
+    "sap-ai-sdk",
+    "generative-ai",
+    "llm",
+    "tokens",
+    "token-optimization",
+    "prompt",
+    "cache",
+    "opentelemetry",
+    "cloud-alm",
+    "mcp",
+    "agents",
+    "langchain",
+    "btp"
+  ],
+  "homepage": "https://github.com/viveka10/npm_token_optimizer/tree/main/cap-ai-token-optimizer#readme",
+  "bugs": {
+    "url": "https://github.com/viveka10/npm_token_optimizer/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/viveka10/npm_token_optimizer.git",
+    "directory": "cap-ai-token-optimizer"
+  },
+  "license": "MIT",
+  "author": "Vivekanandhan Srinivasan",
+  "main": "index.js",
+  "types": "index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "default": "./index.js"
+    },
+    "./cds-plugin": "./cds-plugin.js",
+    "./cds-plugin.js": "./cds-plugin.js",
+    "./db/*": "./db/*",
+    "./srv/*": "./srv/*",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "index.js",
+    "index.d.ts",
+    "cds-plugin.js",
+    "lib/",
+    "srv/",
+    "db/",
+    "docs/",
+    "CHANGELOG.md",
+    "SECURITY.md"
+  ],
+  "engines": {
+    "node": ">=22.12"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage",
+    "test:unit": "node --experimental-vm-modules node_modules/jest/bin/jest.js test/unit",
+    "test:integration": "node --experimental-vm-modules node_modules/jest/bin/jest.js test/integration",
+    "lint": "eslint .",
+    "mock:aicore": "node test/mock-aicore/server.js",
+    "bench": "node bench/run.js",
+    "test:live": "RUN_LIVE_TESTS=1 node --experimental-vm-modules node_modules/jest/bin/jest.js test/live",
+    "test:smoke": "node --test test/smoke/smoke.js",
+    "check:pack": "node scripts/check-pack.js",
+    "prepublishOnly": "npm run check:pack"
+  },
+  "cds": {
+    "requires": {
+      "tokenOptimizer": {
+        "kind": "token-optimizer"
+      }
+    }
+  },
+  "dependencies": {
+    "js-tiktoken": "^1.0.21"
+  },
+  "peerDependencies": {
+    "@sap/cds": "^9 || ^10",
+    "@sap-ai-sdk/orchestration": "^2.8.0",
+    "@sap-ai-sdk/foundation-models": "^2.8.0",
+    "@sap-ai-sdk/langchain": "^2.16.0",
+    "@sap-cloud-sdk/connectivity": "^4",
+    "@opentelemetry/api": "^1.4"
+  },
+  "peerDependenciesMeta": {
+    "@sap/cds": {
+      "optional": true
+    },
+    "@sap-ai-sdk/orchestration": {
+      "optional": true
+    },
+    "@sap-ai-sdk/foundation-models": {
+      "optional": true
+    },
+    "@sap-ai-sdk/langchain": {
+      "optional": true
+    },
+    "@sap-cloud-sdk/connectivity": {
+      "optional": true
+    },
+    "@opentelemetry/api": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.39.5",
+    "@langchain/core": "^1.2.12",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/context-async-hooks": "^2.11.0",
+    "@opentelemetry/sdk-metrics": "^2.11.0",
+    "@opentelemetry/sdk-trace-base": "^2.11.0",
+    "@sap-ai-sdk/foundation-models": "^2.16.0",
+    "@sap-ai-sdk/langchain": "^2.16.0",
+    "@sap-ai-sdk/orchestration": "^2.16.0",
+    "@sap/cds": "^10.1.1",
+    "eslint": "^9.39.5",
+    "express": "^5.3.0",
+    "globals": "^17.13.0",
+    "jest": "^30.5.2",
+    "langchain": "^1.5.12"
+  }
+}

package/srv/admin-service.cds

@@ -0,0 +1,83 @@
+/**
+ * Read-only statistics of @viveka10/cap-ai-token-optimizer for this app instance (in-memory, since
+ * start or the last reset). Served at /odata/v4/token-optimizer when
+ * cds.requires.tokenOptimizer.admin.enabled is true. The required role is
+ * cds.requires.tokenOptimizer.admin.role (default TokenOptimizerAdmin).
+ */
+@path    : 'token-optimizer'
+@requires: 'TokenOptimizerAdmin'
+@readonly
+service TokenOptimizerService {
+
+  type Tokens {
+    prompt     : Integer64;
+    completion : Integer64;
+    total      : Integer64;
+    cached     : Integer64;
+  }
+
+  type Latency {
+    count : Integer;
+    avg   : Integer;
+    p50   : Integer;
+    p95   : Integer;
+    max   : Integer;
+  }
+
+  aspect Totals {
+    calls               : Integer64;
+    success             : Integer64;
+    errors              : Integer64;
+    cacheHits           : Integer64;
+    tokens              : Tokens;
+    savedTokensEstimate : Integer64;
+    costEstimate        : Decimal(20, 6);
+  }
+
+  type Summary : Totals {
+    since          : Timestamp;
+    mode           : String;
+    latencyMs      : Latency;
+    listenerErrors : Integer;
+  }
+
+  /** Totals for this app instance. */
+  function summary() returns Summary;
+
+  @cds.persistence.skip
+  entity UseCases : Totals {
+    key name : String;
+  }
+
+  @cds.persistence.skip
+  entity Models : Totals {
+    key name : String;
+  }
+
+  @cds.persistence.skip
+  entity ErrorCategories {
+    key category : String;
+        count    : Integer64;
+  }
+
+  @cds.persistence.skip
+  entity StepSavings {
+    key step   : String;
+        tokens : Integer64;
+  }
+
+  /** Prompt-caching advisory: calls whose system prompt or tools changed, per use case. */
+  @cds.persistence.skip
+  entity PromptPrefixChanges {
+    key useCase : String;
+        count   : Integer64;
+  }
+
+  @cds.persistence.skip
+  entity Exporters {
+    key name   : String;
+        errors : Integer64;
+        /** JSON from the exporter's stats(), e.g. queued, sent, dropped for http */
+        stats  : LargeString;
+  }
+}

package/srv/admin-service.js

@@ -0,0 +1,41 @@
+'use strict'
+// Use the app's @sap/cds instance (see lib/util/cds.js)
+const cds = globalThis.cds ?? require('@sap/cds')
+const { optimizer } = require('../index')
+
+const totals = (t) => ({
+  calls: t.calls,
+  success: t.success,
+  errors: t.errors,
+  cacheHits: t.cacheHits,
+  tokens: { ...t.tokens },
+  savedTokensEstimate: t.savedTokensEstimate,
+  costEstimate: t.costEstimate
+})
+
+/** Rows for a READ, honouring a key lookup (e.g. UseCases('ask')). */
+function rows (req, list, key) {
+  const wanted = req.data?.[key]
+  if (wanted === undefined) return list
+  return list.find(r => r[key] === wanted) ?? null
+}
+
+module.exports = class TokenOptimizerService extends cds.ApplicationService {
+  init () {
+    this.on('summary', () => {
+      const s = optimizer.stats()
+      return { ...totals(s), since: s.since, mode: optimizer.config.mode, latencyMs: s.latencyMs, listenerErrors: s.listenerErrors }
+    })
+    this.on('READ', 'UseCases', (req) => rows(req, Object.entries(optimizer.stats().byUseCase).map(([name, t]) => ({ name, ...totals(t) })), 'name'))
+    this.on('READ', 'Models', (req) => rows(req, Object.entries(optimizer.stats().byModel).map(([name, t]) => ({ name, ...totals(t) })), 'name'))
+    this.on('READ', 'ErrorCategories', (req) => rows(req, Object.entries(optimizer.stats().errorsByCategory).map(([category, count]) => ({ category, count })), 'category'))
+    this.on('READ', 'StepSavings', (req) => rows(req, Object.entries(optimizer.stats().savingsByStep).map(([step, tokens]) => ({ step, tokens })), 'step'))
+    this.on('READ', 'PromptPrefixChanges', (req) => rows(req, Object.entries(optimizer.stats().promptPrefixChanges).map(([useCase, count]) => ({ useCase, count })), 'useCase'))
+    this.on('READ', 'Exporters', (req) => {
+      const s = optimizer.stats()
+      const names = optimizer.collector.exporters.map(x => x.name ?? 'anonymous')
+      return rows(req, names.map(name => ({ name, errors: s.exporterErrors[name] ?? 0, stats: s.exporters[name] ? JSON.stringify(s.exporters[name]) : null })), 'name')
+    })
+    return super.init()
+  }
+}

package/srv/admin-usage-log.cds

@@ -0,0 +1,8 @@
+using { cap.ai.token_optimizer as db } from '../db/token-usage-log';
+using from './admin-service';
+
+/** Added to the admin service when both the admin service and the db exporter are enabled. */
+extend service TokenOptimizerService with {
+  @readonly
+  entity UsageLog as projection on db.TokenUsageLog;
+}