@viveka10/cap-ai-token-optimizer 0.1.1 → 0.1.2

package/.cdsrc.js

@@ -0,0 +1,27 @@
+'use strict'
+/**
+ * Plugin configuration defaults, read by cds.env. cds.env is also rebuilt from these files
+ * after plugins are loaded (`cds build`, `cds deploy`), so anything cds-plugin.js wrote into
+ * cds.env would be lost there.
+ *
+ * `model` is therefore a getter: CAP's config merge keeps getters, and CAP reads `model`
+ * when it resolves the app's models, after the app's own configuration and profiles have been
+ * merged. Models the app sets explicitly are kept.
+ */
+const { modelsFor } = require('./lib/models')
+
+module.exports = {
+  requires: {
+    tokenOptimizer: {
+      kind: 'token-optimizer',
+      get model () {
+        const own = [].concat(this._model ?? [])
+        const models = [...new Set([...own, ...modelsFor(this)])]
+        return models.length ? models : undefined
+      },
+      set model (value) {
+        Object.defineProperty(this, '_model', { value, writable: true, configurable: true, enumerable: false })
+      }
+    }
+  }
+}

package/CHANGELOG.md

@@ -6,6 +6,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.1.2] - 2026-10-11
+
+### Fixed
+
+- **Token usage of streaming calls.** `stream()` calls (including LangChain models with
+  streaming, as used by `@cap-js/agents`) were recorded when the stream opened, without token
+  usage. They are now recorded when the stream ends, with AI Core's usage, finish reason, request
+  id and cost.
+- **Errors inside a stream** were not recorded, and the call counted as a success. They are now
+  recorded as errors and classified from the error event AI Core sends (for example `SERVER`,
+  `RATE_LIMIT`, `CONTENT_FILTER`).
+- **`cds build` and `cds deploy` left out the plugin's models.** The `TokenUsageLog` entity and
+  the admin service were added to `cds.env` when the plugin loaded, but `cds build` rebuilds
+  `cds.env` afterwards, so the MTA had no usage-log table and no admin service. The models now
+  come from the plugin's `.cdsrc.js` and are computed from the app's final configuration
+  (profiles and environment variables included). Apps that worked around this by listing the
+  models in `cds.requires.tokenOptimizer.model` keep working and can remove the list.
+- **Admin service for Fiori elements:** the in-memory entities (`UseCases`, `Models`,
+  `ErrorCategories`, `StepSavings`, `PromptPrefixChanges`, `Exporters`) answered `$count=true`
+  with 0, so list reports showed empty tables, and `UseCases` and `Models` returned `tokens` as a
+  nested object instead of the `tokens_*` columns of their OData metadata.
+
+### Changed
+
+- `latencyMs` of streaming calls now covers the whole answer instead of the time until the
+  stream opened. A stream closed early by the caller, or not read within 10 minutes, is recorded
+  at that point without usage and with a note in `savings.notes`.
+- The mock AI Core server answers streaming requests as server-sent events and has a
+  `stream_error` scenario.
+
 ## [0.1.1] - 2026-10-10
 
 ### Added
@@ -78,6 +108,7 @@ First public release.
 - The `http` exporter supports Internet destinations only.
 - Not yet verified against SAP Cloud ALM; live AI Core tests are opt-in.
 
-[Unreleased]: https://github.com/viveka10/npm_token_optimizer/compare/v0.1.1...HEAD
+[Unreleased]: https://github.com/viveka10/npm_token_optimizer/compare/v0.1.2...HEAD
+[0.1.2]: https://github.com/viveka10/npm_token_optimizer/compare/v0.1.1...v0.1.2
 [0.1.1]: https://github.com/viveka10/npm_token_optimizer/compare/v0.1.0...v0.1.1
 [0.1.0]: https://github.com/viveka10/npm_token_optimizer/releases/tag/v0.1.0

package/README.md

@@ -104,7 +104,10 @@ Wrap clients once (for example in your service's `init()`) and reuse them.
 
 - **Supported clients:** `OrchestrationClient` (`chatCompletion`, `stream`) and
   `AzureOpenAiChatClient` (`run`, `stream`). Other methods are forwarded untouched.
-- **Streaming:** `stream()` calls are passed through unoptimized and recorded without token usage.
+- **Streaming:** `stream()` calls are passed through unoptimized. They are recorded when the
+  stream has been read to the end, with AI Core's token usage, the finish reason and the latency
+  of the whole answer. A failure inside the stream is recorded as an error. A stream closed early,
+  or not read within 10 minutes, is recorded at that point without usage and with a note.
 - **Errors:** SDK errors are rethrown unchanged. Use `classifyError(err)` to get the failure category.
 
 ```js
@@ -141,7 +144,7 @@ This hooks the `@sap-ai-sdk` client classes `OrchestrationClient` (`chatCompleti
   In short:
   - only these two classes from SDK 2.x, and only the SDK copy resolved from the app root;
   - process-wide, including libraries;
-  - streaming is measured but not optimized.
+  - streaming is measured (with token usage) but not optimized.
 
 ### withUseCase
 
@@ -433,9 +436,10 @@ optimizer.configure({
 })
 ```
 
-The `db` exporter and the admin service also need their CDS models, which the plugin adds
-only when they are enabled in `cds.env` at startup. Enable them in the configuration, not
-only in code.
+The `db` exporter and the admin service also need their CDS models. The plugin adds them to
+the app's model (also in `cds build` and `cds deploy`) when they are enabled in the
+configuration, including profiles and environment variables. Enable them in the
+configuration, not only in code.
 
 `createOptimizer(config)` creates a separate instance with its own stats and exporters.
 
@@ -462,7 +466,8 @@ One event per call:
 
 - Prompt and completion text are never recorded by default; only a SHA-256 hash and the length.
   `user` is a salted hash of the user id.
-- `tokens.prompt`/`completion`/`total`/`cached` come from the AI Core response.
+- `tokens.prompt`/`completion`/`total`/`cached` come from the AI Core response, for streams
+  from the last chunk. Orchestration streams don't report cached tokens (`cached` is 0).
   `promptEstimatedBefore`/`After` are estimates before and after optimization (js-tiktoken for
   OpenAI model families, characters ÷ 4 for other models).
 - `savings.estimatedTokens` is estimated prompt tokens saved plus, for cache hits, the cached
@@ -554,7 +559,8 @@ Writes one row per call to the entity `cap.ai.token_optimizer.TokenUsageLog`
 (table `CAP_AI_TOKEN_OPTIMIZER_TOKENUSAGELOG`) through CAP's database service.
 
 - **Databases:** SAP HANA, SQLite or PostgreSQL. `hana` is an alias for `db`.
-- **Model:** the plugin adds the entity to your model when the exporter is configured.
+- **Model:** the plugin adds the entity to your model when the exporter is configured, also in
+  `cds build`, so it is part of the MTA's db deployer.
   Deploy as usual (`cds deploy`, or the MTA db deployer), and run your MTX upgrade for multitenant apps.
 - **Multitenancy:** rows are inserted in batches on the flush interval with a privileged
   user. Each tenant's rows go to its own container.
@@ -594,6 +600,9 @@ for users with the role `admin.role` (default `TokenOptimizerAdmin`):
 | `UsageLog` | the `TokenUsageLog` rows, if the `db` exporter is enabled (supports `$filter`, `$orderby`, `$top`, …) |
 
 All entries except `UsageLog` are in-memory stats of the app instance that answers, since it started.
+They support key lookups and `$count`, which is enough for a Fiori elements list report, but not
+`$filter`, `$orderby`, `$search` or paging. With flat OData (the default), `tokens` is exposed as
+`tokens_prompt`, `tokens_completion`, `tokens_total` and `tokens_cached`.
 
 To grant access, add the role to `xs-security.json` and a role collection:
 

package/cds-plugin.js

@@ -6,35 +6,21 @@
 const cds = globalThis.cds ?? require('@sap/cds')
 const { optimizer, countText } = require('./index')
 const log = require('./lib/util/log')
-const path = require('node:path')
 
 try {
   optimizer.configure(cds.env.requires?.tokenOptimizer)
   log.debug('configured, mode =', optimizer.config.mode, 'exporters =', optimizer.config.metrics.exporters)
-  setupModels(optimizer.config)
+  setupAdminRole(optimizer.config)
 } catch (e) {
   log.error('could not configure token optimizer:', e?.message)
 }
 
 /**
- * Add our CDS models to the app's model when the features using them are enabled:
- * the TokenUsageLog entity for the db exporter, the read-only admin service, and the
- * UsageLog projection when both are on. Plugins load before the model, so adding a `model`
- * to our cds.requires entry is enough.
+ * The CDS models (usage log entity, admin service) are added through `model` in .cdsrc.js, so
+ * they are also part of `cds build` and `cds deploy`. Here we only set the admin service's role.
  */
-function setupModels (cfg) {
-  const exporters = cfg.metrics.exporters ?? []
-  const dbLog = exporters.includes('db') || exporters.includes('hana')
-  const admin = cfg.admin?.enabled
-  const models = []
-  if (dbLog) models.push('db/token-usage-log')
-  if (admin) models.push('srv/admin-service')
-  if (admin && dbLog) models.push('srv/admin-usage-log')
-  if (!models.length) return
-
-  const req = cds.env.requires.tokenOptimizer ??= {}
-  req.model = [...new Set([].concat(req.model ?? [], models.map(m => path.join(__dirname, m))))]
-  if (!admin) return
+function setupAdminRole (cfg) {
+  if (!cfg.admin?.enabled) return
   const role = cfg.admin.role || 'TokenOptimizerAdmin'
   cds.on('loaded', (csn) => {
     const srv = csn?.definitions?.TokenOptimizerService

package/docs/existing-cap-app-on-btp.md

@@ -179,9 +179,10 @@ the `db` exporter:
 "[production]": { "metrics": { "exporters": ["console", "otel", "db"] }, "admin": { "enabled": true } }
 ```
 
-- The plugin adds the entity `cap.ai.token_optimizer.TokenUsageLog` to your model. It is
-  deployed with your normal HANA deployment (the `db-deployer` module in the MTA), so there's
-  no extra step.
+- The plugin adds the entity `cap.ai.token_optimizer.TokenUsageLog` to your model, also in
+  `cds build --production`. It is deployed with your normal HANA deployment (the `db-deployer`
+  module in the MTA), so there's no extra step. With version 0.1.1 the entity was missing from
+  `cds build`; there, list the models in `cds.requires.tokenOptimizer.model` as a workaround.
 - **Multitenant apps (MTX):** each tenant's rows go to that tenant's container. Run your
   usual tenant upgrade after deploying, so existing tenants get the new table.
 - Rows older than 30 days are deleted (`metrics.db.retentionDays`). No prompt or answer text

package/docs/integration-guide.md

@@ -130,7 +130,8 @@ for the same requests.
 - **Scope:** the hooks apply to every caller in the process, including libraries. If you want
   to choose exactly which calls are optimized, leave `autoInstrument` off and use `wrap()`,
   `withUseCase()` and `wrapLangChain()` instead.
-- **Streaming:** `stream()` calls are measured but not optimized, and have no token usage.
+- **Streaming:** `stream()` calls are measured, with token usage once the stream has been read
+  to the end, but not optimized.
 - **Use cases:** without `withUseCase()`, all hooked calls share the use case set in
   `autoInstrument.useCase` (default `default`), so per-use-case settings apply to all of them.
 - **Start-up:** the hooks are installed when the CAP plugin loads. Outside CAP, call

package/lib/metrics/classifier.js

@@ -7,6 +7,10 @@
  * (orchestration: `{ error: { code, message, location } }`, Azure OpenAI: `{ error: { code, message, innererror } }`).
  * Deployment-resolution and OAuth failures wrap further levels of `cause`. We therefore walk the
  * whole cause chain (and AggregateError.errors) and collect every signal before deciding.
+ *
+ * Errors inside a stream are thrown by the SDK's SSE parser as
+ * `Error("Error received from the server.\n<JSON error>")`, wrapped in
+ * `ErrorWithCause("Error while iterating over SSE stream.")`; the JSON is read like a response body.
  */
 
 const CATEGORIES = Object.freeze([
@@ -24,6 +28,7 @@ const RE_TIMEOUT = /timeout of \d+ms exceeded|timed? ?out|deadline exceeded/i
 const RE_DEPLOYMENT = /no deployment matched|deployment not found|DeploymentNotFound|has no deployment url/i
 const RE_AUTH = /unauthori[sz]ed|forbidden|invalid[_ ]client|could not fetch client credentials token/i
 const RE_NETWORK = /socket hang up|network error|fetch failed|getaddrinfo|connect ECONN/i
+const RE_SSE_ERROR = /^Error received from the server\.\n(\{[\s\S]*\})$/
 
 const TIMEOUT_CODES = new Set(['ECONNABORTED', 'ETIMEDOUT', 'ESOCKETTIMEDOUT', 'ERR_TIMEOUT', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT'])
 const NETWORK_CODES = new Set(['ECONNREFUSED', 'ECONNRESET', 'ENOTFOUND', 'EAI_AGAIN', 'EPIPE', 'EHOSTUNREACH', 'ENETUNREACH', 'ENETDOWN', 'ERR_NETWORK', 'UND_ERR_SOCKET'])
@@ -97,7 +102,7 @@ function collectSignals (root) {
     if (typeof e.message === 'string' && e.message) s.texts.push(e.message)
     s.retryAfter ??= header(e.response?.headers, 'retry-after') ?? header(e.headers, 'retry-after')
 
-    const body = parseBody(e.response?.data ?? e.body)
+    const body = parseBody(e.response?.data ?? e.body ?? sseErrorBody(e.message))
     const errors = providerErrors(body)
     // a non-JSON body (e.g. a gateway's "Payload Too Large") is the most useful message we have
     if (!errors.length && typeof body?.message === 'string' && body.message) {
@@ -132,6 +137,13 @@ function providerErrors (body) {
   return []
 }
 
+/** The `{ error }` body of an error event in a stream, from the SDK's error message. */
+function sseErrorBody (message) {
+  const json = typeof message === 'string' ? RE_SSE_ERROR.exec(message)?.[1] : null
+  if (!json) return undefined
+  try { return { error: JSON.parse(json) } } catch { return undefined }
+}
+
 function parseBody (data) {
   if (typeof data === 'string') {
     try { return JSON.parse(data) } catch { return { message: data } }

package/lib/models.js

@@ -0,0 +1,24 @@
+'use strict'
+/**
+ * The CDS models this plugin adds to the app, depending on its configuration: the
+ * TokenUsageLog entity for the db exporter, the read-only admin service, and the UsageLog
+ * projection when both are on.
+ */
+const { name } = require('../package.json')
+
+/**
+ * @param {object} [cfg] cds.requires.tokenOptimizer (raw, as in cds.env)
+ * @returns {string[]} model ids, resolved from the app root like `using from '<id>'`
+ */
+function modelsFor (cfg) {
+  const exporters = [].concat(cfg?.metrics?.exporters ?? [])
+  const dbLog = exporters.includes('db') || exporters.includes('hana')
+  const admin = cfg?.admin?.enabled === true
+  const models = []
+  if (dbLog) models.push(`${name}/db/token-usage-log`)
+  if (admin) models.push(`${name}/srv/admin-service`)
+  if (admin && dbLog) models.push(`${name}/srv/admin-usage-log`)
+  return models
+}
+
+module.exports = { modelsFor }

package/lib/optimizer.js

@@ -10,6 +10,10 @@
  *
  * Rule: nothing the optimizer does may break the AI call. If the pipeline fails, the original
  * request is sent. Only errors thrown by the SDK itself reach the caller, unchanged.
+ *
+ * Streaming calls are recorded when the caller has read the stream to the end, so the event
+ * has AI Core's token usage, finish reason and the full latency. A stream that fails, is
+ * closed early or is not read within STREAM_RECORD_TIMEOUT_MS is recorded at that point.
  */
 const { resolveConfig, forUseCase } = require('./config')
 const { MetricsCollector } = require('./metrics/collector')
@@ -30,6 +34,7 @@ const WRAPPED = Symbol.for('cap-ai-token-optimizer.wrapped')
 const LANGCHAIN_WRAPPED = Symbol.for('cap-ai-token-optimizer.langchain')
 const MAX_PREFIXES = 1000
 const ADVISORY_LOG_INTERVAL_MS = 10 * 60 * 1000
+const STREAM_RECORD_TIMEOUT_MS = 10 * 60 * 1000
 
 class TokenOptimizer {
   /**
@@ -40,6 +45,7 @@ class TokenOptimizer {
     this.collector = new MetricsCollector()
     this._deps = deps
     this._sleep = deps.sleep ?? ((ms) => new Promise(resolve => setTimeout(resolve, ms)))
+    this._streamRecordTimeoutMs = deps.streamRecordTimeoutMs ?? STREAM_RECORD_TIMEOUT_MS
     this._configured = false
     this.cache = null
     // last prompt prefix (system + tools) per use case and model, for the prompt-caching advisory
@@ -253,10 +259,11 @@ class TokenOptimizer {
         // the original method: a hooked one would route the call through an optimizer again
         const response = await instrument.originalOf(plan.client[methodName]).apply(plan.client, callArgs)
         if (ctx.cacheKey) this._cacheSet(ctx, response)
+        if (method.operation === 'stream' && safe(() => this._trackStream(ctx, { started, retries, response }), logRecordError)) return response
         safe(() => this._record(ctx, { started, retries, response }), logRecordError)
         return response
       } catch (err) {
-        const classified = safe(() => classifyError(err)) ?? { category: 'UNKNOWN', httpStatus: null, code: null, message: String(err?.message ?? err), retryAfterMs: null }
+        const classified = safe(() => classifyError(err)) ?? unknownError(err)
         const delay = this._retryDelay(cfg, classified, retries)
         if (delay != null) {
           retries++
@@ -270,6 +277,44 @@ class TokenOptimizer {
     }
   }
 
+  /**
+   * Record a streaming call when its stream ends: replaces `response.stream` with a stream of
+   * the same class that passes every chunk through unchanged and records the event once, when
+   * the stream is read to the end (with AI Core's usage), fails (the error is classified and
+   * rethrown), is closed early by the caller, or is not read within the record timeout.
+   * @returns {boolean} false if the stream can't be tracked; the caller then records right away
+   */
+  _trackStream (ctx, { started, retries, response }) {
+    const original = response?.stream
+    const Stream = original?.constructor
+    if (typeof original?.[Symbol.asyncIterator] !== 'function' || typeof Stream !== 'function' || Stream === Object) return false
+
+    let recorded = false
+    const record = (outcome = {}) => {
+      if (recorded) return
+      recorded = true
+      clearTimeout(timer)
+      safe(() => this._record(ctx, { started, retries, response, ...outcome }), logRecordError)
+    }
+    const timer = setTimeout(() => record({ note: 'stream: not read to the end within the record timeout' }), this._streamRecordTimeoutMs)
+    timer.unref?.()
+
+    async function * tracked () {
+      let ended = false
+      try {
+        for await (const chunk of original) yield chunk
+        ended = true
+      } catch (err) {
+        record({ error: safe(() => classifyError(err)) ?? unknownError(err) })
+        throw err
+      } finally {
+        record(ended ? {} : { note: 'stream: closed by the caller before the end' })
+      }
+    }
+    response.stream = new Stream(() => tracked(), original.controller)
+    return true
+  }
+
   /**
    * Everything known before the call: call info, estimates, pipeline result and cache key.
    * Never throws: a failure leaves the call unoptimized.
@@ -375,7 +420,7 @@ class TokenOptimizer {
     return Math.min(r.maxDelayMs, Math.round(backoff / 2 + Math.random() * backoff / 2))
   }
 
-  _record (ctx, { started, retries, response, error, cacheEntry }) {
+  _record (ctx, { started, retries, response, error, cacheEntry, note }) {
     const latencyMs = Math.round(performance.now() - started)
     const cfg = ctx.cfg
     const info = ctx.info ?? {}
@@ -398,6 +443,7 @@ class TokenOptimizer {
     const model = info.model ?? null
     const notes = [...(pipeline?.notes ?? [])]
     if (ctx.pipelineFailed) notes.push('pipeline failed; original request sent')
+    if (note) notes.push(note)
     const advisory = cacheEntry ? null : safe(() => this._checkPrefix(ctx))
 
     const event = buildEvent({
@@ -453,6 +499,8 @@ function prefixOf (messages, tools) {
 
 const logRecordError = (e) => log.warn('could not record call metrics:', e?.message)
 
+const unknownError = (err) => ({ category: 'UNKNOWN', httpStatus: null, code: null, message: String(err?.message ?? err), retryAfterMs: null })
+
 /**
  * Run `fn`, returning undefined instead of throwing.
  * @template T

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@viveka10/cap-ai-token-optimizer",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "SAP CAP plugin to reduce, measure and report SAP AI Core LLM token usage: prompt optimization, response cache, cost tracking, and metrics for SAP Cloud ALM / OpenTelemetry",
   "keywords": [
     "sap",
@@ -60,6 +60,7 @@
     "index.js",
     "index.d.ts",
     "cds-plugin.js",
+    ".cdsrc.js",
     "lib/",
     "srv/",
     "db/",

package/srv/admin-service.js

@@ -13,11 +13,30 @@ const totals = (t) => ({
   costEstimate: t.costEstimate
 })
 
-/** Rows for a READ, honouring a key lookup (e.g. UseCases('ask')). */
+/**
+ * OData exposes structured elements (`tokens`) as flat columns (`tokens_prompt`, …) unless the
+ * app uses structured OData (`odata.structs`, e.g. flavor x4). Return rows in the same shape.
+ */
+function shape (row) {
+  if (cds.env.effective?.odata?.structs || !row.tokens) return row
+  const { tokens, ...rest } = row
+  for (const [k, v] of Object.entries(tokens)) rest[`tokens_${k}`] = v
+  return rest
+}
+
+/**
+ * Rows for a READ, honouring a key lookup (e.g. UseCases('ask')). Lists carry `$count`, so
+ * `$count=true` (used by Fiori elements to size its tables) reports the number of rows.
+ */
 function rows (req, list, key) {
   const wanted = req.data?.[key]
-  if (wanted === undefined) return list
-  return list.find(r => r[key] === wanted) ?? null
+  if (wanted !== undefined) {
+    const row = list.find(r => r[key] === wanted)
+    return row ? shape(row) : null
+  }
+  const result = list.map(shape)
+  result.$count = result.length
+  return result
 }
 
 module.exports = class TokenOptimizerService extends cds.ApplicationService {