@viveka10/cap-ai-token-optimizer 0.1.0 → 0.1.2

package/.cdsrc.js

@@ -0,0 +1,27 @@
+'use strict'
+/**
+ * Plugin configuration defaults, read by cds.env. cds.env is also rebuilt from these files
+ * after plugins are loaded (`cds build`, `cds deploy`), so anything cds-plugin.js wrote into
+ * cds.env would be lost there.
+ *
+ * `model` is therefore a getter: CAP's config merge keeps getters, and CAP reads `model`
+ * when it resolves the app's models, after the app's own configuration and profiles have been
+ * merged. Models the app sets explicitly are kept.
+ */
+const { modelsFor } = require('./lib/models')
+
+module.exports = {
+  requires: {
+    tokenOptimizer: {
+      kind: 'token-optimizer',
+      get model () {
+        const own = [].concat(this._model ?? [])
+        const models = [...new Set([...own, ...modelsFor(this)])]
+        return models.length ? models : undefined
+      },
+      set model (value) {
+        Object.defineProperty(this, '_model', { value, writable: true, configurable: true, enumerable: false })
+      }
+    }
+  }
+}

package/CHANGELOG.md

@@ -6,6 +6,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.1.2] - 2026-10-11
+
+### Fixed
+
+- **Token usage of streaming calls.** `stream()` calls (including LangChain models with
+  streaming, as used by `@cap-js/agents`) were recorded when the stream opened, without token
+  usage. They are now recorded when the stream ends, with AI Core's usage, finish reason, request
+  id and cost.
+- **Errors inside a stream** were not recorded, and the call counted as a success. They are now
+  recorded as errors and classified from the error event AI Core sends (for example `SERVER`,
+  `RATE_LIMIT`, `CONTENT_FILTER`).
+- **`cds build` and `cds deploy` left out the plugin's models.** The `TokenUsageLog` entity and
+  the admin service were added to `cds.env` when the plugin loaded, but `cds build` rebuilds
+  `cds.env` afterwards, so the MTA had no usage-log table and no admin service. The models now
+  come from the plugin's `.cdsrc.js` and are computed from the app's final configuration
+  (profiles and environment variables included). Apps that worked around this by listing the
+  models in `cds.requires.tokenOptimizer.model` keep working and can remove the list.
+- **Admin service for Fiori elements:** the in-memory entities (`UseCases`, `Models`,
+  `ErrorCategories`, `StepSavings`, `PromptPrefixChanges`, `Exporters`) answered `$count=true`
+  with 0, so list reports showed empty tables, and `UseCases` and `Models` returned `tokens` as a
+  nested object instead of the `tokens_*` columns of their OData metadata.
+
+### Changed
+
+- `latencyMs` of streaming calls now covers the whole answer instead of the time until the
+  stream opened. A stream closed early by the caller, or not read within 10 minutes, is recorded
+  at that point without usage and with a note in `savings.notes`.
+- The mock AI Core server answers streaming requests as server-sent events and has a
+  `stream_error` scenario.
+
+## [0.1.1] - 2026-10-10
+
+### Added
+
+- Guide "Add to an existing CAP app on SAP BTP" (`docs/existing-cap-app-on-btp.md`): AI Core
+  service binding, `mta.yaml`, secrets, HANA usage log, admin role, and SAP Cloud ALM setup.
+
+### Changed
+
+- Package description, keywords and README introduction, so the package is easier to find.
+
 ## [0.1.0] - 2026-10-10
 
 First public release.
@@ -67,5 +108,7 @@ First public release.
 - The `http` exporter supports Internet destinations only.
 - Not yet verified against SAP Cloud ALM; live AI Core tests are opt-in.
 
-[Unreleased]: https://github.com/viveka10/npm_token_optimizer/compare/v0.1.0...HEAD
+[Unreleased]: https://github.com/viveka10/npm_token_optimizer/compare/v0.1.2...HEAD
+[0.1.2]: https://github.com/viveka10/npm_token_optimizer/compare/v0.1.1...v0.1.2
+[0.1.1]: https://github.com/viveka10/npm_token_optimizer/compare/v0.1.0...v0.1.1
 [0.1.0]: https://github.com/viveka10/npm_token_optimizer/releases/tag/v0.1.0

package/README.md

@@ -5,7 +5,12 @@
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/viveka10/npm_token_optimizer/blob/main/cap-ai-token-optimizer/LICENSE)
 ![Node](https://img.shields.io/node/v/@viveka10/cap-ai-token-optimizer)
 
-Reduce, measure and report the tokens your SAP CAP Node.js app spends on SAP AI Core.
+**SAP CAP plugin** that reduces, measures and reports the LLM tokens your SAP CAP Node.js app
+spends on **SAP AI Core**, and with them your generative AI costs.
+
+```sh
+npm add @viveka10/cap-ai-token-optimizer
+```
 
 - **Optimize** prompts without changing what they mean: whitespace and JSON, duplicate
   instructions, long histories, large tool lists, output limits, and an exact-match response cache.
@@ -20,6 +25,8 @@ Node.js. Wraps the SAP Cloud SDK for AI (`@sap-ai-sdk/orchestration`, `@sap-ai-s
 > **Status: pre-release.** Not affiliated with or endorsed by SAP.
 
 **New here?** [Add it to an existing CAP app in 5 minutes](https://github.com/viveka10/npm_token_optimizer/blob/main/cap-ai-token-optimizer/docs/integration-guide.md): no code changes needed.
+Deploying on SAP BTP with an AI Core binding and SAP Cloud ALM? See
+[Existing CAP app on SAP BTP](https://github.com/viveka10/npm_token_optimizer/blob/main/cap-ai-token-optimizer/docs/existing-cap-app-on-btp.md).
 
 **Contents:** [Requirements](#requirements) · [Setup](#setup) · [Usage](#usage) ·
 [Existing code and LangChain](#instrumenting-existing-code-and-langchain) · [Compatibility](#compatibility) ·
@@ -97,7 +104,10 @@ Wrap clients once (for example in your service's `init()`) and reuse them.
 
 - **Supported clients:** `OrchestrationClient` (`chatCompletion`, `stream`) and
   `AzureOpenAiChatClient` (`run`, `stream`). Other methods are forwarded untouched.
-- **Streaming:** `stream()` calls are passed through unoptimized and recorded without token usage.
+- **Streaming:** `stream()` calls are passed through unoptimized. They are recorded when the
+  stream has been read to the end, with AI Core's token usage, the finish reason and the latency
+  of the whole answer. A failure inside the stream is recorded as an error. A stream closed early,
+  or not read within 10 minutes, is recorded at that point without usage and with a note.
 - **Errors:** SDK errors are rethrown unchanged. Use `classifyError(err)` to get the failure category.
 
 ```js
@@ -134,7 +144,7 @@ This hooks the `@sap-ai-sdk` client classes `OrchestrationClient` (`chatCompleti
   In short:
   - only these two classes from SDK 2.x, and only the SDK copy resolved from the app root;
   - process-wide, including libraries;
-  - streaming is measured but not optimized.
+  - streaming is measured (with token usage) but not optimized.
 
 ### withUseCase
 
@@ -426,9 +436,10 @@ optimizer.configure({
 })
 ```
 
-The `db` exporter and the admin service also need their CDS models, which the plugin adds
-only when they are enabled in `cds.env` at startup. Enable them in the configuration, not
-only in code.
+The `db` exporter and the admin service also need their CDS models. The plugin adds them to
+the app's model (also in `cds build` and `cds deploy`) when they are enabled in the
+configuration, including profiles and environment variables. Enable them in the
+configuration, not only in code.
 
 `createOptimizer(config)` creates a separate instance with its own stats and exporters.
 
@@ -455,7 +466,8 @@ One event per call:
 
 - Prompt and completion text are never recorded by default; only a SHA-256 hash and the length.
   `user` is a salted hash of the user id.
-- `tokens.prompt`/`completion`/`total`/`cached` come from the AI Core response.
+- `tokens.prompt`/`completion`/`total`/`cached` come from the AI Core response, for streams
+  from the last chunk. Orchestration streams don't report cached tokens (`cached` is 0).
   `promptEstimatedBefore`/`After` are estimates before and after optimization (js-tiktoken for
   OpenAI model families, characters ÷ 4 for other models).
 - `savings.estimatedTokens` is estimated prompt tokens saved plus, for cache hits, the cached
@@ -547,7 +559,8 @@ Writes one row per call to the entity `cap.ai.token_optimizer.TokenUsageLog`
 (table `CAP_AI_TOKEN_OPTIMIZER_TOKENUSAGELOG`) through CAP's database service.
 
 - **Databases:** SAP HANA, SQLite or PostgreSQL. `hana` is an alias for `db`.
-- **Model:** the plugin adds the entity to your model when the exporter is configured.
+- **Model:** the plugin adds the entity to your model when the exporter is configured, also in
+  `cds build`, so it is part of the MTA's db deployer.
   Deploy as usual (`cds deploy`, or the MTA db deployer), and run your MTX upgrade for multitenant apps.
 - **Multitenancy:** rows are inserted in batches on the flush interval with a privileged
   user. Each tenant's rows go to its own container.
@@ -587,6 +600,9 @@ for users with the role `admin.role` (default `TokenOptimizerAdmin`):
 | `UsageLog` | the `TokenUsageLog` rows, if the `db` exporter is enabled (supports `$filter`, `$orderby`, `$top`, …) |
 
 All entries except `UsageLog` are in-memory stats of the app instance that answers, since it started.
+They support key lookups and `$count`, which is enough for a Fiori elements list report, but not
+`$filter`, `$orderby`, `$search` or paging. With flat OData (the default), `tokens` is exposed as
+`tokens_prompt`, `tokens_completion`, `tokens_total` and `tokens_cached`.
 
 To grant access, add the role to `xs-security.json` and a role collection:
 

package/cds-plugin.js

@@ -6,35 +6,21 @@
 const cds = globalThis.cds ?? require('@sap/cds')
 const { optimizer, countText } = require('./index')
 const log = require('./lib/util/log')
-const path = require('node:path')
 
 try {
   optimizer.configure(cds.env.requires?.tokenOptimizer)
   log.debug('configured, mode =', optimizer.config.mode, 'exporters =', optimizer.config.metrics.exporters)
-  setupModels(optimizer.config)
+  setupAdminRole(optimizer.config)
 } catch (e) {
   log.error('could not configure token optimizer:', e?.message)
 }
 
 /**
- * Add our CDS models to the app's model when the features using them are enabled:
- * the TokenUsageLog entity for the db exporter, the read-only admin service, and the
- * UsageLog projection when both are on. Plugins load before the model, so adding a `model`
- * to our cds.requires entry is enough.
+ * The CDS models (usage log entity, admin service) are added through `model` in .cdsrc.js, so
+ * they are also part of `cds build` and `cds deploy`. Here we only set the admin service's role.
  */
-function setupModels (cfg) {
-  const exporters = cfg.metrics.exporters ?? []
-  const dbLog = exporters.includes('db') || exporters.includes('hana')
-  const admin = cfg.admin?.enabled
-  const models = []
-  if (dbLog) models.push('db/token-usage-log')
-  if (admin) models.push('srv/admin-service')
-  if (admin && dbLog) models.push('srv/admin-usage-log')
-  if (!models.length) return
-
-  const req = cds.env.requires.tokenOptimizer ??= {}
-  req.model = [...new Set([].concat(req.model ?? [], models.map(m => path.join(__dirname, m))))]
-  if (!admin) return
+function setupAdminRole (cfg) {
+  if (!cfg.admin?.enabled) return
   const role = cfg.admin.role || 'TokenOptimizerAdmin'
   cds.on('loaded', (csn) => {
     const srv = csn?.definitions?.TokenOptimizerService

package/docs/existing-cap-app-on-btp.md

@@ -0,0 +1,426 @@
+# Add @viveka10/cap-ai-token-optimizer to an existing CAP app on SAP BTP
+
+This guide is for a CAP Node.js app on SAP BTP Cloud Foundry that already calls SAP AI Core
+through the SAP Cloud SDK for AI (`@sap-ai-sdk/*`) with an **AI Core service binding**. It
+covers the changes to the app, deployment (`mta.yaml`, `xs-security.json`) and SAP Cloud ALM.
+
+**What stays the same**
+
+- **AI Core binding:** no change. The SDK keeps reading it from `VCAP_SERVICES`.
+- **AI code:** no change needed. Automatic instrumentation measures the existing calls.
+- **Prompts:** you start in `shadow` mode, which sends your original prompts unchanged.
+
+**Contents**
+
+1. [Check the prerequisites](#1-check-the-prerequisites)
+2. [Install the package](#2-install-the-package)
+3. [Configure it](#3-configure-it)
+4. [Code changes (optional)](#4-code-changes-optional)
+5. [Local testing with the AI Core binding](#5-local-testing-with-the-ai-core-binding)
+6. [Secrets and environment](#6-secrets-and-environment)
+7. [Usage log in SAP HANA (optional)](#7-usage-log-in-sap-hana-optional)
+8. [Admin service and roles (optional)](#8-admin-service-and-roles-optional)
+9. [SAP Cloud ALM](#9-sap-cloud-alm)
+10. [Deploy and verify](#10-deploy-and-verify)
+11. [Switch from shadow to active](#11-switch-from-shadow-to-active)
+12. [Turn it off or roll back](#12-turn-it-off-or-roll-back)
+
+---
+
+## 1. Check the prerequisites
+
+In the app's root folder:
+
+```sh
+node --version
+npm ls @sap/cds @sap-ai-sdk/orchestration @sap-ai-sdk/foundation-models @sap-ai-sdk/langchain
+```
+
+| Requirement | Why |
+|---|---|
+| Node.js **22.12 or later**, locally and on Cloud Foundry | The SAP Cloud SDK for AI is ESM-only; Node 22.12+ can load it from CommonJS. |
+| `"engines": { "node": ">=22.12" }` in the app's `package.json` | The Cloud Foundry Node.js buildpack picks the Node version from `engines`. |
+| `@sap/cds` 9 or 10 | Tested versions. |
+| `@sap-ai-sdk/*` **2.8 or later**, all packages on the **same** version, each listed **once** | The packages share internal APIs. Automatic instrumentation only hooks the copy resolved from the app root. If `npm ls` shows nested copies, run `npm dedupe` or align the versions. |
+
+## 2. Install the package
+
+```sh
+npm add @viveka10/cap-ai-token-optimizer
+```
+
+It is a CAP plugin, so there's nothing to register: CAP loads it at startup.
+
+## 3. Configure it
+
+Add the `tokenOptimizer` block to `cds.requires` in the app's `package.json` (or `.cdsrc.json`).
+This starting configuration measures everything without changing any AI call:
+
+```json
+{
+  "cds": {
+    "requires": {
+      "tokenOptimizer": {
+        "mode": "shadow",
+        "autoInstrument": true,
+        "pricing": {
+          "gpt-4.1": { "inputPer1k": 0.0, "outputPer1k": 0.0 }
+        },
+        "metrics": { "exporters": ["console"] },
+        "[production]": {
+          "metrics": { "exporters": ["console", "otel"] },
+          "admin": { "enabled": true }
+        }
+      }
+    }
+  }
+}
+```
+
+| Setting | What it does |
+|---|---|
+| `mode: "shadow"` | Runs the optimizer and records what it *would* save, but sends the original prompts. |
+| `autoInstrument: true` | Hooks the SDK's `OrchestrationClient` and `AzureOpenAiChatClient`, so existing calls are measured without code changes. |
+| `pricing` | Price per 1k tokens per model, from your SAP contract. Leave it out if you don't need cost estimates. |
+| `metrics.exporters` | Where events go. `console` writes one JSON line per AI call to the app log (`cf logs`). `otel` sends OpenTelemetry metrics and spans (see [SAP Cloud ALM](#9-sap-cloud-alm)). `db` writes a usage table (see [step 7](#7-usage-log-in-sap-hana-optional)). |
+| `admin.enabled` | Serves the read-only statistics service at `/odata/v4/token-optimizer` (see [step 8](#8-admin-service-and-roles-optional)). |
+
+All options are described in the [configuration reference](https://github.com/viveka10/npm_token_optimizer/blob/main/cap-ai-token-optimizer/README.md#configuration-reference).
+
+## 4. Code changes (optional)
+
+With `autoInstrument: true`, no code change is needed. All hooked calls are recorded under
+the use case `default`. These optional changes give you better reporting and per-use-case
+settings:
+
+**Name the use cases** where your handlers call AI Core:
+
+```js
+const { optimizer } = require('@viveka10/cap-ai-token-optimizer')
+
+this.on('summarizeTicket', req =>
+  optimizer.withUseCase('ticket-summary', () => this.summarize(req.data.ID)))
+```
+
+**LangChain models** from `@sap-ai-sdk/langchain`: wrap them once where you create them:
+
+```js
+const model = optimizer.wrapLangChain(new OrchestrationClient(config), { useCase: 'support-agent' })
+```
+
+**Better error responses** (optional): the SDK's errors are passed through unchanged, and
+`classifyError` tells you what happened:
+
+```js
+const { classifyError } = require('@viveka10/cap-ai-token-optimizer')
+try { /* AI call */ } catch (err) {
+  const { category } = classifyError(err)   // RATE_LIMIT, CONTENT_FILTER, CONTEXT_LENGTH, TIMEOUT, …
+  return req.reject(category === 'RATE_LIMIT' ? 429 : 502, `AI service error: ${category}`)
+}
+```
+
+Use cases can then get their own settings, for example
+`"useCases": { "ticket-summary": { "steps": { "outputCap": { "default": 300 } } } }`.
+
+## 5. Local testing with the AI Core binding
+
+Nothing changes in how the app gets its AI Core credentials. The SDK reads the `aicore` binding
+from `VCAP_SERVICES`. If you already test locally with `cds bind`, keep doing that:
+
+```sh
+cf login …
+cds bind aicore --to <your-aicore-instance>:<service-key>    # once; writes .cdsrc-private.json (keep it out of git)
+cds bind --exec -- cds watch --profile hybrid                # VCAP_SERVICES is set for the SDK
+```
+
+Call the feature that uses AI Core. The console shows one line per call:
+
+```json
+{"type":"ai-core-call","useCase":"default","mode":"shadow","model":"gpt-4.1",
+ "tokens":{"promptEstimatedBefore":2015,"promptEstimatedAfter":805,"prompt":2010,"completion":41,…},
+ "savings":{"estimatedTokens":1210,"byStep":{"history":774,"dedupe":340,…},…},"status":"success",…}
+```
+
+`savings` shows what `active` mode would have saved on that call.
+
+## 6. Secrets and environment
+
+**User-id hash salt.** User ids are recorded as a hash. Set a random salt per landscape, but
+**not** in `package.json` or `mta.yaml` if they are in git. Use an MTA extension descriptor that
+you keep out of git:
+
+```yaml
+# secrets.mtaext   (do not commit)
+_schema-version: '3.1'
+ID: my-cap-app.secrets
+extends: my-cap-app
+modules:
+  - name: my-cap-app-srv
+    properties:
+      cds_requires_tokenOptimizer_metrics_userHashSalt: <long random string>
+```
+
+```sh
+cf deploy mta_archives/my-cap-app_1.0.0.mtar -e secrets.mtaext
+```
+
+**Overriding settings without a rebuild.** Any setting can be overridden with an environment
+variable in the same format, for example `cds_requires_tokenOptimizer_mode=off`, followed by
+`cf restart`.
+
+Never set `captureContent: true` in production. It adds prompt and answer text to events.
+
+## 7. Usage log in SAP HANA (optional)
+
+To keep one row per AI call in the app's database (for reporting or your own analytics), add
+the `db` exporter:
+
+```json
+"[production]": { "metrics": { "exporters": ["console", "otel", "db"] }, "admin": { "enabled": true } }
+```
+
+- The plugin adds the entity `cap.ai.token_optimizer.TokenUsageLog` to your model, also in
+  `cds build --production`. It is deployed with your normal HANA deployment (the `db-deployer`
+  module in the MTA), so there's no extra step. With version 0.1.1 the entity was missing from
+  `cds build`; there, list the models in `cds.requires.tokenOptimizer.model` as a workaround.
+- **Multitenant apps (MTX):** each tenant's rows go to that tenant's container. Run your
+  usual tenant upgrade after deploying, so existing tenants get the new table.
+- Rows older than 30 days are deleted (`metrics.db.retentionDays`). No prompt or answer text
+  is stored.
+- With the admin service enabled, the rows are readable at `/odata/v4/token-optimizer/UsageLog`.
+- To expose the table in your own service:
+  ```cds
+  using { cap.ai.token_optimizer.TokenUsageLog } from '@viveka10/cap-ai-token-optimizer/db/token-usage-log';
+  service ReportingService { @readonly entity AiUsage as projection on TokenUsageLog; }
+  ```
+
+> The usage log has been tested with SQLite. It hasn't been tested on SAP HANA or with MTX
+> yet, so check it in a development space first.
+
+## 8. Admin service and roles (optional)
+
+With `admin.enabled`, the app serves `/odata/v4/token-optimizer`:
+
+| Path | Content |
+|---|---|
+| `summary()` | totals, mode, latency percentiles |
+| `UseCases`, `Models`, `ErrorCategories`, `StepSavings`, `PromptPrefixChanges`, `Exporters` | statistics of the app instance that answers (in memory, since its start) |
+| `UsageLog` | the usage table, when the `db` exporter is on |
+
+It requires the role `TokenOptimizerAdmin` (configurable with `admin.role`). Add it to
+`xs-security.json`:
+
+```json
+{
+  "scopes": [
+    { "name": "$XSAPPNAME.TokenOptimizerAdmin", "description": "Read AI token usage statistics" }
+  ],
+  "role-templates": [
+    { "name": "TokenOptimizerAdmin", "scope-references": ["$XSAPPNAME.TokenOptimizerAdmin"] }
+  ],
+  "role-collections": [
+    { "name": "AI_Token_Optimizer_Admin", "role-template-references": ["$XSAPPNAME.TokenOptimizerAdmin"] }
+  ]
+}
+```
+
+Merge this into your existing file; don't replace it. After deploying, assign the role
+collection to the right users in the BTP cockpit (Security → Role Collections).
+
+## 9. SAP Cloud ALM
+
+The optimizer has no Cloud ALM-specific code. Its `otel` exporter writes OpenTelemetry metrics
+and one span per AI call through whatever OpenTelemetry setup the app has. For Cloud ALM, that
+setup is SAP's Cloud ALM agent plus `@cap-js/telemetry`. The agent setup below follows SAP's
+documentation:
+[Data Collection Infrastructure](https://support.sap.com/en/alm/sap-cloud-alm/operations/expert-portal/data-collection-infrastructure.html) and
+[Auto-instrumentation for Node.js](https://support.sap.com/en/alm/sap-cloud-alm/operations/expert-portal/data-collection-infrastructure/autoinstrumentation-nodejs.html).
+Check those pages for changes before you start.
+
+### 9.1 Prerequisites
+
+- An SAP Cloud ALM tenant, with the **SAP Cloud ALM API** enabled and a service key for it
+  ([Enabling SAP Cloud ALM API](https://help.sap.com/docs/cloud-alm/setup-administration/enabling-sap-cloud-alm-api)).
+- A technical user for SAP's **Repository Based Shipment Channel** (RBSC), which distributes
+  the agent. Request it at https://ui.repositories.cloud.sap/www/webapp/users. The agent isn't
+  on the public npm registry.
+
+### 9.2 Get access to SAP's package repository
+
+Set `SAP_NPM_AUTH` to the "NPM Base64 Credentials" of the technical user (`base64(user:password)`),
+in your shell and in your CI/build pipeline (as a secret), and add to the app's `.npmrc`:
+
+```
+//73555000100200018064.npmsrv.cdn.repositories.cloud.sap/:_auth=${SAP_NPM_AUTH}
+```
+
+### 9.3 Add the agent and @cap-js/telemetry
+
+In the app's `package.json`:
+
+```json
+{
+  "dependencies": {
+    "@sap/xotel-agent-ext-js": "https://73555000100200018064.npmsrv.cdn.repositories.cloud.sap/@sap/xotel-agent-ext-js/-/xotel-agent-ext-js-2.0.4.tgz",
+    "@cap-js/telemetry": "^2.1.0"
+  },
+  "scripts": {
+    "start": "node ${NODE_ARGS} ./node_modules/.bin/cds-serve"
+  }
+}
+```
+
+- **Agent version:** 2.0.4 or later. That is the version from which `@cap-js/telemetry`
+  supports it, and it uses OpenTelemetry 2.x.
+- **`@cap-js/telemetry` version:** 2.1.0 or later. It detects the agent and adds its span
+  processor, metric reader and log processor to the agent's pipeline instead of starting a
+  second OpenTelemetry SDK. Version 2.0.x doesn't have this integration.
+- **Start script:** loads the agent before CAP via `NODE_ARGS` (set in `mta.yaml` below).
+  After `cds build`, check that `gen/srv/package.json` contains this start script.
+- **Dependency conflicts:** SAP notes that the agent's OpenTelemetry 2.x libraries can
+  conflict with other OpenTelemetry packages in the app. `@viveka10/cap-ai-token-optimizer`
+  itself only uses `@opentelemetry/api`.
+
+### 9.4 Enable the otel exporter
+
+```json
+"tokenOptimizer": { "[production]": { "metrics": { "exporters": ["console", "otel"] } } }
+```
+
+### 9.5 mta.yaml
+
+```yaml
+modules:
+  - name: my-cap-app-srv
+    type: nodejs
+    path: gen/srv
+    properties:
+      SAP_CALM_SERVICE_NAME: my-cap-app-srv     # name shown in Cloud ALM
+      SAP_CALM_SERVICE_TYPE: SAP_CP_CF
+      NODE_ARGS: -r @sap/xotel-agent-ext-js
+    requires:
+      - name: my-cap-app-aicore                 # existing AI Core binding, unchanged
+      - name: my-cap-app-auth                   # existing XSUAA
+      - name: my-cap-app-db                     # existing, only needed for the db exporter
+      - name: my-cap-app-destination            # new: used by the Cloud ALM agent
+
+resources:
+  - name: my-cap-app-destination
+    type: org.cloudfoundry.managed-service
+    parameters:
+      service: destination
+      service-plan: lite
+```
+
+**Staging needs the agent's repository.** Cloud Foundry installs dependencies during staging,
+so it has to reach SAP's repository too. SAP suggests, if staging fails, shipping
+`node_modules` and `package-lock.json` inside the MTA archive: remove them from the module's
+`build-parameters.ignore` list, and install them into `gen/srv` with a `before-all` custom
+build step (`npm install --production` and `npm install --prefix ./gen/srv`).
+
+### 9.6 Destination for Cloud ALM
+
+In the **provider subaccount** (BTP cockpit → Connectivity → Destinations), create:
+
+| Field | Value |
+|---|---|
+| Name | `CALM_datacollector` (exactly this name) |
+| Type | HTTP |
+| URL | the `Api` URL from the Cloud ALM API service key, e.g. `https://eu10.alm.cloud.sap/api` |
+| Proxy Type | Internet |
+| Authentication | OAuth2ClientCredentials |
+| Client ID / Client Secret | from the Cloud ALM API service key |
+| Token Service URL | the `url` from the service key |
+
+SAP notes that "Check Connection" can return 404 even when the destination is correct.
+
+### 9.7 Activate monitoring in Cloud ALM
+
+In SAP Cloud ALM, activate the monitoring use cases you need for the service (for example
+Health Monitoring and Exception Monitoring). Optional agent settings and how to turn the agent
+off (`SAP_CALM_INSTRUMENTATION_ENABLED=false`) are on SAP's auto-instrumentation page.
+
+### 9.8 What you'll see, and what isn't verified
+
+- **App log after deploying:** a line from `@cap-js/telemetry` saying it found
+  `@sap/xotel-agent-ext-js` and added its span processor as a delegate. The optimizer logs a
+  warning if no OpenTelemetry metrics provider is registered.
+- **Spans:** the optimizer's `chat <model>` spans are children of the CAP request spans and
+  carry token usage, savings and error category, never prompt text. They go through the same
+  pipeline as the CAP spans.
+- **Not documented by SAP:** whether Cloud ALM shows **custom** OpenTelemetry metrics (like
+  `gen_ai.client.token.usage` or `cap_ai_optimizer.tokens.saved`) in its UIs. Its documented
+  use cases are Health, Exception, Job & Automation, Business Process and Real User Monitoring.
+- **Not verified:** this setup hasn't been tested against a Cloud ALM tenant.
+
+If you need token dashboards, use the `db` exporter with the admin service or your own
+reporting service, or send the `otel` data to SAP Cloud Logging or Dynatrace via
+`@cap-js/telemetry` (`cds.requires.telemetry.kind` = `to-cloud-logging` / `to-dynatrace`).
+Note that with the agent present, `@cap-js/telemetry` also exports through its own
+configured kind (console by default), so check the log volume after deploying.
+
+## 10. Deploy and verify
+
+```sh
+npm install                 # updates package-lock.json
+mbt build
+cf deploy mta_archives/<your-app>_<version>.mtar -e secrets.mtaext
+```
+
+Then check:
+
+1. **Logs:** `cf logs my-cap-app-srv --recent` shows
+   `autoInstrument: OrchestrationClient … calls are optimized and measured` at startup, and one
+   `"type":"ai-core-call"` line per AI call.
+2. **Admin service:** with the `TokenOptimizerAdmin` role, open
+   `https://<app-url>/odata/v4/token-optimizer/summary()`.
+3. **Cloud ALM:** the service appears under `SAP_CALM_SERVICE_NAME` in the activated
+   monitoring apps.
+4. **Behavior:** your AI features answer exactly as before (shadow mode sends the original prompts).
+
+## 11. Switch from shadow to active
+
+After a few days in shadow mode, compare `savings.estimatedTokens` and the notes in the events
+or the admin service per use case. Then switch one use case at a time:
+
+```json
+"tokenOptimizer": {
+  "mode": "shadow",
+  "useCases": {
+    "ticket-summary": { "mode": "active" }
+  }
+}
+```
+
+In `active` mode the optimized prompts are sent, and identical calls with `temperature` ≤ 0.2
+are answered from the cache, which is per app instance and per tenant. Check answer quality
+for that use case before switching the next one. If an agent needs more context or tools,
+raise `steps.history.maxTurns` or `steps.tools.maxTools`, or pin the tools it needs, for that
+use case.
+
+## 12. Turn it off or roll back
+
+| Goal | How |
+|---|---|
+| Stop optimizing, keep measuring | `cds_requires_tokenOptimizer_mode=off`, then `cf restart` |
+| Stop hooking the SDK | `cds_requires_tokenOptimizer_autoInstrument=false`, then `cf restart` |
+| Turn off the Cloud ALM agent | `SAP_CALM_INSTRUMENTATION_ENABLED=false`, then `cf restart` |
+| Remove the package | `npm rm @viveka10/cap-ai-token-optimizer`, delete the `tokenOptimizer` block, redeploy. If you used the `db` exporter, the `TokenUsageLog` table stays until you drop it. |
+
+## Checklist
+
+- [ ] Node ≥ 22.12 and `engines` set; `@sap-ai-sdk/*` ≥ 2.8, same version, no duplicates
+- [ ] `npm add @viveka10/cap-ai-token-optimizer`
+- [ ] `tokenOptimizer` block with `mode: "shadow"` and `autoInstrument: true`; `pricing` filled in
+- [ ] Salt and other secrets in an `.mtaext` that is not committed
+- [ ] Optional: `withUseCase` / `wrapLangChain` for named use cases
+- [ ] Optional: `db` exporter (HANA table deployed with the app; MTX upgrade)
+- [ ] Optional: admin service plus `TokenOptimizerAdmin` role collection
+- [ ] Cloud ALM:
+  - [ ] RBSC user and `SAP_NPM_AUTH`
+  - [ ] agent ≥ 2.0.4 and `@cap-js/telemetry` ≥ 2.1.0
+  - [ ] `NODE_ARGS`, `SAP_CALM_*` properties and destination service in `mta.yaml`
+  - [ ] `CALM_datacollector` destination
+  - [ ] monitoring use cases activated in Cloud ALM
+  - [ ] `otel` exporter enabled
+- [ ] Deploy, check logs and the admin service, then switch use cases to `active` one by one

package/docs/integration-guide.md

@@ -130,7 +130,8 @@ for the same requests.
 - **Scope:** the hooks apply to every caller in the process, including libraries. If you want
   to choose exactly which calls are optimized, leave `autoInstrument` off and use `wrap()`,
   `withUseCase()` and `wrapLangChain()` instead.
-- **Streaming:** `stream()` calls are measured but not optimized, and have no token usage.
+- **Streaming:** `stream()` calls are measured, with token usage once the stream has been read
+  to the end, but not optimized.
 - **Use cases:** without `withUseCase()`, all hooked calls share the use case set in
   `autoInstrument.useCase` (default `default`), so per-use-case settings apply to all of them.
 - **Start-up:** the hooks are installed when the CAP plugin loads. Outside CAP, call

package/lib/metrics/classifier.js

@@ -7,6 +7,10 @@
  * (orchestration: `{ error: { code, message, location } }`, Azure OpenAI: `{ error: { code, message, innererror } }`).
  * Deployment-resolution and OAuth failures wrap further levels of `cause`. We therefore walk the
  * whole cause chain (and AggregateError.errors) and collect every signal before deciding.
+ *
+ * Errors inside a stream are thrown by the SDK's SSE parser as
+ * `Error("Error received from the server.\n<JSON error>")`, wrapped in
+ * `ErrorWithCause("Error while iterating over SSE stream.")`; the JSON is read like a response body.
  */
 
 const CATEGORIES = Object.freeze([
@@ -24,6 +28,7 @@ const RE_TIMEOUT = /timeout of \d+ms exceeded|timed? ?out|deadline exceeded/i
 const RE_DEPLOYMENT = /no deployment matched|deployment not found|DeploymentNotFound|has no deployment url/i
 const RE_AUTH = /unauthori[sz]ed|forbidden|invalid[_ ]client|could not fetch client credentials token/i
 const RE_NETWORK = /socket hang up|network error|fetch failed|getaddrinfo|connect ECONN/i
+const RE_SSE_ERROR = /^Error received from the server\.\n(\{[\s\S]*\})$/
 
 const TIMEOUT_CODES = new Set(['ECONNABORTED', 'ETIMEDOUT', 'ESOCKETTIMEDOUT', 'ERR_TIMEOUT', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT'])
 const NETWORK_CODES = new Set(['ECONNREFUSED', 'ECONNRESET', 'ENOTFOUND', 'EAI_AGAIN', 'EPIPE', 'EHOSTUNREACH', 'ENETUNREACH', 'ENETDOWN', 'ERR_NETWORK', 'UND_ERR_SOCKET'])
@@ -97,7 +102,7 @@ function collectSignals (root) {
     if (typeof e.message === 'string' && e.message) s.texts.push(e.message)
     s.retryAfter ??= header(e.response?.headers, 'retry-after') ?? header(e.headers, 'retry-after')
 
-    const body = parseBody(e.response?.data ?? e.body)
+    const body = parseBody(e.response?.data ?? e.body ?? sseErrorBody(e.message))
     const errors = providerErrors(body)
     // a non-JSON body (e.g. a gateway's "Payload Too Large") is the most useful message we have
     if (!errors.length && typeof body?.message === 'string' && body.message) {
@@ -132,6 +137,13 @@ function providerErrors (body) {
   return []
 }
 
+/** The `{ error }` body of an error event in a stream, from the SDK's error message. */
+function sseErrorBody (message) {
+  const json = typeof message === 'string' ? RE_SSE_ERROR.exec(message)?.[1] : null
+  if (!json) return undefined
+  try { return { error: JSON.parse(json) } } catch { return undefined }
+}
+
 function parseBody (data) {
   if (typeof data === 'string') {
     try { return JSON.parse(data) } catch { return { message: data } }

package/lib/models.js

@@ -0,0 +1,24 @@
+'use strict'
+/**
+ * The CDS models this plugin adds to the app, depending on its configuration: the
+ * TokenUsageLog entity for the db exporter, the read-only admin service, and the UsageLog
+ * projection when both are on.
+ */
+const { name } = require('../package.json')
+
+/**
+ * @param {object} [cfg] cds.requires.tokenOptimizer (raw, as in cds.env)
+ * @returns {string[]} model ids, resolved from the app root like `using from '<id>'`
+ */
+function modelsFor (cfg) {
+  const exporters = [].concat(cfg?.metrics?.exporters ?? [])
+  const dbLog = exporters.includes('db') || exporters.includes('hana')
+  const admin = cfg?.admin?.enabled === true
+  const models = []
+  if (dbLog) models.push(`${name}/db/token-usage-log`)
+  if (admin) models.push(`${name}/srv/admin-service`)
+  if (admin && dbLog) models.push(`${name}/srv/admin-usage-log`)
+  return models
+}
+
+module.exports = { modelsFor }

package/lib/optimizer.js

@@ -10,6 +10,10 @@
  *
  * Rule: nothing the optimizer does may break the AI call. If the pipeline fails, the original
  * request is sent. Only errors thrown by the SDK itself reach the caller, unchanged.
+ *
+ * Streaming calls are recorded when the caller has read the stream to the end, so the event
+ * has AI Core's token usage, finish reason and the full latency. A stream that fails, is
+ * closed early or is not read within STREAM_RECORD_TIMEOUT_MS is recorded at that point.
  */
 const { resolveConfig, forUseCase } = require('./config')
 const { MetricsCollector } = require('./metrics/collector')
@@ -30,6 +34,7 @@ const WRAPPED = Symbol.for('cap-ai-token-optimizer.wrapped')
 const LANGCHAIN_WRAPPED = Symbol.for('cap-ai-token-optimizer.langchain')
 const MAX_PREFIXES = 1000
 const ADVISORY_LOG_INTERVAL_MS = 10 * 60 * 1000
+const STREAM_RECORD_TIMEOUT_MS = 10 * 60 * 1000
 
 class TokenOptimizer {
   /**
@@ -40,6 +45,7 @@ class TokenOptimizer {
     this.collector = new MetricsCollector()
     this._deps = deps
     this._sleep = deps.sleep ?? ((ms) => new Promise(resolve => setTimeout(resolve, ms)))
+    this._streamRecordTimeoutMs = deps.streamRecordTimeoutMs ?? STREAM_RECORD_TIMEOUT_MS
     this._configured = false
     this.cache = null
     // last prompt prefix (system + tools) per use case and model, for the prompt-caching advisory
@@ -253,10 +259,11 @@ class TokenOptimizer {
         // the original method: a hooked one would route the call through an optimizer again
         const response = await instrument.originalOf(plan.client[methodName]).apply(plan.client, callArgs)
         if (ctx.cacheKey) this._cacheSet(ctx, response)
+        if (method.operation === 'stream' && safe(() => this._trackStream(ctx, { started, retries, response }), logRecordError)) return response
         safe(() => this._record(ctx, { started, retries, response }), logRecordError)
         return response
       } catch (err) {
-        const classified = safe(() => classifyError(err)) ?? { category: 'UNKNOWN', httpStatus: null, code: null, message: String(err?.message ?? err), retryAfterMs: null }
+        const classified = safe(() => classifyError(err)) ?? unknownError(err)
         const delay = this._retryDelay(cfg, classified, retries)
         if (delay != null) {
           retries++
@@ -270,6 +277,44 @@ class TokenOptimizer {
     }
   }
 
+  /**
+   * Record a streaming call when its stream ends: replaces `response.stream` with a stream of
+   * the same class that passes every chunk through unchanged and records the event once, when
+   * the stream is read to the end (with AI Core's usage), fails (the error is classified and
+   * rethrown), is closed early by the caller, or is not read within the record timeout.
+   * @returns {boolean} false if the stream can't be tracked; the caller then records right away
+   */
+  _trackStream (ctx, { started, retries, response }) {
+    const original = response?.stream
+    const Stream = original?.constructor
+    if (typeof original?.[Symbol.asyncIterator] !== 'function' || typeof Stream !== 'function' || Stream === Object) return false
+
+    let recorded = false
+    const record = (outcome = {}) => {
+      if (recorded) return
+      recorded = true
+      clearTimeout(timer)
+      safe(() => this._record(ctx, { started, retries, response, ...outcome }), logRecordError)
+    }
+    const timer = setTimeout(() => record({ note: 'stream: not read to the end within the record timeout' }), this._streamRecordTimeoutMs)
+    timer.unref?.()
+
+    async function * tracked () {
+      let ended = false
+      try {
+        for await (const chunk of original) yield chunk
+        ended = true
+      } catch (err) {
+        record({ error: safe(() => classifyError(err)) ?? unknownError(err) })
+        throw err
+      } finally {
+        record(ended ? {} : { note: 'stream: closed by the caller before the end' })
+      }
+    }
+    response.stream = new Stream(() => tracked(), original.controller)
+    return true
+  }
+
   /**
    * Everything known before the call: call info, estimates, pipeline result and cache key.
    * Never throws: a failure leaves the call unoptimized.
@@ -375,7 +420,7 @@ class TokenOptimizer {
     return Math.min(r.maxDelayMs, Math.round(backoff / 2 + Math.random() * backoff / 2))
   }
 
-  _record (ctx, { started, retries, response, error, cacheEntry }) {
+  _record (ctx, { started, retries, response, error, cacheEntry, note }) {
     const latencyMs = Math.round(performance.now() - started)
     const cfg = ctx.cfg
     const info = ctx.info ?? {}
@@ -398,6 +443,7 @@ class TokenOptimizer {
     const model = info.model ?? null
     const notes = [...(pipeline?.notes ?? [])]
     if (ctx.pipelineFailed) notes.push('pipeline failed; original request sent')
+    if (note) notes.push(note)
     const advisory = cacheEntry ? null : safe(() => this._checkPrefix(ctx))
 
     const event = buildEvent({
@@ -453,6 +499,8 @@ function prefixOf (messages, tools) {
 
 const logRecordError = (e) => log.warn('could not record call metrics:', e?.message)
 
+const unknownError = (err) => ({ category: 'UNKNOWN', httpStatus: null, code: null, message: String(err?.message ?? err), retryAfterMs: null })
+
 /**
  * Run `fn`, returning undefined instead of throwing.
  * @template T

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@viveka10/cap-ai-token-optimizer",
-  "version": "0.1.0",
-  "description": "Reduce, measure and report SAP AI Core token usage in SAP CAP Node.js apps: prompt optimization, response cache, metrics for SAP Cloud ALM / OpenTelemetry, HTTP and HANA",
+  "version": "0.1.2",
+  "description": "SAP CAP plugin to reduce, measure and report SAP AI Core LLM token usage: prompt optimization, response cache, cost tracking, and metrics for SAP Cloud ALM / OpenTelemetry",
   "keywords": [
     "sap",
     "cap",
@@ -20,7 +20,17 @@
     "mcp",
     "agents",
     "langchain",
-    "btp"
+    "btp",
+    "sap-cap",
+    "sap-btp",
+    "sap-ai",
+    "token-optimizer",
+    "ai-token-optimization",
+    "prompt-optimization",
+    "llm-cost",
+    "cost-optimization",
+    "openai",
+    "observability"
   ],
   "homepage": "https://github.com/viveka10/npm_token_optimizer/tree/main/cap-ai-token-optimizer#readme",
   "bugs": {
@@ -50,6 +60,7 @@
     "index.js",
     "index.d.ts",
     "cds-plugin.js",
+    ".cdsrc.js",
     "lib/",
     "srv/",
     "db/",

package/srv/admin-service.js

@@ -13,11 +13,30 @@ const totals = (t) => ({
   costEstimate: t.costEstimate
 })
 
-/** Rows for a READ, honouring a key lookup (e.g. UseCases('ask')). */
+/**
+ * OData exposes structured elements (`tokens`) as flat columns (`tokens_prompt`, …) unless the
+ * app uses structured OData (`odata.structs`, e.g. flavor x4). Return rows in the same shape.
+ */
+function shape (row) {
+  if (cds.env.effective?.odata?.structs || !row.tokens) return row
+  const { tokens, ...rest } = row
+  for (const [k, v] of Object.entries(tokens)) rest[`tokens_${k}`] = v
+  return rest
+}
+
+/**
+ * Rows for a READ, honouring a key lookup (e.g. UseCases('ask')). Lists carry `$count`, so
+ * `$count=true` (used by Fiori elements to size its tables) reports the number of rows.
+ */
 function rows (req, list, key) {
   const wanted = req.data?.[key]
-  if (wanted === undefined) return list
-  return list.find(r => r[key] === wanted) ?? null
+  if (wanted !== undefined) {
+    const row = list.find(r => r[key] === wanted)
+    return row ? shape(row) : null
+  }
+  const result = list.map(shape)
+  result.$count = result.length
+  return result
 }
 
 module.exports = class TokenOptimizerService extends cds.ApplicationService {