@sweetoburrito/backstage-plugin-ai-assistant-backend-module-callback-provider-langfuse 0.0.0-snapshot-20251113134620

package/README.md

@@ -0,0 +1,113 @@
+# @sweetoburrito/backstage-plugin-ai-assistant-backend-module-callback-provider-langfuse
+
+A callback provider module that integrates Langfuse observability into the Backstage AI Assistant backend, enabling tracing, monitoring, and analytics of LLM interactions.
+
+This README explains how the provider works, when to use it, configuration options, and how to wire it into your Backstage backend.
+
+## Features
+
+- Automatically trace all LLM calls, agent executions, and tool invocations through Langfuse.
+- Track token usage, costs, and performance metrics for each conversation.
+- Debug AI interactions with detailed execution traces in the Langfuse dashboard.
+- Monitor user behavior and conversation patterns across your organization.
+- Integrates with both Langfuse Cloud and self-hosted Langfuse instances.
+- Uses OpenTelemetry for comprehensive tracing alongside LangChain callbacks.
+
+## When to use
+
+Use this module if you want to:
+
+- Monitor and debug your AI Assistant interactions in production
+- Track LLM costs and token usage across models and users
+- Analyze conversation patterns and user behavior
+- Maintain audit logs of AI interactions for compliance
+- Optimize prompts and model performance based on real usage data
+- Get visibility into agent decision-making and tool usage
+
+## Configuration
+
+Add the provider configuration to your Backstage `app-config.yaml` or `app-config.local.yaml` under `aiAssistant.callbacks.langfuse`.
+
+Minimum configuration keys (example):
+
+```yaml
+aiAssistant:
+  callbacks:
+    langfuse:
+      secretKey: ${LANGFUSE_SECRET_KEY}
+      publicKey: ${LANGFUSE_PUBLIC_KEY}
+      baseUrl: https://cloud.langfuse.com
+```
+
+### Field descriptions
+
+- `secretKey` - Your Langfuse secret API key (starts with `sk-lf-`). Marked as secret in configuration.
+- `publicKey` - Your Langfuse public API key (starts with `pk-lf-`). Marked as secret in configuration.
+- `baseUrl` - The Langfuse instance URL (e.g., `https://cloud.langfuse.com`, `https://us.cloud.langfuse.com`, or your self-hosted URL)
+
+The exact keys required depend on your Langfuse configuration. Check the provider's `config.d.ts` in the package for the canonical types used by the module.
+
+## Install
+
+Install the module into your Backstage backend workspace:
+
+```sh
+yarn workspace backend add @sweetoburrito/backstage-plugin-ai-assistant-backend-module-callback-provider-langfuse
+```
+
+## Wire the provider into your backend
+
+Add the provider module import to your backend entrypoint (usually `packages/backend/src/index.ts`):
+
+```diff
+// packages/backend/src/index.ts
+
+// other backend modules...
+backend.add(import('@sweetoburrito/backstage-plugin-ai-assistant-backend'));
+
+// Add the Langfuse callback provider
+++backend.add(
+++  import(
+++    '@sweetoburrito/backstage-plugin-ai-assistant-backend-module-callback-provider-langfuse'
+++  ),
+++);
+```
+
+Restart your backend after adding the provider so it registers with the AI Assistant plugin.
+
+## What gets tracked
+
+The Langfuse callback provider automatically captures:
+
+- **Conversations**: Full conversation history with user and assistant messages
+- **Model calls**: All LLM invocations including prompts, completions, and parameters
+- **Tool usage**: When agents use tools like search, catalog lookups, or custom tools
+- **Token metrics**: Input/output tokens for cost and usage tracking
+- **Performance**: Latency and execution time for each operation
+- **Errors**: Failed requests with error messages and stack traces
+- **Metadata**: User entity references, model IDs, session IDs, and custom tags
+
+## Verification
+
+Once configured and running:
+
+1. Use the AI Assistant in Backstage to ask a question
+2. Log in to your Langfuse dashboard
+3. Navigate to the **Traces** section
+4. You should see traces appearing with tags like `backstage-ai-assistant` and `chat`
+
+Each trace includes:
+
+- User queries and AI responses
+- Model calls with prompts and completions
+- Token usage and costs
+- Tool invocations
+- Performance metrics
+- User and session information
+
+## Additional Resources
+
+- [Langfuse Documentation](https://langfuse.com/docs)
+- [Langfuse Setup Guide for AI Assistant](https://github.com/SweetOBurritO/backstage-plugin-ai-assistant/blob/main/docs/langfuse.md)
+- [Creating Custom Callback Providers](https://github.com/SweetOBurritO/backstage-plugin-ai-assistant/blob/main/docs/callbacks/creating-callback-provider.md)
+- [Callback Providers Overview](https://github.com/SweetOBurritO/backstage-plugin-ai-assistant/blob/main/docs/callbacks/index.md)

package/config.d.ts

@@ -0,0 +1,19 @@
+export interface Config {
+  aiAssistant: {
+    callbacks: {
+      langfuse: {
+        baseUrl: string;
+
+        /**
+         * @visibility secret
+         */
+        publicKey: string;
+
+        /**
+         * @visibility secret
+         */
+        secretKey: string;
+      };
+    };
+  };
+}

package/dist/index.cjs.js

@@ -0,0 +1,10 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+var module$1 = require('./module.cjs.js');
+
+
+
+exports.default = module$1.aiAssistantModuleCallbackProviderLangfuse;
+//# sourceMappingURL=index.cjs.js.map

package/dist/index.cjs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.cjs.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;"}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+import * as _backstage_backend_plugin_api from '@backstage/backend-plugin-api';
+
+declare const aiAssistantModuleCallbackProviderLangfuse: _backstage_backend_plugin_api.BackendFeature;
+
+export { aiAssistantModuleCallbackProviderLangfuse as default };

package/dist/module.cjs.js

@@ -0,0 +1,60 @@
+'use strict';
+
+var backendPluginApi = require('@backstage/backend-plugin-api');
+var otel = require('@langfuse/otel');
+var sdkNode = require('@opentelemetry/sdk-node');
+var langchain = require('@langfuse/langchain');
+var backstagePluginAiAssistantNode = require('@sweetoburrito/backstage-plugin-ai-assistant-node');
+
+const aiAssistantModuleCallbackProviderLangfuse = backendPluginApi.createBackendModule({
+  pluginId: "ai-assistant",
+  moduleId: "callback-provider-langfuse",
+  register(reg) {
+    reg.registerInit({
+      deps: {
+        config: backendPluginApi.coreServices.rootConfig,
+        callbackProvider: backstagePluginAiAssistantNode.callbackFactoryExtensionPoint
+      },
+      async init({ config, callbackProvider }) {
+        const secretKey = config.getString(
+          "aiAssistant.callbacks.langfuse.secretKey"
+        );
+        const publicKey = config.getString(
+          "aiAssistant.callbacks.langfuse.publicKey"
+        );
+        const baseUrl = config.getString(
+          "aiAssistant.callbacks.langfuse.baseUrl"
+        );
+        const langfuseSpanProcessor = new otel.LangfuseSpanProcessor({
+          secretKey,
+          publicKey,
+          baseUrl
+        });
+        const sdk = new sdkNode.NodeSDK({
+          spanProcessors: [langfuseSpanProcessor]
+        });
+        sdk.start();
+        callbackProvider.register(async (options) => {
+          const { conversationId, userId, modelId } = options;
+          const callback = new langchain.CallbackHandler({
+            sessionId: conversationId,
+            userId,
+            tags: ["backstage-ai-assistant", "chat", modelId]
+          });
+          const metadata = {
+            langfuseUserId: userId,
+            langfuseSessionId: conversationId,
+            langfuseTags: ["ai-assistant", "chat", modelId]
+          };
+          return {
+            callback,
+            metadata
+          };
+        });
+      }
+    });
+  }
+});
+
+exports.aiAssistantModuleCallbackProviderLangfuse = aiAssistantModuleCallbackProviderLangfuse;
+//# sourceMappingURL=module.cjs.js.map

package/dist/module.cjs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"module.cjs.js","sources":["../src/module.ts"],"sourcesContent":["import {\n  coreServices,\n  createBackendModule,\n} from '@backstage/backend-plugin-api';\nimport { LangfuseSpanProcessor } from '@langfuse/otel';\nimport { NodeSDK } from '@opentelemetry/sdk-node';\nimport { CallbackHandler } from '@langfuse/langchain';\nimport { callbackFactoryExtensionPoint } from '@sweetoburrito/backstage-plugin-ai-assistant-node';\n\nexport const aiAssistantModuleCallbackProviderLangfuse = createBackendModule({\n  pluginId: 'ai-assistant',\n  moduleId: 'callback-provider-langfuse',\n  register(reg) {\n    reg.registerInit({\n      deps: {\n        config: coreServices.rootConfig,\n        callbackProvider: callbackFactoryExtensionPoint,\n      },\n      async init({ config, callbackProvider }) {\n        const secretKey = config.getString(\n          'aiAssistant.callbacks.langfuse.secretKey',\n        );\n        const publicKey = config.getString(\n          'aiAssistant.callbacks.langfuse.publicKey',\n        );\n        const baseUrl = config.getString(\n          'aiAssistant.callbacks.langfuse.baseUrl',\n        );\n\n        const langfuseSpanProcessor = new LangfuseSpanProcessor({\n          secretKey,\n          publicKey,\n          baseUrl,\n        });\n\n        const sdk = new NodeSDK({\n          spanProcessors: [langfuseSpanProcessor],\n        });\n\n        sdk.start();\n\n        callbackProvider.register(async options => {\n          const { conversationId, userId, modelId } = options;\n          const callback = new CallbackHandler({\n            sessionId: conversationId,\n            userId,\n            tags: ['backstage-ai-assistant', 'chat', modelId],\n          });\n\n          const metadata = {\n            langfuseUserId: userId,\n            langfuseSessionId: conversationId,\n            langfuseTags: ['ai-assistant', 'chat', modelId],\n          };\n\n          return {\n            callback,\n            metadata,\n          };\n        });\n      },\n    });\n  },\n});\n"],"names":["createBackendModule","coreServices","callbackFactoryExtensionPoint","LangfuseSpanProcessor","NodeSDK","CallbackHandler"],"mappings":";;;;;;;;AASO,MAAM,4CAA4CA,oCAAA,CAAoB;AAAA,EAC3E,QAAA,EAAU,cAAA;AAAA,EACV,QAAA,EAAU,4BAAA;AAAA,EACV,SAAS,GAAA,EAAK;AACZ,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM;AAAA,QACJ,QAAQC,6BAAA,CAAa,UAAA;AAAA,QACrB,gBAAA,EAAkBC;AAAA,OACpB;AAAA,MACA,MAAM,IAAA,CAAK,EAAE,MAAA,EAAQ,kBAAiB,EAAG;AACvC,QAAA,MAAM,YAAY,MAAA,CAAO,SAAA;AAAA,UACvB;AAAA,SACF;AACA,QAAA,MAAM,YAAY,MAAA,CAAO,SAAA;AAAA,UACvB;AAAA,SACF;AACA,QAAA,MAAM,UAAU,MAAA,CAAO,SAAA;AAAA,UACrB;AAAA,SACF;AAEA,QAAA,MAAM,qBAAA,GAAwB,IAAIC,0BAAA,CAAsB;AAAA,UACtD,SAAA;AAAA,UACA,SAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAA,MAAM,GAAA,GAAM,IAAIC,eAAA,CAAQ;AAAA,UACtB,cAAA,EAAgB,CAAC,qBAAqB;AAAA,SACvC,CAAA;AAED,QAAA,GAAA,CAAI,KAAA,EAAM;AAEV,QAAA,gBAAA,CAAiB,QAAA,CAAS,OAAM,OAAA,KAAW;AACzC,UAAA,MAAM,EAAE,cAAA,EAAgB,MAAA,EAAQ,OAAA,EAAQ,GAAI,OAAA;AAC5C,UAAA,MAAM,QAAA,GAAW,IAAIC,yBAAA,CAAgB;AAAA,YACnC,SAAA,EAAW,cAAA;AAAA,YACX,MAAA;AAAA,YACA,IAAA,EAAM,CAAC,wBAAA,EAA0B,MAAA,EAAQ,OAAO;AAAA,WACjD,CAAA;AAED,UAAA,MAAM,QAAA,GAAW;AAAA,YACf,cAAA,EAAgB,MAAA;AAAA,YAChB,iBAAA,EAAmB,cAAA;AAAA,YACnB,YAAA,EAAc,CAAC,cAAA,EAAgB,MAAA,EAAQ,OAAO;AAAA,WAChD;AAEA,UAAA,OAAO;AAAA,YACL,QAAA;AAAA,YACA;AAAA,WACF;AAAA,QACF,CAAC,CAAA;AAAA,MACH;AAAA,KACD,CAAA;AAAA,EACH;AACF,CAAC;;;;"}

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@sweetoburrito/backstage-plugin-ai-assistant-backend-module-callback-provider-langfuse",
+  "version": "0.0.0-snapshot-20251113134620",
+  "license": "Apache-2.0",
+  "description": "The callback-provider-langfuse backend module for the ai-assistant plugin.",
+  "main": "dist/index.cjs.js",
+  "types": "dist/index.d.ts",
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin-module",
+    "pluginId": "ai-assistant",
+    "pluginPackage": "@sweetoburrito/backstage-plugin-ai-assistant-backend",
+    "features": {
+      ".": "@backstage/BackendFeature"
+    }
+  },
+  "scripts": {
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "clean": "backstage-cli package clean",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack"
+  },
+  "dependencies": {
+    "@backstage/backend-plugin-api": "backstage:^",
+    "@langfuse/langchain": "^4.4.2",
+    "@langfuse/otel": "^4.4.2",
+    "@opentelemetry/sdk-node": "^0.208.0",
+    "@sweetoburrito/backstage-plugin-ai-assistant-node": "workspace:^"
+  },
+  "devDependencies": {
+    "@backstage/backend-test-utils": "backstage:^",
+    "@backstage/cli": "backstage:^"
+  },
+  "configSchema": "config.d.ts",
+  "files": [
+    "dist",
+    "config.d.ts"
+  ],
+  "typesVersions": {
+    "*": {
+      "package.json": [
+        "package.json"
+      ]
+    }
+  }
+}