@botonic/plugin-ai-agents 0.46.0-alpha.0 → 0.46.0-alpha.1

package/lib/esm/guardrails/input.js

@@ -3,7 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createInputGuardrail = createInputGuardrail;
 const agents_1 = require("@openai/agents");
 const zod_1 = require("zod");
-function createInputGuardrail(rules, llmConfig) {
+const constants_1 = require("../constants");
+const hubtype_api_client_1 = require("../services/hubtype-api-client");
+const types_1 = require("../services/types");
+function createInputGuardrail(rules, llmConfig, trackingContext) {
     const outputType = zod_1.z.object(Object.fromEntries(rules.map(rule => [rule.name, zod_1.z.boolean().describe(rule.description)])));
     const agent = new agents_1.Agent({
         name: 'InputGuardrail',
@@ -14,12 +17,18 @@ function createInputGuardrail(rules, llmConfig) {
         name: 'InputGuardrail',
         execute: async ({ input, context }) => {
             const lastMessage = input[input.length - 1];
+            const modelProvider = llmConfig.modelProvider;
+            const modelSettings = llmConfig.modelSettings;
+            modelSettings.toolChoice = undefined;
             const runner = new agents_1.Runner({
-                modelSettings: llmConfig.modelSettings,
-                modelProvider: llmConfig.modelProvider,
+                modelSettings,
+                modelProvider,
                 tracingDisabled: true,
             });
+            const startTime = Date.now();
             const result = await runner.run(agent, [lastMessage], { context });
+            const endTime = Date.now();
+            void sendGuardrailLlmRunTracking(result, trackingContext, llmConfig, startTime, endTime);
             const finalOutput = result.finalOutput;
             if (finalOutput === undefined) {
                 throw new Error('Guardrail agent failed to produce output');
@@ -33,4 +42,36 @@ function createInputGuardrail(rules, llmConfig) {
         },
     };
 }
+async function sendGuardrailLlmRunTracking(result, trackingContext, llmConfig, startTime, endTime) {
+    if (!constants_1.isProd) {
+        return;
+    }
+    const rawResponses = result.rawResponses ?? [];
+    if (rawResponses.length === 0) {
+        return;
+    }
+    const totalDuration = endTime - startTime;
+    const durationPerCall = Math.round(totalDuration / rawResponses.length);
+    const temperature = llmConfig.modelSettings.temperature ?? 0;
+    const apiVersion = constants_1.OPENAI_PROVIDER === 'azure' ? constants_1.AZURE_OPENAI_API_VERSION : '';
+    const llmRuns = rawResponses.map(response => ({
+        inference_id: trackingContext.inferenceId,
+        is_test: trackingContext.isTest,
+        product_name: types_1.TrackProductName.AI_AGENT,
+        deployment_name: llmConfig.modelName,
+        model_name: response.providerData?.['model'] ??
+            llmConfig.modelName,
+        feature: types_1.TrackFeature.AI_AGENT_GUARDRAIL,
+        api_version: apiVersion,
+        num_prompt_tokens: response.usage.inputTokens,
+        num_completion_tokens: response.usage.outputTokens,
+        duration_in_milliseconds: durationPerCall,
+        temperature,
+        error: null,
+    }));
+    const client = new hubtype_api_client_1.HubtypeApiClient(trackingContext.authToken);
+    await client.trackLlmRuns(trackingContext.botId, {
+        llm_runs: llmRuns,
+    });
+}
 //# sourceMappingURL=input.js.map

package/lib/esm/guardrails/input.js.map

@@ -1 +1 @@
-{"version":3,"file":"input.js","sourceRoot":"","sources":["../../../src/guardrails/input.ts"],"names":[],"mappings":";;AAUA,oDA0CC;AApDD,2CAKuB;AACvB,6BAAuB;AAIvB,SAAgB,oBAAoB,CAClC,KAAsB,EACtB,SAAoB;IAEpB,MAAM,UAAU,GAAG,OAAC,CAAC,MAAM,CACzB,MAAM,CAAC,WAAW,CAChB,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CACvE,CACF,CAAA;IAED,MAAM,KAAK,GAAG,IAAI,cAAK,CAAC;QACtB,IAAI,EAAE,gBAAgB;QACtB,YAAY,EACV,8DAA8D;QAChE,UAAU;KACX,CAAC,CAAA;IAEF,OAAO;QACL,IAAI,EAAE,gBAAgB;QACtB,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE;YACpC,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAoB,CAAA;YAC9D,MAAM,MAAM,GAAG,IAAI,eAAM,CAAC;gBACxB,aAAa,EAAE,SAAS,CAAC,aAAa;gBACtC,aAAa,EAAE,SAAS,CAAC,aAAa;gBACtC,eAAe,EAAE,IAAI;aACtB,CAAC,CAAA;YACF,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;YAElE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAsC,CAAA;YACjE,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;YAC7D,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAA;YAC1E,MAAM,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CACzD,GAAG,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,IAAI,CACjC,CAAA;YACD,OAAO;gBACL,UAAU,EAAE,mBAAmB;gBAC/B,iBAAiB,EAAE,SAAS;aAC7B,CAAA;QACH,CAAC;KACF,CAAA;AACH,CAAC"}
+{"version":3,"file":"input.js","sourceRoot":"","sources":["../../../src/guardrails/input.ts"],"names":[],"mappings":";;AAoBA,oDAwDC;AA5ED,2CAKuB;AACvB,6BAAuB;AACvB,4CAAgF;AAEhF,uEAAiE;AACjE,6CAAkE;AAUlE,SAAgB,oBAAoB,CAClC,KAAsB,EACtB,SAAoB,EACpB,eAAyC;IAEzC,MAAM,UAAU,GAAG,OAAC,CAAC,MAAM,CACzB,MAAM,CAAC,WAAW,CAChB,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CACvE,CACF,CAAA;IAED,MAAM,KAAK,GAAG,IAAI,cAAK,CAAC;QACtB,IAAI,EAAE,gBAAgB;QACtB,YAAY,EACV,8DAA8D;QAChE,UAAU;KACX,CAAC,CAAA;IAEF,OAAO;QACL,IAAI,EAAE,gBAAgB;QACtB,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE;YACpC,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAoB,CAAA;YAC9D,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,CAAA;YAC7C,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,CAAA;YAC7C,aAAa,CAAC,UAAU,GAAG,SAAS,CAAA;YACpC,MAAM,MAAM,GAAG,IAAI,eAAM,CAAC;gBACxB,aAAa;gBACb,aAAa;gBACb,eAAe,EAAE,IAAI;aACtB,CAAC,CAAA;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;YAClE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAE1B,KAAK,2BAA2B,CAC9B,MAAM,EACN,eAAe,EACf,SAAS,EACT,SAAS,EACT,OAAO,CACR,CAAA;YAED,MAAM,WAAW,GAAG,MAAM,CAAC,WAAsC,CAAA;YACjE,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;YAC7D,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAA;YAC1E,MAAM,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CACzD,GAAG,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,IAAI,CACjC,CAAA;YACD,OAAO;gBACL,UAAU,EAAE,mBAAmB;gBAC/B,iBAAiB,EAAE,SAAS;aAC7B,CAAA;QACH,CAAC;KACF,CAAA;AACH,CAAC;AAED,KAAK,UAAU,2BAA2B,CACxC,MAEC,EACD,eAAyC,EACzC,SAAoB,EACpB,SAAiB,EACjB,OAAe;IAEf,IAAI,CAAC,kBAAM,EAAE,CAAC;QACZ,OAAM;IACR,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,EAAE,CAAA;IAC9C,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAM;IACR,CAAC;IACD,MAAM,aAAa,GAAG,OAAO,GAAG,SAAS,CAAA;IACzC,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,CAAA;IACvE,MAAM,WAAW,GACd,SAAS,CAAC,aAAa,CAAC,WAAkC,IAAI,CAAC,CAAA;IAClE,MAAM,UAAU,GAAG,2BAAe,KAAK,OAAO,CAAC,CAAC,CAAC,oCAAwB,CAAC,CAAC,CAAC,EAAE,CAAA;IAE9E,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC5C,YAAY,EAAE,eAAe,CAAC,WAAW;QACzC,OAAO,EAAE,eAAe,CAAC,MAAM;QAC/B,YAAY,EAAE,wBAAgB,CAAC,QAAQ;QACvC,eAAe,EAAE,SAAS,CAAC,SAAS;QACpC,UAAU,EACP,QAAQ,CAAC,YAAY,EAAE,CAAC,OAAO,CAAwB;YACxD,SAAS,CAAC,SAAS;QACrB,OAAO,EAAE,oBAAY,CAAC,kBAAkB;QACxC,WAAW,EAAE,UAAU;QACvB,iBAAiB,EAAE,QAAQ,CAAC,KAAK,CAAC,WAAW;QAC7C,qBAAqB,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;QAClD,wBAAwB,EAAE,eAAe;QACzC,WAAW;QACX,KAAK,EAAE,IAAI;KACZ,CAAC,CAAC,CAAA;IAEH,MAAM,MAAM,GAAG,IAAI,qCAAgB,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;IAC9D,MAAM,MAAM,CAAC,YAAY,CAAC,eAAe,CAAC,KAAK,EAAE;QAC/C,QAAQ,EAAE,OAAO;KAClB,CAAC,CAAA;AACJ,CAAC"}

package/lib/esm/index.js

@@ -2,12 +2,13 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const tslib_1 = require("tslib");
 const agents_1 = require("@openai/agents");
+const uuid_1 = require("uuid");
 const agent_builder_1 = require("./agent-builder");
 const constants_1 = require("./constants");
 const debug_logger_1 = require("./debug-logger");
-const hubtype_api_client_1 = require("./hubtype-api-client");
 const llm_config_1 = require("./llm-config");
 const runner_1 = require("./runner");
+const hubtype_api_client_1 = require("./services/hubtype-api-client");
 class BotonicPluginAiAgents {
     constructor(options) {
         this.toolDefinitions = [];
@@ -41,6 +42,7 @@ class BotonicPluginAiAgents {
             if (!authToken) {
                 throw new Error('Auth token is required');
             }
+            const inferenceId = (0, uuid_1.v7)();
             // Create client for OpenAI/Azure OpenAI
             const llmConfig = new llm_config_1.LLMConfig(this.maxRetries, this.timeout, aiAgentArgs.model, aiAgentArgs.verbosity);
             // Build tools
@@ -56,6 +58,12 @@ class BotonicPluginAiAgents {
                 campaignsContext: botContext.input.context?.campaigns_v2,
                 logger: this.logger,
                 llmConfig,
+                guardrailTrackingContext: {
+                    botId: botContext.session.bot.id,
+                    isTest: botContext.session.is_test_integration,
+                    authToken,
+                    inferenceId,
+                },
             }).build();
             // Get messages
             const messages = await this.getMessages(botContext, authToken, constants_1.MAX_MEMORY_LENGTH);
@@ -74,7 +82,7 @@ class BotonicPluginAiAgents {
             // Log agent debug info
             this.logger.logAgentDebugInfo(aiAgentArgs, tools.map(t => t.name), messages);
             // Run agent
-            const runner = new runner_1.AIAgentRunner(agent, llmConfig, this.logger);
+            const runner = new runner_1.AIAgentRunner(agent, llmConfig, inferenceId, this.logger);
             return await runner.run(messages, context);
         }
         catch (error) {

package/lib/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;AAMA,2CAAqC;AAErC,mDAAgD;AAChD,2CAKoB;AACpB,iDAAoE;AACpE,6DAAuD;AACvD,6CAAwC;AACxC,qCAAwC;AAYxC,MAAqB,qBAAqB;IAaxC,YAAY,OAAoD;QAFzD,oBAAe,GAAuC,EAAE,CAAA;QAG7D,IAAI,OAAO,EAAE,wBAAwB,KAAK,IAAI,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CACb,2DAA2D;gBACzD,+DAA+D,CAClE,CAAA;QACH,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,CAAA;QACnC,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,WAAW,IAAI,EAAE,CAAA;QACjD,IAAI,CAAC,wBAAwB,GAAG,OAAO,EAAE,wBAAwB,IAAI,IAAI,CAAA;QACzE,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,EAAE,CAAA;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,sCAA0B,CAAA;QAC7D,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,+BAAmB,CAAA;QAC5D,IAAI,CAAC,MAAM,GAAG,IAAA,gCAAiB,EAAC,OAAO,EAAE,WAAW,IAAI,KAAK,CAAC,CAAA;QAE9D,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC;YAC3B,wBAAwB,EAAE,IAAI,CAAC,wBAAwB;YACvD,UAAU,EAAE,OAAO,EAAE,UAAU,IAAI,CAAC;YACpC,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,KAAK;YAClC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YACtD,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAA;IACJ,CAAC;IAED,GAAG;QACD,OAAM;IACR,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,UAA4C,EAC5C,WAAwB;QAExB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,kBAAM;gBACtB,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,aAAa;gBAClC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAA;YAClB,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;YAC3C,CAAC;YAED,wCAAwC;YACxC,MAAM,SAAS,GAAG,IAAI,sBAAS,CAC7B,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,OAAO,EACZ,WAAW,CAAC,KAAK,EACjB,WAAW,CAAC,SAAS,CACtB,CAAA;YAED,cAAc;YACd,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAC3B,WAAW,CAAC,WAAW,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CACtD,CAAA;YAED,cAAc;YACd,MAAM,KAAK,GAAG,IAAI,8BAAc,CAAuB;gBACrD,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,YAAY,EAAE,WAAW,CAAC,YAAY;gBACtC,KAAK,EAAE,KAAK;gBACZ,WAAW,EAAE,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE;gBACvD,mBAAmB,EAAE,WAAW,CAAC,mBAAmB,IAAI,EAAE;gBAC1D,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,EAAE;gBACtC,gBAAgB,EAAE,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,YAAY;gBACxD,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS;aACV,CAAC,CAAC,KAAK,EAAE,CAAA;YAEV,eAAe;YACf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CACrC,UAAU,EACV,SAAS,EACT,6BAAiB,CAClB,CAAA;YAED,gBAAgB;YAChB,MAAM,OAAO,GAAkC;gBAC7C,SAAS;gBACT,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,EAAE;gBACtC,aAAa,EAAE;oBACb,KAAK,EAAE,EAAE;oBACT,SAAS,EAAE,EAAE;oBACb,SAAS,EAAE,EAAE;oBACb,UAAU,EAAE,EAAE;iBACf;gBACD,OAAO,EAAE,UAAU;aACpB,CAAA;YAED,uBAAuB;YACvB,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAC3B,WAAW,EACX,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EACtB,QAAQ,CACT,CAAA;YAED,YAAY;YACZ,MAAM,MAAM,GAAG,IAAI,sBAAa,CAC9B,KAAK,EACL,SAAS,EACT,IAAI,CAAC,MAAM,CACZ,CAAA;YACD,OAAO,MAAM,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;QAC5C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAA;YACtD,OAAO;gBACL,QAAQ,EAAE,EAAE;gBACZ,aAAa,EAAE,EAAE;gBACjB,YAAY,EAAE,CAAC;gBACf,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,IAAI;gBACX,wBAAwB,EAAE,EAAE;gBAC5B,yBAAyB,EAAE,EAAE;aAC9B,CAAA;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,UAAsB,EACtB,SAAiB,EACjB,YAAoB;QAEpB,MAAM,aAAa,GAAG,IAAI,qCAAgB,CAAC,SAAS,CAAC,CAAA;QAErD,IAAI,CAAC,kBAAM,EAAE,CAAC;YACZ,OAAO,MAAM,aAAa,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAA;QAC3D,CAAC;QAED,IAAI,IAAI,CAAC,wBAAwB,KAAK,IAAI,EAAE,CAAC;YAC3C,OAAO,MAAM,aAAa,CAAC,WAAW,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;QAClE,CAAC;QAED,gBAAgB;QAChB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,aAAa,CAAC,UAAU,EAAE;YAC3D,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,YAAY;YACpD,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI;YACtD,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,CAAC;YACvD,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,KAAK;SAC1C,CAAC,CAAA;QACF,OAAO,MAAM,CAAC,QAAQ,CAAA;IACxB,CAAC;IAEO,UAAU,CAAC,eAAyB;QAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CACxD,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CACpC,CAAA;QACD,OAAO,cAAc,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACzC,OAAO,IAAA,aAAI,EAA0C;gBACnD,IAAI,EAAE,cAAc,CAAC,IAAI;gBACzB,WAAW,EAAE,cAAc,CAAC,WAAW;gBACvC,UAAU,EAAE,cAAc,CAAC,MAAM;gBACjC,OAAO,EAAE,cAAc,CAAC,IAAI;aAC7B,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC;CACF;AAtKD,wCAsKC;AAED,6DAAkC;AAClC,kDAAuB"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;AAMA,2CAAqC;AACrC,+BAAmC;AACnC,mDAAgD;AAChD,2CAKoB;AACpB,iDAAoE;AACpE,6CAAwC;AACxC,qCAAwC;AACxC,sEAAgE;AAYhE,MAAqB,qBAAqB;IAaxC,YAAY,OAAoD;QAFzD,oBAAe,GAAuC,EAAE,CAAA;QAG7D,IAAI,OAAO,EAAE,wBAAwB,KAAK,IAAI,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CACb,2DAA2D;gBACzD,+DAA+D,CAClE,CAAA;QACH,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,CAAA;QACnC,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,WAAW,IAAI,EAAE,CAAA;QACjD,IAAI,CAAC,wBAAwB,GAAG,OAAO,EAAE,wBAAwB,IAAI,IAAI,CAAA;QACzE,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,EAAE,CAAA;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,sCAA0B,CAAA;QAC7D,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,+BAAmB,CAAA;QAC5D,IAAI,CAAC,MAAM,GAAG,IAAA,gCAAiB,EAAC,OAAO,EAAE,WAAW,IAAI,KAAK,CAAC,CAAA;QAE9D,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC;YAC3B,wBAAwB,EAAE,IAAI,CAAC,wBAAwB;YACvD,UAAU,EAAE,OAAO,EAAE,UAAU,IAAI,CAAC;YACpC,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,KAAK;YAClC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YACtD,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAA;IACJ,CAAC;IAED,GAAG;QACD,OAAM;IACR,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,UAA4C,EAC5C,WAAwB;QAExB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,kBAAM;gBACtB,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,aAAa;gBAClC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAA;YAClB,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;YAC3C,CAAC;YAED,MAAM,WAAW,GAAG,IAAA,SAAM,GAAE,CAAA;YAE5B,wCAAwC;YACxC,MAAM,SAAS,GAAG,IAAI,sBAAS,CAC7B,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,OAAO,EACZ,WAAW,CAAC,KAAK,EACjB,WAAW,CAAC,SAAS,CACtB,CAAA;YAED,cAAc;YACd,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAC3B,WAAW,CAAC,WAAW,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CACtD,CAAA;YAED,cAAc;YACd,MAAM,KAAK,GAAG,IAAI,8BAAc,CAAuB;gBACrD,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,YAAY,EAAE,WAAW,CAAC,YAAY;gBACtC,KAAK,EAAE,KAAK;gBACZ,WAAW,EAAE,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE;gBACvD,mBAAmB,EAAE,WAAW,CAAC,mBAAmB,IAAI,EAAE;gBAC1D,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,EAAE;gBACtC,gBAAgB,EAAE,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,YAAY;gBACxD,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS;gBACT,wBAAwB,EAAE;oBACxB,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;oBAChC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,mBAAmB;oBAC9C,SAAS;oBACT,WAAW;iBACZ;aACF,CAAC,CAAC,KAAK,EAAE,CAAA;YAEV,eAAe;YACf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CACrC,UAAU,EACV,SAAS,EACT,6BAAiB,CAClB,CAAA;YAED,gBAAgB;YAChB,MAAM,OAAO,GAAkC;gBAC7C,SAAS;gBACT,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,EAAE;gBACtC,aAAa,EAAE;oBACb,KAAK,EAAE,EAAE;oBACT,SAAS,EAAE,EAAE;oBACb,SAAS,EAAE,EAAE;oBACb,UAAU,EAAE,EAAE;iBACf;gBACD,OAAO,EAAE,UAAU;aACpB,CAAA;YAED,uBAAuB;YACvB,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAC3B,WAAW,EACX,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EACtB,QAAQ,CACT,CAAA;YAED,YAAY;YACZ,MAAM,MAAM,GAAG,IAAI,sBAAa,CAC9B,KAAK,EACL,SAAS,EACT,WAAW,EACX,IAAI,CAAC,MAAM,CACZ,CAAA;YACD,OAAO,MAAM,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;QAC5C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAA;YACtD,OAAO;gBACL,QAAQ,EAAE,EAAE;gBACZ,aAAa,EAAE,EAAE;gBACjB,YAAY,EAAE,CAAC;gBACf,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,IAAI;gBACX,wBAAwB,EAAE,EAAE;gBAC5B,yBAAyB,EAAE,EAAE;aAC9B,CAAA;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,UAAsB,EACtB,SAAiB,EACjB,YAAoB;QAEpB,MAAM,aAAa,GAAG,IAAI,qCAAgB,CAAC,SAAS,CAAC,CAAA;QAErD,IAAI,CAAC,kBAAM,EAAE,CAAC;YACZ,OAAO,MAAM,aAAa,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAA;QAC3D,CAAC;QAED,IAAI,IAAI,CAAC,wBAAwB,KAAK,IAAI,EAAE,CAAC;YAC3C,OAAO,MAAM,aAAa,CAAC,WAAW,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;QAClE,CAAC;QAED,gBAAgB;QAChB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,aAAa,CAAC,UAAU,EAAE;YAC3D,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,YAAY;YACpD,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI;YACtD,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,CAAC;YACvD,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,KAAK;SAC1C,CAAC,CAAA;QACF,OAAO,MAAM,CAAC,QAAQ,CAAA;IACxB,CAAC;IAEO,UAAU,CAAC,eAAyB;QAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CACxD,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CACpC,CAAA;QACD,OAAO,cAAc,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACzC,OAAO,IAAA,aAAI,EAA0C;gBACnD,IAAI,EAAE,cAAc,CAAC,IAAI;gBACzB,WAAW,EAAE,cAAc,CAAC,WAAW;gBACvC,UAAU,EAAE,cAAc,CAAC,MAAM;gBACjC,OAAO,EAAE,cAAc,CAAC,IAAI;aAC7B,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC;CACF;AA/KD,wCA+KC;AAED,6DAAkC;AAClC,kDAAuB"}

package/lib/esm/llm-config.js

@@ -56,7 +56,7 @@ class LLMConfig {
         if (model.includes('gpt-4')) {
             return {
                 temperature: 0,
-                text: { verbosity },
+                text: { verbosity: 'low' },
             };
         }
         throw new Error(`Unsupported model: ${model}`);

package/lib/esm/llm-config.js.map

@@ -1 +1 @@
-{"version":3,"file":"llm-config.js","sourceRoot":"","sources":["../../src/llm-config.ts"],"names":[],"mappings":";;;;AACA,2CAIuB;AACvB,yDAA4C;AAC5C,2CAQoB;AAEpB,MAAa,SAAS;IAOpB,YACE,UAAkB,EAClB,OAAe,EACf,SAAiB,EACjB,SAAyB;QAEzB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAC5B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QACtB,IAAI,CAAC,SAAS,GAAG,2BAAe,KAAK,QAAQ,CAAC,CAAC,CAAC,wBAAY,CAAC,CAAC,CAAC,SAAS,CAAA;QACxE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAA;QAC5C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IAClE,CAAC;IAEO,gBAAgB;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAC/B,OAAO,IAAI,uBAAc,CAAC;YACxB,YAAY,EAAE,MAAM;YACpB,YAAY,EAAE,KAAK;SACpB,CAAC,CAAA;IACJ,CAAC;IAEO,SAAS;QACf,IAAI,2BAAe,KAAK,QAAQ,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC,eAAe,EAAE,CAAA;QAC/B,CAAC;QAED,OAAO,IAAI,CAAC,cAAc,EAAE,CAAA;IAC9B,CAAC;IAEO,eAAe;QACrB,OAAO,IAAI,gBAAM,CAAC;YAChB,MAAM,EAAE,0BAAc;YACtB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,uBAAuB,EAAE,CAAC,kBAAM;SACjC,CAAC,CAAA;IACJ,CAAC;IAEO,cAAc;QACpB,OAAO,IAAI,oBAAW,CAAC;YACrB,MAAM,EAAE,gCAAoB;YAC5B,UAAU,EAAE,oCAAwB;YACpC,UAAU,EAAE,IAAI,CAAC,SAAS;YAC1B,OAAO,EAAE,iCAAqB;YAC9B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,uBAAuB,EAAE,CAAC,kBAAM;SACjC,CAAC,CAAA;IACJ,CAAC;IAEO,gBAAgB,CACtB,KAAa,EACb,SAAyB;QAEzB,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO;gBACL,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE;gBAC7B,WAAW,EAAE,CAAC;gBACd,IAAI,EAAE,EAAE,SAAS,EAAE;aACpB,CAAA;QACH,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO;gBACL,WAAW,EAAE,CAAC;gBACd,IAAI,EAAE,EAAE,SAAS,EAAE;aACpB,CAAA;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAA;IAChD,CAAC;CACF;AA9ED,8BA8EC"}
+{"version":3,"file":"llm-config.js","sourceRoot":"","sources":["../../src/llm-config.ts"],"names":[],"mappings":";;;;AACA,2CAIuB;AACvB,yDAA4C;AAC5C,2CAQoB;AAEpB,MAAa,SAAS;IAOpB,YACE,UAAkB,EAClB,OAAe,EACf,SAAiB,EACjB,SAAyB;QAEzB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAC5B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QACtB,IAAI,CAAC,SAAS,GAAG,2BAAe,KAAK,QAAQ,CAAC,CAAC,CAAC,wBAAY,CAAC,CAAC,CAAC,SAAS,CAAA;QACxE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAA;QAC5C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IAClE,CAAC;IAEO,gBAAgB;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAC/B,OAAO,IAAI,uBAAc,CAAC;YACxB,YAAY,EAAE,MAAM;YACpB,YAAY,EAAE,KAAK;SACpB,CAAC,CAAA;IACJ,CAAC;IAEO,SAAS;QACf,IAAI,2BAAe,KAAK,QAAQ,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC,eAAe,EAAE,CAAA;QAC/B,CAAC;QAED,OAAO,IAAI,CAAC,cAAc,EAAE,CAAA;IAC9B,CAAC;IAEO,eAAe;QACrB,OAAO,IAAI,gBAAM,CAAC;YAChB,MAAM,EAAE,0BAAc;YACtB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,uBAAuB,EAAE,CAAC,kBAAM;SACjC,CAAC,CAAA;IACJ,CAAC;IAEO,cAAc;QACpB,OAAO,IAAI,oBAAW,CAAC;YACrB,MAAM,EAAE,gCAAoB;YAC5B,UAAU,EAAE,oCAAwB;YACpC,UAAU,EAAE,IAAI,CAAC,SAAS;YAC1B,OAAO,EAAE,iCAAqB;YAC9B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,uBAAuB,EAAE,CAAC,kBAAM;SACjC,CAAC,CAAA;IACJ,CAAC;IAEO,gBAAgB,CACtB,KAAa,EACb,SAAyB;QAEzB,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO;gBACL,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE;gBAC7B,WAAW,EAAE,CAAC;gBACd,IAAI,EAAE,EAAE,SAAS,EAAE;aACpB,CAAA;QACH,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO;gBACL,WAAW,EAAE,CAAC;gBACd,IAAI,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;aAC3B,CAAA;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAA;IAChD,CAAC;CACF;AA9ED,8BA8EC"}

package/lib/esm/runner.d.ts

@@ -4,10 +4,12 @@ import type { LLMConfig } from './llm-config';
 import type { AgenticInputMessage, AIAgent, Context, RunResult } from './types';
 export declare class AIAgentRunner<TPlugins extends ResolvedPlugins = ResolvedPlugins, TExtraData = any> {
     private agent;
-    private logger;
     private llmConfig;
-    constructor(agent: AIAgent<TPlugins, TExtraData>, openAiClient: LLMConfig, logger: DebugLogger);
+    private inferenceId;
+    private logger;
+    constructor(agent: AIAgent<TPlugins, TExtraData>, openAiClient: LLMConfig, inferenceId: string, logger: DebugLogger);
     run(messages: AgenticInputMessage[], context: Context<TPlugins, TExtraData>): Promise<RunResult>;
+    private sendLlmRunTracking;
     private getToolsExecuted;
     private getToolResultsByCallId;
     private extractToolResult;

package/lib/esm/runner.js

@@ -3,11 +3,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.AIAgentRunner = void 0;
 const agents_1 = require("@openai/agents");
 const constants_1 = require("./constants");
+const hubtype_api_client_1 = require("./services/hubtype-api-client");
+const types_1 = require("./services/types");
 const tools_1 = require("./tools");
 class AIAgentRunner {
-    constructor(agent, openAiClient, logger) {
+    constructor(agent, openAiClient, inferenceId, logger) {
         this.agent = agent;
         this.llmConfig = openAiClient;
+        this.inferenceId = inferenceId;
         this.logger = logger;
     }
     async run(messages, context) {
@@ -30,6 +33,10 @@ class AIAgentRunner {
             const result = (await runner.run(this.agent, messages, {
                 context,
             }));
+            const endTime = Date.now();
+            void this.sendLlmRunTracking(result, context, startTime, endTime);
+            const { _context, ...restResult } = result.state;
+            console.log('Runner result:', restResult);
             const outputMessages = result.finalOutput?.messages || [];
             const hasExit = outputMessages.length === 0 ||
                 outputMessages.some(message => message.type === 'exit');
@@ -67,6 +74,40 @@ class AIAgentRunner {
             throw error;
         }
     }
+    async sendLlmRunTracking(result, context, startTime, endTime) {
+        if (!constants_1.isProd) {
+            return;
+        }
+        const rawResponses = result.rawResponses ?? [];
+        if (rawResponses.length === 0) {
+            return;
+        }
+        const botId = context.request.session.bot.id;
+        const isTest = context.request.session.is_test_integration;
+        const totalDuration = endTime - startTime;
+        const durationPerCall = Math.round(totalDuration / rawResponses.length);
+        const temperature = this.llmConfig.modelSettings.temperature ?? 0;
+        const apiVersion = constants_1.OPENAI_PROVIDER === 'azure' ? constants_1.AZURE_OPENAI_API_VERSION : '';
+        const llmRuns = rawResponses.map(response => ({
+            inference_id: this.inferenceId,
+            is_test: isTest,
+            product_name: types_1.TrackProductName.AI_AGENT,
+            deployment_name: this.llmConfig.modelName,
+            model_name: response.providerData?.['model'] ??
+                this.llmConfig.modelName,
+            feature: types_1.TrackFeature.AI_AGENT_RUN,
+            api_version: apiVersion,
+            num_prompt_tokens: response.usage.inputTokens,
+            num_completion_tokens: response.usage.outputTokens,
+            duration_in_milliseconds: durationPerCall,
+            temperature,
+            error: null,
+        }));
+        const client = new hubtype_api_client_1.HubtypeApiClient(context.authToken);
+        await client.trackLlmRuns(botId, {
+            llm_runs: llmRuns,
+        });
+    }
     getToolsExecuted(result, context) {
         const toolResultsByCallId = this.getToolResultsByCallId(result.newItems);
         return (result.newItems

package/lib/esm/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":";;;AAKA,2CAKuB;AACvB,2CAA6C;AAG7C,mCAA2C;AAkB3C,MAAa,aAAa;IAQxB,YACE,KAAoC,EACpC,YAAuB,EACvB,MAAmB;QAEnB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,IAAI,CAAC,SAAS,GAAG,YAAY,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,GAAG,CACP,QAA+B,EAC/B,OAAsC;QAEtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE5B,IAAI,CAAC,MAAM,CAAC,cAAc,CACxB,IAAI,CAAC,SAAS,CAAC,SAAS,EACxB,IAAI,CAAC,SAAS,CAAC,aAAa,CAC7B,CAAA;QAED,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAAA;YAClD,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAAA;YAElD,MAAM,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,yBAAiB,CAAC,CAAA;YACzE,IAAI,oBAAoB,IAAI,2BAAe,KAAK,OAAO,EAAE,CAAC;gBACxD,aAAa,CAAC,UAAU,GAAG,yBAAiB,CAAC,IAAI,CAAA;YACnD,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,eAAM,CAAC;gBACxB,aAAa;gBACb,aAAa;gBACb,eAAe,EAAE,IAAI;aACtB,CAAC,CAAA;YACF,2FAA2F;YAC3F,qFAAqF;YACrF,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE;gBACrD,OAAO;aACR,CAAC,CAAwB,CAAA;YAE1B,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAA;YACzD,MAAM,OAAO,GACX,cAAc,CAAC,MAAM,KAAK,CAAC;gBAC3B,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;YACzD,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;YAE5D,MAAM,SAAS,GAAc;gBAC3B,QAAQ,EAAE,OAAO;oBACf,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAE,cAAc,CAAC,MAAM,CACpB,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CACR;gBAChC,aAAa;gBACb,IAAI,EAAE,OAAO;gBACb,YAAY,EAAE,QAAQ,CAAC,MAAM;gBAC7B,KAAK,EAAE,KAAK;gBACZ,wBAAwB,EAAE,EAAE;gBAC5B,yBAAyB,EAAE,EAAE;aAC9B,CAAA;YAED,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;YAE9C,OAAO,SAAS,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,wCAA+B,EAAE,CAAC;gBACrD,MAAM,SAAS,GAAc;oBAC3B,QAAQ,EAAE,EAAE;oBACZ,YAAY,EAAE,CAAC;oBACf,aAAa,EAAE,EAAE;oBACjB,IAAI,EAAE,IAAI;oBACV,KAAK,EAAE,KAAK;oBACZ,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU;oBACxD,yBAAyB,EAAE,EAAE;iBAC9B,CAAA;gBAED,IAAI,CAAC,MAAM,CAAC,qBAAqB,EAAE,CAAA;gBACnC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;gBAE9C,OAAO,SAAS,CAAA;YAClB,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;YAE5C,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAEO,gBAAgB,CACtB,MAAM,EACN,OAAsC;QAEtC,MAAM,mBAAmB,GAAG,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAExE,OAAO,CACL,MAAM,CAAC,QAAQ;YACb,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,YAAY,wBAAe,CAAC;aAChD,GAAG,CAAC,CAAC,IAAqB,EAAE,EAAE,CAC7B,IAAI,CAAC,oBAAoB,CACvB,IAAuB,EACvB,OAAO,EACP,mBAAmB,CACpB,CACF;aACA,MAAM,CACL,CAAC,aAA4B,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,KAAK,EAAE,CAChE,IAAI,EAAE,CACV,CAAA;IACH,CAAC;IAEO,sBAAsB,CAAC,QAAmB;QAChD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAA;QACrC,KAAK,MAAM,IAAI,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,CAAC,IAAI,YAAY,8BAAqB,CAAC,EAAE,CAAC;gBAC7C,SAAQ;YACV,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAGpB,CAAA;YACD,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAA;YAC9B,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAA;YAC9B,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBACrC,SAAQ;YACV,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAA;YAC3C,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YACvB,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC;IAEO,iBAAiB,CACvB,MAAiD;QAEjD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,MAAM,CAAA;QACf,CAAC;QACD,OAAO,MAAM,EAAE,IAAI,KAAK,MAAM,IAAI,OAAO,MAAM,EAAE,IAAI,KAAK,QAAQ;YAChE,CAAC,CAAC,MAAM,CAAC,IAAI;YACb,CAAC,CAAC,SAAS,CAAA;IACf,CAAC;IAEO,oBAAoB,CAC1B,IAAqB,EACrB,OAAsC,EACtC,mBAAwC;QAExC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;YAC1C,OAAO;gBACL,QAAQ,EAAE,EAAE;gBACZ,aAAa,EAAE,EAAE;aAClB,CAAA;QACH,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAA;QAClC,MAAM,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACvE,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;YACrC,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAC9C,CAAC,CAAC,SAAS,CAAA;QAEb,MAAM,aAAa,GAAkB;YACnC,QAAQ;YACR,aAAa;YACb,WAAW;SACZ,CAAA;QAED,IAAI,QAAQ,KAAK,yBAAiB,CAAC,IAAI,EAAE,CAAC;YACxC,OAAO;gBACL,GAAG,aAAa;gBAChB,uBAAuB,EAAE,OAAO,CAAC,aAAa,CAAC,SAAS;gBACxD,sBAAsB,EAAE,OAAO,CAAC,aAAa,CAAC,SAAS;aACxD,CAAA;QACH,CAAC;QAED,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,oBAAoB,CAAC,gBAAwB;QACnD,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAA;QACrC,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YAChB,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;CACF;AAjMD,sCAiMC"}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":";;;AAKA,2CAKuB;AACvB,2CAA+E;AAG/E,sEAAgE;AAChE,4CAAiE;AACjE,mCAA2C;AAsB3C,MAAa,aAAa;IASxB,YACE,KAAoC,EACpC,YAAuB,EACvB,WAAmB,EACnB,MAAmB;QAEnB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,IAAI,CAAC,SAAS,GAAG,YAAY,CAAA;QAC7B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAA;QAC9B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,GAAG,CACP,QAA+B,EAC/B,OAAsC;QAEtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE5B,IAAI,CAAC,MAAM,CAAC,cAAc,CACxB,IAAI,CAAC,SAAS,CAAC,SAAS,EACxB,IAAI,CAAC,SAAS,CAAC,aAAa,CAC7B,CAAA;QAED,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAAA;YAClD,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAAA;YAElD,MAAM,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,yBAAiB,CAAC,CAAA;YACzE,IAAI,oBAAoB,IAAI,2BAAe,KAAK,OAAO,EAAE,CAAC;gBACxD,aAAa,CAAC,UAAU,GAAG,yBAAiB,CAAC,IAAI,CAAA;YACnD,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,eAAM,CAAC;gBACxB,aAAa;gBACb,aAAa;gBACb,eAAe,EAAE,IAAI;aACtB,CAAC,CAAA;YACF,2FAA2F;YAC3F,qFAAqF;YACrF,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE;gBACrD,OAAO;aACR,CAAC,CAAwB,CAAA;YAE1B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAE1B,KAAK,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAA;YAEjE,MAAM,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC,KAAK,CAAA;YAChD,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAA;YAEzC,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,EAAE,QAAQ,IAAI,EAAE,CAAA;YACzD,MAAM,OAAO,GACX,cAAc,CAAC,MAAM,KAAK,CAAC;gBAC3B,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;YACzD,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;YAE5D,MAAM,SAAS,GAAc;gBAC3B,QAAQ,EAAE,OAAO;oBACf,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAE,cAAc,CAAC,MAAM,CACpB,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CACR;gBAChC,aAAa;gBACb,IAAI,EAAE,OAAO;gBACb,YAAY,EAAE,QAAQ,CAAC,MAAM;gBAC7B,KAAK,EAAE,KAAK;gBACZ,wBAAwB,EAAE,EAAE;gBAC5B,yBAAyB,EAAE,EAAE;aAC9B,CAAA;YAED,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;YAE9C,OAAO,SAAS,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,wCAA+B,EAAE,CAAC;gBACrD,MAAM,SAAS,GAAc;oBAC3B,QAAQ,EAAE,EAAE;oBACZ,YAAY,EAAE,CAAC;oBACf,aAAa,EAAE,EAAE;oBACjB,IAAI,EAAE,IAAI;oBACV,KAAK,EAAE,KAAK;oBACZ,wBAAwB,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU;oBACxD,yBAAyB,EAAE,EAAE;iBAC9B,CAAA;gBAED,IAAI,CAAC,MAAM,CAAC,qBAAqB,EAAE,CAAA;gBACnC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;gBAE9C,OAAO,SAAS,CAAA;YAClB,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;YAE5C,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,MAA2B,EAC3B,OAAsC,EACtC,SAAiB,EACjB,OAAe;QAEf,IAAI,CAAC,kBAAM,EAAE,CAAC;YACZ,OAAM;QACR,CAAC;QACD,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,EAAE,CAAA;QAC9C,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAM;QACR,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAA;QAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAA;QAC1D,MAAM,aAAa,GAAG,OAAO,GAAG,SAAS,CAAA;QACzC,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,CAAA;QACvE,MAAM,WAAW,GACd,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,WAAkC,IAAI,CAAC,CAAA;QACvE,MAAM,UAAU,GACd,2BAAe,KAAK,OAAO,CAAC,CAAC,CAAC,oCAAwB,CAAC,CAAC,CAAC,EAAE,CAAA;QAE7D,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC5C,YAAY,EAAE,IAAI,CAAC,WAAW;YAC9B,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,wBAAgB,CAAC,QAAQ;YACvC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;YACzC,UAAU,EACP,QAAQ,CAAC,YAAY,EAAE,CAAC,OAAO,CAAwB;gBACxD,IAAI,CAAC,SAAS,CAAC,SAAS;YAC1B,OAAO,EAAE,oBAAY,CAAC,YAAY;YAClC,WAAW,EAAE,UAAU;YACvB,iBAAiB,EAAE,QAAQ,CAAC,KAAK,CAAC,WAAW;YAC7C,qBAAqB,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;YAClD,wBAAwB,EAAE,eAAe;YACzC,WAAW;YACX,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC,CAAA;QAEH,MAAM,MAAM,GAAG,IAAI,qCAAgB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACtD,MAAM,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE;YAC/B,QAAQ,EAAE,OAAO;SAClB,CAAC,CAAA;IACJ,CAAC;IAEO,gBAAgB,CACtB,MAAM,EACN,OAAsC;QAEtC,MAAM,mBAAmB,GAAG,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAExE,OAAO,CACL,MAAM,CAAC,QAAQ;YACb,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,YAAY,wBAAe,CAAC;aAChD,GAAG,CAAC,CAAC,IAAqB,EAAE,EAAE,CAC7B,IAAI,CAAC,oBAAoB,CACvB,IAAuB,EACvB,OAAO,EACP,mBAAmB,CACpB,CACF;aACA,MAAM,CACL,CAAC,aAA4B,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,KAAK,EAAE,CAChE,IAAI,EAAE,CACV,CAAA;IACH,CAAC;IAEO,sBAAsB,CAAC,QAAmB;QAChD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAA;QACrC,KAAK,MAAM,IAAI,IAAI,QAAQ,IAAI,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,CAAC,IAAI,YAAY,8BAAqB,CAAC,EAAE,CAAC;gBAC7C,SAAQ;YACV,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAGpB,CAAA;YACD,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAA;YAC9B,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAA;YAC9B,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBACrC,SAAQ;YACV,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAA;YAC3C,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YACvB,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC;IAEO,iBAAiB,CACvB,MAAiD;QAEjD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,MAAM,CAAA;QACf,CAAC;QACD,OAAO,MAAM,EAAE,IAAI,KAAK,MAAM,IAAI,OAAO,MAAM,EAAE,IAAI,KAAK,QAAQ;YAChE,CAAC,CAAC,MAAM,CAAC,IAAI;YACb,CAAC,CAAC,SAAS,CAAA;IACf,CAAC;IAEO,oBAAoB,CAC1B,IAAqB,EACrB,OAAsC,EACtC,mBAAwC;QAExC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;YAC1C,OAAO;gBACL,QAAQ,EAAE,EAAE;gBACZ,aAAa,EAAE,EAAE;aAClB,CAAA;QACH,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAA;QAClC,MAAM,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACvE,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;YACrC,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAC9C,CAAC,CAAC,SAAS,CAAA;QAEb,MAAM,aAAa,GAAkB;YACnC,QAAQ;YACR,aAAa;YACb,WAAW;SACZ,CAAA;QAED,IAAI,QAAQ,KAAK,yBAAiB,CAAC,IAAI,EAAE,CAAC;YACxC,OAAO;gBACL,GAAG,aAAa;gBAChB,uBAAuB,EAAE,OAAO,CAAC,aAAa,CAAC,SAAS;gBACxD,sBAAsB,EAAE,OAAO,CAAC,aAAa,CAAC,SAAS;aACxD,CAAA;QACH,CAAC;QAED,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,oBAAoB,CAAC,gBAAwB;QACnD,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAA;QACrC,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YAChB,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;CACF;AAxPD,sCAwPC"}

package/lib/esm/{hubtype-api-client.d.ts → services/hubtype-api-client.d.ts}

@@ -1,16 +1,6 @@
 import type { BotContext } from '@botonic/core';
-import type { AgenticInputMessage, Chunk } from './types';
-export interface GetMessagesV2Options {
-    maxMessages?: number;
-    includeToolCalls?: boolean;
-    maxFullToolResults?: number;
-    debugMode?: boolean;
-}
-export interface GetMessagesV2Result {
-    messages: AgenticInputMessage[];
-    conversationId: string | null;
-    truncated: boolean;
-}
+import type { AgenticInputMessage, Chunk } from '../types';
+import type { GetMessagesV2Options, GetMessagesV2Result, TrackLlmRunsData } from './types';
 export declare class HubtypeApiClient {
     private readonly authToken;
     constructor(authToken: string);
@@ -19,5 +9,6 @@ export declare class HubtypeApiClient {
     getMessages(botContext: BotContext, maxMemoryLength: number): Promise<AgenticInputMessage[]>;
     getMessagesV2(botContext: BotContext, options?: GetMessagesV2Options): Promise<GetMessagesV2Result>;
     private formatMessageV2;
+    trackLlmRuns(botId: string, data: TrackLlmRunsData): Promise<void>;
     private formatMessage;
 }

package/lib/{cjs → esm/services}/hubtype-api-client.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.HubtypeApiClient = void 0;
 const tslib_1 = require("tslib");
 const axios_1 = tslib_1.__importDefault(require("axios"));
-const constants_1 = require("./constants");
+const constants_1 = require("../constants");
 class HubtypeApiClient {
     constructor(authToken) {
         this.authToken = authToken;
@@ -164,6 +164,20 @@ class HubtypeApiClient {
                 throw new Error(`Invalid message role: ${message.role}`);
         }
     }
+    async trackLlmRuns(botId, data) {
+        // Not stop the bot execution if the tracking fails, just log the error
+        try {
+            const url = `${constants_1.HUBTYPE_API_URL}/external/v2/conversational_apps/${botId}/track_llm_runs/`;
+            const headers = {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${this.authToken}`,
+            };
+            await axios_1.default.post(url, data, { headers });
+        }
+        catch (error) {
+            console.error('Failed to track LLM runs:', error);
+        }
+    }
     formatMessage(message) {
         if (message.role === 'user') {
             return {

package/lib/esm/services/hubtype-api-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hubtype-api-client.js","sourceRoot":"","sources":["../../../src/services/hubtype-api-client.ts"],"names":[],"mappings":";;;;AAEA,0DAAyB;AAEzB,4CAA8C;AAc9C,MAAa,gBAAgB;IAG3B,YAAY,SAAiB;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,qBAAqB,CACzB,KAAa,EACb,SAAmB;QAEnB,MAAM,GAAG,GAAG,GAAG,2BAAe,gDAAgD,CAAA;QAC9E,MAAM,OAAO,GAAG;YACd,cAAc,EAAE,kBAAkB;YAClC,aAAa,EAAE,UAAU,IAAI,CAAC,SAAS,EAAE;SAC1C,CAAA;QACD,MAAM,MAAM,GAAG;YACb,KAAK;YACL,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,CAAC;SACd,CAAA;QACD,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;QAC3D,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,CAAC;YACzD,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,IAAI,EAAE,KAAK,CAAC,IAAI;SACjB,CAAC,CAAC,CAAA;QACH,OAAO,MAAM,CAAA;IACf,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,eAAuB;QAEvB,MAAM,iBAAiB,GACrB,OAAO,UAAU,KAAK,WAAW,IAAI,UAAU,CAAC,YAAY;YAC1D,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,OAAO,CAAC,cAAc,CAAC;YACjD,CAAC,CAAC,IAAI,CAAA;QACV,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,IAAI,CAAC,CAAA;QAC1D,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAA;QACtC,MAAM,gBAAgB,GAAG,QAAQ;aAC9B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;aACpC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACf,IAAI,EAAE,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW;YACtD,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI;SAC3B,CAAC,CAAC;aACF,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAA;QAC9C,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAA;IACjD,CAAC;IAED,KAAK,CAAC,WAAW,CACf,UAAsB,EACtB,eAAuB;QAEvB,MAAM,GAAG,GAAG,GAAG,2BAAe,wCAAwC,CAAA;QACtE,MAAM,OAAO,GAAG;YACd,cAAc,EAAE,kBAAkB;YAClC,aAAa,EAAE,UAAU,IAAI,CAAC,SAAS,EAAE;SAC1C,CAAA;QACD,MAAM,MAAM,GAAG;YACb,eAAe,EAAE,UAAU,CAAC,KAAK,CAAC,UAAU;YAC5C,YAAY,EAAE,eAAe;SAC9B,CAAA;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,GAAG,CAAiC,GAAG,EAAE;gBACpE,OAAO;gBACP,MAAM;aACP,CAAC,CAAA;YACF,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAA;YACvC,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAA;QAC7D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YACpB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,UAAsB,EACtB,UAAgC,EAAE;QAElC,MAAM,GAAG,GAAG,GAAG,2BAAe,wCAAwC,CAAA;QACtE,MAAM,OAAO,GAAG;YACd,cAAc,EAAE,kBAAkB;YAClC,aAAa,EAAE,UAAU,IAAI,CAAC,SAAS,EAAE;SAC1C,CAAA;QACD,MAAM,MAAM,GAA2B;YACrC,eAAe,EAAE,UAAU,CAAC,KAAK,CAAC,UAAU;YAC5C,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI;gBACvC,YAAY,EAAE,OAAO,CAAC,WAAW;aAClC,CAAC;YACF,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,IAAI;gBAC5C,kBAAkB,EAAE,OAAO,CAAC,gBAAgB;aAC7C,CAAC;YACF,GAAG,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,IAAI;gBAC9C,qBAAqB,EAAE,OAAO,CAAC,kBAAkB;aAClD,CAAC;YACF,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI;gBACrC,UAAU,EAAE,OAAO,CAAC,SAAS;aAC9B,CAAC;SACH,CAAA;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,GAAG,CAA2B,GAAG,EAAE;gBAC9D,OAAO;gBACP,MAAM;aACP,CAAC,CAAA;YACF,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAA;YAC9D,MAAM,iBAAiB,GAAG,QAAQ;iBAC/B,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;iBAC7C,MAAM,CAAC,CAAC,OAAO,EAAkC,EAAE,CAAC,OAAO,KAAK,IAAI,CAAC,CAAA;YACxE,OAAO;gBACL,QAAQ,EAAE,iBAAiB;gBAC3B,cAAc,EAAE,eAAe;gBAC/B,SAAS;aACV,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YACpB,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;QAC/D,CAAC;IACH,CAAC;IAEO,eAAe,CACrB,OAAyB;QAEzB,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,MAAM;gBACT,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;iBAC/B,CAAA;YACH,KAAK,WAAW,CAAC,CAAC,CAAC;gBACjB,MAAM,gBAAgB,GAAG,OAAoC,CAAA;gBAC7D,gEAAgE;gBAChE,iEAAiE;gBACjE,gDAAgD;gBAChD,IACE,gBAAgB,CAAC,UAAU;oBAC3B,gBAAgB,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EACtC,CAAC;oBACD,OAAO;wBACL,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,gBAAgB,CAAC,OAAO,IAAI,EAAE;wBACvC,UAAU,EAAE,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;4BACjD,EAAE,EAAE,EAAE,CAAC,EAAE;4BACT,IAAI,EAAE,EAAE,CAAC,IAAI;4BACb,QAAQ,EAAE;gCACR,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;gCACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,SAAS;6BACjC;yBACF,CAAC,CAAC;qBAC8B,CAAA;gBACrC,CAAC;gBACD,+CAA+C;gBAC/C,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,aAAa;4BACnB,IAAI,EAAE,gBAAgB,CAAC,OAAO,IAAI,EAAE;yBACrC;qBACF;oBACD,MAAM,EAAE,WAAW;iBACpB,CAAA;YACH,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,+DAA+D;gBAC/D,iEAAiE;gBACjE,gDAAgD;gBAChD,MAAM,WAAW,GAAG,OAA+B,CAAA;gBACnD,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,YAAY,EAAE,WAAW,CAAC,YAAY;oBACtC,OAAO,EAAE,WAAW,CAAC,OAAO,IAAI,EAAE;iBACD,CAAA;YACrC,CAAC;YACD,KAAK,QAAQ;gBACX,OAAO;oBACL,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;iBAC/B,CAAA;YACH;gBACE,MAAM,IAAI,KAAK,CACb,yBAA0B,OAA4B,CAAC,IAAI,EAAE,CAC9D,CAAA;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,IAAsB;QACtD,uEAAuE;QACvE,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,GAAG,2BAAe,oCAAoC,KAAK,kBAAkB,CAAA;YACzF,MAAM,OAAO,GAAG;gBACd,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,SAAS,EAAE;aAC1C,CAAA;YACD,MAAM,eAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAA;QACnD,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,OAAuB;QAC3C,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC5B,OAAO;gBACL,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,IAAI,EAAE,MAAM;aACb,CAAA;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACxC,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,aAAa;wBACnB,IAAI,EAAE,OAAO,CAAC,OAAO;qBACtB;iBACF;gBACD,MAAM,EAAE,WAAW;aACpB,CAAA;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAA;QACzC,CAAC;IACH,CAAC;CACF;AA7ND,4CA6NC"}

package/lib/esm/services/types.d.ts

@@ -0,0 +1,85 @@
+import type { AgenticInputMessage } from '../types';
+export declare enum TrackProductName {
+    AI_AGENT = "ai_agent"
+}
+export declare enum TrackFeature {
+    AI_AGENT_RUN = "ai_agent_run",
+    AI_AGENT_GUARDRAIL = "ai_agent_guardrail"
+}
+export interface LlmRunData {
+    inference_id: string;
+    is_test: boolean;
+    product_name: string;
+    deployment_name: string;
+    model_name: string;
+    feature: string;
+    api_version: string;
+    num_prompt_tokens: number;
+    num_completion_tokens: number;
+    duration_in_milliseconds: number;
+    temperature: number;
+    error: string | null;
+}
+export interface TrackLlmRunsData {
+    llm_runs: LlmRunData[];
+}
+interface HubtypeAssistantMessage {
+    role: 'assistant';
+    content: string;
+}
+interface HubtypeUserMessage {
+    role: 'user';
+    content: string;
+}
+export type HubtypeMessage = HubtypeAssistantMessage | HubtypeUserMessage;
+interface HubtypeToolCall {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        arguments: string;
+    };
+}
+interface HubtypeUserMessageV2 {
+    role: 'user';
+    content: string | null;
+}
+export interface HubtypeAssistantMessageV2 {
+    role: 'assistant';
+    content: string | null;
+    tool_calls?: HubtypeToolCall[] | null;
+}
+export interface HubtypeToolMessageV2 {
+    role: 'tool';
+    content: string | null;
+    tool_call_id: string;
+}
+interface HubtypeSystemMessageV2 {
+    role: 'system';
+    content: string | null;
+}
+export type HubtypeMessageV2 = HubtypeUserMessageV2 | HubtypeAssistantMessageV2 | HubtypeToolMessageV2 | HubtypeSystemMessageV2;
+export interface MessageHistoryResponseV2 {
+    messages: HubtypeMessageV2[];
+    conversation_id: string | null;
+    truncated: boolean;
+}
+export interface GetMessagesV2Options {
+    maxMessages?: number;
+    includeToolCalls?: boolean;
+    maxFullToolResults?: number;
+    debugMode?: boolean;
+}
+export interface GetMessagesV2Result {
+    messages: AgenticInputMessage[];
+    conversationId: string | null;
+    truncated: boolean;
+}
+export interface MessageHistoryV2Params {
+    last_message_id: string;
+    max_messages?: number;
+    include_tool_calls?: boolean;
+    max_full_tool_results?: number;
+    debug_mode?: boolean;
+}
+export {};

package/lib/esm/services/types.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TrackFeature = exports.TrackProductName = void 0;
+var TrackProductName;
+(function (TrackProductName) {
+    TrackProductName["AI_AGENT"] = "ai_agent";
+})(TrackProductName || (exports.TrackProductName = TrackProductName = {}));
+var TrackFeature;
+(function (TrackFeature) {
+    TrackFeature["AI_AGENT_RUN"] = "ai_agent_run";
+    TrackFeature["AI_AGENT_GUARDRAIL"] = "ai_agent_guardrail";
+})(TrackFeature || (exports.TrackFeature = TrackFeature = {}));
+//# sourceMappingURL=types.js.map

package/lib/esm/services/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/services/types.ts"],"names":[],"mappings":";;;AAEA,IAAY,gBAEX;AAFD,WAAY,gBAAgB;IAC1B,yCAAqB,CAAA;AACvB,CAAC,EAFW,gBAAgB,gCAAhB,gBAAgB,QAE3B;AAED,IAAY,YAGX;AAHD,WAAY,YAAY;IACtB,6CAA6B,CAAA;IAC7B,yDAAyC,CAAA;AAC3C,CAAC,EAHW,YAAY,4BAAZ,YAAY,QAGvB"}

package/lib/esm/tools/retrieve-knowledge.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.retrieveKnowledge = void 0;
 const agents_1 = require("@openai/agents");
 const zod_1 = require("zod");
-const hubtype_api_client_1 = require("../hubtype-api-client");
+const hubtype_api_client_1 = require("../services/hubtype-api-client");
 exports.retrieveKnowledge = (0, agents_1.tool)({
     name: 'retrieve_knowledge',
     description: 'Consult the knowledge base for information before answering. Use this tool to make sure the information you provide is faithful.',

package/lib/esm/tools/retrieve-knowledge.js.map

@@ -1 +1 @@
-{"version":3,"file":"retrieve-knowledge.js","sourceRoot":"","sources":["../../../src/tools/retrieve-knowledge.ts"],"names":[],"mappings":";;;AAAA,2CAAsD;AACtD,6BAAuB;AAEvB,8DAAwD;AAG3C,QAAA,iBAAiB,GAAG,IAAA,aAAI,EAAC;IACpC,IAAI,EAAE,oBAAoB;IAC1B,WAAW,EACT,kIAAkI;IACpI,UAAU,EAAE,OAAC,CAAC,MAAM,CAAC;QACnB,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4CAA4C,CAAC;KACzE,CAAC;IACF,OAAO,EAAE,KAAK,EACZ,KAAwB,EACxB,UAAgC,EACb,EAAE;QACrB,MAAM,OAAO,GAAG,UAAU,EAAE,OAAO,CAAA;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;QACzB,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;QACxC,CAAC;QACD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QACnC,MAAM,MAAM,GAAG,IAAI,qCAAgB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACtD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QACnE,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAElD,OAAO,CAAC,aAAa,GAAG;YACtB,KAAK;YACL,SAAS;YACT,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;YACxC,UAAU;SACX,CAAA;QAED,OAAO,UAAU,CAAA;IACnB,CAAC;CACF,CAAC,CAAA"}
+{"version":3,"file":"retrieve-knowledge.js","sourceRoot":"","sources":["../../../src/tools/retrieve-knowledge.ts"],"names":[],"mappings":";;;AAAA,2CAAsD;AACtD,6BAAuB;AAEvB,uEAAiE;AAGpD,QAAA,iBAAiB,GAAG,IAAA,aAAI,EAAC;IACpC,IAAI,EAAE,oBAAoB;IAC1B,WAAW,EACT,kIAAkI;IACpI,UAAU,EAAE,OAAC,CAAC,MAAM,CAAC;QACnB,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4CAA4C,CAAC;KACzE,CAAC;IACF,OAAO,EAAE,KAAK,EACZ,KAAwB,EACxB,UAAgC,EACb,EAAE;QACrB,MAAM,OAAO,GAAG,UAAU,EAAE,OAAO,CAAA;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;QACzB,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;QACxC,CAAC;QACD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QACnC,MAAM,MAAM,GAAG,IAAI,qCAAgB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACtD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QACnE,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAElD,OAAO,CAAC,aAAa,GAAG;YACtB,KAAK;YACL,SAAS;YACT,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;YACxC,UAAU;SACX,CAAA;QAED,OAAO,UAAU,CAAA;IACnB,CAAC;CACF,CAAC,CAAA"}

package/lib/esm/types.d.ts

@@ -45,5 +45,12 @@ export interface PluginAiAgentOptions<TPlugins extends ResolvedPlugins = Resolve
     /** Enable debug logging for AI agent configuration and execution details. */
     enableDebug?: boolean;
 }
+export interface ResultRawResponse {
+    usage: {
+        inputTokens: number;
+        outputTokens: number;
+    };
+    providerData?: Record<string, unknown>;
+}
 export type AgenticInputMessage = AgentInputItem;
 export { AgenticOutputMessage, AiAgentArgs, GuardrailRule, InferenceResponse, RunResult, };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botonic/plugin-ai-agents",
-  "version": "0.46.0-alpha.0",
+  "version": "0.46.0-alpha.1",
   "main": "./lib/cjs/index.js",
   "module": "./lib/esm/index.js",
   "description": "Use AI Agents to generate your contents",
@@ -14,7 +14,7 @@
     "format": "biome format --write src/ tests/"
   },
   "dependencies": {
-    "@botonic/core": "^0.46.0-alpha.0",
+    "@botonic/core": "^0.46.0-alpha.1",
     "@openai/agents": "^0.3.9",
     "axios": "^1.13.6",
     "openai": "^6.0.0",

package/src/agent-builder.ts

@@ -8,6 +8,7 @@ import {
 import { OPENAI_PROVIDER } from './constants'
 import type { DebugLogger } from './debug-logger'
 import { createInputGuardrail } from './guardrails'
+import type { GuardrailTrackingContext } from './guardrails/input'
 import type { LLMConfig } from './llm-config'
 import { OutputSchema } from './structured-output'
 import { mandatoryTools, retrieveKnowledge } from './tools'
@@ -26,6 +27,7 @@ interface AIAgentBuilderOptions<
   sourceIds: string[]
   llmConfig: LLMConfig
   logger: DebugLogger
+  guardrailTrackingContext: GuardrailTrackingContext
 }
 
 export class AIAgentBuilder<
@@ -53,7 +55,8 @@ export class AIAgentBuilder<
     if (options.inputGuardrailRules.length > 0) {
       const inputGuardrail = createInputGuardrail(
         options.inputGuardrailRules,
-        options.llmConfig
+        options.llmConfig,
+        options.guardrailTrackingContext
       )
       this.inputGuardrails.push(inputGuardrail)
     }

package/src/guardrails/input.ts

@@ -5,12 +5,23 @@ import {
   type UserMessageItem,
 } from '@openai/agents'
 import { z } from 'zod'
+import { AZURE_OPENAI_API_VERSION, isProd, OPENAI_PROVIDER } from '../constants'
 import type { LLMConfig } from '../llm-config'
-import type { GuardrailRule } from '../types'
+import { HubtypeApiClient } from '../services/hubtype-api-client'
+import { TrackFeature, TrackProductName } from '../services/types'
+import type { GuardrailRule, ResultRawResponse } from '../types'
+
+export interface GuardrailTrackingContext {
+  botId: string
+  isTest: boolean
+  authToken: string
+  inferenceId: string
+}
 
 export function createInputGuardrail(
   rules: GuardrailRule[],
-  llmConfig: LLMConfig
+  llmConfig: LLMConfig,
+  trackingContext: GuardrailTrackingContext
 ): InputGuardrail {
   const outputType = z.object(
     Object.fromEntries(
@@ -29,12 +40,25 @@ export function createInputGuardrail(
     name: 'InputGuardrail',
     execute: async ({ input, context }) => {
       const lastMessage = input[input.length - 1] as UserMessageItem
+      const modelProvider = llmConfig.modelProvider
+      const modelSettings = llmConfig.modelSettings
+      modelSettings.toolChoice = undefined
       const runner = new Runner({
-        modelSettings: llmConfig.modelSettings,
-        modelProvider: llmConfig.modelProvider,
+        modelSettings,
+        modelProvider,
         tracingDisabled: true,
       })
+      const startTime = Date.now()
       const result = await runner.run(agent, [lastMessage], { context })
+      const endTime = Date.now()
+
+      void sendGuardrailLlmRunTracking(
+        result,
+        trackingContext,
+        llmConfig,
+        startTime,
+        endTime
+      )
 
       const finalOutput = result.finalOutput as Record<string, boolean>
       if (finalOutput === undefined) {
@@ -51,3 +75,48 @@ export function createInputGuardrail(
     },
   }
 }
+
+async function sendGuardrailLlmRunTracking(
+  result: {
+    rawResponses?: ResultRawResponse[]
+  },
+  trackingContext: GuardrailTrackingContext,
+  llmConfig: LLMConfig,
+  startTime: number,
+  endTime: number
+): Promise<void> {
+  if (!isProd) {
+    return
+  }
+  const rawResponses = result.rawResponses ?? []
+  if (rawResponses.length === 0) {
+    return
+  }
+  const totalDuration = endTime - startTime
+  const durationPerCall = Math.round(totalDuration / rawResponses.length)
+  const temperature =
+    (llmConfig.modelSettings.temperature as number | undefined) ?? 0
+  const apiVersion = OPENAI_PROVIDER === 'azure' ? AZURE_OPENAI_API_VERSION : ''
+
+  const llmRuns = rawResponses.map(response => ({
+    inference_id: trackingContext.inferenceId,
+    is_test: trackingContext.isTest,
+    product_name: TrackProductName.AI_AGENT,
+    deployment_name: llmConfig.modelName,
+    model_name:
+      (response.providerData?.['model'] as string | undefined) ??
+      llmConfig.modelName,
+    feature: TrackFeature.AI_AGENT_GUARDRAIL,
+    api_version: apiVersion,
+    num_prompt_tokens: response.usage.inputTokens,
+    num_completion_tokens: response.usage.outputTokens,
+    duration_in_milliseconds: durationPerCall,
+    temperature,
+    error: null,
+  }))
+
+  const client = new HubtypeApiClient(trackingContext.authToken)
+  await client.trackLlmRuns(trackingContext.botId, {
+    llm_runs: llmRuns,
+  })
+}