@contractspec/lib.ai-agent 2.0.0 → 2.1.1

package/dist/node/agent/contract-spec-agent.js

@@ -2536,24 +2536,78 @@ function generateSessionId() {
 
 // src/telemetry/adapter.ts
 function parseAgentId(agentId) {
-  const match = agentId.match(/^(.+)\.v(\s+)$/);
+  const match = agentId.match(/^(.+)\.v(\d+)$/);
   if (match) {
     return { name: match[1], version: match[2] };
   }
   return { name: agentId, version: "1.0.0" };
 }
-async function trackAgentStep(collector, agentId, step, durationMs) {
+function getRecord(value) {
+  return value && typeof value === "object" ? value : undefined;
+}
+function getStepResponse(step) {
+  const response = getRecord(step.response);
+  if (!response)
+    return;
+  return {
+    id: response["id"],
+    modelId: response["modelId"],
+    timestamp: response["timestamp"],
+    headers: response["headers"],
+    body: response["body"],
+    messages: response["messages"]
+  };
+}
+function getRequestBodyValue(request, key) {
+  const body = request?.["body"];
+  if (!body)
+    return;
+  if (typeof body === "string") {
+    try {
+      const parsed = JSON.parse(body);
+      return typeof parsed[key] === "string" ? parsed[key] : undefined;
+    } catch {
+      return;
+    }
+  }
+  const record = getRecord(body);
+  return typeof record?.[key] === "string" ? record[key] : undefined;
+}
+async function trackAgentStep(collector, agentId, step, durationMs, context) {
   const { name, version } = parseAgentId(agentId);
+  const response = getStepResponse(step);
+  const providerMetadata = step.providerMetadata;
+  const warnings = step.warnings;
+  const rawFinishReason = step.rawFinishReason;
+  const request = getRecord(step.request);
+  const traceId = context?.traceId ?? getRequestBodyValue(request, "$ai_trace_id");
+  const sessionId = context?.sessionId ?? getRequestBodyValue(request, "$ai_session_id");
+  const parentSpanId = response?.id;
   for (const toolCall of step.toolCalls ?? []) {
+    const toolResult = step.toolResults?.find((r) => r.toolCallId === toolCall.toolCallId);
+    const toolError = getRecord(toolResult?.output)?.["error"];
     const toolSample = {
       operation: { name: `${name}.${toolCall.toolName}`, version },
       durationMs: durationMs ?? 0,
       success: step.toolResults?.some((r) => r.toolCallId === toolCall.toolCallId && r.output !== undefined) ?? false,
       timestamp: new Date,
       metadata: {
+        telemetryEvent: "span",
+        traceId,
+        sessionId,
+        spanId: toolCall.toolCallId,
+        parentSpanId,
+        spanName: `tool.${toolCall.toolName}`,
         agentId,
+        actorId: context?.actorId,
+        tenantId: context?.tenantId,
+        stepIndex: context?.stepIndex,
         toolName: toolCall.toolName,
-        finishReason: step.finishReason
+        toolCallArgs: toolCall.input,
+        toolResultOutput: toolResult?.output,
+        errorMessage: typeof toolError === "string" ? toolError : typeof toolResult?.output === "string" ? toolResult.output : undefined,
+        finishReason: step.finishReason,
+        rawFinishReason
       }
     };
     await collector.collect(toolSample);
@@ -2564,10 +2618,36 @@ async function trackAgentStep(collector, agentId, step, durationMs) {
     success: step.finishReason !== "error",
     timestamp: new Date,
     metadata: {
+      telemetryEvent: "generation",
+      traceId,
+      sessionId,
+      spanId: response?.id,
+      spanName: `agent.${name}.step`,
       agentId,
+      actorId: context?.actorId,
+      tenantId: context?.tenantId,
+      stepIndex: context?.stepIndex,
+      stepStartedAt: context?.stepStartedAt,
       finishReason: step.finishReason,
+      rawFinishReason,
       tokenUsage: step.usage,
-      toolCallCount: step.toolCalls?.length ?? 0
+      totalUsage: step.totalUsage,
+      providerMetadata,
+      warnings,
+      stepText: step.text,
+      stepReasoningText: step.reasoningText,
+      responseId: response?.id,
+      responseModelId: response?.modelId,
+      responseTimestamp: response?.timestamp,
+      responseHeaders: response?.headers,
+      responseBody: response?.body,
+      responseMessages: response?.messages,
+      requestBody: request?.["body"],
+      requestHeaders: request?.["headers"],
+      toolCallCount: step.toolCalls?.length ?? 0,
+      toolCalls: step.toolCalls,
+      toolResults: step.toolResults,
+      errorMessage: step.finishReason === "error" ? step.text : undefined
     }
   };
   await collector.collect(stepSample);
@@ -2627,23 +2707,60 @@ class PostHogTelemetryCollector {
   }
   async collect(sample) {
     const client = await this.getClient();
-    const distinctId = this.config.defaults?.posthogDistinctId ?? sample.metadata?.["actorId"] ?? "system";
+    const metadata = asRecord(sample.metadata);
+    const distinctId = this.config.defaults?.posthogDistinctId ?? asString(metadata["actorId"]) ?? "system";
+    const traceId = asString(metadata["traceId"]) ?? this.config.defaults?.posthogTraceId;
+    const sessionId = asString(metadata["sessionId"]);
+    const telemetryEvent = asString(metadata["telemetryEvent"]);
+    const event = telemetryEvent === "span" ? "$ai_span" : "$ai_generation";
+    const tokenUsage = metadata["tokenUsage"];
+    const totalUsage = metadata["totalUsage"];
+    const errorMessage = asString(metadata["errorMessage"]);
     client.capture({
       distinctId,
-      event: "$ai_generation",
+      event,
       properties: {
-        $ai_model: sample.operation.name,
-        $ai_provider: "contractspec",
+        $ai_model: asString(metadata["responseModelId"]) ?? sample.operation.name,
+        $ai_provider: asString(metadata["provider"]) ?? "contractspec",
         $ai_latency: sample.durationMs / 1000,
         $ai_is_error: !sample.success,
-        $ai_trace_id: this.config.defaults?.posthogTraceId,
-        ...sample.metadata?.["tokenUsage"] ? mapTokenUsage(sample.metadata["tokenUsage"]) : {},
+        $ai_error: !sample.success ? errorMessage : undefined,
+        $ai_trace_id: traceId,
+        $ai_session_id: sessionId,
+        $ai_span_id: asString(metadata["spanId"]),
+        $ai_parent_id: asString(metadata["parentSpanId"]),
+        $ai_span_name: asString(metadata["spanName"]) ?? sample.operation.name,
+        ...tokenUsage ? mapTokenUsage(tokenUsage) : {},
+        ...totalUsage ? mapTotalUsage(totalUsage) : {},
+        $ai_http_status: asNumber(metadata["httpStatus"]),
+        $ai_request_url: asString(metadata["requestUrl"]),
+        $ai_base_url: asString(metadata["baseUrl"]),
         ...this.config.defaults?.posthogProperties,
         contractspec_operation: sample.operation.name,
         contractspec_version: sample.operation.version,
-        contractspec_agent_id: sample.metadata?.["agentId"],
-        contractspec_finish_reason: sample.metadata?.["finishReason"],
-        contractspec_tool_count: sample.metadata?.["toolCallCount"]
+        contractspec_agent_id: asString(metadata["agentId"]),
+        contractspec_tenant_id: asString(metadata["tenantId"]),
+        contractspec_actor_id: asString(metadata["actorId"]),
+        contractspec_step_index: asNumber(metadata["stepIndex"]),
+        contractspec_step_started_at: asDateIso(metadata["stepStartedAt"]),
+        contractspec_finish_reason: asString(metadata["finishReason"]),
+        contractspec_finish_reason_raw: asString(metadata["rawFinishReason"]),
+        contractspec_tool_count: asNumber(metadata["toolCallCount"]),
+        contractspec_tool_name: asString(metadata["toolName"]),
+        contractspec_tool_call_args: metadata["toolCallArgs"],
+        contractspec_tool_result_output: metadata["toolResultOutput"],
+        contractspec_provider_metadata: metadata["providerMetadata"],
+        contractspec_step_warnings: metadata["warnings"],
+        contractspec_response_id: asString(metadata["responseId"]),
+        contractspec_response_model_id: asString(metadata["responseModelId"]),
+        contractspec_response_timestamp: asDateIso(metadata["responseTimestamp"]),
+        contractspec_response_headers: metadata["responseHeaders"],
+        contractspec_response_body: this.config.defaults?.posthogPrivacyMode ? undefined : metadata["responseBody"],
+        contractspec_response_messages: this.config.defaults?.posthogPrivacyMode ? undefined : metadata["responseMessages"],
+        contractspec_request_headers: metadata["requestHeaders"],
+        contractspec_request_body: this.config.defaults?.posthogPrivacyMode ? undefined : metadata["requestBody"],
+        contractspec_step_text: this.config.defaults?.posthogPrivacyMode ? undefined : metadata["stepText"],
+        contractspec_step_reasoning_text: this.config.defaults?.posthogPrivacyMode ? undefined : metadata["stepReasoningText"]
       },
       groups: this.config.defaults?.posthogGroups
     });
@@ -2704,11 +2821,44 @@ async function resolvePostHogClient(config) {
   }
 }
 function mapTokenUsage(usage) {
+  const usageRecord = asRecord(usage);
+  const inputTokenDetails = asRecord(usageRecord["inputTokenDetails"]);
+  const outputTokenDetails = asRecord(usageRecord["outputTokenDetails"]);
   return {
-    $ai_input_tokens: usage["promptTokens"],
-    $ai_output_tokens: usage["completionTokens"]
+    $ai_input_tokens: asNumber(usageRecord["inputTokens"]) ?? asNumber(usageRecord["promptTokens"]),
+    $ai_output_tokens: asNumber(usageRecord["outputTokens"]) ?? asNumber(usageRecord["completionTokens"]),
+    $ai_reasoning_tokens: asNumber(outputTokenDetails["reasoningTokens"]) ?? asNumber(usageRecord["reasoningTokens"]),
+    $ai_cache_read_input_tokens: asNumber(inputTokenDetails["cacheReadTokens"]) ?? asNumber(usageRecord["cachedInputTokens"]),
+    $ai_cache_creation_input_tokens: asNumber(inputTokenDetails["cacheWriteTokens"]),
+    $ai_usage: maybeRecord(usageRecord["raw"]) ?? usageRecord
   };
 }
+function mapTotalUsage(usage) {
+  const usageRecord = asRecord(usage);
+  return {
+    contractspec_total_input_tokens: asNumber(usageRecord["inputTokens"]),
+    contractspec_total_output_tokens: asNumber(usageRecord["outputTokens"]),
+    contractspec_total_tokens: asNumber(usageRecord["totalTokens"]),
+    contractspec_total_usage_raw: maybeRecord(usageRecord["raw"]) ?? usageRecord
+  };
+}
+function asRecord(value) {
+  return value && typeof value === "object" ? value : {};
+}
+function maybeRecord(value) {
+  return value && typeof value === "object" ? value : undefined;
+}
+function asString(value) {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+function asNumber(value) {
+  return typeof value === "number" ? value : undefined;
+}
+function asDateIso(value) {
+  if (value instanceof Date)
+    return value.toISOString();
+  return;
+}
 
 // src/agent/contract-spec-agent.ts
 var exports_contract_spec_agent = {};
@@ -2719,6 +2869,7 @@ import {
   Experimental_Agent as ToolLoopAgent,
   stepCountIs
 } from "ai";
+import { randomUUID } from "node:crypto";
 import * as z3 from "zod";
 
 class ContractSpecAgent {
@@ -2726,33 +2877,18 @@ class ContractSpecAgent {
   id;
   spec;
   tools;
-  inner;
   config;
   instructions;
+  activeStepContexts = new Map;
   constructor(config, instructions, tools) {
     this.config = config;
     this.spec = config.spec;
     this.id = agentKey(config.spec.meta);
     this.tools = tools;
     this.instructions = instructions;
-    this.inner = new ToolLoopAgent({
-      model: config.model,
-      instructions,
-      tools,
-      stopWhen: stepCountIs(config.spec.maxSteps ?? 10),
-      callOptionsSchema: ContractSpecCallOptionsSchema,
-      onStepFinish: async (step) => {
-        await this.handleStepFinish(step);
-      }
-    });
   }
   static async create(config) {
-    let effectiveConfig = config;
-    if (config.posthogConfig) {
-      const { createPostHogTracedModel: createPostHogTracedModel2 } = await Promise.resolve().then(() => exports_posthog);
-      const tracedModel = await createPostHogTracedModel2(config.model, config.posthogConfig, config.posthogConfig.tracingOptions);
-      effectiveConfig = { ...config, model: tracedModel };
-    }
+    const effectiveConfig = config;
     const instructions = await injectStaticKnowledge(effectiveConfig.spec.instructions, effectiveConfig.spec.knowledge ?? [], effectiveConfig.knowledgeRetriever);
     const specTools = specToolsToAISDKTools(effectiveConfig.spec.tools, effectiveConfig.toolHandlers, { agentId: agentKey(effectiveConfig.spec.meta) });
     const knowledgeTool = effectiveConfig.knowledgeRetriever ? createKnowledgeQueryTool(effectiveConfig.knowledgeRetriever, effectiveConfig.spec.knowledge ?? []) : null;
@@ -2765,6 +2901,14 @@ class ContractSpecAgent {
   }
   async generate(params) {
     const sessionId = params.options?.sessionId ?? generateSessionId();
+    const traceId = params.options?.metadata?.["traceId"] ?? this.config.posthogConfig?.tracingOptions?.posthogTraceId ?? randomUUID();
+    this.activeStepContexts.set(sessionId, {
+      traceId,
+      tenantId: params.options?.tenantId,
+      actorId: params.options?.actorId,
+      stepIndex: 0,
+      stepStartedAt: new Date
+    });
     if (this.config.sessionStore) {
       const existing = await this.config.sessionStore.get(sessionId);
       if (!existing) {
@@ -2785,7 +2929,13 @@ class ContractSpecAgent {
 ${params.systemOverride}
 
 ${params.prompt}` : params.prompt;
-    const result = await this.inner.generate({
+    const model = await this.resolveModelForCall({
+      sessionId,
+      traceId,
+      options: params.options
+    });
+    const inner = this.createInnerAgent(model);
+    const result = await inner.generate({
       prompt,
       abortSignal: params.signal,
       options: {
@@ -2794,6 +2944,8 @@ ${params.prompt}` : params.prompt;
         sessionId,
         metadata: params.options?.metadata
       }
+    }).finally(() => {
+      this.activeStepContexts.delete(sessionId);
     });
     if (this.config.sessionStore) {
       await this.config.sessionStore.update(sessionId, {
@@ -2821,12 +2973,26 @@ ${params.prompt}` : params.prompt;
   }
   async stream(params) {
     const sessionId = params.options?.sessionId ?? generateSessionId();
+    const traceId = params.options?.metadata?.["traceId"] ?? this.config.posthogConfig?.tracingOptions?.posthogTraceId ?? randomUUID();
+    this.activeStepContexts.set(sessionId, {
+      traceId,
+      tenantId: params.options?.tenantId,
+      actorId: params.options?.actorId,
+      stepIndex: 0,
+      stepStartedAt: new Date
+    });
     const prompt = params.systemOverride ? `${this.instructions}
 
 ${params.systemOverride}
 
 ${params.prompt}` : params.prompt;
-    return this.inner.stream({
+    const model = await this.resolveModelForCall({
+      sessionId,
+      traceId,
+      options: params.options
+    });
+    const inner = this.createInnerAgent(model);
+    return inner.stream({
       prompt,
       abortSignal: params.signal,
       options: {
@@ -2843,9 +3009,63 @@ ${params.prompt}` : params.prompt;
       await this.config.sessionStore.appendStep(sessionId, step);
     }
     if (this.config.telemetryCollector) {
-      await trackAgentStep(this.config.telemetryCollector, this.id, step);
+      const now = new Date;
+      const context = sessionId ? this.activeStepContexts.get(sessionId) : undefined;
+      const stepStartedAt = context?.stepStartedAt ?? now;
+      const durationMs = Math.max(now.getTime() - stepStartedAt.getTime(), 0);
+      if (context) {
+        context.stepIndex += 1;
+        context.stepStartedAt = now;
+      }
+      await trackAgentStep(this.config.telemetryCollector, this.id, step, durationMs, {
+        sessionId,
+        tenantId: context?.tenantId,
+        actorId: context?.actorId,
+        traceId: context?.traceId,
+        stepIndex: context?.stepIndex,
+        stepStartedAt
+      });
+      if (sessionId && step.finishReason !== "tool-calls") {
+        this.activeStepContexts.delete(sessionId);
+      }
     }
   }
+  createInnerAgent(model) {
+    return new ToolLoopAgent({
+      model,
+      instructions: this.instructions,
+      tools: this.tools,
+      stopWhen: stepCountIs(this.spec.maxSteps ?? 10),
+      callOptionsSchema: ContractSpecCallOptionsSchema,
+      onStepFinish: async (step) => {
+        await this.handleStepFinish(step);
+      }
+    });
+  }
+  async resolveModelForCall(params) {
+    const posthogConfig = this.config.posthogConfig;
+    if (!posthogConfig) {
+      return this.config.model;
+    }
+    const mergedProperties = {
+      ...posthogConfig.defaults?.posthogProperties,
+      ...posthogConfig.tracingOptions?.posthogProperties,
+      $ai_session_id: params.sessionId,
+      contractspec_session_id: params.sessionId,
+      contractspec_trace_id: params.traceId,
+      contractspec_agent_id: this.id,
+      contractspec_tenant_id: params.options?.tenantId,
+      contractspec_actor_id: params.options?.actorId
+    };
+    const tracingOptions = {
+      ...posthogConfig.tracingOptions,
+      posthogDistinctId: posthogConfig.tracingOptions?.posthogDistinctId ?? params.options?.actorId,
+      posthogTraceId: params.traceId,
+      posthogProperties: mergedProperties
+    };
+    const { createPostHogTracedModel: createPostHogTracedModel2 } = await Promise.resolve().then(() => exports_posthog);
+    return createPostHogTracedModel2(this.config.model, posthogConfig, tracingOptions);
+  }
 }
 var ContractSpecCallOptionsSchema;
 var init_contract_spec_agent = __esm(() => {