@townco/agent 0.1.51 → 0.1.53

package/dist/acp-server/adapter.d.ts

@@ -24,6 +24,8 @@ export declare class AgentAcpAdapter implements acp.Agent {
     private agentVersion;
     private agentDescription;
     private agentSuggestedPrompts;
+    private currentToolOverheadTokens;
+    private currentMcpOverheadTokens;
     constructor(agent: AgentRunner, connection: acp.AgentSideConnection, agentDir?: string, agentName?: string);
     /**
      * Helper to save session to disk

package/dist/acp-server/adapter.js

@@ -103,6 +103,8 @@ export class AgentAcpAdapter {
     agentVersion;
     agentDescription;
     agentSuggestedPrompts;
+    currentToolOverheadTokens = 0; // Track tool overhead for current turn
+    currentMcpOverheadTokens = 0; // Track MCP overhead for current turn
     constructor(agent, connection, agentDir, agentName) {
         this.connection = connection;
         this.sessions = new Map();
@@ -338,6 +340,9 @@ export class AgentAcpAdapter {
         }
         session.pendingPrompt?.abort();
         session.pendingPrompt = new AbortController();
+        // Reset tool overhead for new turn (will be set by harness)
+        this.currentToolOverheadTokens = 0;
+        this.currentMcpOverheadTokens = 0;
         // Generate a unique messageId for this assistant response
         const messageId = Math.random().toString(36).substring(2);
         // Extract and store the user message
@@ -397,7 +402,9 @@ export class AgentAcpAdapter {
                 }
             }
             // Calculate context size - no LLM call yet, so only estimated values
-            const context_size = calculateContextSize(contextMessages, this.agent.definition.systemPrompt ?? undefined, undefined);
+            const context_size = calculateContextSize(contextMessages, this.agent.definition.systemPrompt ?? undefined, undefined, // No LLM-reported tokens yet
+            this.currentToolOverheadTokens, // Include tool overhead
+            this.currentMcpOverheadTokens);
             const contextSnapshot = createContextSnapshot(session.messages.length, new Date().toISOString(), previousContext, context_size);
             session.context.push(contextSnapshot);
             await this.saveSessionToDisk(params.sessionId, session);
@@ -461,6 +468,20 @@ export class AgentAcpAdapter {
             let iterResult = await generator.next();
             while (!iterResult.done) {
                 const msg = iterResult.value;
+                // Capture tool overhead info if provided by harness
+                if ("sessionUpdate" in msg &&
+                    msg.sessionUpdate === "tool_overhead_info") {
+                    const overheadInfo = msg;
+                    this.currentToolOverheadTokens = overheadInfo.toolOverheadTokens;
+                    this.currentMcpOverheadTokens = overheadInfo.mcpOverheadTokens;
+                    logger.debug("Received tool overhead info from harness", {
+                        toolOverheadTokens: this.currentToolOverheadTokens,
+                        mcpOverheadTokens: this.currentMcpOverheadTokens,
+                    });
+                    // Don't send this update to client, it's internal metadata
+                    iterResult = await generator.next();
+                    continue;
+                }
                 // Extract and accumulate token usage from message chunks
                 if ("sessionUpdate" in msg &&
                     msg.sessionUpdate === "agent_message_chunk" &&
@@ -659,7 +680,9 @@ export class AgentAcpAdapter {
                                 }
                             }
                             // Calculate context size - tool result is now in the message, but hasn't been sent to LLM yet
-                            const context_size = calculateContextSize(contextMessages, this.agent.definition.systemPrompt ?? undefined, undefined);
+                            const context_size = calculateContextSize(contextMessages, this.agent.definition.systemPrompt ?? undefined, undefined, // Tool result hasn't been sent to LLM yet, so no new LLM-reported tokens
+                            this.currentToolOverheadTokens, // Include tool overhead
+                            this.currentMcpOverheadTokens);
                             // Create snapshot with a pointer to the partial message (not a full copy!)
                             const midTurnSnapshot = {
                                 timestamp: new Date().toISOString(),
@@ -779,7 +802,9 @@ export class AgentAcpAdapter {
                 }
             }
             // Calculate context size with LLM-reported tokens from this turn
-            const context_size = calculateContextSize(contextMessages, this.agent.definition.systemPrompt ?? undefined, turnTokenUsage.inputTokens);
+            const context_size = calculateContextSize(contextMessages, this.agent.definition.systemPrompt ?? undefined, turnTokenUsage.inputTokens, // Final LLM-reported tokens from this turn
+            this.currentToolOverheadTokens, // Include tool overhead
+            this.currentMcpOverheadTokens);
             const contextSnapshot = createContextSnapshot(session.messages.length, new Date().toISOString(), previousContext, context_size);
             session.context.push(contextSnapshot);
             await this.saveSessionToDisk(params.sessionId, session);

package/dist/acp-server/index.js

@@ -1,2 +1,7 @@
+import { initializeOpenTelemetryFromEnv } from "../telemetry/setup.js";
+// Initialize OpenTelemetry when this module is imported (if enabled)
+if (process.env.ENABLE_TELEMETRY === "true") {
+    initializeOpenTelemetryFromEnv();
+}
 export { makeStdioTransport } from "./cli";
 export { makeHttpTransport } from "./http";

package/dist/acp-server/session-storage.d.ts

@@ -62,6 +62,8 @@ export interface ContextEntry {
      */
     context_size: {
         systemPromptTokens: number;
+        toolOverheadTokens?: number | undefined;
+        mcpOverheadTokens?: number | undefined;
         userMessagesTokens: number;
         assistantMessagesTokens: number;
         toolInputTokens: number;

package/dist/acp-server/session-storage.js

@@ -58,6 +58,8 @@ const contextEntrySchema = z.object({
     compactedUpTo: z.number().optional(),
     context_size: z.object({
         systemPromptTokens: z.number(),
+        toolOverheadTokens: z.number().optional(),
+        mcpOverheadTokens: z.number().optional(),
         userMessagesTokens: z.number(),
         assistantMessagesTokens: z.number(),
         toolInputTokens: z.number(),

package/dist/runner/agent-runner.d.ts

@@ -109,6 +109,10 @@ export type ExtendedSessionUpdate = (SessionNotification["update"] & {
         contextInputTokens?: number;
         [key: string]: unknown;
     };
+} | {
+    sessionUpdate: "tool_overhead_info";
+    toolOverheadTokens: number;
+    mcpOverheadTokens: number;
 } | AgentMessageChunkWithTokens | HookNotificationUpdate;
 /** Describes an object that can run an agent definition */
 export interface AgentRunner {

package/dist/runner/langchain/index.d.ts

@@ -10,7 +10,6 @@ type MakeLazy<T> = T extends LangchainTool ? () => T : never;
 export declare const TOOL_REGISTRY: Record<BuiltInToolType, LangchainTool | LazyLangchainTool | LazyLangchainTools>;
 export declare class LangchainAgent implements AgentRunner {
     definition: CreateAgentRunnerParams;
-    private toolSpans;
     constructor(params: CreateAgentRunnerParams);
     invoke(req: InvokeRequest): AsyncGenerator<ExtendedSessionUpdate, PromptResponse, undefined>;
 }

package/dist/runner/langchain/index.js

@@ -54,7 +54,6 @@ async function loadCustomTools(modulePaths) {
 }
 export class LangchainAgent {
     definition;
-    toolSpans = new Map();
     constructor(params) {
         this.definition = params;
     }
@@ -157,10 +156,41 @@ export class LangchainAgent {
                 const customTools = await loadCustomTools(customToolPaths);
                 enabledTools.push(...customTools);
             }
-            // MCP tools
+            // Calculate tool overhead tokens for non-MCP tools
+            const { countTokens } = await import("../../utils/token-counter.js");
+            const { extractToolMetadata, estimateAllToolsOverhead } = await import("../../utils/tool-overhead-calculator.js");
+            // Calculate overhead for non-MCP tools (built-in, custom, filesystem)
+            const nonMcpToolMetadata = enabledTools.map(extractToolMetadata);
+            const nonMcpToolDefinitionsTokens = estimateAllToolsOverhead(nonMcpToolMetadata);
+            // Calculate TODO_WRITE_INSTRUCTIONS overhead if applicable
+            const hasTodoWriteTool = builtInNames.includes("todo_write");
+            const todoInstructionsTokens = hasTodoWriteTool
+                ? countTokens(TODO_WRITE_INSTRUCTIONS)
+                : 0;
+            // Total non-MCP tool overhead: tool definitions + TODO instructions
+            const toolOverheadTokens = nonMcpToolDefinitionsTokens + todoInstructionsTokens;
+            // MCP tools - calculate overhead separately
+            let mcpOverheadTokens = 0;
             if ((this.definition.mcps?.length ?? 0) > 0) {
-                enabledTools.push(...(await makeMcpToolsClient(this.definition.mcps).getTools()));
+                const mcpTools = await makeMcpToolsClient(this.definition.mcps).getTools();
+                const mcpToolMetadata = mcpTools.map(extractToolMetadata);
+                mcpOverheadTokens = estimateAllToolsOverhead(mcpToolMetadata);
+                enabledTools.push(...mcpTools);
             }
+            _logger.debug("Calculated tool overhead for context sizing", {
+                enabledToolCount: enabledTools.length,
+                nonMcpToolDefinitionsTokens,
+                mcpToolDefinitionsTokens: mcpOverheadTokens,
+                todoInstructionsTokens,
+                totalNonMcpOverheadTokens: toolOverheadTokens,
+                totalMcpOverheadTokens: mcpOverheadTokens,
+            });
+            // Yield tool overhead info to adapter early in the turn
+            yield {
+                sessionUpdate: "tool_overhead_info",
+                toolOverheadTokens,
+                mcpOverheadTokens,
+            };
             // Wrap tools with response compaction if hook is configured
             const hooks = this.definition.hooks ?? [];
             const hasToolResponseHook = hooks.some((h) => h.type === "tool_response");
@@ -255,9 +285,12 @@ export class LangchainAgent {
             }
             // Filter tools if running in subagent mode
             const isSubagent = req.sessionMeta?.[SUBAGENT_MODE_KEY] === true;
-            const finalTools = isSubagent
+            const filteredTools = isSubagent
                 ? wrappedTools.filter((t) => t.name !== TODO_WRITE_TOOL_NAME && t.name !== TASK_TOOL_NAME)
                 : wrappedTools;
+            // Wrap tools with tracing so each tool executes within its own span context.
+            // This ensures subagent spans are children of the Task tool span.
+            const finalTools = filteredTools.map((t) => wrapToolWithTracing(t, req.sessionId));
             // Create the model instance using the factory
             // This detects the provider from the model string:
             // - "gemini-2.0-flash" → Google Generative AI
@@ -362,13 +395,6 @@ export class LangchainAgent {
                             if (toolCall.id == null) {
                                 throw new Error(`Tool call is missing id: ${JSON.stringify(toolCall)}`);
                             }
-                            // Create tool span within the invocation context
-                            // This makes the tool span a child of the invocation span
-                            const toolSpan = context.with(invocationContext, () => telemetry.startSpan("agent.tool_call", {
-                                "tool.name": toolCall.name,
-                                "tool.id": toolCall.id,
-                            }));
-                            this.toolSpans.set(toolCall.id, toolSpan);
                             telemetry.log("info", `Tool call started: ${toolCall.name}`, {
                                 toolCallId: toolCall.id,
                                 toolName: toolCall.name,
@@ -550,15 +576,9 @@ export class LangchainAgent {
                                 // Skip tool_call_update for todo_write tools
                                 continue;
                             }
-                            // End telemetry span for this tool call
-                            const toolSpan = this.toolSpans.get(aiMessage.tool_call_id);
-                            if (toolSpan) {
-                                telemetry.log("info", "Tool call completed", {
-                                    toolCallId: aiMessage.tool_call_id,
-                                });
-                                telemetry.endSpan(toolSpan);
-                                this.toolSpans.delete(aiMessage.tool_call_id);
-                            }
+                            telemetry.log("info", "Tool call completed", {
+                                toolCallId: aiMessage.tool_call_id,
+                            });
                             // Send status update (metadata only, no content)
                             yield {
                                 sessionUpdate: "tool_call_update",
@@ -708,3 +728,51 @@ I've found some existing telemetry code. Let me mark the first todo as in_progre
 `.trim();
 // Re-export subagent tool utility
 export { makeSubagentsTool } from "./tools/subagent.js";
+/**
+ * Wraps a LangChain tool with OpenTelemetry tracing.
+ * This ensures the tool executes within its own span context,
+ * so any child operations (like subagent spawning) become children
+ * of the tool span rather than the parent invocation span.
+ */
+function wrapToolWithTracing(originalTool, sessionId) {
+    const wrappedFunc = async (input) => {
+        const toolInputJson = JSON.stringify(input);
+        const toolSpan = telemetry.startSpan("agent.tool_call", {
+            "tool.name": originalTool.name,
+            "tool.input": toolInputJson,
+            "agent.session_id": sessionId,
+        });
+        // Create a context with the tool span as active
+        const spanContext = toolSpan
+            ? trace.setSpan(context.active(), toolSpan)
+            : context.active();
+        try {
+            // Execute within the tool span's context
+            const result = await context.with(spanContext, () => originalTool.invoke(input));
+            const resultStr = typeof result === "string" ? result : JSON.stringify(result);
+            if (toolSpan) {
+                telemetry.setSpanAttributes(toolSpan, {
+                    "tool.output": resultStr,
+                });
+                telemetry.endSpan(toolSpan);
+            }
+            return result;
+        }
+        catch (error) {
+            if (toolSpan) {
+                telemetry.endSpan(toolSpan, error);
+            }
+            throw error;
+        }
+    };
+    // Create new tool with wrapped function
+    const wrappedTool = tool(wrappedFunc, {
+        name: originalTool.name,
+        description: originalTool.description,
+        schema: originalTool.schema,
+    });
+    // Preserve metadata
+    wrappedTool.prettyName = originalTool.prettyName;
+    wrappedTool.icon = originalTool.icon;
+    return wrappedTool;
+}

package/dist/runner/langchain/otel-callbacks.js

@@ -1,4 +1,4 @@
-import { context } from "@opentelemetry/api";
+import { context, trace } from "@opentelemetry/api";
 import { telemetry } from "../../telemetry/index.js";
 /**
  * OpenTelemetry callback handler for LangChain LLM calls.
@@ -45,6 +45,38 @@ function extractSystemPrompt(messages) {
         return undefined;
     }
 }
+/**
+ * Serializes LLM output to a string for logging.
+ * Preserves the raw provider format (content blocks, tool_calls, etc.)
+ */
+function serializeOutput(output) {
+    try {
+        const generations = output.generations.flat();
+        const serialized = generations.map((gen) => {
+            // ChatGeneration has a message property with the full AIMessage
+            const chatGen = gen;
+            if (chatGen.message) {
+                const msg = chatGen.message;
+                const result = {
+                    role: msg._getType?.() ?? "assistant",
+                    content: msg.content, // Keep as-is: string or ContentBlock[]
+                };
+                // Include tool_calls if present (LangChain's normalized format)
+                const aiMsg = msg;
+                if (aiMsg.tool_calls && aiMsg.tool_calls.length > 0) {
+                    result.tool_calls = aiMsg.tool_calls;
+                }
+                return result;
+            }
+            // Fallback for non-chat generations
+            return { text: gen.text };
+        });
+        return JSON.stringify(serialized);
+    }
+    catch (error) {
+        return `[Error serializing output: ${error}]`;
+    }
+}
 /**
  * Creates OpenTelemetry callback handlers for LangChain LLM calls.
  * These handlers instrument model invocations with OTEL spans and record token usage.
@@ -84,6 +116,18 @@ export function makeOtelCallbacks(opts) {
             }));
             if (span) {
                 spansByRunId.set(runId, span);
+                // Emit log for LLM request with trace context
+                const spanContext = span.spanContext();
+                telemetry.log("info", "LLM Request", {
+                    "gen_ai.operation.name": "chat",
+                    "gen_ai.provider.name": opts.provider,
+                    "gen_ai.request.model": opts.model,
+                    "gen_ai.input.messages": serializedMessages,
+                    "langchain.run_id": runId,
+                    // Include trace context for correlation
+                    trace_id: spanContext.traceId,
+                    span_id: spanContext.spanId,
+                });
             }
         },
         /**
@@ -105,6 +149,28 @@ export function makeOtelCallbacks(opts) {
                         : 0);
                 telemetry.recordTokenUsage(inputTokens, outputTokens, span);
             }
+            // Serialize output and attach to span
+            const serializedOutput = serializeOutput(output);
+            telemetry.setSpanAttributes(span, {
+                "gen_ai.output.messages": serializedOutput,
+            });
+            // Emit log for LLM response with trace context
+            const spanContext = span.spanContext();
+            telemetry.log("info", "LLM Response", {
+                "gen_ai.operation.name": "chat",
+                "gen_ai.output.messages": serializedOutput,
+                "langchain.run_id": runId,
+                // Include token usage in log
+                ...(tokenUsage
+                    ? {
+                        "gen_ai.usage.input_tokens": tokenUsage.inputTokens ?? 0,
+                        "gen_ai.usage.output_tokens": tokenUsage.outputTokens ?? 0,
+                    }
+                    : {}),
+                // Include trace context for correlation
+                trace_id: spanContext.traceId,
+                span_id: spanContext.spanId,
+            });
             telemetry.endSpan(span);
             spansByRunId.delete(runId);
         },

package/dist/telemetry/setup.d.ts

@@ -1,7 +1,8 @@
 /**
  * OpenTelemetry provider setup for @townco/agent
- * Initializes the trace provider, exporter, and propagator
+ * Initializes the trace provider, log provider, exporters, and propagator
  */
+import { LoggerProvider } from "@opentelemetry/sdk-logs";
 import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
 export interface TelemetrySetupOptions {
     serviceName?: string;
@@ -14,6 +15,7 @@ export interface TelemetrySetupOptions {
  */
 export declare function initializeOpenTelemetry(options?: TelemetrySetupOptions): {
     provider: NodeTracerProvider;
+    loggerProvider: LoggerProvider;
     shutdown: () => Promise<void>;
 };
 /**

package/dist/telemetry/setup.js

@@ -1,11 +1,14 @@
 /**
  * OpenTelemetry provider setup for @townco/agent
- * Initializes the trace provider, exporter, and propagator
+ * Initializes the trace provider, log provider, exporters, and propagator
  */
 import { propagation } from "@opentelemetry/api";
+import { logs } from "@opentelemetry/api-logs";
 import { W3CTraceContextPropagator } from "@opentelemetry/core";
+import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 import { Resource } from "@opentelemetry/resources";
+import { BatchLogRecordProcessor, LoggerProvider, } from "@opentelemetry/sdk-logs";
 import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
 import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
 import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
@@ -83,13 +86,38 @@ export function initializeOpenTelemetry(options = {}) {
     const batchProcessor = new BatchSpanProcessor(loggingExporter, {
         maxQueueSize: 100,
         maxExportBatchSize: 10,
-        scheduledDelayMillis: 5000, // Export every 5 seconds (default)
+        scheduledDelayMillis: 2000, // Export every 2 seconds
     });
     provider.addSpanProcessor(batchProcessor);
     // Register the provider globally
     provider.register();
     // Configure W3C Trace Context propagator for cross-process traces
     propagation.setGlobalPropagator(new W3CTraceContextPropagator());
+    // Set up LoggerProvider for OTLP log export
+    const logUrl = otlpEndpoint.endsWith("/")
+        ? `${otlpEndpoint}v1/logs`
+        : `${otlpEndpoint}/v1/logs`;
+    if (debug) {
+        console.log(`OTLP log URL: ${logUrl}`);
+    }
+    const logExporter = new OTLPLogExporter({
+        url: logUrl,
+    });
+    const loggerProvider = new LoggerProvider({
+        resource: new Resource({
+            [ATTR_SERVICE_NAME]: serviceName,
+        }),
+    });
+    loggerProvider.addLogRecordProcessor(new BatchLogRecordProcessor(logExporter, {
+        maxQueueSize: 100,
+        maxExportBatchSize: 10,
+        scheduledDelayMillis: 2000,
+    }));
+    // Register the logger provider globally
+    logs.setGlobalLoggerProvider(loggerProvider);
+    if (debug) {
+        console.log("✓ Log exporter configured");
+    }
     // Now configure our telemetry wrapper
     configureTelemetry({
         enabled: true,
@@ -106,6 +134,8 @@ export function initializeOpenTelemetry(options = {}) {
         try {
             await provider.forceFlush();
             await provider.shutdown();
+            await loggerProvider.forceFlush();
+            await loggerProvider.shutdown();
             if (debug) {
                 console.log("✓ Telemetry flushed");
             }
@@ -114,7 +144,7 @@ export function initializeOpenTelemetry(options = {}) {
             console.error("Error flushing telemetry:", error);
         }
     };
-    return { provider, shutdown };
+    return { provider, loggerProvider, shutdown };
 }
 /**
  * Initialize OpenTelemetry from environment variables and register shutdown handlers