@townco/agent 0.1.84 → 0.1.87

package/dist/runner/hooks/predefined/tool-response-compactor.js

@@ -2,6 +2,7 @@ import Anthropic from "@anthropic-ai/sdk";
 import { createLogger } from "../../../logger.js";
 import { telemetry } from "../../../telemetry/index.js";
 import { countToolResultTokens } from "../../../utils/token-counter.js";
+import { extractDocumentContext } from "./document-context-extractor/index.js";
 const logger = createLogger("tool-response-compactor");
 // Create Anthropic client directly (not using LangChain)
 // This ensures compaction LLM calls don't get captured by LangGraph's streaming
@@ -32,7 +33,6 @@ export const toolResponseCompactor = async (ctx) => {
     // Get settings from hook configuration
     const settings = ctx.session.requestParams.hookSettings;
     const maxTokensSize = settings?.maxTokensSize ?? 20000; // Default: 20000 tokens
-    const responseTruncationThreshold = settings?.responseTruncationThreshold ?? 30;
     // Use maxTokensSize directly as it's now in tokens
     const maxAllowedResponseSize = maxTokensSize;
     // Calculate available space in context
@@ -41,7 +41,9 @@ export const toolResponseCompactor = async (ctx) => {
     const effectiveMaxResponseSize = availableSpace < maxAllowedResponseSize
         ? Math.floor(availableSpace * 0.9)
         : maxAllowedResponseSize;
-    const compactionLimit = COMPACTION_MODEL_CONTEXT * (responseTruncationThreshold / 100);
+    // Calculate compaction limit: max response size that can fit in a single LLM compaction call
+    const COMPACTION_OVERHEAD = 10000;
+    const compactionLimit = Math.floor((COMPACTION_MODEL_CONTEXT - COMPACTION_OVERHEAD) * 0.9); // ~175K tokens
     logger.info("Tool response compaction hook triggered", {
         toolCallId,
         toolName,
@@ -66,14 +68,12 @@ export const toolResponseCompactor = async (ctx) => {
             },
         };
     }
-    // Response would exceed threshold, need to compact or truncate
-    // Determine target size: use effectiveMaxResponseSize, but cap at compactionLimit for truncation
+    // Response would exceed threshold, need to compact or extract
+    // Target size is the effectiveMaxResponseSize (what we want the final output to be)
     // IMPORTANT: If context is already very full, availableSpace might be very small
     // In that case, use a minimum reasonable target size (e.g., 10% of the output or 1000 tokens)
     const minTargetSize = Math.max(Math.floor(outputTokens * 0.1), 1000);
-    const targetSize = effectiveMaxResponseSize > 0
-        ? Math.min(effectiveMaxResponseSize, compactionLimit)
-        : minTargetSize;
+    const targetSize = effectiveMaxResponseSize > 0 ? effectiveMaxResponseSize : minTargetSize;
     logger.info("Calculated target size for compaction", {
         availableSpace,
         effectiveMaxResponseSize,
@@ -82,73 +82,79 @@ export const toolResponseCompactor = async (ctx) => {
         targetSize,
         contextAlreadyOverThreshold: availableSpace <= maxAllowedResponseSize,
     });
-    // Case 2: Huge response, must truncate (too large for LLM compaction)
+    // Case 2: Huge response - use document context extractor (with truncation fallback)
     if (outputTokens >= compactionLimit) {
-        logger.warn("Tool response exceeds compaction capacity, truncating", {
+        logger.info("Tool response exceeds compaction capacity, using document context extractor", {
             outputTokens,
             compactionLimit,
             targetSize,
             availableSpace,
         });
-        const truncated = truncateToolResponse(rawOutput, targetSize);
-        const finalTokens = countToolResultTokens(truncated);
-        // Verify truncation stayed within boundaries
-        if (finalTokens > targetSize) {
-            logger.error("Truncation exceeded target size - this should not happen!", {
-                finalTokens,
-                targetSize,
-                excess: finalTokens - targetSize,
-            });
-            // Try more aggressive truncation (70% of target as emergency measure)
-            const emergencySize = Math.floor(targetSize * 0.7);
-            const emergencyTruncated = truncateToolResponse(rawOutput, emergencySize);
-            const emergencyTokens = countToolResultTokens(emergencyTruncated);
-            // Final safety check - if emergency truncation STILL exceeded target, use ultra-conservative fallback
-            if (emergencyTokens > targetSize) {
-                logger.error("Emergency truncation STILL exceeded target - using ultra-conservative fallback", {
-                    emergencyTokens,
-                    targetSize,
-                    emergencySize,
+        // Build conversation context for extraction
+        const recentMessages = ctx.session.messages.slice(-5);
+        const conversationContext = recentMessages
+            .map((msg) => {
+            const text = msg.content
+                .filter((b) => b.type === "text")
+                .map((b) => (b.type === "text" ? b.text : ""))
+                .join("\n");
+            return `${msg.role}: ${text}`;
+        })
+            .join("\n\n");
+        // Try document context extraction
+        try {
+            const extractionResult = await extractDocumentContext(rawOutput, toolName, toolCallId, toolInput, conversationContext, targetSize, ctx.sessionId ?? "unknown", ctx.storage);
+            if (extractionResult.success && extractionResult.extractedData) {
+                logger.info("Document context extraction succeeded", {
+                    originalTokens: outputTokens,
+                    finalTokens: extractionResult.extractedTokens,
+                    chunksProcessed: extractionResult.metadata.chunksProcessed,
+                    chunksExtractedFrom: extractionResult.metadata.chunksExtractedFrom,
                 });
-                // Ultra-conservative: just return a simple error structure with the raw data sliced to 50% of target
-                const ultraConservativeSize = Math.floor(targetSize * 0.5);
                 return {
                     newContextEntry: null,
                     metadata: {
-                        action: "truncated",
+                        action: "compacted",
                         originalTokens: outputTokens,
-                        finalTokens: ultraConservativeSize, // Conservative estimate
-                        modifiedOutput: {
-                            _truncation_error: "Tool response was too large and could not be reliably truncated",
-                            _original_token_count: outputTokens,
-                            _target_token_count: targetSize,
-                            _partial_data: JSON.stringify(rawOutput).slice(0, ultraConservativeSize * 3),
-                        },
-                        truncationWarning: `Tool response was severely truncated from ${outputTokens.toLocaleString()} to ~${ultraConservativeSize.toLocaleString()} tokens (emergency truncation failed - data may be incomplete)`,
+                        finalTokens: extractionResult.extractedTokens,
+                        tokensSaved: outputTokens - (extractionResult.extractedTokens ?? 0),
+                        modifiedOutput: extractionResult.extractedData,
+                        compactionMethod: "document_context_extraction",
+                        extractionMetadata: extractionResult.metadata,
                     },
                 };
             }
-            return {
-                newContextEntry: null,
-                metadata: {
-                    action: "truncated",
-                    originalTokens: outputTokens,
-                    finalTokens: emergencyTokens,
-                    modifiedOutput: emergencyTruncated,
-                    truncationWarning: `Tool response was aggressively truncated from ${outputTokens.toLocaleString()} to ${emergencyTokens.toLocaleString()} tokens to fit within context limit (emergency truncation)`,
-                },
-            };
+            // Extraction failed - throw error to terminate agent loop
+            logger.error("Document context extraction failed", {
+                error: extractionResult.error,
+                phase: extractionResult.metadata.phase,
+                toolName,
+                toolCallId,
+                outputTokens,
+            });
+            throw new Error(`Document context extraction failed for tool "${toolName}": ${extractionResult.error}. ` +
+                `Original response was ${outputTokens.toLocaleString()} tokens. ` +
+                `Full response saved to artifacts.`);
+        }
+        catch (extractionError) {
+            // Re-throw if it's already our error
+            if (extractionError instanceof Error &&
+                extractionError.message.includes("Document context extraction failed")) {
+                throw extractionError;
+            }
+            // Extraction threw an unexpected error - terminate agent loop
+            logger.error("Document context extraction threw an error", {
+                error: extractionError instanceof Error
+                    ? extractionError.message
+                    : String(extractionError),
+                toolName,
+                toolCallId,
+                outputTokens,
+            });
+            throw new Error(`Document context extraction failed for tool "${toolName}": ${extractionError instanceof Error
+                ? extractionError.message
+                : String(extractionError)}. Original response was ${outputTokens.toLocaleString()} tokens.`);
         }
-        return {
-            newContextEntry: null,
-            metadata: {
-                action: "truncated",
-                originalTokens: outputTokens,
-                finalTokens,
-                modifiedOutput: truncated,
-                truncationWarning: `Tool response was truncated from ${outputTokens.toLocaleString()} to ${finalTokens.toLocaleString()} tokens to fit within max response size limit (max allowed: ${effectiveMaxResponseSize.toLocaleString()} tokens)`,
-            },
-        };
     }
     // Case 1: Medium response, intelligent compaction
     logger.info("Tool response requires intelligent compaction", {
@@ -171,28 +177,19 @@ export const toolResponseCompactor = async (ctx) => {
         })
             .join("\n\n");
         const compacted = await compactWithLLM(rawOutput, toolName, toolInput, conversationContext, targetSize);
-        let finalTokens = countToolResultTokens(compacted);
+        const finalTokens = countToolResultTokens(compacted);
         // Verify compaction stayed within boundaries
         if (finalTokens > targetSize) {
-            logger.warn("LLM compaction exceeded target, falling back to truncation", {
+            logger.error("LLM compaction exceeded target", {
                 finalTokens,
                 targetSize,
                 excess: finalTokens - targetSize,
+                toolName,
+                toolCallId,
             });
-            // Fallback to truncation
-            const truncated = truncateToolResponse(compacted, targetSize);
-            finalTokens = countToolResultTokens(truncated);
-            return {
-                newContextEntry: null,
-                metadata: {
-                    action: "compacted_then_truncated",
-                    originalTokens: outputTokens,
-                    finalTokens,
-                    tokensSaved: outputTokens - finalTokens,
-                    modifiedOutput: truncated,
-                    truncationWarning: `Tool response was compacted then truncated from ${outputTokens.toLocaleString()} to ${finalTokens.toLocaleString()} tokens to fit within context limit`,
-                },
-            };
+            throw new Error(`LLM compaction for tool "${toolName}" exceeded target size. ` +
+                `Compacted to ${finalTokens.toLocaleString()} tokens but target was ${targetSize.toLocaleString()}. ` +
+                `Original response was ${outputTokens.toLocaleString()} tokens.`);
         }
         logger.info("Successfully compacted tool response", {
             originalTokens: outputTokens,
@@ -212,62 +209,13 @@ export const toolResponseCompactor = async (ctx) => {
         };
     }
     catch (error) {
-        logger.error("Compaction failed, falling back to truncation", {
+        logger.error("Compaction failed", {
             error: error instanceof Error ? error.message : String(error),
+            toolName,
+            toolCallId,
+            outputTokens,
         });
-        // Fallback to truncation with the same target size
-        const truncated = truncateToolResponse(rawOutput, targetSize);
-        let finalTokens = countToolResultTokens(truncated);
-        // Verify truncation stayed within boundaries
-        if (finalTokens > targetSize) {
-            logger.error("Fallback truncation exceeded target, using emergency truncation", {
-                finalTokens,
-                targetSize,
-            });
-            const emergencySize = Math.floor(targetSize * 0.7);
-            const emergencyTruncated = truncateToolResponse(rawOutput, emergencySize);
-            finalTokens = countToolResultTokens(emergencyTruncated);
-            // Final safety check
-            if (finalTokens > targetSize) {
-                logger.error("Emergency truncation STILL exceeded target - using ultra-conservative fallback");
-                const ultraConservativeSize = Math.floor(targetSize * 0.5);
-                return {
-                    newContextEntry: null,
-                    metadata: {
-                        action: "truncated",
-                        originalTokens: outputTokens,
-                        finalTokens: ultraConservativeSize,
-                        modifiedOutput: {
-                            _truncation_error: "Tool response was too large and could not be reliably truncated (compaction failed)",
-                            _original_token_count: outputTokens,
-                            _target_token_count: targetSize,
-                            _partial_data: JSON.stringify(rawOutput).slice(0, ultraConservativeSize * 3),
-                        },
-                        truncationWarning: `Tool response was severely truncated from ${outputTokens.toLocaleString()} to ~${ultraConservativeSize.toLocaleString()} tokens (compaction+emergency truncation failed)`,
-                    },
-                };
-            }
-            return {
-                newContextEntry: null,
-                metadata: {
-                    action: "truncated",
-                    originalTokens: outputTokens,
-                    finalTokens,
-                    modifiedOutput: emergencyTruncated,
-                    truncationWarning: `Tool response was truncated from ${outputTokens.toLocaleString()} to ${finalTokens.toLocaleString()} tokens (compaction failed, emergency truncation applied)`,
-                },
-            };
-        }
-        return {
-            newContextEntry: null,
-            metadata: {
-                action: "truncated",
-                originalTokens: outputTokens,
-                finalTokens,
-                modifiedOutput: truncated,
-                truncationWarning: `Tool response was truncated from ${outputTokens.toLocaleString()} to ${finalTokens.toLocaleString()} tokens (compaction failed)`,
-            },
-        };
+        throw new Error(`LLM compaction failed for tool "${toolName}": ${error instanceof Error ? error.message : String(error)}. Original response was ${outputTokens.toLocaleString()} tokens.`);
     }
 };
 /**
@@ -456,86 +404,3 @@ Return ONLY valid JSON (no explanation text).`;
     });
     return currentData;
 }
-/**
- * Truncate tool response to target token count
- * Uses iterative approach to ensure we stay under the target
- */
-function truncateToolResponse(rawOutput, targetTokens) {
-    const currentTokens = countToolResultTokens(rawOutput);
-    if (currentTokens <= targetTokens) {
-        return rawOutput; // Already within limit
-    }
-    const outputString = JSON.stringify(rawOutput);
-    // Start with 70% of target to leave significant room for closing braces and metadata
-    let ratio = 0.7;
-    let lastResult = null;
-    // Iteratively truncate until we meet the target
-    for (let attempt = 0; attempt < 15; attempt++) {
-        // Calculate character limit based on ratio
-        const targetChars = Math.floor((targetTokens * ratio * outputString.length) / currentTokens);
-        // Truncate the JSON string
-        let truncated = outputString.slice(0, targetChars);
-        // Try to close any open JSON structures
-        const openBraces = (truncated.match(/{/g) || []).length;
-        const closeBraces = (truncated.match(/}/g) || []).length;
-        const openBrackets = (truncated.match(/\[/g) || []).length;
-        const closeBrackets = (truncated.match(/\]/g) || []).length;
-        truncated += "}".repeat(Math.max(0, openBraces - closeBraces));
-        truncated += "]".repeat(Math.max(0, openBrackets - closeBrackets));
-        try {
-            // Try to parse as valid JSON
-            const parsed = JSON.parse(truncated);
-            const parsedTokens = countToolResultTokens(parsed);
-            // Store the result
-            lastResult = { parsed, tokens: parsedTokens };
-            if (parsedTokens <= targetTokens) {
-                // Success! Add truncation notice
-                return {
-                    ...parsed,
-                    _truncation_notice: "... [TRUNCATED - response exceeded size limit]",
-                    _original_token_count: currentTokens,
-                    _truncated_token_count: parsedTokens,
-                };
-            }
-            // Still too large - calculate how much we need to reduce
-            // If we overshot, reduce ratio proportionally to how much we exceeded
-            const overshootRatio = parsedTokens / targetTokens; // e.g., 1.03 if we're 3% over
-            ratio = (ratio / overshootRatio) * 0.95; // Reduce by overshoot amount plus 5% safety margin
-            logger.debug("Truncation attempt resulted in overshoot, retrying", {
-                attempt,
-                targetTokens,
-                parsedTokens,
-                overshootRatio,
-                newRatio: ratio,
-            });
-        }
-        catch {
-            // JSON parse failed, try more aggressive truncation
-            ratio *= 0.85;
-        }
-    }
-    // If we exhausted all attempts, return the last successful parse (if any)
-    // or a very conservative fallback
-    if (lastResult && lastResult.tokens <= targetTokens * 1.1) {
-        // Within 10% of target - good enough
-        logger.warn("Truncation reached attempt limit but result is close enough", {
-            targetTokens,
-            actualTokens: lastResult.tokens,
-        });
-        return {
-            ...lastResult.parsed,
-            _truncation_notice: "... [TRUNCATED - response exceeded size limit]",
-            _original_token_count: currentTokens,
-            _truncated_token_count: lastResult.tokens,
-        };
-    }
-    // If all attempts failed, return a simple truncated structure
-    const safeChars = Math.floor(targetTokens * 3); // Very conservative
-    return {
-        truncated: true,
-        originalSize: currentTokens,
-        targetSize: targetTokens,
-        content: outputString.slice(0, safeChars),
-        warning: "Response was truncated due to size constraints (JSON parsing failed)",
-    };
-}

package/dist/runner/hooks/types.d.ts

@@ -1,5 +1,11 @@
 import type { ContextEntry } from "../../acp-server/session-storage";
 import type { SessionMessage } from "../agent-runner";
+/**
+ * Storage interface for hooks that need to persist data
+ */
+export interface HookStorageInterface {
+    getArtifactsDir(sessionId: string): string;
+}
 /**
  * Hook types supported by the agent system
  */
@@ -19,17 +25,10 @@ export interface ContextSizeSettings {
 export interface ToolResponseSettings {
     /**
      * Maximum size of a tool response in tokens.
-     * Tool responses larger than this will trigger compaction.
+     * Tool responses larger than this will trigger compaction/extraction.
      * Default: 20000
      */
     maxTokensSize?: number | undefined;
-    /**
-     * Maximum % of compaction model context (Haiku: 200k) that a tool response can be
-     * to attempt LLM-based compaction. Larger responses are truncated instead.
-     * The truncation limit is also this percentage.
-     * Default: 30
-     */
-    responseTruncationThreshold?: number | undefined;
 }
 /**
  * Hook configuration in agent definition
@@ -90,6 +89,14 @@ export interface HookContext {
      * The model being used
      */
     model: string;
+    /**
+     * Session ID for the current session
+     */
+    sessionId?: string | undefined;
+    /**
+     * Storage interface for hooks that need to persist data
+     */
+    storage?: HookStorageInterface | undefined;
     /**
      * Tool response data (only for tool_response hooks)
      */

package/dist/runner/index.d.ts

@@ -1,4 +1,53 @@
-import type { AgentDefinition } from "../definition";
 import { type AgentRunner } from "./agent-runner";
 export type { AgentRunner };
-export declare const makeRunnerFromDefinition: (definition: AgentDefinition) => AgentRunner;
+export declare const makeRunnerFromDefinition: (definition: {
+    displayName?: string | undefined;
+    version?: string | undefined;
+    description?: string | undefined;
+    suggestedPrompts?: string[] | undefined;
+    systemPrompt: string | null;
+    model: string;
+    tools?: (string | {
+        type: "custom";
+        modulePath: string;
+    } | {
+        type: "filesystem";
+        working_directory?: string | undefined;
+    } | {
+        type: "direct";
+        name: string;
+        description: string;
+        fn: import("zod/v4/core").$InferOuterFunctionType<import("zod/v4/core").$ZodFunctionArgs, import("zod/v4/core").$ZodFunctionOut>;
+        schema: any;
+        prettyName?: string | undefined;
+        icon?: string | undefined;
+    })[] | undefined;
+    mcps?: (string | {
+        name: string;
+        transport: "stdio";
+        command: string;
+        args?: string[] | undefined;
+    } | {
+        name: string;
+        transport: "http";
+        url: string;
+        headers?: Record<string, string> | undefined;
+    })[] | undefined;
+    harnessImplementation?: "langchain" | undefined;
+    hooks?: {
+        type: "context_size" | "tool_response";
+        setting?: {
+            threshold: number;
+        } | {
+            maxTokensSize?: number | undefined;
+        } | undefined;
+        callback: string;
+    }[] | undefined;
+    initialMessage?: {
+        enabled: boolean;
+        content: string;
+    } | undefined;
+    uiConfig?: {
+        hideTopBar?: boolean | undefined;
+    } | undefined;
+}) => AgentRunner;

package/dist/runner/langchain/index.js

@@ -14,6 +14,7 @@ import { createModelFromString, detectProvider } from "./model-factory.js";
 import { makeOtelCallbacks } from "./otel-callbacks.js";
 import { makeArtifactsTools } from "./tools/artifacts";
 import { makeBrowserTools } from "./tools/browser";
+import { makeDocumentExtractTool } from "./tools/document_extract";
 import { makeFilesystemTools } from "./tools/filesystem";
 import { makeGenerateImageTool, makeTownGenerateImageTool, } from "./tools/generate_image";
 import { SUBAGENT_TOOL_NAME } from "./tools/subagent";
@@ -42,6 +43,7 @@ export const TOOL_REGISTRY = {
     generate_image: () => makeGenerateImageTool(),
     town_generate_image: () => makeTownGenerateImageTool(),
     browser: () => makeBrowserTools(),
+    document_extract: () => makeDocumentExtractTool(),
 };
 // ============================================================================
 // Custom tool loading
@@ -530,6 +532,14 @@ export class LangchainAgent {
             if (hasTodoWrite) {
                 agentConfig.systemPrompt = `${agentConfig.systemPrompt ?? ""}\n\n${TODO_WRITE_INSTRUCTIONS}`;
             }
+            // Process template variables in system prompt and inject current date/time
+            if (agentConfig.systemPrompt) {
+                const currentDateTime = getCurrentDateTimeString();
+                // Replace {{.CurrentDate}} template variable
+                agentConfig.systemPrompt = agentConfig.systemPrompt.replace(/\{\{\.CurrentDate\}\}/g, currentDateTime);
+                // Replace {{.CurrentDateTime}} template variable (alias)
+                agentConfig.systemPrompt = agentConfig.systemPrompt.replace(/\{\{\.CurrentDateTime\}\}/g, currentDateTime);
+            }
             const agent = createAgent(agentConfig);
             // Build messages from context history if available, otherwise use just the prompt
             // Type includes tool messages for sending tool results
@@ -1270,6 +1280,23 @@ I've found some existing telemetry code. Let me mark the first todo as in_progre
 [Assistant continues implementing the feature step by step, marking todos as in_progress and completed as they go]
 </example>
 `.trim();
+/**
+ * Returns a human-readable string of the current date and time.
+ * Example: "Monday, December 9, 2024 at 2:30 PM PST"
+ */
+const getCurrentDateTimeString = () => {
+    const now = new Date();
+    const options = {
+        weekday: "long",
+        year: "numeric",
+        month: "long",
+        day: "numeric",
+        hour: "numeric",
+        minute: "2-digit",
+        timeZoneName: "short",
+    };
+    return now.toLocaleString("en-US", options);
+};
 // Re-export subagent tool utility
 export { makeSubagentsTool } from "./tools/subagent.js";
 /**

package/dist/runner/langchain/tools/artifacts.d.ts

@@ -19,19 +19,19 @@ export declare function makeArtifactsTools(): (import("langchain").DynamicStruct
     source: z.ZodString;
     destination: z.ZodString;
     direction: z.ZodEnum<{
-        upload: "upload";
         download: "download";
+        upload: "upload";
     }>;
 }, z.core.$strip>, {
     session_id: string;
     source: string;
     destination: string;
-    direction: "upload" | "download";
+    direction: "download" | "upload";
 }, {
+    session_id?: string | undefined;
     source: string;
     destination: string;
-    direction: "upload" | "download";
-    session_id?: string | undefined;
+    direction: "download" | "upload";
 }, string> | import("langchain").DynamicStructuredTool<z.ZodObject<{
     session_id: z.ZodOptional<z.ZodString>;
     path: z.ZodString;
@@ -39,8 +39,8 @@ export declare function makeArtifactsTools(): (import("langchain").DynamicStruct
     session_id: string;
     path: string;
 }, {
-    path: string;
     session_id?: string | undefined;
+    path: string;
 }, string> | import("langchain").DynamicStructuredTool<z.ZodObject<{
     session_id: z.ZodOptional<z.ZodString>;
     path: z.ZodOptional<z.ZodString>;
@@ -62,7 +62,7 @@ export declare function makeArtifactsTools(): (import("langchain").DynamicStruct
     path: string;
     expires_in?: number;
 }, {
-    path: string;
     session_id?: string | undefined;
+    path: string;
     expires_in?: number | undefined;
 }, string>)[];