@compilr-dev/agents 0.3.11 → 0.3.12

package/dist/rate-limit/provider-wrapper.js

@@ -5,6 +5,7 @@
  */
 import { createRateLimiter } from './limiter.js';
 import { withRetry } from './retry.js';
+import { countMessageTokens } from '../utils/tokenizer.js';
 /**
  * Wrapper that adds rate limiting and retry to any LLMProvider
  *
@@ -105,35 +106,10 @@ export class RateLimitedProvider {
         return withRetry(() => countTokensFn(messages), this.retryConfig);
     }
     /**
-     * Estimate tokens from messages (rough approximation)
+     * Estimate tokens from messages using tiktoken
      */
     estimateTokens(messages) {
-        let charCount = 0;
-        for (const msg of messages) {
-            if (typeof msg.content === 'string') {
-                charCount += msg.content.length;
-            }
-            else {
-                for (const block of msg.content) {
-                    switch (block.type) {
-                        case 'text':
-                            charCount += block.text.length;
-                            break;
-                        case 'tool_use':
-                            charCount += JSON.stringify(block.input).length;
-                            break;
-                        case 'tool_result':
-                            charCount += block.content.length;
-                            break;
-                        case 'thinking':
-                            charCount += block.thinking.length;
-                            break;
-                    }
-                }
-            }
-        }
-        // Rough estimate: 4 characters per token
-        return Math.ceil(charCount / 4);
+        return countMessageTokens(messages);
     }
 }
 /**

package/dist/tools/builtin/index.d.ts

@@ -33,6 +33,8 @@ export { askUserTool, createAskUserTool } from './ask-user.js';
 export type { AskUserInput, AskUserResult, AskUserQuestion, AskUserOption, AskUserToolOptions, } from './ask-user.js';
 export { askUserSimpleTool, createAskUserSimpleTool } from './ask-user-simple.js';
 export type { AskUserSimpleInput, AskUserSimpleResult, AskUserSimpleToolOptions, } from './ask-user-simple.js';
+export { createRecallResultTool } from './recall-result.js';
+export type { RecallResultInput, RecallResultToolOptions } from './recall-result.js';
 export { backlogReadTool, backlogWriteTool, createBacklogTools } from './backlog.js';
 export type { BacklogItem, BacklogStatus, BacklogItemType, BacklogPriority, BacklogReadInput, BacklogReadResult, BacklogWriteInput, BacklogWriteResult, BacklogToolOptions, } from './backlog.js';
 export declare const builtinTools: {

package/dist/tools/builtin/index.js

@@ -32,6 +32,8 @@ export { suggestTool, createSuggestTool } from './suggest.js';
 // Ask user tools (user interaction)
 export { askUserTool, createAskUserTool } from './ask-user.js';
 export { askUserSimpleTool, createAskUserSimpleTool } from './ask-user-simple.js';
+// Recall result tool (tool result delegation)
+export { createRecallResultTool } from './recall-result.js';
 // Backlog tools (file-based project backlog)
 export { backlogReadTool, backlogWriteTool, createBacklogTools } from './backlog.js';
 /**

package/dist/tools/builtin/recall-result.d.ts

@@ -0,0 +1,29 @@
+/**
+ * recall_full_result tool
+ *
+ * Retrieves the full output of a previously delegated tool result.
+ * Use when a summary is insufficient and you need the complete data.
+ */
+import type { Tool } from '../types.js';
+import type { DelegatedResultStore } from '../../context/delegated-result-store.js';
+import type { DelegationEvent } from '../../context/delegation-types.js';
+/**
+ * Input for the recall_full_result tool.
+ */
+export interface RecallResultInput {
+    /** Delegation ID from a previously delegated result (e.g., "dr_1707900000_0") */
+    id: string;
+}
+/**
+ * Options for creating the recall_full_result tool.
+ */
+export interface RecallResultToolOptions {
+    /** The delegation store to retrieve results from */
+    store: DelegatedResultStore;
+    /** Optional event callback for recall events */
+    onEvent?: (event: DelegationEvent) => void;
+}
+/**
+ * Create a recall_full_result tool that retrieves delegated results from the store.
+ */
+export declare function createRecallResultTool(options: RecallResultToolOptions): Tool<RecallResultInput>;

package/dist/tools/builtin/recall-result.js

@@ -0,0 +1,48 @@
+/**
+ * recall_full_result tool
+ *
+ * Retrieves the full output of a previously delegated tool result.
+ * Use when a summary is insufficient and you need the complete data.
+ */
+import { defineTool, createSuccessResult, createErrorResult } from '../define.js';
+/**
+ * Create a recall_full_result tool that retrieves delegated results from the store.
+ */
+export function createRecallResultTool(options) {
+    const { store, onEvent } = options;
+    return defineTool({
+        name: 'recall_full_result',
+        description: 'Retrieve the full output of a previously delegated tool result. ' +
+            'Use when a summary is insufficient and you need the complete data. ' +
+            'Results expire after 10 minutes.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                id: {
+                    type: 'string',
+                    description: 'Delegation ID (e.g., "dr_1707900000_0")',
+                },
+            },
+            required: ['id'],
+        },
+        execute: (input) => {
+            const stored = store.get(input.id);
+            onEvent?.({
+                type: 'delegation:recall',
+                delegationId: input.id,
+                found: stored !== undefined,
+            });
+            if (!stored) {
+                return Promise.resolve(createErrorResult('Delegation ID not found or expired. Re-run the original tool to get the result.'));
+            }
+            return Promise.resolve(createSuccessResult({
+                delegationId: stored.id,
+                toolName: stored.toolName,
+                fullContent: stored.fullContent,
+                fullTokens: stored.fullTokens,
+            }));
+        },
+        parallel: true,
+        silent: true,
+    });
+}

package/dist/tools/index.d.ts

@@ -6,5 +6,5 @@ export { defineTool, createSuccessResult, createErrorResult, wrapToolExecute } f
 export type { DefineToolOptions } from './define.js';
 export { DefaultToolRegistry, createToolRegistry } from './registry.js';
 export type { ToolRegistryOptions } from './registry.js';
-export { readFileTool, createReadFileTool, writeFileTool, createWriteFileTool, bashTool, createBashTool, execStream, detectFifoUsage, bashOutputTool, createBashOutputTool, killShellTool, createKillShellTool, ShellManager, getDefaultShellManager, setDefaultShellManager, grepTool, createGrepTool, globTool, createGlobTool, editTool, createEditTool, todoWriteTool, todoReadTool, createTodoTools, TodoStore, resetDefaultTodoStore, getDefaultTodoStore, createIsolatedTodoStore, cleanupTodoContextMessages, getTodoContextStats, webFetchTool, createWebFetchTool, createTaskTool, defaultAgentTypes, suggestTool, createSuggestTool, builtinTools, allBuiltinTools, TOOL_NAMES, TOOL_SETS, } from './builtin/index.js';
-export type { ReadFileInput, WriteFileInput, BashInput, BashResult, FifoDetectionResult, BashOutputInput, BashOutputResult, KillShellInput, KillShellResult, ShellStatus, BackgroundShell, ShellOutput, ShellManagerOptions, GrepInput, GlobInput, EditInput, TodoWriteInput, TodoReadInput, TodoItem, TodoStatus, TodoContextCleanupOptions, WebFetchInput, WebFetchResult, WebFetchOptions, TaskInput, TaskResult, AgentTypeConfig, TaskToolOptions, ContextMode, ThoroughnessLevel, SubAgentEventInfo, SuggestInput, SuggestToolOptions, } from './builtin/index.js';
+export { readFileTool, createReadFileTool, writeFileTool, createWriteFileTool, bashTool, createBashTool, execStream, detectFifoUsage, bashOutputTool, createBashOutputTool, killShellTool, createKillShellTool, ShellManager, getDefaultShellManager, setDefaultShellManager, grepTool, createGrepTool, globTool, createGlobTool, editTool, createEditTool, todoWriteTool, todoReadTool, createTodoTools, TodoStore, resetDefaultTodoStore, getDefaultTodoStore, createIsolatedTodoStore, cleanupTodoContextMessages, getTodoContextStats, webFetchTool, createWebFetchTool, createTaskTool, defaultAgentTypes, suggestTool, createSuggestTool, createRecallResultTool, builtinTools, allBuiltinTools, TOOL_NAMES, TOOL_SETS, } from './builtin/index.js';
+export type { ReadFileInput, WriteFileInput, BashInput, BashResult, FifoDetectionResult, BashOutputInput, BashOutputResult, KillShellInput, KillShellResult, ShellStatus, BackgroundShell, ShellOutput, ShellManagerOptions, GrepInput, GlobInput, EditInput, TodoWriteInput, TodoReadInput, TodoItem, TodoStatus, TodoContextCleanupOptions, WebFetchInput, WebFetchResult, WebFetchOptions, TaskInput, TaskResult, AgentTypeConfig, TaskToolOptions, ContextMode, ThoroughnessLevel, SubAgentEventInfo, SuggestInput, SuggestToolOptions, RecallResultInput, RecallResultToolOptions, } from './builtin/index.js';

package/dist/tools/index.js

@@ -35,6 +35,8 @@ webFetchTool, createWebFetchTool,
 createTaskTool, defaultAgentTypes, 
 // Suggest (next action suggestions)
 suggestTool, createSuggestTool, 
+// Recall result (delegation)
+createRecallResultTool, 
 // Collections
 builtinTools, allBuiltinTools, 
 // Tool names - single source of truth

package/dist/utils/index.d.ts

@@ -1,6 +1,7 @@
 /**
  * Utility functions for @compilr-dev/agents
  */
+export { countTokens, countMessageTokens } from './tokenizer.js';
 /**
  * Generate a unique ID for tool uses
  */

package/dist/utils/index.js

@@ -1,6 +1,8 @@
 /**
  * Utility functions for @compilr-dev/agents
  */
+// Token counting
+export { countTokens, countMessageTokens } from './tokenizer.js';
 /**
  * Generate a unique ID for tool uses
  */

package/dist/utils/tokenizer.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Token counting utility using tiktoken (cl100k_base encoding)
+ *
+ * Replaces the rough `Math.ceil(charCount / 4)` heuristic with actual
+ * BPE tokenization for accurate context window management.
+ */
+import type { Message } from '../providers/types.js';
+/**
+ * Count tokens in a text string using tiktoken
+ */
+export declare function countTokens(text: string): number;
+/**
+ * Count tokens across an array of messages
+ *
+ * Extracts all text content from messages (text blocks, tool inputs/results,
+ * thinking blocks) and counts tokens using tiktoken.
+ */
+export declare function countMessageTokens(messages: Message[]): number;

package/dist/utils/tokenizer.js

@@ -0,0 +1,56 @@
+/**
+ * Token counting utility using tiktoken (cl100k_base encoding)
+ *
+ * Replaces the rough `Math.ceil(charCount / 4)` heuristic with actual
+ * BPE tokenization for accurate context window management.
+ */
+import { getEncoding } from 'js-tiktoken';
+// Lazy-initialized encoder (loaded on first use)
+let encoder = null;
+function getEncoder() {
+    if (!encoder) {
+        encoder = getEncoding('cl100k_base');
+    }
+    return encoder;
+}
+/**
+ * Count tokens in a text string using tiktoken
+ */
+export function countTokens(text) {
+    if (!text)
+        return 0;
+    return getEncoder().encode(text).length;
+}
+/**
+ * Count tokens across an array of messages
+ *
+ * Extracts all text content from messages (text blocks, tool inputs/results,
+ * thinking blocks) and counts tokens using tiktoken.
+ */
+export function countMessageTokens(messages) {
+    const parts = [];
+    for (const msg of messages) {
+        if (typeof msg.content === 'string') {
+            parts.push(msg.content);
+        }
+        else {
+            for (const block of msg.content) {
+                switch (block.type) {
+                    case 'text':
+                        parts.push(block.text);
+                        break;
+                    case 'tool_use':
+                        parts.push(JSON.stringify(block.input));
+                        break;
+                    case 'tool_result':
+                        parts.push(typeof block.content === 'string' ? block.content : JSON.stringify(block.content));
+                        break;
+                    case 'thinking':
+                        parts.push(block.thinking);
+                        break;
+                }
+            }
+        }
+    }
+    return countTokens(parts.join(' '));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@compilr-dev/agents",
-  "version": "0.3.11",
+  "version": "0.3.12",
   "description": "Lightweight multi-LLM agent library for building CLI AI assistants",
   "type": "module",
   "main": "dist/index.js",
@@ -77,7 +77,8 @@
     "vitest": "^4.0.18"
   },
   "dependencies": {
-    "@google/genai": "^1.38.0"
+    "@google/genai": "^1.38.0",
+    "js-tiktoken": "^1.0.21"
   },
   "overrides": {
     "hono": "^4.11.7"