@tyvm/knowhow 0.0.21 → 0.0.22

package/src/agents/tools/executeScript/examples/test-runner.ts

@@ -0,0 +1,204 @@
+#!/usr/bin/env ts-node
+/**
+ * Test runner for the executeScript tool
+ * Usage: npx ts-node src/agents/tools/executeScript/examples/test-runner.ts
+ */
+
+import { executeScript } from "../../executeScript";
+import { Tools } from "../../../../services";
+import { Clients } from "../../../../clients";
+import { includedTools } from "../../../tools/list";
+import * as allTools from "../../../tools";
+
+// Sample script to test with
+const testScript = `
+// Test script that demonstrates various executeScript capabilities
+console.log("Starting test script execution...");
+
+async function main() {
+  // Test 1: Simple console output
+  console.log("Test 1: Basic logging works");
+
+  // Test 2: Call a tool (file search)
+  try {
+    console.log("Test 2: Calling fileSearch tool...");
+    const searchResult = await callTool("fileSearch", {
+      searchTerm: "package.json"
+    });
+    console.log("File search result:", searchResult);
+  } catch (error) {
+    console.error("Tool call failed:", error.message);
+  }
+
+  // Test 3: Call another tool (text search)
+  try {
+    console.log("Test 3: Calling textSearch tool...");
+    const textResult = await callTool("textSearch", {
+      searchTerm: "executeScript"
+    });
+    console.log("Text search found", textResult?.length || 0, "matches");
+  } catch (error) {
+    console.error("Text search failed:", error.message);
+  }
+
+  // Test 4: Make an LLM call
+  try {
+    console.log("Test 4: Making LLM call...");
+    const llmResponse = await llm([
+      {
+        role: "system",
+        content: "You are a helpful assistant. Respond with exactly one sentence."
+      },
+      {
+        role: "user",
+        content: "What is 2+2? Just give the answer briefly."
+      }
+    ], {
+      model: "gpt-4o-mini",
+      max_tokens: 50
+    });
+
+    console.log("LLM Response:", llmResponse.choices[0].message.content);
+  } catch (error) {
+    console.error("LLM call failed:", error.message);
+  }
+
+  // Test 5: Create an artifact
+  try {
+    console.log("Test 5: Creating artifact...");
+    createArtifact("test-results.md", \`# Test Results
+
+Script executed successfully at: \${new Date().toISOString()}
+
+This is a test artifact created by the executeScript tool.
+
+## Test Summary
+- Console logging: ✓
+- Tool calls: ✓
+- LLM calls: ✓
+- Artifact creation: ✓
+\`, "markdown");
+    console.log("Artifact created successfully");
+  } catch (error) {
+    console.error("Artifact creation failed:", error.message);
+  }
+
+  // Return final result
+  return {
+    success: true,
+    message: "All tests completed successfully",
+    timestamp: new Date().toISOString(),
+    testsRun: 5
+  };
+}
+
+// Execute the main function
+await main().then(result => {
+  console.log("=== SCRIPT COMPLETED ===");
+  console.log("Final result:", JSON.stringify(result, null, 2));
+}).catch(error => {
+  console.error("=== SCRIPT FAILED ===");
+  console.error("Error:", error);
+  throw error;
+});
+`;
+
+async function runTest() {
+  console.log("🚀 Starting executeScript test...\n");
+
+  try {
+    Tools.defineTools(includedTools, allTools);
+
+    const context = {
+      tools: Tools,
+      clients: Clients,
+    };
+
+    console.log("📋 Test Parameters:");
+    console.log("- Max Tool Calls: 10");
+    console.log("- Max Tokens: 1000");
+    console.log("- Max Execution Time: 60s");
+    console.log("- Max Cost: $0.50\n");
+
+    const startTime = Date.now();
+
+    // Execute the test script
+    const result = await executeScript(
+      {
+        script: testScript,
+        maxToolCalls: 10,
+        maxTokens: 1000,
+        maxExecutionTimeMs: 60000,
+        maxCostUsd: 0.5,
+      },
+      context
+    );
+
+    const executionTime = Date.now() - startTime;
+
+    console.log("\n" + "=".repeat(60));
+    console.log("🎯 TEST RESULTS");
+    console.log("=".repeat(60));
+    console.log(`⏱️  Execution Time: ${executionTime}ms`);
+    console.log(`✅ Success: ${result.success}`);
+
+    if (result.success) {
+      console.log(`📊 Result:`, result.result);
+      console.log(`🔧 Tool Calls Made: ${result.quotaUsage.toolCalls}`);
+      console.log(`🎯 Tokens Used: ${result.quotaUsage.tokens}`);
+      console.log(`💰 Cost: $${result.quotaUsage.costUsd.toFixed(4)}`);
+
+      if (result.artifacts.length > 0) {
+        console.log(`📁 Artifacts Created: ${result.artifacts.length}`);
+        result.artifacts.forEach((artifact) => {
+          console.log(
+            `   - ${artifact.name} (${artifact.type}, ${artifact.contentLength} bytes)`
+          );
+        });
+      }
+
+      if (result.consoleOutput.length > 0) {
+        console.log(
+          `\n📝 Console Output (${result.consoleOutput.length} entries):`
+        );
+        result.consoleOutput.forEach((entry) => {
+          console.log(`   ${entry}`);
+        });
+      }
+
+      if (result.violations.length > 0) {
+        console.log(`\n⚠️  Policy Violations: ${result.violations.length}`);
+        result.violations.forEach((violation) => {
+          console.log(`   - ${JSON.stringify(violation)}`);
+        });
+      }
+    } else {
+      console.log(`❌ Error: ${result.error}`);
+
+      if (result.consoleOutput.length > 0) {
+        console.log(`\n📝 Console Output Before Failure:`);
+        result.consoleOutput.forEach((entry) => {
+          console.log(`   ${entry}`);
+        });
+      }
+    }
+
+    console.log("\n" + "=".repeat(60));
+    console.log(result.success ? "🎉 TEST PASSED!" : "💥 TEST FAILED!");
+    console.log("=".repeat(60));
+  } catch (error) {
+    console.error("\n💥 TEST RUNNER ERROR:");
+    console.error(error);
+    process.exit(1);
+  }
+}
+
+// Run the test if this file is executed directly
+if (require.main === module) {
+  runTest().catch((error) => {
+    console.error("Unhandled error:", error);
+    process.exit(1);
+  });
+}
+
+export { runTest, testScript };

package/src/agents/tools/executeScript/index.ts

@@ -0,0 +1,74 @@
+import { ScriptExecutor } from "../../../services/script-execution/ScriptExecutor";
+import { Tools } from "../../../services";
+import { Clients } from "../../../clients";
+import {
+  ExecutionRequest,
+  ExecutionResult,
+} from "../../../services/script-execution/types";
+
+
+export const executeScript = async (
+  { script, maxToolCalls, maxTokens, maxExecutionTimeMs, maxCostUsd },
+  context
+) => {
+  try {
+    // Create script executor with access to tools and clients
+    const executor = new ScriptExecutor(Tools, Clients);
+
+    // Execute the script
+    const result = await executor.execute({
+      script,
+      quotas: {
+        maxToolCalls: maxToolCalls || 50,
+        maxTokens: maxTokens || 10000,
+        maxExecutionTimeMs: maxExecutionTimeMs || 30000,
+        maxCostUsd: maxCostUsd || 1.0,
+        maxMemoryMb: 100,
+      },
+    });
+
+    // If there were policy violations, include them in the response
+    const violations = result.trace.events
+      .filter((e) => e.type.includes("violation") || e.type.includes("error"))
+      .map((e) => e.data);
+
+    // Format the response
+    return {
+      success: result.success,
+      result: result.result,
+      error: result.error,
+      artifacts: result.artifacts.map((a) => ({
+        id: a.id,
+        name: a.name,
+        type: a.type,
+        contentLength: a.content.length,
+        createdAt: a.createdAt,
+      })),
+      consoleOutput: result.consoleOutput,
+      metrics: result.trace.metrics,
+      violations,
+      executionTimeMs: result.trace.endTime - result.trace.startTime,
+      quotaUsage: {
+        toolCalls: result.trace.metrics.toolCallCount,
+        tokens: result.trace.metrics.tokenUsage.total,
+        costUsd: result.trace.metrics.costUsd,
+      },
+    };
+  } catch (error) {
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : String(error),
+      result: null,
+      artifacts: [],
+      consoleOutput: [],
+      metrics: null,
+      violations: [],
+      executionTimeMs: 0,
+      quotaUsage: {
+        toolCalls: 0,
+        tokens: 0,
+        costUsd: 0,
+      },
+    };
+  }
+};

package/src/agents/tools/index.ts

@@ -23,3 +23,4 @@ export * from "./aiClient";
 export * from "./googleSearch";
 export * from "./loadWebpage";
 export * from "./stringReplace";
+export * from "./executeScript";

package/src/agents/tools/list.ts

@@ -7,6 +7,7 @@ import * as github from "./github/definitions";
 import * as asana from "./asana/definitions";
 import * as language from "./language/definitions";
 import { googleSearchDefinition } from "./googleSearch";
+import { executeScriptDefinition } from "./executeScript/definition";
 
 export const includedTools = [
   {
@@ -552,7 +553,7 @@ export const includedTools = [
       },
     },
   },
-
+  executeScriptDefinition,
   googleSearchDefinition,
   ...asana.definitions,
   ...github.definitions,

package/src/cli.ts

@@ -11,7 +11,7 @@ import { Vimmer } from "./agents/vim/vim";
 import { Developer } from "./agents/developer/developer";
 import { Tools } from "./services";
 import { includedTools } from "./agents/tools/list";
-import * as allTools from "./agents/tools/index";
+import * as allTools from "./agents/tools";
 import { Mcp } from "./services/Mcp";
 import { login } from "./login";
 import { worker } from "./worker";
@@ -24,12 +24,8 @@ async function main() {
   Agents.registerAgent(Patcher);
   Agents.registerAgent(Developer);
   Agents.loadAgentsFromConfig();
-  Tools.addTools(includedTools);
 
-  const toolFunctions = Object.entries(allTools)
-    .filter(([_, value]) => typeof value === 'function')
-    .reduce((acc, [key, value]) => ({ ...acc, [key]: value }), {});
-  Tools.addFunctions(toolFunctions);
+  Tools.defineTools(includedTools, allTools);
 
   await Mcp.connectToConfigured(Tools);
   await Clients.registerConfiguredModels();

package/src/clients/index.ts

@@ -153,13 +153,24 @@ export class AIClient {
     );
   }
 
-  private providerHasModel(provider: string, model: string): boolean {
+  providerHasModel(provider: string, model: string): boolean {
     const models = this.clientModels[provider];
     if (!models) return false;
     return models.includes(model);
   }
 
-  private detectProviderModel(provider: string, model?: string) {
+  findModel(modelPrefix: string) {
+    for (const provider of Object.keys(this.clientModels)) {
+      const models = this.clientModels[provider];
+      const foundModel = models.find((m) => m.startsWith(modelPrefix));
+      if (foundModel) {
+        return { provider, model: foundModel };
+      }
+    }
+    return undefined;
+  }
+
+  detectProviderModel(provider: string, model?: string) {
     if (this.providerHasModel(provider, model)) {
       return { provider, model };
     }
@@ -170,18 +181,21 @@ export class AIClient {
       const inferredProvider = split[0];
       const inferredModel = split.slice(1).join("/");
 
+      // Exact match
       if (this.providerHasModel(inferredProvider, inferredModel)) {
         return { provider: inferredProvider, model: inferredModel };
       }
-    }
 
-    const providers = Object.keys(this.clientModels);
-    const foundProvider = providers.find((p) =>
-      this.providerHasModel(p, model)
-    );
+      // Starts with match
+      const foundBySplit = this.findModel(inferredModel);
+      if (foundBySplit) {
+        return foundBySplit;
+      }
+    }
 
-    if (foundProvider) {
-      return { provider: foundProvider, model };
+    const foundByModel = this.findModel(model);
+    if (foundByModel) {
+      return foundByModel;
     }
 
     return { provider, model };

package/src/services/Tools.ts

@@ -97,23 +97,39 @@ export class ToolsService {
   }
 
   addTools(tools: Tool[]) {
-    this.tools.push(...tools);
+    // Prevent duplicate tool names
+    const existingTools = this.getToolNames();
+    const filteredTools = tools.filter(
+      (tool) => !existingTools.includes(tool.function.name)
+    );
+
+    this.tools.push(...filteredTools);
   }
 
   addFunctions(fns: { [fnName: string]: (...args: any) => any }) {
     for (const fnName of Object.keys(fns)) {
+      if (typeof fns[fnName] !== "function") {
+        // Skip non-function entries
+        continue;
+      }
       this.setFunction(fnName, fns[fnName]);
     }
   }
 
+  defineTools(
+    tools: Tool[],
+    functions: { [fnName: string]: ((...args: any) => any) | any }
+  ) {
+    this.addTools(tools);
+    this.addFunctions(functions);
+  }
+
   async callTool(toolCall: ToolCall, enabledTools = this.getToolNames()) {
     const functionName = toolCall.function.name;
     const functionArgs = JSON.parse(
       restoreEscapedNewLines(toolCall.function.arguments)
     );
 
-    console.log(toolCall);
-
     try {
       // Check if tool is enabled
       if (!enabledTools.includes(functionName)) {

package/src/services/script-execution/SandboxContext.ts

@@ -0,0 +1,278 @@
+import { Tools } from "../../services";
+import { Clients } from "../../clients";
+import { ScriptTracer } from "./ScriptTracer";
+import { ScriptPolicyEnforcer } from "./ScriptPolicy";
+import { Artifact, QuotaUsage } from "./types";
+import { Message } from "../../clients/types";
+
+/**
+ * Provides the execution context for scripts with controlled access to tools and AI
+ */
+export class SandboxContext {
+  private artifacts: Artifact[] = [];
+  private consoleOutput: string[] = [];
+
+  constructor(
+    private toolsService: typeof Tools = Tools,
+    private clients: typeof Clients = Clients,
+    private tracer: ScriptTracer,
+    private policyEnforcer: ScriptPolicyEnforcer
+  ) {}
+
+  /**
+   * Console implementation that captures output
+   */
+  console = {
+    log: (...args: any[]) => {
+      const message = args
+        .map((arg) =>
+          typeof arg === "object" ? JSON.stringify(arg) : String(arg)
+        )
+        .join(" ");
+      this.consoleOutput.push(`[LOG] ${message}`);
+      this.tracer.emitEvent("console_log", { message, args });
+    },
+
+    error: (...args: any[]) => {
+      const message = args
+        .map((arg) =>
+          typeof arg === "object" ? JSON.stringify(arg) : String(arg)
+        )
+        .join(" ");
+      this.consoleOutput.push(`[ERROR] ${message}`);
+      this.tracer.emitEvent("console_error", { message, args });
+    },
+
+    warn: (...args: any[]) => {
+      const message = args
+        .map((arg) =>
+          typeof arg === "object" ? JSON.stringify(arg) : String(arg)
+        )
+        .join(" ");
+      this.consoleOutput.push(`[WARN] ${message}`);
+      this.tracer.emitEvent("console_warn", { message, args });
+    },
+
+    info: (...args: any[]) => {
+      const message = args
+        .map((arg) =>
+          typeof arg === "object" ? JSON.stringify(arg) : String(arg)
+        )
+        .join(" ");
+      this.consoleOutput.push(`[INFO] ${message}`);
+      this.tracer.emitEvent("console_info", { message, args });
+    },
+  };
+
+  /**
+   * Call a tool through the tools service
+   */
+  async callTool(toolName: string, parameters: any): Promise<any> {
+    // Check policy first
+    if (!this.policyEnforcer.checkToolCall(toolName)) {
+      throw new Error(`Tool call '${toolName}' blocked by policy`);
+    }
+
+    this.tracer.emitEvent("tool_call_start", {
+      toolName,
+      parameters: this.sanitizeForLogging(parameters),
+    });
+
+    try {
+      // Record the tool call
+      this.policyEnforcer.recordToolCall();
+
+      // Create a proper ToolCall object
+      const toolCall = {
+        id: `script-tool-${Date.now()}-${Math.random()
+          .toString(36)
+          .substr(2, 9)}`,
+        type: "function" as const,
+        function: {
+          name: toolName,
+          arguments: JSON.stringify(parameters),
+        },
+      };
+
+      // Call the actual tool through the Tools service
+      const result = await this.toolsService.callTool(toolCall);
+
+      this.tracer.emitEvent("tool_call_success", {
+        toolName,
+        result: this.sanitizeForLogging(result),
+      });
+
+      return result;
+    } catch (error) {
+      this.tracer.emitEvent("tool_call_error", {
+        toolName,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+  }
+
+  /**
+   * Call LLM through the clients service
+   */
+  async llm(
+    messages: Message[],
+    options: {
+      model?: string;
+      maxTokens?: number;
+      temperature?: number;
+    } = {}
+  ) {
+    const estimatedTokens = this.estimateTokens(messages);
+
+    // Check token quota
+    if (!this.policyEnforcer.checkTokenUsage(estimatedTokens)) {
+      throw new Error("Token quota would be exceeded");
+    }
+
+    this.tracer.emitEvent("llm_call_start", {
+      messageCount: messages.length,
+      estimatedTokens,
+      model: options.model,
+      options: this.sanitizeForLogging(options),
+    });
+
+    try {
+      // Record token usage
+      this.policyEnforcer.recordTokenUsage(estimatedTokens);
+
+      // Use the actual Clients service to make LLM calls
+      const completionOptions = {
+        model: options.model,
+        messages,
+        max_tokens: options.maxTokens,
+      };
+
+      // Detect provider from model or use default
+      const response = await this.clients.createCompletion(
+        "",
+        completionOptions
+      );
+
+      this.tracer.emitEvent("llm_call_success", {
+        model: response.model,
+        usage: response.usage,
+        usdCost: response.usd_cost,
+      });
+
+      return response;
+    } catch (error) {
+      this.tracer.emitEvent("llm_call_error", {
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+  }
+
+  /**
+   * Get current quota usage
+   */
+  getQuotaUsage(): QuotaUsage {
+    return this.policyEnforcer.getUsage();
+  }
+
+  /**
+   * Create an artifact
+   */
+  async createArtifact(
+    name: string,
+    content: string,
+    type: "text" | "json" | "csv" | "html" | "markdown" = "text"
+  ): Promise<Artifact> {
+    const artifact: Artifact = {
+      id: `artifact-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
+      name,
+      type,
+      content,
+      createdAt: new Date().toISOString(),
+    };
+
+    this.artifacts.push(artifact);
+
+    this.tracer.emitEvent("artifact_created", {
+      artifactId: artifact.id,
+      name,
+      type,
+      contentLength: content.length,
+    });
+
+    return artifact;
+  }
+
+  async sleep(ms: number): Promise<void> {
+    if (typeof ms !== "number" || ms < 0 || ms > 2000) {
+      throw new Error("Invalid sleep duration");
+    }
+    await new Promise((res) => setTimeout(res, ms));
+    this.tracer.emitEvent("sleep", { durationMs: ms });
+  }
+
+  /**
+   * Get all created artifacts
+   */
+  getArtifacts(): Artifact[] {
+    return [...this.artifacts];
+  }
+
+  /**
+   * Get console output
+   */
+  getConsoleOutput(): string[] {
+    return [...this.consoleOutput];
+  }
+
+  /**
+   * Estimate tokens for text (rough approximation)
+   */
+  private estimateTokens(messages: any[]): number {
+    let totalText = "";
+    for (const message of messages) {
+      if (typeof message === "string") {
+        totalText += message;
+      } else if (message && typeof message.content === "string") {
+        totalText += message.content;
+      }
+    }
+    // Rough estimation: ~4 characters per token
+    return Math.ceil(totalText.length / 4);
+  }
+
+  /**
+   * Sanitize data for logging (remove sensitive information)
+   */
+  private sanitizeForLogging(data: any): any {
+    if (data === null || data === undefined) {
+      return data;
+    }
+
+    if (typeof data === "string") {
+      // Truncate very long strings
+      return data.length > 500 ? data.substring(0, 500) + "..." : data;
+    }
+
+    if (typeof data === "object") {
+      const sanitized: any = {};
+      for (const [key, value] of Object.entries(data)) {
+        // Skip potentially sensitive keys
+        if (
+          key.toLowerCase().includes("password") ||
+          key.toLowerCase().includes("token") ||
+          key.toLowerCase().includes("secret") ||
+          key.toLowerCase().includes("key")
+        ) {
+          sanitized[key] = "[REDACTED]";
+        } else {
+          sanitized[key] = this.sanitizeForLogging(value);
+        }
+      }
+      return sanitized;
+    }
+
+    return data;
+  }
+}