@tyvm/knowhow 0.0.21 → 0.0.23

package/src/agents/tools/executeScript/examples/quick-test.ts

@@ -0,0 +1,74 @@
+#!/usr/bin/env ts-node
+/**
+ * Quick test for executeScript - minimal example
+ * Usage: npx ts-node src/agents/tools/executeScript/examples/quick-test.ts
+ */
+
+import { executeScript } from "../../executeScript";
+import { services } from "../../../../services";
+
+// Simple test script
+const simpleScript = `
+console.log("Hello from executeScript!");
+
+async function main() {
+  // Test basic functionality
+  console.log("Running simple test...");
+
+  // Try a simple tool call
+  const files = await callTool("fileSearch", { searchTerm: "*.ts" });
+  console.log("Found", files?.length || 0, "TypeScript files");
+
+  return {
+    message: "Simple test completed!",
+    filesFound: files?.length || 0,
+    timestamp: new Date().toISOString()
+  };
+}
+
+await main().then(result => {
+  console.log("Result:", result);
+}).catch(error => {
+  console.error("Error:", error);
+});
+`;
+
+async function quickTest() {
+  console.log("🧪 Quick executeScript test\n");
+
+  try {
+    const { Tools, Clients } = services();
+    const result = await executeScript({
+      script: simpleScript,
+      maxToolCalls: 5,
+      maxTokens: 100,
+      maxExecutionTimeMs: 10000,
+      maxCostUsd: 0.1,
+    });
+
+    console.log("\n📊 QUICK TEST RESULT:");
+    console.log("Success:", result.success);
+    console.log("Result:", result.result);
+    console.log("Tool calls:", result.quotaUsage.toolCalls);
+    console.log("Cost: $" + result.quotaUsage.costUsd.toFixed(4));
+
+    if (result.consoleOutput.length > 0) {
+      console.log("\n📝 Console Output:");
+      result.consoleOutput.forEach((entry) => {
+        console.log(`  ${entry}`);
+      });
+    }
+
+    if (!result.success) {
+      console.log("❌ Error:", result.error);
+    }
+  } catch (error) {
+    console.error("💥 Test failed:", error);
+  }
+}
+
+if (require.main === module) {
+  quickTest();
+}
+
+export { quickTest };

package/src/agents/tools/executeScript/examples/serialization-test.ts

@@ -0,0 +1,321 @@
+#!/usr/bin/env ts-node
+/**
+ * Serialization Test for executeScript - demonstrates transfer issues
+ * Usage: npx ts-node src/agents/tools/executeScript/examples/serialization-test.ts
+ *
+ * This test demonstrates the "A non-transferable value was passed" errors
+ * that occur when trying to return complex objects from executeScript.
+ */
+
+import { executeScript } from "../../executeScript";
+import { services } from "../../../../services";
+
+interface TestCase {
+  name: string;
+  script: string;
+  expectedToWork: boolean;
+  description: string;
+}
+
+const testCases: TestCase[] = [
+  {
+    name: "primitive-string",
+    expectedToWork: true,
+    description: "Simple string return - should work",
+    script: `
+      console.log("Testing primitive string return");
+      return "Hello World";
+    `,
+  },
+
+  {
+    name: "primitive-number",
+    expectedToWork: true,
+    description: "Simple number return - should work",
+    script: `
+      console.log("Testing primitive number return");
+      return 42;
+    `,
+  },
+
+  {
+    name: "primitive-boolean",
+    expectedToWork: true,
+    description: "Simple boolean return - should work",
+    script: `
+      console.log("Testing primitive boolean return");
+      return true;
+    `,
+  },
+
+  {
+    name: "simple-array",
+    expectedToWork: true, // You mentioned this works
+    description: "Simple array return - you said this works",
+    script: `
+      console.log("Testing simple array return");
+      return [1, 2, 3, "hello"];
+    `,
+  },
+
+  {
+    name: "simple-object",
+    expectedToWork: false, // This is where I got errors
+    description:
+      "Simple object return - expected to fail with transferable error",
+    script: `
+      console.log("Testing simple object return");
+      return {
+        message: "Hello",
+        count: 42,
+        success: true
+      };
+    `,
+  },
+
+  {
+    name: "nested-object",
+    expectedToWork: false,
+    description: "Nested object return - expected to fail",
+    script: `
+      console.log("Testing nested object return");
+      return {
+        data: {
+          items: [1, 2, 3],
+          metadata: { timestamp: new Date().toISOString() }
+        },
+        status: "success"
+      };
+    `,
+  },
+
+  {
+    name: "object-with-functions",
+    expectedToWork: false,
+    description: "Object with functions - definitely should fail",
+    script: `
+      console.log("Testing object with functions");
+      return {
+        data: [1, 2, 3],
+        transform: function(x) { return x * 2; },
+        helper: () => "test"
+      };
+    `,
+  },
+
+  {
+    name: "array-of-objects",
+    expectedToWork: false, // Based on my experience
+    description: "Array containing objects - expected to fail",
+    script: `
+      console.log("Testing array of objects");
+      return [
+        { id: 1, name: "Alice" },
+        { id: 2, name: "Bob" },
+        { id: 3, name: "Charlie" }
+      ];
+    `,
+  },
+
+  {
+    name: "json-stringify-workaround",
+    expectedToWork: true,
+    description: "Using JSON.stringify as workaround - should work",
+    script: `
+      console.log("Testing JSON.stringify workaround");
+      const data = {
+        message: "Hello",
+        items: [1, 2, 3],
+        nested: { key: "value" }
+      };
+      return JSON.stringify(data);
+    `,
+  },
+
+  {
+    name: "tool-call-result",
+    expectedToWork: false, // Based on my experience with news aggregation
+    description: "Tool call result object - expected to fail",
+    script: `
+      console.log("Testing tool call result return");
+
+      try {
+        const searchResult = await callTool("fileSearch", { searchTerm: "package.json" });
+
+        // Try to return a structured response with the tool result
+        return {
+          success: true,
+          toolResult: searchResult,
+          timestamp: new Date().toISOString()
+        };
+      } catch (error) {
+        return {
+          success: false,
+          error: error.message
+        };
+      }
+    `,
+  },
+
+  {
+    name: "date-object",
+    expectedToWork: false,
+    description: "Date object return - expected to fail",
+    script: `
+      console.log("Testing Date object return");
+      return new Date();
+    `,
+  },
+
+  {
+    name: "map-object",
+    expectedToWork: false,
+    description: "Map object return - expected to fail",
+    script: `
+      console.log("Testing Map object return");
+      const map = new Map();
+      map.set("key1", "value1");
+      map.set("key2", "value2");
+      return map;
+    `,
+  },
+];
+
+async function runSerializationTests() {
+  console.log("🧪 Running executeScript Serialization Tests\\n");
+  console.log("=".repeat(80));
+
+  const results = {
+    passed: 0,
+    failed: 0,
+    unexpected: 0,
+    details: [] as any[],
+  };
+
+  for (const testCase of testCases) {
+    console.log(`\\n📋 Testing: ${testCase.name}`);
+    console.log(`📝 Description: ${testCase.description}`);
+    console.log(`🎯 Expected to work: ${testCase.expectedToWork}`);
+
+    try {
+      const result = await executeScript({
+        script: testCase.script,
+        maxToolCalls: 5,
+        maxTokens: 500,
+        maxExecutionTimeMs: 10000,
+        maxCostUsd: 0.1,
+      });
+
+      const actualWorked = result.success;
+      const matchesExpectation = actualWorked === testCase.expectedToWork;
+
+      if (matchesExpectation) {
+        results.passed++;
+        console.log(`✅ PASS - Behaved as expected`);
+      } else {
+        results.unexpected++;
+        console.log(
+          `⚠️  UNEXPECTED - Expected ${
+            testCase.expectedToWork ? "success" : "failure"
+          }, got ${actualWorked ? "success" : "failure"}`
+        );
+      }
+
+      results.details.push({
+        name: testCase.name,
+        expected: testCase.expectedToWork,
+        actual: actualWorked,
+        matches: matchesExpectation,
+        result: actualWorked ? result.result : null,
+        error: actualWorked ? null : result.error,
+        consoleOutput: result.consoleOutput,
+      });
+
+      if (actualWorked) {
+        console.log(`📊 Result type: ${typeof result.result}`);
+        console.log(
+          `📊 Result: ${JSON.stringify(result.result).substring(0, 200)}${
+            JSON.stringify(result.result).length > 200 ? "..." : ""
+          }`
+        );
+      } else {
+        console.log(`❌ Error: ${result.error}`);
+      }
+
+      if (result.consoleOutput.length > 0) {
+        console.log(`📝 Console: ${result.consoleOutput.join(", ")}`);
+      }
+    } catch (error) {
+      results.failed++;
+      console.log(`💥 TEST FRAMEWORK ERROR: ${error.message}`);
+
+      results.details.push({
+        name: testCase.name,
+        expected: testCase.expectedToWork,
+        actual: false,
+        matches: !testCase.expectedToWork,
+        result: null,
+        error: error.message,
+        consoleOutput: [],
+      });
+    }
+  }
+
+  // Print summary
+  console.log("\\n" + "=".repeat(80));
+  console.log("📊 SERIALIZATION TEST SUMMARY");
+  console.log("=".repeat(80));
+  console.log(`✅ Tests matching expectations: ${results.passed}`);
+  console.log(`⚠️  Unexpected behaviors: ${results.unexpected}`);
+  console.log(`💥 Framework failures: ${results.failed}`);
+  console.log(`📋 Total tests: ${testCases.length}`);
+
+  if (results.unexpected > 0) {
+    console.log("\\n🔍 UNEXPECTED RESULTS:");
+    results.details
+      .filter((d) => !d.matches)
+      .forEach((detail) => {
+        console.log(
+          `  - ${detail.name}: Expected ${
+            detail.expected ? "success" : "failure"
+          }, got ${detail.actual ? "success" : "failure"}`
+        );
+        if (detail.error) {
+          console.log(`    Error: ${detail.error}`);
+        }
+      });
+  }
+
+  // Analysis and recommendations
+  console.log("\\n🔬 ANALYSIS:");
+
+  const workingTypes = results.details
+    .filter((d) => d.actual)
+    .map((d) => d.name);
+  const failingTypes = results.details
+    .filter((d) => !d.actual)
+    .map((d) => d.name);
+
+  console.log("\\n✅ Types that work:");
+  workingTypes.forEach((name) => console.log(`  - ${name}`));
+
+  console.log("\\n❌ Types that fail:");
+  failingTypes.forEach((name) => console.log(`  - ${name}`));
+
+  console.log("\\n💡 RECOMMENDATIONS:");
+  console.log("  1. Use JSON.stringify() for complex objects");
+  console.log("  2. Return primitive values when possible");
+  console.log("  3. Consider createArtifact() for structured data");
+  console.log("  4. Test your return types with this suite");
+
+  return results;
+}
+
+if (require.main === module) {
+  runSerializationTests().catch((error) => {
+    console.error("Test suite failed:", error);
+    process.exit(1);
+  });
+}
+
+export { runSerializationTests, testCases };

package/src/agents/tools/executeScript/examples/test-runner.ts

@@ -0,0 +1,197 @@
+#!/usr/bin/env ts-node
+/**
+ * Test runner for the executeScript tool
+ * Usage: npx ts-node src/agents/tools/executeScript/examples/test-runner.ts
+ */
+
+import { executeScript } from "../../executeScript";
+import { services } from "../../../../services";
+import { Clients } from "../../../../clients";
+import { includedTools } from "../../../tools/list";
+import * as allTools from "../../../tools";
+
+// Sample script to test with
+const testScript = `
+// Test script that demonstrates various executeScript capabilities
+console.log("Starting test script execution...");
+
+async function main() {
+  // Test 1: Simple console output
+  console.log("Test 1: Basic logging works");
+
+  // Test 2: Call a tool (file search)
+  try {
+    console.log("Test 2: Calling fileSearch tool...");
+    const searchResult = await callTool("fileSearch", {
+      searchTerm: "package.json"
+    });
+    console.log("File search result:", searchResult);
+  } catch (error) {
+    console.error("Tool call failed:", error.message);
+  }
+
+  // Test 3: Call another tool (text search)
+  try {
+    console.log("Test 3: Calling textSearch tool...");
+    const textResult = await callTool("textSearch", {
+      searchTerm: "executeScript"
+    });
+    console.log("Text search found", textResult?.length || 0, "matches");
+  } catch (error) {
+    console.error("Text search failed:", error.message);
+  }
+
+  // Test 4: Make an LLM call
+  try {
+    console.log("Test 4: Making LLM call...");
+    const llmResponse = await llm([
+      {
+        role: "system",
+        content: "You are a helpful assistant. Respond with exactly one sentence."
+      },
+      {
+        role: "user",
+        content: "What is 2+2? Just give the answer briefly."
+      }
+    ], {
+      model: "gpt-4o-mini",
+      max_tokens: 50
+    });
+
+    console.log("LLM Response:", llmResponse.choices[0].message.content);
+  } catch (error) {
+    console.error("LLM call failed:", error.message);
+  }
+
+  // Test 5: Create an artifact
+  try {
+    console.log("Test 5: Creating artifact...");
+    createArtifact("test-results.md", \`# Test Results
+
+Script executed successfully at: \${new Date().toISOString()}
+
+This is a test artifact created by the executeScript tool.
+
+## Test Summary
+- Console logging: ✓
+- Tool calls: ✓
+- LLM calls: ✓
+- Artifact creation: ✓
+\`, "markdown");
+    console.log("Artifact created successfully");
+  } catch (error) {
+    console.error("Artifact creation failed:", error.message);
+  }
+
+  // Return final result
+  return {
+    success: true,
+    message: "All tests completed successfully",
+    timestamp: new Date().toISOString(),
+    testsRun: 5
+  };
+}
+
+// Execute the main function
+await main().then(result => {
+  console.log("=== SCRIPT COMPLETED ===");
+  console.log("Final result:", JSON.stringify(result, null, 2));
+}).catch(error => {
+  console.error("=== SCRIPT FAILED ===");
+  console.error("Error:", error);
+  throw error;
+});
+`;
+
+async function runTest() {
+  console.log("🚀 Starting executeScript test...\n");
+  const { Tools } = services();
+
+  try {
+    Tools.defineTools(includedTools, allTools);
+
+    console.log("📋 Test Parameters:");
+    console.log("- Max Tool Calls: 10");
+    console.log("- Max Tokens: 1000");
+    console.log("- Max Execution Time: 60s");
+    console.log("- Max Cost: $0.50\n");
+
+    const startTime = Date.now();
+
+    // Execute the test script
+    const result = await executeScript({
+      script: testScript,
+      maxToolCalls: 10,
+      maxTokens: 1000,
+      maxExecutionTimeMs: 60000,
+      maxCostUsd: 0.5,
+    });
+
+    const executionTime = Date.now() - startTime;
+
+    console.log("\n" + "=".repeat(60));
+    console.log("🎯 TEST RESULTS");
+    console.log("=".repeat(60));
+    console.log(`⏱️  Execution Time: ${executionTime}ms`);
+    console.log(`✅ Success: ${result.success}`);
+
+    if (result.success) {
+      console.log(`📊 Result:`, result.result);
+      console.log(`🔧 Tool Calls Made: ${result.quotaUsage.toolCalls}`);
+      console.log(`🎯 Tokens Used: ${result.quotaUsage.tokens}`);
+      console.log(`💰 Cost: $${result.quotaUsage.costUsd.toFixed(4)}`);
+
+      if (result.artifacts.length > 0) {
+        console.log(`📁 Artifacts Created: ${result.artifacts.length}`);
+        result.artifacts.forEach((artifact) => {
+          console.log(
+            `   - ${artifact.name} (${artifact.type}, ${artifact.contentLength} bytes)`
+          );
+        });
+      }
+
+      if (result.consoleOutput.length > 0) {
+        console.log(
+          `\n📝 Console Output (${result.consoleOutput.length} entries):`
+        );
+        result.consoleOutput.forEach((entry) => {
+          console.log(`   ${entry}`);
+        });
+      }
+
+      if (result.violations.length > 0) {
+        console.log(`\n⚠️  Policy Violations: ${result.violations.length}`);
+        result.violations.forEach((violation) => {
+          console.log(`   - ${JSON.stringify(violation)}`);
+        });
+      }
+    } else {
+      console.log(`❌ Error: ${result.error}`);
+
+      if (result.consoleOutput.length > 0) {
+        console.log(`\n📝 Console Output Before Failure:`);
+        result.consoleOutput.forEach((entry) => {
+          console.log(`   ${entry}`);
+        });
+      }
+    }
+
+    console.log("\n" + "=".repeat(60));
+    console.log(result.success ? "🎉 TEST PASSED!" : "💥 TEST FAILED!");
+    console.log("=".repeat(60));
+  } catch (error) {
+    console.error("\n💥 TEST RUNNER ERROR:");
+    console.error(error);
+    process.exit(1);
+  }
+}
+
+// Run the test if this file is executed directly
+if (require.main === module) {
+  runTest().catch((error) => {
+    console.error("Unhandled error:", error);
+    process.exit(1);
+  });
+}
+
+export { runTest, testScript };

package/src/agents/tools/executeScript/index.ts

@@ -0,0 +1,93 @@
+import { ScriptExecutor } from "../../../services/script-execution/ScriptExecutor";
+import { ToolsService } from "../../../services/Tools";
+import {
+  ExecutionRequest,
+  ExecutionResult,
+} from "../../../services/script-execution/types";
+import { services } from "src/services";
+
+export async function executeScript({
+  script,
+  maxToolCalls,
+  maxTokens,
+  maxExecutionTimeMs,
+  maxCostUsd,
+}: {
+  script: string;
+  maxToolCalls?: number;
+  maxTokens?: number;
+  maxExecutionTimeMs?: number;
+  maxCostUsd?: number;
+}) {
+  try {
+    // Get context from bound ToolsService
+    const toolService = (
+      this instanceof ToolsService ? this : services().Tools
+    ) as ToolsService;
+    const toolContext = toolService.getContext();
+    const { Clients, Tools } = toolContext;
+
+    if (!Clients) {
+      throw new Error("Clients not available in tool context");
+    }
+
+    // Create script executor with access to tools and clients
+    const executor = new ScriptExecutor(Tools, Clients);
+
+    // Execute the script
+    const result = await executor.execute({
+      script,
+      quotas: {
+        maxToolCalls: maxToolCalls || 50,
+        maxTokens: maxTokens || 10000,
+        maxExecutionTimeMs: maxExecutionTimeMs || 30000,
+        maxCostUsd: maxCostUsd || 1.0,
+        maxMemoryMb: 100,
+      },
+    });
+
+    // If there were policy violations, include them in the response
+    const violations = result.trace.events
+      .filter((e) => e.type.includes("violation") || e.type.includes("error"))
+      .map((e) => e.data);
+
+    // Format the response
+    return {
+      success: result.success,
+      result: result.result,
+      error: result.error,
+      artifacts: result.artifacts.map((a) => ({
+        id: a.id,
+        name: a.name,
+        type: a.type,
+        contentLength: a.content.length,
+        createdAt: a.createdAt,
+      })),
+      consoleOutput: result.consoleOutput,
+      metrics: result.trace.metrics,
+      violations,
+      executionTimeMs: result.trace.endTime - result.trace.startTime,
+      quotaUsage: {
+        toolCalls: result.trace.metrics.toolCallCount,
+        tokens: result.trace.metrics.tokenUsage.total,
+        costUsd: result.trace.metrics.costUsd,
+      },
+    };
+  } catch (error) {
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : String(error),
+      result: null,
+      artifacts: [],
+      consoleOutput: [],
+      metrics: null,
+      violations: [],
+      executionTimeMs: 0,
+      quotaUsage: {
+        toolCalls: 0,
+        tokens: 0,
+        costUsd: 0,
+      },
+    };
+  }
+}

package/src/agents/tools/index.ts

@@ -23,3 +23,4 @@ export * from "./aiClient";
 export * from "./googleSearch";
 export * from "./loadWebpage";
 export * from "./stringReplace";
+export * from "./executeScript";

package/src/agents/tools/list.ts

@@ -7,6 +7,7 @@ import * as github from "./github/definitions";
 import * as asana from "./asana/definitions";
 import * as language from "./language/definitions";
 import { googleSearchDefinition } from "./googleSearch";
+import { executeScriptDefinition } from "./executeScript/definition";
 
 export const includedTools = [
   {
@@ -552,7 +553,7 @@ export const includedTools = [
       },
     },
   },
-
+  executeScriptDefinition,
   googleSearchDefinition,
   ...asana.definitions,
   ...github.definitions,