@tyvm/knowhow 0.0.21 → 0.0.22

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tyvm/knowhow",
-  "version": "0.0.21",
+  "version": "0.0.22",
   "description": "ai cli with plugins and agents",
   "main": "ts_build/src/index.js",
   "bin": {
@@ -51,10 +51,12 @@
     "axios": "^1.5.0",
     "cheerio": "^1.0.0",
     "diff": "^5.2.0",
+    "esbuild": "^0.25.8",
     "express": "^4.19.2",
     "figma-js": "^1.16.1-0",
     "gitignore-to-glob": "^0.3.0",
     "ink": "^6.0.1",
+    "isolated-vm": "^5.0.4",
     "jira-client": "^8.2.2",
     "marked": "^10.0.0",
     "marked-terminal": "^6.2.0",

package/src/agents/tools/executeScript/README.md

@@ -0,0 +1,78 @@
+# Execute Script Tool
+
+The `executeScript` tool provides secure, isolated execution of TypeScript scripts with access to the Knowhow platform's tools and AI capabilities.
+
+## Features
+
+- **Secure Sandbox**: Scripts run in isolated-vm with no access to Node.js APIs
+- **Tool Integration**: Access to all existing Knowhow tools via `callTool()`
+- **AI Integration**: Direct access to AI completions via `llm()`
+- **Resource Quotas**: Configurable limits on execution time, memory, tool calls, and tokens
+- **Comprehensive Tracing**: Full audit trail of all script activities
+- **Policy Enforcement**: Fine-grained security controls and access restrictions
+
+## Usage
+
+```typescript
+// Basic usage
+await executeScript({
+  script: `
+    console.log("Hello from script!");
+    
+    const files = await callTool("fileSearch", { searchTerm: "*.ts" });
+    console.log("Found TypeScript files:", files);
+    
+    const response = await llm([
+      { role: "user", content: "What is TypeScript?" }
+    ]);
+    
+    return { message: "Script completed successfully" };
+  `
+});
+
+// With custom policy
+await executeScript({
+  script: "...",
+  policy: {
+    maxExecutionTimeMs: 10000,
+    maxMemoryMB: 64,
+    maxToolCalls: 5,
+    maxTokens: 1000,
+    allowedTools: ["fileSearch", "textSearch"],
+    deniedTools: ["execCommand"]
+  }
+});
+```
+
+## Available Functions in Scripts
+
+### `callTool(name: string, args: any): Promise<any>`
+Call any available Knowhow tool by name with arguments.
+
+### `llm(messages: ChatMessage[], options?: LLMOptions): Promise<ChatCompletion>`
+Make AI completion requests with message history and options.
+
+## Security Features
+
+- **No Node.js Access**: Scripts cannot access filesystem, network, or system APIs directly
+- **Tool Authorization**: All external access goes through existing authorization systems
+- **Resource Limits**: Prevents runaway scripts with time, memory, and usage quotas
+- **Trace Sanitization**: Sensitive data is redacted from execution logs
+- **Policy Enforcement**: Granular control over what tools and resources scripts can access
+
+## Implementation Details
+
+- **ScriptExecutor**: Handles compilation and isolated execution
+- **SandboxContext**: Provides `callTool` and `llm` function implementations
+- **ScriptTracer**: Records all script activities for debugging and audit
+- **ScriptPolicyEnforcer**: Enforces resource quotas and security policies
+
+## Files
+
+- `types.ts` - TypeScript interfaces and type definitions
+- `ScriptExecutor.ts` - Core execution engine with isolated-vm
+- `SandboxContext.ts` - Script execution context with tool/AI access
+- `ScriptTracer.ts` - Event tracing and monitoring system
+- `ScriptPolicyEnforcer.ts` - Security policy enforcement
+- `executeScript.ts` - Tool handler and main entry point
+- `examples/` - Example scripts demonstrating capabilities

package/src/agents/tools/executeScript/definition.ts

@@ -0,0 +1,73 @@
+import { Tool } from "../../../clients";
+
+/**
+ * Tool for executing TypeScript scripts in a secure sandbox
+ */
+export const executeScriptDefinition: Tool = {
+  type: "function",
+  function: {
+    name: "executeScript",
+    description: `Execute TypeScript code in a secure sandbox environment with access to tools and AI models.
+
+  The script has access to:
+  - callTool(toolName, parameters): Call any available tool
+  - llm(messages, options): Make LLM calls
+  - createArtifact(name, content, type): Create downloadable artifacts
+  - console: Standard console logging
+  - getQuotaUsage(): Check resource usage
+  - sleep(ms): Pause execution for a specified time
+
+  Example:
+  \`\`\`typescript
+  // Call a tool
+  const searchResult = await callTool('textSearch', { searchTerm: 'hello world' });
+  console.log('Search found:', searchResult);
+
+  // Call LLM
+  const response = await llm([
+    { role: 'user', content: 'Explain quantum computing' }
+  ], { model: 'gpt-4o-mini', maxTokens: 100 });
+  console.log('LLM response:', response.choices[0].message.content);
+
+  // Create an artifact
+  createArtifact('summary.md', '# Summary\\nThis is a test', 'markdown');
+
+  return { message: 'Script completed successfully' };
+  \`\`\`
+
+  Must either return or await the a top level function
+
+  Test tools yourself to know the return type when scripting. Can pass JSON.stringified data into llm call if you don't need to know the type.
+  You cannot use isolation breaking methods like: setTimeout setInterval setImmediate clearTimeout clearInterval
+
+  Security: Scripts run in isolation with quotas on tool calls, tokens, time, and cost.`,
+
+    parameters: {
+      type: "object",
+      properties: {
+        script: {
+          type: "string",
+          description: "The TypeScript code to execute",
+        },
+        maxToolCalls: {
+          type: "number",
+          description: "Maximum number of tool calls allowed (default: 50)",
+        },
+        maxTokens: {
+          type: "number",
+          description: "Maximum tokens for LLM calls (default: 10000)",
+        },
+        maxExecutionTimeMs: {
+          type: "number",
+          description:
+            "Maximum execution time in milliseconds (default: 30000)",
+        },
+        maxCostUsd: {
+          type: "number",
+          description: "Maximum cost in USD (default: 1.0)",
+        },
+      },
+      required: ["script"],
+    },
+  },
+};

package/src/agents/tools/executeScript/examples/quick-test.ts

@@ -0,0 +1,80 @@
+#!/usr/bin/env ts-node
+/**
+ * Quick test for executeScript - minimal example
+ * Usage: npx ts-node src/agents/tools/executeScript/examples/quick-test.ts
+ */
+
+import { executeScript } from "../../executeScript";
+import { Tools } from "../../../../services";
+import { Clients } from "../../../../clients";
+
+// Simple test script
+const simpleScript = `
+console.log("Hello from executeScript!");
+
+async function main() {
+  // Test basic functionality
+  console.log("Running simple test...");
+
+  // Try a simple tool call
+  const files = await callTool("fileSearch", { searchTerm: "*.ts" });
+  console.log("Found", files?.length || 0, "TypeScript files");
+
+  return {
+    message: "Simple test completed!",
+    filesFound: files?.length || 0,
+    timestamp: new Date().toISOString()
+  };
+}
+
+await main().then(result => {
+  console.log("Result:", result);
+}).catch(error => {
+  console.error("Error:", error);
+});
+`;
+
+async function quickTest() {
+  console.log("🧪 Quick executeScript test\n");
+
+  try {
+    const result = await executeScript(
+      {
+        script: simpleScript,
+        maxToolCalls: 5,
+        maxTokens: 100,
+        maxExecutionTimeMs: 10000,
+        maxCostUsd: 0.1,
+      },
+      {
+        tools: Tools,
+        clients: Clients,
+      }
+    );
+
+    console.log("\n📊 QUICK TEST RESULT:");
+    console.log("Success:", result.success);
+    console.log("Result:", result.result);
+    console.log("Tool calls:", result.quotaUsage.toolCalls);
+    console.log("Cost: $" + result.quotaUsage.costUsd.toFixed(4));
+
+    if (result.consoleOutput.length > 0) {
+      console.log("\n📝 Console Output:");
+      result.consoleOutput.forEach((entry) => {
+        console.log(`  ${entry}`);
+      });
+    }
+
+    if (!result.success) {
+      console.log("❌ Error:", result.error);
+    }
+  } catch (error) {
+    console.error("💥 Test failed:", error);
+  }
+}
+
+if (require.main === module) {
+  quickTest();
+}
+
+export { quickTest };

package/src/agents/tools/executeScript/examples/serialization-test.ts

@@ -0,0 +1,309 @@
+#!/usr/bin/env ts-node
+/**
+ * Serialization Test for executeScript - demonstrates transfer issues
+ * Usage: npx ts-node src/agents/tools/executeScript/examples/serialization-test.ts
+ * 
+ * This test demonstrates the "A non-transferable value was passed" errors
+ * that occur when trying to return complex objects from executeScript.
+ */
+
+import { executeScript } from "../../executeScript";
+import { Tools } from "../../../../services";
+import { Clients } from "../../../../clients";
+
+interface TestCase {
+  name: string;
+  script: string;
+  expectedToWork: boolean;
+  description: string;
+}
+
+const testCases: TestCase[] = [
+  {
+    name: "primitive-string",
+    expectedToWork: true,
+    description: "Simple string return - should work",
+    script: `
+      console.log("Testing primitive string return");
+      return "Hello World";
+    `
+  },
+  
+  {
+    name: "primitive-number", 
+    expectedToWork: true,
+    description: "Simple number return - should work",
+    script: `
+      console.log("Testing primitive number return");
+      return 42;
+    `
+  },
+
+  {
+    name: "primitive-boolean",
+    expectedToWork: true,
+    description: "Simple boolean return - should work", 
+    script: `
+      console.log("Testing primitive boolean return");
+      return true;
+    `
+  },
+
+  {
+    name: "simple-array",
+    expectedToWork: true, // You mentioned this works
+    description: "Simple array return - you said this works",
+    script: `
+      console.log("Testing simple array return");
+      return [1, 2, 3, "hello"];
+    `
+  },
+
+  {
+    name: "simple-object",
+    expectedToWork: false, // This is where I got errors
+    description: "Simple object return - expected to fail with transferable error",
+    script: `
+      console.log("Testing simple object return");
+      return { 
+        message: "Hello", 
+        count: 42, 
+        success: true 
+      };
+    `
+  },
+
+  {
+    name: "nested-object",
+    expectedToWork: false,
+    description: "Nested object return - expected to fail",
+    script: `
+      console.log("Testing nested object return");
+      return {
+        data: {
+          items: [1, 2, 3],
+          metadata: { timestamp: new Date().toISOString() }
+        },
+        status: "success"
+      };
+    `
+  },
+
+  {
+    name: "object-with-functions",
+    expectedToWork: false,
+    description: "Object with functions - definitely should fail",
+    script: `
+      console.log("Testing object with functions");
+      return {
+        data: [1, 2, 3],
+        transform: function(x) { return x * 2; },
+        helper: () => "test"
+      };
+    `
+  },
+
+  {
+    name: "array-of-objects",
+    expectedToWork: false, // Based on my experience
+    description: "Array containing objects - expected to fail",
+    script: `
+      console.log("Testing array of objects");
+      return [
+        { id: 1, name: "Alice" },
+        { id: 2, name: "Bob" },
+        { id: 3, name: "Charlie" }
+      ];
+    `
+  },
+
+  {
+    name: "json-stringify-workaround",
+    expectedToWork: true,
+    description: "Using JSON.stringify as workaround - should work",
+    script: `
+      console.log("Testing JSON.stringify workaround");
+      const data = {
+        message: "Hello",
+        items: [1, 2, 3],
+        nested: { key: "value" }
+      };
+      return JSON.stringify(data);
+    `
+  },
+
+  {
+    name: "tool-call-result",
+    expectedToWork: false, // Based on my experience with news aggregation
+    description: "Tool call result object - expected to fail",
+    script: `
+      console.log("Testing tool call result return");
+      
+      try {
+        const searchResult = await callTool("fileSearch", { searchTerm: "package.json" });
+        
+        // Try to return a structured response with the tool result
+        return {
+          success: true,
+          toolResult: searchResult,
+          timestamp: new Date().toISOString()
+        };
+      } catch (error) {
+        return {
+          success: false,
+          error: error.message
+        };
+      }
+    `
+  },
+
+  {
+    name: "date-object",
+    expectedToWork: false,
+    description: "Date object return - expected to fail",
+    script: `
+      console.log("Testing Date object return");
+      return new Date();
+    `
+  },
+
+  {
+    name: "map-object",
+    expectedToWork: false,
+    description: "Map object return - expected to fail",
+    script: `
+      console.log("Testing Map object return");
+      const map = new Map();
+      map.set("key1", "value1");
+      map.set("key2", "value2");
+      return map;
+    `
+  }
+];
+
+async function runSerializationTests() {
+  console.log("🧪 Running executeScript Serialization Tests\\n");
+  console.log("=" .repeat(80));
+
+  const results = {
+    passed: 0,
+    failed: 0,
+    unexpected: 0,
+    details: [] as any[]
+  };
+
+  for (const testCase of testCases) {
+    console.log(`\\n📋 Testing: ${testCase.name}`);
+    console.log(`📝 Description: ${testCase.description}`);
+    console.log(`🎯 Expected to work: ${testCase.expectedToWork}`);
+    
+    try {
+      const result = await executeScript({
+        script: testCase.script,
+        maxToolCalls: 5,
+        maxTokens: 500,
+        maxExecutionTimeMs: 10000,
+        maxCostUsd: 0.1
+      }, {
+        tools: Tools,
+        clients: Clients,
+      });
+
+      const actualWorked = result.success;
+      const matchesExpectation = actualWorked === testCase.expectedToWork;
+
+      if (matchesExpectation) {
+        results.passed++;
+        console.log(`✅ PASS - Behaved as expected`);
+      } else {
+        results.unexpected++;
+        console.log(`⚠️  UNEXPECTED - Expected ${testCase.expectedToWork ? 'success' : 'failure'}, got ${actualWorked ? 'success' : 'failure'}`);
+      }
+
+      results.details.push({
+        name: testCase.name,
+        expected: testCase.expectedToWork,
+        actual: actualWorked,
+        matches: matchesExpectation,
+        result: actualWorked ? result.result : null,
+        error: actualWorked ? null : result.error,
+        consoleOutput: result.consoleOutput
+      });
+
+      if (actualWorked) {
+        console.log(`📊 Result type: ${typeof result.result}`);
+        console.log(`📊 Result: ${JSON.stringify(result.result).substring(0, 200)}${JSON.stringify(result.result).length > 200 ? '...' : ''}`);
+      } else {
+        console.log(`❌ Error: ${result.error}`);
+      }
+
+      if (result.consoleOutput.length > 0) {
+        console.log(`📝 Console: ${result.consoleOutput.join(', ')}`);
+      }
+
+    } catch (error) {
+      results.failed++;
+      console.log(`💥 TEST FRAMEWORK ERROR: ${error.message}`);
+      
+      results.details.push({
+        name: testCase.name,
+        expected: testCase.expectedToWork,
+        actual: false,
+        matches: !testCase.expectedToWork,
+        result: null,
+        error: error.message,
+        consoleOutput: []
+      });
+    }
+  }
+
+  // Print summary
+  console.log("\\n" + "=".repeat(80));
+  console.log("📊 SERIALIZATION TEST SUMMARY");
+  console.log("=".repeat(80));
+  console.log(`✅ Tests matching expectations: ${results.passed}`);
+  console.log(`⚠️  Unexpected behaviors: ${results.unexpected}`);
+  console.log(`💥 Framework failures: ${results.failed}`);
+  console.log(`📋 Total tests: ${testCases.length}`);
+
+  if (results.unexpected > 0) {
+    console.log("\\n🔍 UNEXPECTED RESULTS:");
+    results.details
+      .filter(d => !d.matches)
+      .forEach(detail => {
+        console.log(`  - ${detail.name}: Expected ${detail.expected ? 'success' : 'failure'}, got ${detail.actual ? 'success' : 'failure'}`);
+        if (detail.error) {
+          console.log(`    Error: ${detail.error}`);
+        }
+      });
+  }
+
+  // Analysis and recommendations
+  console.log("\\n🔬 ANALYSIS:");
+  
+  const workingTypes = results.details.filter(d => d.actual).map(d => d.name);
+  const failingTypes = results.details.filter(d => !d.actual).map(d => d.name);
+  
+  console.log("\\n✅ Types that work:");
+  workingTypes.forEach(name => console.log(`  - ${name}`));
+  
+  console.log("\\n❌ Types that fail:");
+  failingTypes.forEach(name => console.log(`  - ${name}`));
+
+  console.log("\\n💡 RECOMMENDATIONS:");
+  console.log("  1. Use JSON.stringify() for complex objects");
+  console.log("  2. Return primitive values when possible");
+  console.log("  3. Consider createArtifact() for structured data");
+  console.log("  4. Test your return types with this suite");
+
+  return results;
+}
+
+if (require.main === module) {
+  runSerializationTests().catch(error => {
+    console.error("Test suite failed:", error);
+    process.exit(1);
+  });
+}
+
+export { runSerializationTests, testCases };