@tyvm/knowhow 0.0.21 → 0.0.22

package/src/services/script-execution/ScriptTracer.ts

@@ -0,0 +1,249 @@
+import { TraceEvent, TraceMetrics, ExecutionTrace, QuotaUsage } from "./types";
+
+/**
+ * Handles tracing and monitoring of script execution
+ */
+export class ScriptTracer {
+  private events: TraceEvent[] = [];
+  private metrics: TraceMetrics;
+  private startTime: number;
+
+  constructor() {
+    this.startTime = Date.now();
+    this.metrics = {
+      executionTimeMs: 0,
+      toolCallCount: 0,
+      llmCallCount: 0,
+      tokenUsage: {
+        prompt: 0,
+        completion: 0,
+        total: 0,
+      },
+      memoryUsage: {
+        heapUsed: 0,
+        heapTotal: 0,
+      },
+      costUsd: 0,
+    };
+  }
+
+  /**
+   * Emit a trace event
+   */
+  emitEvent(type: string, data: any): void {
+    const event: TraceEvent = {
+      id: `evt-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
+      type,
+      timestamp: Date.now(),
+      data: this.sanitizeEventData(data),
+    };
+
+    this.events.push(event);
+    this.updateMetrics(event);
+  }
+
+  /**
+   * Record cost for tracking
+   */
+  recordCost(costUsd: number): void {
+    this.metrics.costUsd += costUsd;
+    this.emitEvent("cost_recorded", {
+      costUsd,
+      totalCost: this.metrics.costUsd,
+    });
+  }
+
+  /**
+   * Get current quota usage
+   */
+  getCurrentQuota(): QuotaUsage {
+    return {
+      toolCalls: this.metrics.toolCallCount,
+      tokens: this.metrics.tokenUsage.total,
+      executionTimeMs: Date.now() - this.startTime,
+      costUsd: this.metrics.costUsd,
+    };
+  }
+
+  /**
+   * Get all trace events
+   */
+  getEvents(): TraceEvent[] {
+    return [...this.events];
+  }
+
+  /**
+   * Get current metrics
+   */
+  getMetrics(): TraceMetrics {
+    return {
+      ...this.metrics,
+      executionTimeMs: Date.now() - this.startTime,
+    };
+  }
+
+  /**
+   * Generate execution trace
+   */
+  getTrace(): ExecutionTrace {
+    return {
+      id: `trace-${Date.now()}`,
+      startTime: this.startTime,
+      endTime: Date.now(),
+      events: this.getEvents(),
+      metrics: this.getMetrics(),
+      success: !this.events.some((e) => e.type.includes("error")),
+      error: this.getLastError(),
+    };
+  }
+
+  /**
+   * Clear all events and reset metrics
+   */
+  reset(): void {
+    this.events = [];
+    this.startTime = Date.now();
+    this.metrics = {
+      executionTimeMs: 0,
+      toolCallCount: 0,
+      llmCallCount: 0,
+      tokenUsage: {
+        prompt: 0,
+        completion: 0,
+        total: 0,
+      },
+      memoryUsage: {
+        heapUsed: 0,
+        heapTotal: 0,
+      },
+      costUsd: 0,
+    };
+  }
+
+  /**
+   * Update metrics based on events
+   */
+  private updateMetrics(event: TraceEvent): void {
+    switch (event.type) {
+      case "tool_call_start":
+        this.metrics.toolCallCount++;
+        break;
+
+      case "llm_call_start":
+        this.metrics.llmCallCount++;
+        break;
+
+      case "llm_call_success":
+        if (event.data && event.data.usage) {
+          const usage = event.data.usage;
+          this.metrics.tokenUsage.prompt += usage.prompt_tokens || 0;
+          this.metrics.tokenUsage.completion += usage.completion_tokens || 0;
+          this.metrics.tokenUsage.total += usage.total_tokens || 0;
+        }
+        break;
+    }
+
+    // Update memory usage if available
+    if (typeof process !== "undefined" && process.memoryUsage) {
+      const memUsage = process.memoryUsage();
+      this.metrics.memoryUsage.heapUsed = memUsage.heapUsed;
+      this.metrics.memoryUsage.heapTotal = memUsage.heapTotal;
+    }
+  }
+
+  /**
+   * Get the last error from events
+   */
+  private getLastError(): string | undefined {
+    const errorEvents = this.events
+      .filter((e) => e.type.includes("error"))
+      .reverse();
+
+    if (errorEvents.length > 0) {
+      const lastError = errorEvents[0];
+      return (
+        lastError.data?.error || lastError.data?.message || "Unknown error"
+      );
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Sanitize event data to prevent sensitive information leaks
+   */
+  private sanitizeEventData(data: any): any {
+    if (data === null || data === undefined) {
+      return data;
+    }
+
+    if (typeof data === "string") {
+      // Truncate very long strings
+      return data.length > 1000
+        ? data.substring(0, 1000) + "...[TRUNCATED]"
+        : data;
+    }
+
+    if (Array.isArray(data)) {
+      return data.map((item) => this.sanitizeEventData(item));
+    }
+
+    if (typeof data === "object") {
+      const sanitized: any = {};
+      for (const [key, value] of Object.entries(data)) {
+        // Redact sensitive keys
+        if (this.isSensitiveKey(key)) {
+          sanitized[key] = "[REDACTED]";
+        } else if (key === "parameters" && typeof value === "object") {
+          // Special handling for tool parameters
+          sanitized[key] = this.sanitizeParameters(value);
+        } else {
+          sanitized[key] = this.sanitizeEventData(value);
+        }
+      }
+      return sanitized;
+    }
+
+    return data;
+  }
+
+  /**
+   * Check if a key contains sensitive information
+   */
+  private isSensitiveKey(key: string): boolean {
+    const lowerKey = key.toLowerCase();
+    const sensitivePatterns = [
+      "password",
+      "secret",
+      "token",
+      "key",
+      "auth",
+      "credential",
+      "private",
+      "confidential",
+    ];
+
+    return sensitivePatterns.some((pattern) => lowerKey.includes(pattern));
+  }
+
+  /**
+   * Sanitize tool parameters
+   */
+  private sanitizeParameters(params: any): any {
+    if (!params || typeof params !== "object") {
+      return params;
+    }
+
+    const sanitized: any = {};
+    for (const [key, value] of Object.entries(params)) {
+      if (this.isSensitiveKey(key)) {
+        sanitized[key] = "[REDACTED]";
+      } else if (typeof value === "string" && value.length > 500) {
+        sanitized[key] = value.substring(0, 500) + "...[TRUNCATED]";
+      } else {
+        sanitized[key] = this.sanitizeEventData(value);
+      }
+    }
+    return sanitized;
+  }
+}

package/src/services/script-execution/types.ts

@@ -0,0 +1,134 @@
+import { Message } from '../../clients/types';
+
+// Basic trace event interface
+export interface TraceEvent {
+  id: string;
+  type: string;
+  timestamp: number;
+  data: any;
+}
+
+// Trace metrics for performance monitoring
+export interface TraceMetrics {
+  executionTimeMs: number;
+  toolCallCount: number;
+  llmCallCount: number;
+  tokenUsage: {
+    prompt: number;
+    completion: number;
+    total: number;
+  };
+  memoryUsage: {
+    heapUsed: number;
+    heapTotal: number;
+  };
+  costUsd: number;
+}
+
+// Complete execution trace
+export interface ExecutionTrace {
+  id: string;
+  startTime: number;
+  endTime: number;
+  events: TraceEvent[];
+  metrics: TraceMetrics;
+  success: boolean;
+  error?: string;
+}
+
+// Resource quotas for script execution
+export interface ResourceQuotas {
+  maxToolCalls: number;
+  maxTokens: number;
+  maxExecutionTimeMs: number;
+  maxCostUsd: number;
+  maxMemoryMb: number;
+}
+
+// Security policy configuration
+export interface SecurityPolicy {
+  allowlistedTools: string[];
+  denylistedTools: string[];
+  maxScriptLength: number;
+  allowNetworkAccess: boolean;
+  allowFileSystemAccess: boolean;
+}
+
+// Current quota usage tracking
+export interface QuotaUsage {
+  toolCalls: number;
+  tokens: number;
+  executionTimeMs: number;
+  costUsd: number;
+}
+
+// Policy violation record
+export interface PolicyViolation {
+  id: string;
+  type: 'quota_exceeded' | 'tool_denied' | 'tool_not_allowed' | 'script_validation';
+  message: string;
+  timestamp: number;
+  usage: QuotaUsage;
+}
+
+// Script execution request
+export interface ExecutionRequest {
+  script: string;
+  context?: Record<string, any>;
+  quotas?: Partial<ResourceQuotas>;
+  policy?: Partial<SecurityPolicy>;
+}
+
+// Script execution result
+export interface ExecutionResult {
+  success: boolean;
+  error: string | null;
+  result: any;
+  trace: ExecutionTrace;
+  artifacts: Artifact[];
+  consoleOutput: string[];
+}
+
+// Artifact created by script
+export interface Artifact {
+  id: string;
+  name: string;
+  type: 'text' | 'json' | 'csv' | 'html' | 'markdown';
+  content: string;
+  createdAt: string;
+}
+
+// Tool execution result
+export interface ToolResult {
+  success: boolean;
+  result?: any;
+  error?: string;
+}
+
+// Tool call record
+export interface ToolCall {
+  id: string;
+  name: string;
+  args: Record<string, any>;
+  result?: any;
+  error?: string;
+  timestamp: number;
+  duration?: number;
+}
+
+// LLM call record
+export interface LLMCall {
+  id: string;
+  model: string;
+  messages: Message[];
+  response?: any;
+  error?: string;
+  timestamp: number;
+  duration?: number;
+  tokenUsage?: {
+    prompt: number;
+    completion: number;
+    total: number;
+  };
+  cost?: number;
+}

package/ts_build/src/agents/tools/executeScript/definition.d.ts

@@ -0,0 +1,2 @@
+import { Tool } from "../../../clients";
+export declare const executeScriptDefinition: Tool;

package/ts_build/src/agents/tools/executeScript/definition.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeScriptDefinition = void 0;
+exports.executeScriptDefinition = {
+    type: "function",
+    function: {
+        name: "executeScript",
+        description: `Execute TypeScript code in a secure sandbox environment with access to tools and AI models.
+
+  The script has access to:
+  - callTool(toolName, parameters): Call any available tool
+  - llm(messages, options): Make LLM calls
+  - createArtifact(name, content, type): Create downloadable artifacts
+  - console: Standard console logging
+  - getQuotaUsage(): Check resource usage
+  - sleep(ms): Pause execution for a specified time
+
+  Example:
+  \`\`\`typescript
+  // Call a tool
+  const searchResult = await callTool('textSearch', { searchTerm: 'hello world' });
+  console.log('Search found:', searchResult);
+
+  // Call LLM
+  const response = await llm([
+    { role: 'user', content: 'Explain quantum computing' }
+  ], { model: 'gpt-4o-mini', maxTokens: 100 });
+  console.log('LLM response:', response.choices[0].message.content);
+
+  // Create an artifact
+  createArtifact('summary.md', '# Summary\\nThis is a test', 'markdown');
+
+  return { message: 'Script completed successfully' };
+  \`\`\`
+
+  Must either return or await the a top level function
+
+  Test tools yourself to know the return type when scripting. Can pass JSON.stringified data into llm call if you don't need to know the type.
+  You cannot use isolation breaking methods like: setTimeout setInterval setImmediate clearTimeout clearInterval
+
+  Security: Scripts run in isolation with quotas on tool calls, tokens, time, and cost.`,
+        parameters: {
+            type: "object",
+            properties: {
+                script: {
+                    type: "string",
+                    description: "The TypeScript code to execute",
+                },
+                maxToolCalls: {
+                    type: "number",
+                    description: "Maximum number of tool calls allowed (default: 50)",
+                },
+                maxTokens: {
+                    type: "number",
+                    description: "Maximum tokens for LLM calls (default: 10000)",
+                },
+                maxExecutionTimeMs: {
+                    type: "number",
+                    description: "Maximum execution time in milliseconds (default: 30000)",
+                },
+                maxCostUsd: {
+                    type: "number",
+                    description: "Maximum cost in USD (default: 1.0)",
+                },
+            },
+            required: ["script"],
+        },
+    },
+};
+//# sourceMappingURL=definition.js.map

package/ts_build/src/agents/tools/executeScript/definition.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"definition.js","sourceRoot":"","sources":["../../../../../src/agents/tools/executeScript/definition.ts"],"names":[],"mappings":";;;AAKa,QAAA,uBAAuB,GAAS;IAC3C,IAAI,EAAE,UAAU;IAChB,QAAQ,EAAE;QACR,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wFAiCuE;QAEpF,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,gCAAgC;iBAC9C;gBACD,YAAY,EAAE;oBACZ,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,oDAAoD;iBAClE;gBACD,SAAS,EAAE;oBACT,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,+CAA+C;iBAC7D;gBACD,kBAAkB,EAAE;oBAClB,IAAI,EAAE,QAAQ;oBACd,WAAW,EACT,yDAAyD;iBAC5D;gBACD,UAAU,EAAE;oBACV,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,oCAAoC;iBAClD;aACF;YACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;SACrB;KACF;CACF,CAAC"}

package/ts_build/src/agents/tools/executeScript/examples/quick-test.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env ts-node
+declare function quickTest(): Promise<void>;
+export { quickTest };

package/ts_build/src/agents/tools/executeScript/examples/quick-test.js

@@ -0,0 +1,68 @@
+#!/usr/bin/env ts-node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.quickTest = void 0;
+const executeScript_1 = require("../../executeScript");
+const services_1 = require("../../../../services");
+const clients_1 = require("../../../../clients");
+const simpleScript = `
+console.log("Hello from executeScript!");
+
+async function main() {
+  // Test basic functionality
+  console.log("Running simple test...");
+
+  // Try a simple tool call
+  const files = await callTool("fileSearch", { searchTerm: "*.ts" });
+  console.log("Found", files?.length || 0, "TypeScript files");
+
+  return {
+    message: "Simple test completed!",
+    filesFound: files?.length || 0,
+    timestamp: new Date().toISOString()
+  };
+}
+
+await main().then(result => {
+  console.log("Result:", result);
+}).catch(error => {
+  console.error("Error:", error);
+});
+`;
+async function quickTest() {
+    console.log("🧪 Quick executeScript test\n");
+    try {
+        const result = await (0, executeScript_1.executeScript)({
+            script: simpleScript,
+            maxToolCalls: 5,
+            maxTokens: 100,
+            maxExecutionTimeMs: 10000,
+            maxCostUsd: 0.1,
+        }, {
+            tools: services_1.Tools,
+            clients: clients_1.Clients,
+        });
+        console.log("\n📊 QUICK TEST RESULT:");
+        console.log("Success:", result.success);
+        console.log("Result:", result.result);
+        console.log("Tool calls:", result.quotaUsage.toolCalls);
+        console.log("Cost: $" + result.quotaUsage.costUsd.toFixed(4));
+        if (result.consoleOutput.length > 0) {
+            console.log("\n📝 Console Output:");
+            result.consoleOutput.forEach((entry) => {
+                console.log(`  ${entry}`);
+            });
+        }
+        if (!result.success) {
+            console.log("❌ Error:", result.error);
+        }
+    }
+    catch (error) {
+        console.error("💥 Test failed:", error);
+    }
+}
+exports.quickTest = quickTest;
+if (require.main === module) {
+    quickTest();
+}
+//# sourceMappingURL=quick-test.js.map

package/ts_build/src/agents/tools/executeScript/examples/quick-test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"quick-test.js","sourceRoot":"","sources":["../../../../../../src/agents/tools/executeScript/examples/quick-test.ts"],"names":[],"mappings":";;;;AAMA,uDAAoD;AACpD,mDAA6C;AAC7C,iDAA8C;AAG9C,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;CAuBpB,CAAC;AAEF,KAAK,UAAU,SAAS;IACtB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAE7C,IAAI;QACF,MAAM,MAAM,GAAG,MAAM,IAAA,6BAAa,EAChC;YACE,MAAM,EAAE,YAAY;YACpB,YAAY,EAAE,CAAC;YACf,SAAS,EAAE,GAAG;YACd,kBAAkB,EAAE,KAAK;YACzB,UAAU,EAAE,GAAG;SAChB,EACD;YACE,KAAK,EAAE,gBAAK;YACZ,OAAO,EAAE,iBAAO;SACjB,CACF,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9D,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;YACnC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACpC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBACrC,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;YAC5B,CAAC,CAAC,CAAC;SACJ;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;YACnB,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;SACvC;KACF;IAAC,OAAO,KAAK,EAAE;QACd,OAAO,CAAC,KAAK,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;KACzC;AACH,CAAC;AAMQ,8BAAS;AAJlB,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE;IAC3B,SAAS,EAAE,CAAC;CACb"}

package/ts_build/src/agents/tools/executeScript/examples/serialization-test.d.ts

@@ -0,0 +1,15 @@
+#!/usr/bin/env ts-node
+interface TestCase {
+    name: string;
+    script: string;
+    expectedToWork: boolean;
+    description: string;
+}
+declare const testCases: TestCase[];
+declare function runSerializationTests(): Promise<{
+    passed: number;
+    failed: number;
+    unexpected: number;
+    details: any[];
+}>;
+export { runSerializationTests, testCases };