@tyvm/knowhow 0.0.20 → 0.0.22

package/src/services/script-execution/ScriptPolicy.ts

@@ -0,0 +1,236 @@
+import { 
+  ResourceQuotas, 
+  SecurityPolicy,
+  QuotaUsage,
+  PolicyViolation 
+} from './types';
+
+/**
+ * Enforces security policies and resource quotas for script execution
+ */
+export class ScriptPolicyEnforcer {
+  private usage: QuotaUsage;
+  private violations: PolicyViolation[] = [];
+
+  constructor(
+    private quotas: ResourceQuotas,
+    private policy: SecurityPolicy
+  ) {
+    this.usage = {
+      toolCalls: 0,
+      tokens: 0,
+      executionTimeMs: 0,
+      costUsd: 0
+    };
+  }
+
+  /**
+   * Check if a tool call is allowed
+   */
+  checkToolCall(toolName: string): boolean {
+    // Check if tool is in denylist
+    if (this.policy.denylistedTools && this.policy.denylistedTools.includes(toolName)) {
+      this.recordViolation('tool_denied', `Tool '${toolName}' is in denylist`);
+      return false;
+    }
+
+    // Check if tool is in allowlist (if allowlist is defined and not empty)
+    if (this.policy.allowlistedTools && this.policy.allowlistedTools.length > 0 && 
+        !this.policy.allowlistedTools.includes(toolName)) {
+      this.recordViolation('tool_not_allowed', `Tool '${toolName}' is not in allowlist`);
+      return false;
+    }
+
+    // Check quota
+    if (this.usage.toolCalls >= this.quotas.maxToolCalls) {
+      this.recordViolation('quota_exceeded', 'Maximum tool calls exceeded');
+      return false;
+    }
+
+    return true;
+  }
+
+  /**
+   * Record a tool call
+   */
+  recordToolCall(): void {
+    this.usage.toolCalls++;
+  }
+
+  /**
+   * Check if token usage is allowed
+   */
+  checkTokenUsage(tokens: number): boolean {
+    if (this.usage.tokens + tokens > this.quotas.maxTokens) {
+      this.recordViolation('quota_exceeded', 'Maximum tokens would be exceeded');
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Record token usage
+   */
+  recordTokenUsage(tokens: number): void {
+    this.usage.tokens += tokens;
+  }
+
+  /**
+   * Check if execution time limit is exceeded
+   */
+  checkExecutionTime(currentTimeMs: number): boolean {
+    if (currentTimeMs > this.quotas.maxExecutionTimeMs) {
+      this.recordViolation('quota_exceeded', 'Maximum execution time exceeded');
+      return false;
+    }
+    this.usage.executionTimeMs = currentTimeMs;
+    return true;
+  }
+
+  /**
+   * Check if cost limit is exceeded
+   */
+  checkCost(additionalCost: number): boolean {
+    if (this.usage.costUsd + additionalCost > this.quotas.maxCostUsd) {
+      this.recordViolation('quota_exceeded', 'Maximum cost would be exceeded');
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Record cost usage
+   */
+  recordCost(cost: number): void {
+    this.usage.costUsd += cost;
+  }
+
+  /**
+   * Get current usage
+   */
+  getUsage(): QuotaUsage {
+    return { ...this.usage };
+  }
+
+  /**
+   * Get current quotas
+   */
+  getQuotas(): ResourceQuotas {
+    return { ...this.quotas };
+  }
+
+  /**
+   * Get all policy violations
+   */
+  getViolations(): PolicyViolation[] {
+    return [...this.violations];
+  }
+
+  /**
+   * Check if there are any violations
+   */
+  hasViolations(): boolean {
+    return this.violations.length > 0;
+  }
+
+  /**
+   * Get the most recent violation
+   */
+  getLastViolation(): PolicyViolation | undefined {
+    return this.violations[this.violations.length - 1];
+  }
+
+  /**
+   * Reset usage counters
+   */
+  resetUsage(): void {
+    this.usage = {
+      toolCalls: 0,
+      tokens: 0,
+      executionTimeMs: 0,
+      costUsd: 0
+    };
+  }
+
+  /**
+   * Reset violations
+   */
+  resetViolations(): void {
+    this.violations = [];
+  }
+
+  /**
+   * Validate script content for security issues
+   */
+  validateScript(scriptContent: string): { valid: boolean; issues: string[] } {
+    const issues: string[] = [];
+
+    // Check for dangerous patterns
+    const dangerousPatterns = [
+      /require\s*\(/gi,           // Node.js require
+      /import\s+.*\s+from/gi,     // ES6 imports (should be handled by bundler)
+      /process\./gi,              // Process access
+      /global\./gi,               // Global object access
+      /eval\s*\(/gi,              // eval calls
+      /Function\s*\(/gi,          // Function constructor
+      /setTimeout/gi,             // setTimeout
+      /setInterval/gi,            // setInterval
+      /fetch\s*\(/gi,             // Direct fetch calls
+      /XMLHttpRequest/gi,         // XHR
+      /WebSocket/gi,              // WebSocket
+      /location\./gi,             // Location object
+      /document\./gi,             // Document object
+      /window\./gi,               // Window object
+    ];
+
+    for (const pattern of dangerousPatterns) {
+      if (pattern.test(scriptContent)) {
+        issues.push(`Potentially dangerous pattern detected: ${pattern.source}`);
+      }
+    }
+
+    // Check script length
+    if (scriptContent.length > this.policy.maxScriptLength) {
+      issues.push(`Script too long: ${scriptContent.length} > ${this.policy.maxScriptLength}`);
+    }
+
+    // Check for excessive complexity (rough heuristic)
+    const complexityIndicators = [
+      /for\s*\(/gi,
+      /while\s*\(/gi,
+      /function\s+\w+/gi,
+      /=>\s*{/gi,
+      /if\s*\(/gi,
+    ];
+
+    let complexityScore = 0;
+    for (const indicator of complexityIndicators) {
+      const matches = scriptContent.match(indicator);
+      complexityScore += matches ? matches.length : 0;
+    }
+
+    if (complexityScore > 50) {
+      issues.push(`Script complexity too high: ${complexityScore} constructs detected`);
+    }
+
+    return {
+      valid: issues.length === 0,
+      issues
+    };
+  }
+
+  /**
+   * Record a policy violation
+   */
+  private recordViolation(type: 'quota_exceeded' | 'tool_denied' | 'tool_not_allowed' | 'script_validation', message: string): void {
+    const violation: PolicyViolation = {
+      id: `violation-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
+      type,
+      message,
+      timestamp: Date.now(),
+      usage: { ...this.usage }
+    };
+
+    this.violations.push(violation);
+  }
+}

package/src/services/script-execution/ScriptTracer.ts

@@ -0,0 +1,249 @@
+import { TraceEvent, TraceMetrics, ExecutionTrace, QuotaUsage } from "./types";
+
+/**
+ * Handles tracing and monitoring of script execution
+ */
+export class ScriptTracer {
+  private events: TraceEvent[] = [];
+  private metrics: TraceMetrics;
+  private startTime: number;
+
+  constructor() {
+    this.startTime = Date.now();
+    this.metrics = {
+      executionTimeMs: 0,
+      toolCallCount: 0,
+      llmCallCount: 0,
+      tokenUsage: {
+        prompt: 0,
+        completion: 0,
+        total: 0,
+      },
+      memoryUsage: {
+        heapUsed: 0,
+        heapTotal: 0,
+      },
+      costUsd: 0,
+    };
+  }
+
+  /**
+   * Emit a trace event
+   */
+  emitEvent(type: string, data: any): void {
+    const event: TraceEvent = {
+      id: `evt-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
+      type,
+      timestamp: Date.now(),
+      data: this.sanitizeEventData(data),
+    };
+
+    this.events.push(event);
+    this.updateMetrics(event);
+  }
+
+  /**
+   * Record cost for tracking
+   */
+  recordCost(costUsd: number): void {
+    this.metrics.costUsd += costUsd;
+    this.emitEvent("cost_recorded", {
+      costUsd,
+      totalCost: this.metrics.costUsd,
+    });
+  }
+
+  /**
+   * Get current quota usage
+   */
+  getCurrentQuota(): QuotaUsage {
+    return {
+      toolCalls: this.metrics.toolCallCount,
+      tokens: this.metrics.tokenUsage.total,
+      executionTimeMs: Date.now() - this.startTime,
+      costUsd: this.metrics.costUsd,
+    };
+  }
+
+  /**
+   * Get all trace events
+   */
+  getEvents(): TraceEvent[] {
+    return [...this.events];
+  }
+
+  /**
+   * Get current metrics
+   */
+  getMetrics(): TraceMetrics {
+    return {
+      ...this.metrics,
+      executionTimeMs: Date.now() - this.startTime,
+    };
+  }
+
+  /**
+   * Generate execution trace
+   */
+  getTrace(): ExecutionTrace {
+    return {
+      id: `trace-${Date.now()}`,
+      startTime: this.startTime,
+      endTime: Date.now(),
+      events: this.getEvents(),
+      metrics: this.getMetrics(),
+      success: !this.events.some((e) => e.type.includes("error")),
+      error: this.getLastError(),
+    };
+  }
+
+  /**
+   * Clear all events and reset metrics
+   */
+  reset(): void {
+    this.events = [];
+    this.startTime = Date.now();
+    this.metrics = {
+      executionTimeMs: 0,
+      toolCallCount: 0,
+      llmCallCount: 0,
+      tokenUsage: {
+        prompt: 0,
+        completion: 0,
+        total: 0,
+      },
+      memoryUsage: {
+        heapUsed: 0,
+        heapTotal: 0,
+      },
+      costUsd: 0,
+    };
+  }
+
+  /**
+   * Update metrics based on events
+   */
+  private updateMetrics(event: TraceEvent): void {
+    switch (event.type) {
+      case "tool_call_start":
+        this.metrics.toolCallCount++;
+        break;
+
+      case "llm_call_start":
+        this.metrics.llmCallCount++;
+        break;
+
+      case "llm_call_success":
+        if (event.data && event.data.usage) {
+          const usage = event.data.usage;
+          this.metrics.tokenUsage.prompt += usage.prompt_tokens || 0;
+          this.metrics.tokenUsage.completion += usage.completion_tokens || 0;
+          this.metrics.tokenUsage.total += usage.total_tokens || 0;
+        }
+        break;
+    }
+
+    // Update memory usage if available
+    if (typeof process !== "undefined" && process.memoryUsage) {
+      const memUsage = process.memoryUsage();
+      this.metrics.memoryUsage.heapUsed = memUsage.heapUsed;
+      this.metrics.memoryUsage.heapTotal = memUsage.heapTotal;
+    }
+  }
+
+  /**
+   * Get the last error from events
+   */
+  private getLastError(): string | undefined {
+    const errorEvents = this.events
+      .filter((e) => e.type.includes("error"))
+      .reverse();
+
+    if (errorEvents.length > 0) {
+      const lastError = errorEvents[0];
+      return (
+        lastError.data?.error || lastError.data?.message || "Unknown error"
+      );
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Sanitize event data to prevent sensitive information leaks
+   */
+  private sanitizeEventData(data: any): any {
+    if (data === null || data === undefined) {
+      return data;
+    }
+
+    if (typeof data === "string") {
+      // Truncate very long strings
+      return data.length > 1000
+        ? data.substring(0, 1000) + "...[TRUNCATED]"
+        : data;
+    }
+
+    if (Array.isArray(data)) {
+      return data.map((item) => this.sanitizeEventData(item));
+    }
+
+    if (typeof data === "object") {
+      const sanitized: any = {};
+      for (const [key, value] of Object.entries(data)) {
+        // Redact sensitive keys
+        if (this.isSensitiveKey(key)) {
+          sanitized[key] = "[REDACTED]";
+        } else if (key === "parameters" && typeof value === "object") {
+          // Special handling for tool parameters
+          sanitized[key] = this.sanitizeParameters(value);
+        } else {
+          sanitized[key] = this.sanitizeEventData(value);
+        }
+      }
+      return sanitized;
+    }
+
+    return data;
+  }
+
+  /**
+   * Check if a key contains sensitive information
+   */
+  private isSensitiveKey(key: string): boolean {
+    const lowerKey = key.toLowerCase();
+    const sensitivePatterns = [
+      "password",
+      "secret",
+      "token",
+      "key",
+      "auth",
+      "credential",
+      "private",
+      "confidential",
+    ];
+
+    return sensitivePatterns.some((pattern) => lowerKey.includes(pattern));
+  }
+
+  /**
+   * Sanitize tool parameters
+   */
+  private sanitizeParameters(params: any): any {
+    if (!params || typeof params !== "object") {
+      return params;
+    }
+
+    const sanitized: any = {};
+    for (const [key, value] of Object.entries(params)) {
+      if (this.isSensitiveKey(key)) {
+        sanitized[key] = "[REDACTED]";
+      } else if (typeof value === "string" && value.length > 500) {
+        sanitized[key] = value.substring(0, 500) + "...[TRUNCATED]";
+      } else {
+        sanitized[key] = this.sanitizeEventData(value);
+      }
+    }
+    return sanitized;
+  }
+}

package/src/services/script-execution/types.ts

@@ -0,0 +1,134 @@
+import { Message } from '../../clients/types';
+
+// Basic trace event interface
+export interface TraceEvent {
+  id: string;
+  type: string;
+  timestamp: number;
+  data: any;
+}
+
+// Trace metrics for performance monitoring
+export interface TraceMetrics {
+  executionTimeMs: number;
+  toolCallCount: number;
+  llmCallCount: number;
+  tokenUsage: {
+    prompt: number;
+    completion: number;
+    total: number;
+  };
+  memoryUsage: {
+    heapUsed: number;
+    heapTotal: number;
+  };
+  costUsd: number;
+}
+
+// Complete execution trace
+export interface ExecutionTrace {
+  id: string;
+  startTime: number;
+  endTime: number;
+  events: TraceEvent[];
+  metrics: TraceMetrics;
+  success: boolean;
+  error?: string;
+}
+
+// Resource quotas for script execution
+export interface ResourceQuotas {
+  maxToolCalls: number;
+  maxTokens: number;
+  maxExecutionTimeMs: number;
+  maxCostUsd: number;
+  maxMemoryMb: number;
+}
+
+// Security policy configuration
+export interface SecurityPolicy {
+  allowlistedTools: string[];
+  denylistedTools: string[];
+  maxScriptLength: number;
+  allowNetworkAccess: boolean;
+  allowFileSystemAccess: boolean;
+}
+
+// Current quota usage tracking
+export interface QuotaUsage {
+  toolCalls: number;
+  tokens: number;
+  executionTimeMs: number;
+  costUsd: number;
+}
+
+// Policy violation record
+export interface PolicyViolation {
+  id: string;
+  type: 'quota_exceeded' | 'tool_denied' | 'tool_not_allowed' | 'script_validation';
+  message: string;
+  timestamp: number;
+  usage: QuotaUsage;
+}
+
+// Script execution request
+export interface ExecutionRequest {
+  script: string;
+  context?: Record<string, any>;
+  quotas?: Partial<ResourceQuotas>;
+  policy?: Partial<SecurityPolicy>;
+}
+
+// Script execution result
+export interface ExecutionResult {
+  success: boolean;
+  error: string | null;
+  result: any;
+  trace: ExecutionTrace;
+  artifacts: Artifact[];
+  consoleOutput: string[];
+}
+
+// Artifact created by script
+export interface Artifact {
+  id: string;
+  name: string;
+  type: 'text' | 'json' | 'csv' | 'html' | 'markdown';
+  content: string;
+  createdAt: string;
+}
+
+// Tool execution result
+export interface ToolResult {
+  success: boolean;
+  result?: any;
+  error?: string;
+}
+
+// Tool call record
+export interface ToolCall {
+  id: string;
+  name: string;
+  args: Record<string, any>;
+  result?: any;
+  error?: string;
+  timestamp: number;
+  duration?: number;
+}
+
+// LLM call record
+export interface LLMCall {
+  id: string;
+  model: string;
+  messages: Message[];
+  response?: any;
+  error?: string;
+  timestamp: number;
+  duration?: number;
+  tokenUsage?: {
+    prompt: number;
+    completion: number;
+    total: number;
+  };
+  cost?: number;
+}

package/ts_build/src/agents/base/base.js

@@ -141,62 +141,11 @@ class BaseAgent {
         return this.summaries;
     }
     async processToolMessages(toolCall) {
-        const functionName = toolCall.function.name;
-        const functionToCall = this.tools.getFunction(functionName);
-        console.log(toolCall);
-        const functionArgs = JSON.parse(this.formatAiResponse(toolCall.function.arguments));
-        const toJsonIfObject = (arg) => {
-            if (typeof arg === "object") {
-                return JSON.stringify(arg, null, 2);
-            }
-            return arg;
-        };
-        const toolDefinition = this.tools.getTool(functionName);
-        const properties = toolDefinition?.function?.parameters?.properties || {};
-        const isPositional = toolDefinition?.function?.parameters?.positional || false;
-        const fnArgs = isPositional
-            ? Object.keys(properties).map((p) => functionArgs[p])
-            : functionArgs;
-        console.log(`Calling function ${functionName} with args:`, JSON.stringify(fnArgs, null, 2));
-        if (!functionToCall) {
-            const options = this.getEnabledToolNames().join(", ");
-            const error = `Function ${functionName} not found, options are ${options}`;
-            console.log(error);
-            return [
-                {
-                    tool_call_id: toolCall.id,
-                    role: "tool",
-                    name: "error",
-                    content: error,
-                },
-            ];
-        }
-        const functionResponse = await Promise.resolve(isPositional ? functionToCall(...fnArgs) : functionToCall(fnArgs)).catch((e) => "ERROR: " + e.message);
+        const { functionResp, toolMessages } = await this.tools.callTool(toolCall, this.getEnabledToolNames());
         this.agentEvents.emit(this.eventTypes.toolUsed, {
             toolCall,
-            functionResponse,
+            functionResp,
         });
-        let toolMessages = [];
-        if (functionName === "multi_tool_use.parallel") {
-            const args = fnArgs[0];
-            toolMessages = args.map((call, index) => {
-                return {
-                    tool_call_id: toolCall.id + "_" + index,
-                    role: "tool",
-                    name: call.recipient_name.split(".").pop(),
-                    content: toJsonIfObject(functionResponse[index]) || "Done",
-                };
-            });
-        }
-        toolMessages = [
-            {
-                tool_call_id: toolCall.id,
-                role: "tool",
-                name: functionName,
-                content: toJsonIfObject(functionResponse) || "Done",
-            },
-        ];
-        console.log(toolMessages);
         return toolMessages;
     }
     logMessages(messages) {