language-operator 0.0.1 → 0.1.31

data/lib/language_operator/agent/metrics_tracker.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module LanguageOperator
+  module Agent
+    class MetricsTracker
+      attr_reader :metrics
+
+      def initialize
+        @mutex = Mutex.new
+        @metrics = {
+          total_input_tokens: 0,
+          total_output_tokens: 0,
+          total_cached_tokens: 0,
+          total_cache_creation_tokens: 0,
+          request_count: 0,
+          total_cost: 0.0,
+          requests: []
+        }
+        @pricing_cache = {}
+      end
+
+      # Record token usage from an LLM response
+      # @param response [Object] RubyLLM response object
+      # @param model_id [String] Model identifier
+      def record_request(response, model_id)
+        return unless response
+
+        @mutex.synchronize do
+          # Extract token counts with defensive checks
+          input_tokens = extract_token_count(response, :input_tokens)
+          output_tokens = extract_token_count(response, :output_tokens)
+          cached_tokens = extract_token_count(response, :cached_tokens)
+          cache_creation_tokens = extract_token_count(response, :cache_creation_tokens)
+
+          # Calculate cost for this request
+          cost = calculate_cost(input_tokens, output_tokens, model_id)
+
+          # Update cumulative metrics
+          @metrics[:total_input_tokens] += input_tokens
+          @metrics[:total_output_tokens] += output_tokens
+          @metrics[:total_cached_tokens] += cached_tokens
+          @metrics[:total_cache_creation_tokens] += cache_creation_tokens
+          @metrics[:request_count] += 1
+          @metrics[:total_cost] += cost
+
+          # Store per-request history (limited to last 100 requests)
+          @metrics[:requests] << {
+            timestamp: Time.now.iso8601,
+            model: model_id,
+            input_tokens: input_tokens,
+            output_tokens: output_tokens,
+            cached_tokens: cached_tokens,
+            cache_creation_tokens: cache_creation_tokens,
+            cost: cost.round(6)
+          }
+          @metrics[:requests].shift if @metrics[:requests].size > 100
+        end
+      end
+
+      # Get cumulative statistics
+      # @return [Hash] Hash with totalTokens, estimatedCost, requestCount
+      def cumulative_stats
+        @mutex.synchronize do
+          {
+            totalTokens: @metrics[:total_input_tokens] + @metrics[:total_output_tokens],
+            inputTokens: @metrics[:total_input_tokens],
+            outputTokens: @metrics[:total_output_tokens],
+            cachedTokens: @metrics[:total_cached_tokens],
+            cacheCreationTokens: @metrics[:total_cache_creation_tokens],
+            requestCount: @metrics[:request_count],
+            estimatedCost: @metrics[:total_cost].round(6)
+          }
+        end
+      end
+
+      # Get recent request history
+      # @param limit [Integer] Number of recent requests to return
+      # @return [Array<Hash>] Array of request details
+      def recent_requests(limit = 10)
+        @mutex.synchronize do
+          @metrics[:requests].last(limit)
+        end
+      end
+
+      # Reset all metrics (for testing)
+      def reset!
+        @mutex.synchronize do
+          @metrics = {
+            total_input_tokens: 0,
+            total_output_tokens: 0,
+            total_cached_tokens: 0,
+            total_cache_creation_tokens: 0,
+            request_count: 0,
+            total_cost: 0.0,
+            requests: []
+          }
+          @pricing_cache = {}
+        end
+      end
+
+      private
+
+      # Extract token count from response with defensive checks
+      # @param response [Object] Response object
+      # @param method [Symbol] Method name to call
+      # @return [Integer] Token count or 0 if unavailable
+      def extract_token_count(response, method)
+        return 0 unless response.respond_to?(method)
+
+        value = response.public_send(method)
+        value.is_a?(Integer) ? value : 0
+      rescue StandardError
+        0
+      end
+
+      # Calculate cost based on token usage and model pricing
+      # @param input_tokens [Integer] Number of input tokens
+      # @param output_tokens [Integer] Number of output tokens
+      # @param model_id [String] Model identifier
+      # @return [Float] Estimated cost in USD
+      def calculate_cost(input_tokens, output_tokens, model_id)
+        pricing = get_pricing(model_id)
+        return 0.0 unless pricing
+
+        input_cost = (input_tokens / 1_000_000.0) * pricing[:input]
+        output_cost = (output_tokens / 1_000_000.0) * pricing[:output]
+        input_cost + output_cost
+      rescue StandardError => e
+        LanguageOperator.logger.warn('Cost calculation failed',
+                                     model: model_id,
+                                     error: e.message)
+        0.0
+      end
+
+      # Get pricing for a model (with caching)
+      # @param model_id [String] Model identifier
+      # @return [Hash, nil] Hash with :input and :output prices per million tokens
+      def get_pricing(model_id)
+        # Return cached pricing if available
+        return @pricing_cache[model_id] if @pricing_cache.key?(model_id)
+
+        # Try to fetch from RubyLLM registry
+        pricing = fetch_ruby_llm_pricing(model_id)
+
+        # Cache and return
+        @pricing_cache[model_id] = pricing
+        pricing
+      rescue StandardError => e
+        LanguageOperator.logger.warn('Pricing lookup failed',
+                                     model: model_id,
+                                     error: e.message)
+        # Cache nil to avoid repeated failures
+        @pricing_cache[model_id] = nil
+        nil
+      end
+
+      # Fetch pricing from RubyLLM registry
+      # @param model_id [String] Model identifier
+      # @return [Hash, nil] Pricing hash or nil
+      def fetch_ruby_llm_pricing(model_id)
+        # Check if RubyLLM is available
+        return nil unless defined?(RubyLLM)
+
+        # Try to find model in registry
+        model_info = RubyLLM.models.find(model_id)
+        return nil unless model_info
+
+        # Extract pricing (assuming RubyLLM provides these attributes)
+        if model_info.respond_to?(:input_price_per_million) &&
+           model_info.respond_to?(:output_price_per_million)
+          {
+            input: model_info.input_price_per_million,
+            output: model_info.output_price_per_million
+          }
+        else
+          nil
+        end
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/language_operator/agent/safety/ast_validator.rb

@@ -0,0 +1,272 @@
+# frozen_string_literal: true
+
+require 'parser/current'
+
+module LanguageOperator
+  module Agent
+    module Safety
+      # Validates synthesized Ruby code for security before execution
+      # Performs static analysis to detect dangerous method calls
+      class ASTValidator
+        # Gems that are safe to require (allowlist)
+        # These are required for agent execution and are safe
+        ALLOWED_REQUIRES = %w[
+          language_operator
+        ].freeze
+
+        # Dangerous methods that should never be called in synthesized code
+        DANGEROUS_METHODS = %w[
+          system exec spawn fork ` eval instance_eval class_eval module_eval
+          require load autoload require_relative
+          send __send__ public_send method __method__
+          const_set const_get remove_const
+          define_method define_singleton_method
+          undef_method remove_method alias_method
+          exit exit! abort raise fail throw
+          trap at_exit
+          open
+        ].freeze
+
+        # Dangerous constants that should not be accessed
+        DANGEROUS_CONSTANTS = %w[
+          File Dir IO FileUtils Pathname
+          Process Kernel ObjectSpace GC
+          Thread Fiber Mutex ConditionVariable
+          Socket TCPSocket UDPSocket TCPServer UDPServer
+          STDIN STDOUT STDERR
+        ].freeze
+
+        # Safe DSL methods that are allowed in agent definitions
+        SAFE_AGENT_METHODS = %w[
+          agent description persona schedule objectives objective
+          workflow step tool params depends_on prompt
+          constraints budget max_requests rate_limit content_filter
+          output mode webhook as_mcp_server as_chat_endpoint
+        ].freeze
+
+        # Safe DSL methods for tool definitions
+        SAFE_TOOL_METHODS = %w[
+          tool description parameter type required default
+          execute
+        ].freeze
+
+        # Safe helper methods available in execute blocks
+        SAFE_HELPER_METHODS = %w[
+          HTTP Shell
+          validate_url validate_phone validate_email
+          env_required env_get
+          truncate parse_csv
+          error success
+        ].freeze
+
+        # Safe Ruby built-in methods and classes
+        SAFE_BUILTINS = %w[
+          String Array Hash Integer Float Symbol
+          puts print p pp warn
+          true false nil
+          if unless case when then else elsif end
+          while until for break next redo retry return
+          begin rescue ensure
+          lambda proc block_given? yield
+          attr_reader attr_writer attr_accessor
+          private protected public
+          initialize new
+        ].freeze
+
+        class SecurityError < StandardError; end
+
+        def initialize
+          @parser = Parser::CurrentRuby.new
+        end
+
+        # Validate code and raise SecurityError if dangerous methods found
+        # @param code [String] Ruby code to validate
+        # @param file_path [String] Path to file (for error messages)
+        # @raise [SecurityError] if code contains dangerous methods
+        def validate!(code, file_path = '(eval)')
+          ast = parse_code(code, file_path)
+          return if ast.nil? # Empty code is safe
+
+          violations = scan_ast(ast)
+
+          return if violations.empty?
+
+          raise SecurityError, format_violations(violations, file_path)
+        end
+
+        # Validate code and return array of violations (non-raising version)
+        # @param code [String] Ruby code to validate
+        # @param file_path [String] Path to file (for error messages)
+        # @return [Array<Hash>] Array of violation hashes
+        def validate(code, file_path = '(eval)')
+          begin
+            ast = parse_code(code, file_path)
+          rescue SecurityError => e
+            # Convert SecurityError (which wraps Parser::SyntaxError) to violation
+            return [{ type: :syntax_error, message: e.message }]
+          end
+
+          return [] if ast.nil?
+
+          scan_ast(ast)
+        rescue Parser::SyntaxError => e
+          [{ type: :syntax_error, message: e.message }]
+        end
+
+        private
+
+        def parse_code(code, file_path)
+          buffer = Parser::Source::Buffer.new(file_path)
+          buffer.source = code
+          @parser.parse(buffer)
+        rescue Parser::SyntaxError => e
+          raise SecurityError, "Syntax error in #{file_path}: #{e.message}"
+        end
+
+        def scan_ast(node, violations = [])
+          return violations if node.nil?
+
+          case node.type
+          when :send
+            check_method_call(node, violations)
+          when :const
+            check_constant(node, violations)
+          when :gvar
+            check_global_variable(node, violations)
+          when :xstr
+            # Backtick string execution (e.g., `command`)
+            violations << {
+              type: :backtick_execution,
+              location: node.location.line,
+              message: 'Backtick command execution is not allowed'
+            }
+          end
+
+          # Recursively scan all child nodes
+          node.children.each do |child|
+            scan_ast(child, violations) if child.is_a?(Parser::AST::Node)
+          end
+
+          violations
+        end
+
+        def check_method_call(node, violations)
+          receiver, method_name, *args = node.children
+
+          method_str = method_name.to_s
+
+          # Special handling for require - check if it's in the allowlist
+          if %w[require require_relative].include?(method_str)
+            required_gem = extract_require_argument(args)
+
+            # Allow if in the allowlist
+            return if required_gem && ALLOWED_REQUIRES.include?(required_gem)
+
+            # Otherwise, add violation
+            violations << {
+              type: :dangerous_method,
+              method: method_str,
+              location: node.location.line,
+              message: "Dangerous method '#{method_str}' is not allowed"
+            }
+            return
+          end
+
+          # Check for other dangerous methods
+          if DANGEROUS_METHODS.include?(method_str)
+            violations << {
+              type: :dangerous_method,
+              method: method_str,
+              location: node.location.line,
+              message: "Dangerous method '#{method_str}' is not allowed"
+            }
+          end
+
+          # Check for File/Dir/IO operations
+          if receiver && receiver.type == :const
+            const_name = receiver.children[1].to_s
+            if DANGEROUS_CONSTANTS.include?(const_name)
+              violations << {
+                type: :dangerous_constant,
+                constant: const_name,
+                method: method_str,
+                location: node.location.line,
+                message: "Access to #{const_name}.#{method_str} is not allowed"
+              }
+            end
+          end
+
+          # Check for backtick execution (e.g., `command`)
+          # Note: backticks are represented as send with method name :`
+          return unless method_str == '`'
+
+          violations << {
+            type: :backtick_execution,
+            location: node.location.line,
+            message: 'Backtick command execution is not allowed'
+          }
+        end
+
+        def check_constant(node, violations)
+          _, const_name = node.children
+          const_str = const_name.to_s
+
+          # Check for dangerous constants being accessed directly
+          return unless DANGEROUS_CONSTANTS.include?(const_str)
+
+          violations << {
+            type: :dangerous_constant_access,
+            constant: const_str,
+            location: node.location.line,
+            message: "Direct access to #{const_str} constant is not allowed"
+          }
+        end
+
+        def check_global_variable(node, violations)
+          var_name = node.children[0].to_s
+
+          # Block access to dangerous global variables
+          dangerous_globals = %w[$0 $PROGRAM_NAME $LOAD_PATH $: $LOADED_FEATURES $"]
+
+          return unless dangerous_globals.include?(var_name)
+
+          violations << {
+            type: :dangerous_global,
+            variable: var_name,
+            location: node.location.line,
+            message: "Access to global variable #{var_name} is not allowed"
+          }
+        end
+
+        def extract_require_argument(args)
+          # args is an array of AST nodes representing the arguments to require
+          # We're looking for a string literal like 'language_operator' or "language_operator"
+          return nil if args.empty?
+
+          arg_node = args.first
+          return nil unless arg_node
+
+          # Check if it's a string literal (:str node)
+          return arg_node.children[0] if arg_node.type == :str
+
+          # If it's not a string literal (e.g., dynamic require), we can't verify it
+          nil
+        end
+
+        def format_violations(violations, file_path)
+          header = "Security violations detected in #{file_path}:\n\n"
+
+          violation_messages = violations.map do |v|
+            "  Line #{v[:location]}: #{v[:message]}"
+          end
+
+          footer = "\n\nSynthesized code must only use safe DSL methods and approved helpers."
+          footer += "\nSafe methods include: #{SAFE_AGENT_METHODS.join(', ')}, #{SAFE_TOOL_METHODS.join(', ')}"
+          footer += "\nSafe helpers include: HTTP.*, Shell.run, validate_*, env_*"
+
+          header + violation_messages.join("\n") + footer
+        end
+      end
+    end
+  end
+end

data/lib/language_operator/agent/safety/audit_logger.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative '../../logger'
+require_relative '../../loggable'
+
+module LanguageOperator
+  module Agent
+    module Safety
+      # Audit Logger for safety events
+      #
+      # Logs all safety-related events (blocked requests, budget exceeded, etc.)
+      # for compliance and debugging.
+      #
+      # @example
+      #   audit = AuditLogger.new
+      #   audit.log_blocked_request(reason: 'Budget exceeded', details: {...})
+      class AuditLogger
+        include LanguageOperator::Loggable
+
+        def initialize
+          @audit_log_path = ENV['AUDIT_LOG_PATH'] || '/tmp/langop-audit.jsonl'
+          logger.info('Audit logger initialized', log_path: @audit_log_path)
+        end
+
+        # Log a blocked request
+        #
+        # @param reason [String] Reason for blocking
+        # @param details [Hash] Additional details
+        def log_blocked_request(reason:, details: {})
+          log_event(
+            event_type: 'blocked_request',
+            reason: reason,
+            details: details
+          )
+        end
+
+        # Log a budget event
+        #
+        # @param event [String] Event description
+        # @param details [Hash] Budget details
+        def log_budget_event(event:, details: {})
+          log_event(
+            event_type: 'budget_event',
+            event: event,
+            details: details
+          )
+        end
+
+        # Log a rate limit event
+        #
+        # @param event [String] Event description
+        # @param details [Hash] Rate limit details
+        def log_rate_limit_event(event:, details: {})
+          log_event(
+            event_type: 'rate_limit_event',
+            event: event,
+            details: details
+          )
+        end
+
+        # Log a content filter event
+        #
+        # @param event [String] Event description
+        # @param details [Hash] Filter details
+        def log_content_filter_event(event:, details: {})
+          log_event(
+            event_type: 'content_filter_event',
+            event: event,
+            details: details
+          )
+        end
+
+        private
+
+        def logger_component
+          'Safety::AuditLogger'
+        end
+
+        def log_event(event_data)
+          event = {
+            timestamp: Time.now.utc.iso8601,
+            agent_name: ENV['AGENT_NAME'] || 'unknown',
+            **event_data
+          }
+
+          # Log to standard logger
+          logger.info('Audit event', **event_data)
+
+          # Append to audit log file
+          begin
+            File.open(@audit_log_path, 'a') do |f|
+              f.puts(JSON.generate(event))
+            end
+          rescue StandardError => e
+            logger.error('Failed to write audit log',
+                         error: e.message,
+                         log_path: @audit_log_path)
+          end
+        end
+      end
+    end
+  end
+end