ruby-skill-bench 0.1.0

data/lib/skill_bench/tools/read_file.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require_relative '../error_logger'
+
+module SkillBench
+  module Tools
+    # Handles reading the contents of a file within the working directory.
+    class ReadFile < Base
+      # @return [Hash] The tool definition for the LLM API.
+      def self.definition
+        {
+          type: 'function',
+          function: {
+            name: 'read_file',
+            description: 'Read the contents of a file.',
+            parameters: {
+              type: 'object',
+              properties: {
+                path: { type: 'string', description: 'Relative path to the file to read.' }
+              },
+              required: ['path'],
+              additionalProperties: false
+            }
+          }
+        }
+      end
+
+      # Reads the contents of a file.
+      #
+      # @param path [String] The relative path to the file.
+      # @param working_dir_path [Pathname] The working directory to resolve the path against.
+      # @return [String] The file contents, or an error message if not found.
+      def self.call(path, working_dir_path)
+        validation_error = validate_read_file_path(path)
+        return validation_error if validation_error
+
+        target = secure_path(path, working_dir_path)
+        return 'Error: File not found' unless target.exist? && target.file?
+        return 'Error: File is not readable' unless target.readable?
+
+        target.read
+      rescue ArgumentError
+        raise
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'ReadFile Error')
+        "Error reading file: #{e.message}"
+      end
+
+      class << self
+        private
+
+        def validate_read_file_path(path)
+          return 'Error: Invalid path. Path must be a string.' unless path.is_a?(String)
+
+          normalized = path.strip
+          return 'Error: Invalid path. Path must not be empty.' if normalized.empty?
+          raise ArgumentError, "Path traversal attempt: #{path}" if normalized.include?('..') || normalized.include?('\\')
+          return 'Error: Invalid path. Allowed characters are letters, numbers, dot, underscore, hyphen, and slash.' unless normalized.match?(%r{\A[a-zA-Z0-9._\-/]+\z})
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/tools/registry.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative 'read_file'
+require_relative 'write_file'
+require_relative 'run_command'
+
+module SkillBench
+  module Tools
+    # Registry for all available tools, providing their definitions to the LLM.
+    class Registry
+      # Returns an array of tool definitions in the format expected by the LLM API.
+      #
+      # @return [Array<Hash>] The list of available tools with their names, descriptions, and schemas.
+      def self.definitions
+        [
+          ReadFile.definition,
+          WriteFile.definition,
+          RunCommand.definition
+        ]
+      end
+    end
+  end
+end

data/lib/skill_bench/tools/run_command.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'timeout'
+require 'shellwords'
+require_relative '../config'
+
+module SkillBench
+  module Tools
+    # Handles executing a shell command within the working directory.
+    class RunCommand
+      # Commands that are always blocked even if listed in allowed_commands,
+      # because they can be used to escape the sandbox or execute arbitrary code.
+      DANGEROUS_COMMANDS = %w[
+        bash sh zsh fish dash ksh csh tcsh
+        python python3 python2 ruby perl node
+        php lua tcl wish
+        curl wget nc ncat socat
+        eval exec
+        sudo su doas
+        chmod chown mount umount
+        dd mkfs fdisk parted
+        insmod rmmod modprobe
+        systemctl service
+        passwd useradd userdel groupadd groupdel
+      ].freeze
+
+      # @return [Hash] The tool definition for the LLM API.
+      def self.definition
+        {
+          type: 'function',
+          function: {
+            name: 'run_command',
+            description: 'Execute a shell command (e.g., rspec).',
+            parameters: {
+              type: 'object',
+              properties: {
+                command: { type: 'string', description: 'The shell command to run.' }
+              },
+              required: ['command'],
+              additionalProperties: false
+            }
+          }
+        }
+      end
+
+      # Executes a shell command within the working directory (host or container).
+      #
+      # Tokenizes the command string before execution so that arguments are passed
+      # directly to the OS without shell interpretation, preventing shell injection.
+      #
+      # @param command [String] The command to run (e.g. "rspec spec/models").
+      # @param working_dir_path [Pathname] The host directory (ignored if container_id present).
+      # @param container_id [String, nil] The Docker container ID for isolated execution.
+      # @return [String] A formatted string containing the exit status, STDOUT, and STDERR.
+      # @raise [Timeout::Error] Internally rescued; returns a timeout message string.
+      def self.call(command, working_dir_path, container_id = nil)
+        argv = command.shellsplit
+        return 'Error: Empty command.' if argv.empty?
+
+        base_cmd = argv.first
+        return "Error: Command '#{base_cmd}' is blocked for security reasons." if DANGEROUS_COMMANDS.include?(base_cmd)
+
+        allowed = SkillBench::Config.allowed_commands
+        return 'Error: No allowed commands configured. Set allowed_commands in skill-bench.json or use --mode mock.' if allowed.nil?
+        return "Error: Command '#{base_cmd}' is not permitted." unless allowed.include?(base_cmd)
+
+        max_time = SkillBench::Config.max_execution_time
+        Timeout.timeout(max_time) do
+          stdout_str, stderr_str, status = if container_id
+                                             docker_cmd = ['docker', 'exec', '-w', '/sandbox', container_id] + argv
+                                             Open3.capture3(*docker_cmd)
+                                           else
+                                             Open3.capture3(*argv, chdir: working_dir_path.to_s)
+                                           end
+          <<~RESULT
+            Exit Status: #{status.exitstatus}
+            STDOUT:
+            #{stdout_str}
+            STDERR:
+            #{stderr_str}
+          RESULT
+        end
+      rescue Timeout::Error
+        "Error: Command execution timed out after #{max_time} seconds."
+      end
+    end
+  end
+end

data/lib/skill_bench/tools/write_file.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+# Top-level namespace for the Rails Agent Evaluator.
+module SkillBench
+  # Contains tool implementations for the evaluator.
+  module Tools
+    # Handles writing content to a file within the working directory.
+    class WriteFile < Base
+      # @return [Hash] The tool definition for the LLM API.
+      def self.definition
+        {
+          type: 'function',
+          function: {
+            name: 'write_file',
+            description: 'Write content to a file. Overwrites the file if it exists.',
+            parameters: {
+              type: 'object',
+              properties: {
+                path: { type: 'string', description: 'Relative path to the file to write.' },
+                content: { type: 'string', description: 'The content to write into the file.' }
+              },
+              required: %w[path content],
+              additionalProperties: false
+            }
+          }
+        }
+      end
+
+      # Writes content to a file. Creates missing parent directories.
+      #
+      # @param path [String] The relative path to the file.
+      # @param content [String] The content to write.
+      # @param working_dir_path [Pathname] The working directory to resolve the path against.
+      # @return [String] A success message.
+      def self.call(path, content, working_dir_path)
+        validate_write_path!(path)
+
+        target = secure_path(path, working_dir_path)
+        target.dirname.mkpath
+        # Re-verify path after mkpath to mitigate TOCTOU vulnerabilities
+        target = secure_path(path, working_dir_path)
+
+        File.open(target, File::WRONLY | File::CREAT | File::TRUNC, 0o644) do |f|
+          f.write(content)
+        end
+        "Successfully wrote to #{path}"
+      end
+
+      class << self
+        private
+
+        # Validates the path against strict security rules to prevent traversal.
+        # Following recommendations to disallow directory separators and multiple dots.
+        #
+        # @param path [String] The relative path to validate.
+        # @raise [ArgumentError] if the path is invalid, empty, or attempts traversal.
+        # @return [void]
+        def validate_write_path!(path)
+          raise ArgumentError, 'Path must be a string' unless path.is_a?(String)
+
+          normalized = path.strip
+          raise ArgumentError, 'Path cannot be empty' if normalized.empty?
+
+          # Allow forward slashes for nested directories, but reject '..'
+          raise ArgumentError, "Path traversal attempt: #{path}" if normalized.include?('..')
+
+          raise ArgumentError, "Backslashes are not allowed in path: #{path}" if normalized.include?('\\')
+
+          return if normalized.match?(%r{\A[a-zA-Z0-9._\-/]+\z})
+
+          raise ArgumentError, "Invalid characters in path: #{path}"
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/tools.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative 'tools/base'
+require_relative 'tools/argument_parser'
+require_relative 'tools/read_file'
+require_relative 'tools/write_file'
+require_relative 'tools/run_command'
+require_relative 'tools/dispatcher'
+require_relative 'tools/registry'
+
+module SkillBench
+  # Provides the definitions and execution logic for tools available to the ReAct agent.
+  # Currently supports reading files, writing files, and running shell commands.
+  module Tools
+    # Returns an array of tool definitions in the format expected by the LLM API.
+    #
+    # @return [Array<Hash>] The list of available tools with their names, descriptions, and schemas.
+    def self.definitions
+      Registry.definitions
+    end
+
+    # Executes a specified tool with the given arguments within a working directory.
+    #
+    # @param name [String] The name of the tool to execute (e.g., 'read_file').
+    # @param arguments [String] A JSON string containing the arguments for the tool.
+    # @param working_dir [String] The base directory in which the tool should operate.
+    # @param container_id [String, nil] The Docker container ID for isolated execution.
+    # @return [String] The result of the tool execution, or an error message.
+    def self.execute(name, arguments, working_dir, container_id = nil)
+      Dispatcher.call(name, arguments, working_dir, container_id)
+    end
+  end
+end

data/lib/skill_bench/trend_tracker/persistence.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'pathname'
+
+module SkillBench
+  class TrendTracker
+    # Handles history file persistence operations including backup management
+    class Persistence
+      # @param history_file [String] Path to the history JSON file
+      def initialize(history_file)
+        @history_file = File.expand_path(history_file)
+      end
+
+      # Loads history from file with corruption recovery
+      #
+      # @return [Array<Hash>] List of historical entries
+      def load
+        return [] unless File.exist?(history_file)
+
+        JSON.parse(File.read(history_file), symbolize_names: true)
+      rescue JSON::ParserError => e
+        backup = read_backup
+        return backup if backup
+
+        SkillBench::ErrorLogger.log_error(e, "History file #{history_file} corrupted")
+        []
+      end
+
+      # Writes history to file with atomic operation and backup.
+      # Returns a result hash so callers do not need to rescue SystemCallError.
+      #
+      # @param history [Array<Hash>] History entries to write
+      # @return [Hash] { success: true } on success, { success: false, error: { message: '...' } } on failure
+      def write(history)
+        json = JSON.pretty_generate(history)
+        temp_file = "#{history_file}.tmp"
+        File.write(temp_file, json)
+        File.rename(temp_file, history_file)
+
+        begin
+          File.write("#{history_file}.bak", json)
+        rescue SystemCallError => e
+          warn "Backup write failed for #{history_file}: #{e.message}"
+        end
+
+        { success: true }
+      rescue SystemCallError => e
+        { success: false, error: { message: e.message } }
+      end
+
+      private
+
+      attr_reader :history_file
+
+      # Reads backup file if it exists
+      #
+      # @return [Array<Hash>, nil] Backup data or nil if unavailable
+      def read_backup
+        backup_path = "#{history_file}.bak"
+        return nil unless File.exist?(backup_path)
+
+        JSON.parse(File.read(backup_path), symbolize_names: true)
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/skill_bench/trend_tracker/trend_calculator.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module SkillBench
+  class TrendTracker
+    # Calculates performance trends between evaluation results
+    class TrendCalculator
+      # Computes trend comparison between current result and historical entries
+      #
+      # @param entries [Array<Hash>] Historical entries
+      # @param current_entry [Hash] Current evaluation entry
+      # @return [Hash, nil] Trend data or nil if no matching history exists
+      def self.compute_trend(entries, current_entry)
+        matching = filter_matching_entries(entries, current_entry)
+        return nil if matching.empty?
+
+        previous = matching.last
+        current_baseline = current_entry[:baseline_total]
+        current_context = current_entry[:context_total]
+        previous_baseline = previous[:baseline_total]
+        previous_context = previous[:context_total]
+        return nil unless current_baseline && current_context && previous_baseline && previous_context
+
+        {
+          baseline_trend: trend_direction(current_baseline, previous_baseline),
+          context_trend: trend_direction(current_context, previous_context),
+          baseline_delta: current_baseline - previous_baseline,
+          context_delta: current_context - previous_context,
+          previous_run: previous[:timestamp]
+        }
+      end
+
+      class << self
+        private
+
+        # Filters historical entries to match current evaluation configuration
+        #
+        # @param entries [Array<Hash>] Historical entries
+        # @param current_entry [Hash] Current evaluation entry
+        # @return [Array<Hash>] Matching entries
+        def filter_matching_entries(entries, current_entry)
+          entries.select do |entry|
+            entry[:eval_name] == current_entry[:eval_name] &&
+              entry[:skill_names].sort == current_entry[:skill_names].sort
+          end
+        end
+
+        # Determines trend direction between two values
+        #
+        # @param current [Numeric] Current value
+        # @param previous [Numeric] Previous value
+        # @return [Symbol] :improved, :regressed, or :unchanged
+        def trend_direction(current, previous)
+          return :unchanged if current == previous
+
+          current > previous ? :improved : :regressed
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/trend_tracker.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'time'
+require_relative 'trend_tracker/persistence'
+require_relative 'trend_tracker/trend_calculator'
+
+module SkillBench
+  # Tracks evaluation results over time and computes trend deltas.
+  class TrendTracker
+    DEFAULT_HISTORY_FILE = '.skill-bench-trends.json'
+
+    # @param history_file [String] Path to the history JSON file.
+    def initialize(history_file: DEFAULT_HISTORY_FILE)
+      @persistence = Persistence.new(history_file)
+    end
+
+    # Records an evaluation result.
+    #
+    # @param result [Hash] The evaluation result from EvaluationRunner.
+    # @return [Hash] Service response.
+    def record(result)
+      history = @persistence.load
+      history << extract_entry(result)
+      write_result = @persistence.write(history)
+
+      return { success: false, response: { error: write_result[:error] } } unless write_result[:success]
+
+      { success: true, response: { recorded: true } }
+    rescue StandardError => e
+      SkillBench::ErrorLogger.log_error(e, 'TrendTracker Error')
+      { success: false, response: { error: { message: e.message } } }
+    end
+
+    # Loads the full history.
+    #
+    # @return [Array<Hash>] List of historical entries.
+    def history
+      @persistence.load
+    end
+
+    # Computes the trend of the given result against the most recent matching history entry.
+    #
+    # @param result [Hash] The current evaluation result.
+    # @return [Hash, nil] Trend data or nil if no matching history exists.
+    def trend_for(result)
+      entries = @persistence.load
+      current = extract_entry(result)
+      TrendCalculator.compute_trend(entries, current)
+    end
+
+    private
+
+    def extract_entry(result)
+      report = result.dig(:response, :report)
+      {
+        timestamp: Time.now.iso8601,
+        eval_name: result[:eval_name],
+        skill_names: result[:skill_names],
+        verdict: report&.verdict,
+        baseline_total: report&.baseline_total,
+        context_total: report&.context_total,
+        deltas: report&.deltas
+      }
+    end
+  end
+end

data/lib/skill_bench/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module SkillBench
+  # The current gem version.
+  VERSION = '0.1.0'
+end

data/lib/skill_bench.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+# Ruby Skill Bench - AI Agent Skills Evaluation Engine
+#
+# @example Basic usage
+#   require 'skill_bench'
+#   SkillBench::CLI.call(ARGV)
+
+# Core modules
+require_relative 'skill_bench/version'
+require_relative 'skill_bench/dimension'
+require_relative 'skill_bench/criteria'
+require_relative 'skill_bench/delta_report'
+require_relative 'skill_bench/cli'
+require_relative 'skill_bench/config'
+require_relative 'skill_bench/output_formatter'
+require_relative 'skill_bench/client'
+
+# Judge subsystem
+require_relative 'skill_bench/judge'
+require_relative 'skill_bench/judge/judge'
+require_relative 'skill_bench/judge/prompt'
+require_relative 'skill_bench/judge/response'
+
+# Agent subsystem
+require_relative 'skill_bench/agent'
+require_relative 'skill_bench/agent/runner'
+require_relative 'skill_bench/agent/summary'
+require_relative 'skill_bench/agent/react_agent'
+
+# Task subsystem
+require_relative 'skill_bench/task'
+require_relative 'skill_bench/task/evaluator'
+require_relative 'skill_bench/task/file_reader'
+
+# Evaluation orchestration
+require_relative 'skill_bench/evaluation'
+require_relative 'skill_bench/evaluation/runner'
+require_relative 'skill_bench/evaluation/generator'
+
+# Execution environment
+require_relative 'skill_bench/execution'
+require_relative 'skill_bench/execution/context_hydrator'
+require_relative 'skill_bench/execution/sandbox'
+require_relative 'skill_bench/execution/source_path_resolver'
+
+# Clients
+require_relative 'skill_bench/clients/all'
+require_relative 'skill_bench/clients/provider_schemas'
+
+# Config management
+require_relative 'skill_bench/config/store'
+require_relative 'skill_bench/config/defaults'
+require_relative 'skill_bench/config/applier'
+require_relative 'skill_bench/config/env_overrides'
+require_relative 'skill_bench/config/json_loader'
+require_relative 'skill_bench/config/facade_readers'
+require_relative 'skill_bench/config/facade_writers'
+
+# Models
+require_relative 'skill_bench/models/config'
+require_relative 'skill_bench/models/criteria_validator'
+require_relative 'skill_bench/models/eval'
+require_relative 'skill_bench/models/skill'
+require_relative 'skill_bench/models/provider'
+
+# Commands
+require_relative 'skill_bench/commands/init'
+require_relative 'skill_bench/commands/run'
+require_relative 'skill_bench/commands/skill_new'
+require_relative 'skill_bench/commands/eval_new'
+
+# Services
+require_relative 'skill_bench/services/runner_service'
+require_relative 'skill_bench/services/template_registry'
+
+# Tools
+require_relative 'skill_bench/tools'
+
+# History recording
+require_relative 'skill_bench/history_recorder'
+require_relative 'skill_bench/history_recorder/persistence_service'
+require_relative 'skill_bench/history_recorder/summary_service'
+
+# Trend tracking
+require_relative 'skill_bench/trend_tracker'
+require_relative 'skill_bench/trend_tracker/persistence'
+require_relative 'skill_bench/trend_tracker/trend_calculator'
+
+# Rails integrations
+require_relative 'skill_bench/rails/skill_templates'
+
+# Migration utilities
+require_relative 'skill_bench/migration/provider_migrator'
+
+# Interactive mode
+require_relative 'skill_bench/interactive'
+
+# Package verification
+require_relative 'skill_bench/package_verifier'
+
+# Utility modules
+require_relative 'skill_bench/error_logger'