ruby-skill-bench 0.1.0 → 1.0.1

data/lib/skill_bench/services/trend_recorder_service.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative '../trend_tracker'
+
+module SkillBench
+  module Services
+    # Records evaluation results and computes trends.
+    class TrendRecorderService
+      # Records evaluation results and computes trends.
+      #
+      # @param result [Hash] The evaluation result from Evaluation::Runner
+      # @param eval_name [String] Name of the eval
+      # @param skill_names [Array<String>] Names of the skills
+      # @return [Hash] Result with success status and trend data
+      def self.call(result, eval_name, skill_names)
+        new(result, eval_name, skill_names).call
+      end
+
+      # @param result [Hash] The evaluation result from Evaluation::Runner
+      # @param eval_name [String] Name of the eval
+      # @param skill_names [Array<String>] Names of the skills
+      def initialize(result, eval_name, skill_names)
+        @result = result
+        @eval_name = eval_name
+        @skill_names = skill_names
+      end
+
+      # Records evaluation results and computes trends.
+      #
+      # @return [Hash] Result with success status and trend data
+      def call
+        tracker = TrendTracker.new
+        enriched = @result.merge(eval_name: @eval_name, skill_names: @skill_names)
+        trend = tracker.trend_for(enriched)
+        record_result = tracker.record(enriched)
+
+        record_success = record_result.is_a?(Hash) && record_result[:success]
+        unless record_success
+          message = if record_result.is_a?(Hash)
+                      record_result.dig(:response, :error, :message) ||
+                        record_result.dig(:error, :message) ||
+                        'Unknown error'
+                    else
+                      'Unexpected record response'
+                    end
+          SkillBench::ErrorLogger.log_error(
+            StandardError.new(message),
+            "Trend tracking record failed for eval #{@eval_name}"
+          )
+          return {
+            success: false,
+            response: {
+              error: {
+                message: "Trend tracking record failed: #{message}",
+                record_result: record_result
+              }
+            }
+          }
+        end
+        { success: true, trend: trend }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Trend tracking failed')
+        { success: false, response: { error: { message: e.message } } }
+      end
+    end
+  end
+end

data/lib/skill_bench/services/variant_parser.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Parses variant specifications for skill comparison.
+    class VariantParser
+      # Parses a variant specification string.
+      #
+      # @param spec [String] Variant spec (e.g., "pack:rails" or "/path/to/skill")
+      # @return [Hash] Parsed variant with :type (:pack or :path) and corresponding key
+      def self.call(spec)
+        new(spec).call
+      end
+
+      # @param spec [String] Variant spec (e.g., "pack:rails" or "/path/to/skill")
+      def initialize(spec)
+        @spec = spec
+      end
+
+      # Parses the variant specification.
+      #
+      # @return [Hash] Parsed variant with :type (:pack or :path) and corresponding key
+      def call
+        if @spec.start_with?('pack:')
+          { type: :pack, name: @spec.sub('pack:', '') }
+        else
+          { type: :path, path: @spec }
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/services/variant_resolver.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../registry/pack_resolver'
+require_relative 'runner_service'
+require_relative 'manifest_finder'
+
+module SkillBench
+  module Services
+    # Resolves skill paths from variant specifications.
+    class VariantResolver
+      # Resolves skill paths from a variant specification.
+      #
+      # @param variant [Hash] Parsed variant from VariantParser
+      # @param skill_name [String] Name of the skill to resolve
+      # @param manifest_path [String, nil] Optional path to registry manifest
+      # @return [Array<String>] Array of skill paths
+      # @raise [ArgumentError] when skill cannot be resolved
+      def self.call(variant, skill_name, manifest_path: nil)
+        new(variant, skill_name, manifest_path: manifest_path).call
+      end
+
+      # @param variant [Hash] Parsed variant from VariantParser
+      # @param skill_name [String] Name of the skill to resolve
+      # @param manifest_path [String, nil] Optional path to registry manifest
+      def initialize(variant, skill_name, manifest_path: nil)
+        @variant = variant
+        @skill_name = skill_name
+        @manifest_path = manifest_path
+      end
+
+      # Resolves skill paths from the variant specification.
+      #
+      # @return [Array<String>] Array of skill paths
+      # @raise [ArgumentError] when skill cannot be resolved or variant type is unknown
+      def call
+        case @variant[:type]
+        when :pack
+          resolve_pack_skill
+        when :path
+          [@variant[:path]]
+        else
+          raise ArgumentError, "Unknown variant type: #{@variant[:type]}, variant: #{@variant.inspect}"
+        end
+      end
+
+      private
+
+      # Resolves a skill from a pack.
+      #
+      # @return [Array<String>] Array containing the resolved skill path
+      # @raise [ArgumentError] when skill is not found in pack
+      def resolve_pack_skill
+        pack_name = @variant[:name]
+        manifest = @manifest_path || ManifestFinder.call
+        resolver = Registry::PackResolver.new(manifest)
+        resolved = resolver.resolve_skill(pack_name, @skill_name)
+        raise ArgumentError, "Skill '#{@skill_name}' not found in pack '#{pack_name}'" unless resolved
+
+        [resolved]
+      end
+    end
+  end
+end

data/lib/skill_bench/version.rb

@@ -2,5 +2,5 @@
 
 module SkillBench
   # The current gem version.
-  VERSION = '0.1.0'
+  VERSION = '1.0.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby-skill-bench
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Ismael Marin
@@ -119,6 +119,7 @@ files:
 - lib/skill_bench/agent/runner.rb
 - lib/skill_bench/agent/summary.rb
 - lib/skill_bench/cli.rb
+- lib/skill_bench/cli/compare_command.rb
 - lib/skill_bench/cli/eval/eval_command_registry.rb
 - lib/skill_bench/cli/eval/eval_commands.rb
 - lib/skill_bench/cli/eval/eval_options.rb
@@ -139,6 +140,7 @@ files:
 - lib/skill_bench/clients/providers/deepseek.rb
 - lib/skill_bench/clients/providers/gemini.rb
 - lib/skill_bench/clients/providers/groq.rb
+- lib/skill_bench/clients/providers/mock.rb
 - lib/skill_bench/clients/providers/null_client.rb
 - lib/skill_bench/clients/providers/ollama.rb
 - lib/skill_bench/clients/providers/openai.rb
@@ -191,21 +193,40 @@ files:
 - lib/skill_bench/output_formatter.rb
 - lib/skill_bench/package_verifier.rb
 - lib/skill_bench/rails/skill_templates.rb
+- lib/skill_bench/registry/pack_resolver.rb
 - lib/skill_bench/runner.rb
+- lib/skill_bench/services/agent_spawner_service.rb
+- lib/skill_bench/services/compare_option_parser.rb
+- lib/skill_bench/services/comparison_reporter.rb
+- lib/skill_bench/services/comparison_runner.rb
+- lib/skill_bench/services/context_loader_service.rb
 - lib/skill_bench/services/delta_table_formatter.rb
+- lib/skill_bench/services/error_response_builder.rb
+- lib/skill_bench/services/eval_resolver.rb
+- lib/skill_bench/services/exit_code_calculator.rb
 - lib/skill_bench/services/feedback_generator.rb
 - lib/skill_bench/services/formatting_helpers.rb
 - lib/skill_bench/services/iteration_formatter.rb
 - lib/skill_bench/services/json_formatter.rb
+- lib/skill_bench/services/judge_params_builder.rb
 - lib/skill_bench/services/judge_score_parser_service.rb
 - lib/skill_bench/services/junit_formatter.rb
+- lib/skill_bench/services/manifest_finder.rb
 - lib/skill_bench/services/option_parser_service.rb
+- lib/skill_bench/services/output_formatter.rb
 - lib/skill_bench/services/output_persistence_service.rb
+- lib/skill_bench/services/prompt_builder_service.rb
+- lib/skill_bench/services/provider_resolver.rb
 - lib/skill_bench/services/result_printer_service.rb
 - lib/skill_bench/services/runner_service.rb
 - lib/skill_bench/services/skill_resolver.rb
+- lib/skill_bench/services/skill_resolver_service.rb
+- lib/skill_bench/services/source_path_resolver_service.rb
 - lib/skill_bench/services/template_registry.rb
 - lib/skill_bench/services/template_registry/category_data.rb
+- lib/skill_bench/services/trend_recorder_service.rb
+- lib/skill_bench/services/variant_parser.rb
+- lib/skill_bench/services/variant_resolver.rb
 - lib/skill_bench/task.rb
 - lib/skill_bench/task/evaluator.rb
 - lib/skill_bench/task/file_reader.rb
@@ -241,7 +262,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.11
+rubygems_version: 4.0.12
 specification_version: 4
 summary: The evaluation engine for AI Agent Skills benchmarking.
 test_files: []