serialbench 0.3.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ed5cf6538b0a64c26b9d9c0739b943f750f30476aa27a94c5b87aae98c203b55
-  data.tar.gz: 3d1e7eb5c8f71c4052014729bd6dbbf4417a68339065a7a934ea1e5a3ef82a3b
+  metadata.gz: 7267d278bec24e38ca217bfbf164c10411f03b4fb333295f05c617c56f91a8b8
+  data.tar.gz: fcf4034a75ee9615197e2fd0cd3e7b63a795c7f3f0b1ba2c3b1cd0170babecb6
 SHA512:
-  metadata.gz: 8bd0e7a6052712f3a49c4656e6aa663cd4db2215c9f9ebc3ac8379b7f258f78e488fdfc46bb52a8e6e9d89cc07ef78fa4fbebe6f0471881b679d40d5ab42284d
-  data.tar.gz: 5441d7d92266df86976a78dd4fb68acf8c491147c799b06aa9e1a67ac5bd382dfd27547505e4566483a9657f983bc7bb0ffb5280cc17089de61a8b070fe5677b
+  metadata.gz: 4acd59cfc6d0b8d47d1d09172ea2f034063633ba5435fccc2fcfbd35ec11d96d41080210d84d149423979b3e14873cf3833512d36917515dfdb0ccdb698dfc1f
+  data.tar.gz: a913c26df13272e009996925fc6dd4789f74c6f968d27cd02da6155b71f1026d03e8bc2bb1cef1ba400ca19170b3c5a7f0d04ab125658ebf8a54c14551c5607b

data/.github/workflows/benchmark.yml

@@ -3,11 +3,7 @@ name: benchmark-weekly
 on:
   push:
     branches: [main]
-    paths-ignore:
-      - '.github/workflows/windows-debug.yml'
   pull_request:
-    paths-ignore:
-      - '.github/workflows/windows-debug.yml'
   schedule:
     # Run benchmarks weekly on Sundays at 2 AM UTC
     - cron: '0 2 * * 0'
@@ -111,26 +107,6 @@ jobs:
       - name: Install gems
         run: bundle install
 
-      - name: Build libleptris from source (Ubuntu 22.04)
-        if: matrix.platform == 'ubuntu-22.04' || matrix.platform == 'ubuntu-22.04-arm'
-        run: |
-          LEPTRIS_VERSION=$(bundle exec ruby -e "print Gem::Specification.find_by_name('leptris').version")
-          LEPTRIS_TAG=$(curl -s "https://api.github.com/repos/leptris/leptris/tags?per_page=100" \
-            | grep -oE '"v[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"' | sort -V \
-            | awk -v v="v${LEPTRIS_VERSION}" '$0 <= v' | tail -1)
-          echo "Building libleptris ${LEPTRIS_VERSION} from tag ${LEPTRIS_TAG}"
-          test -n "${LEPTRIS_TAG}"
-          curl -sL "https://github.com/leptris/leptris/archive/refs/tags/${LEPTRIS_TAG}.tar.gz" | tar xz
-          ln -s "leptris-${LEPTRIS_TAG#v}" "leptris-${LEPTRIS_VERSION}"
-          cmake -S "leptris-${LEPTRIS_VERSION}" -B /tmp/leptris-build \
-            -DLEPTRIS_BUILD_SHARED=ON -DLEPTRIS_BUILD_STATIC=OFF -DBUILD_TESTING=OFF \
-            -DLEPTRIS_BUILD_CLI=OFF -DLEPTRIS_BUILD_BENCHMARKS=OFF -DLEPTRIS_BUILD_MAN_PAGES=OFF \
-            -DLEPTRIS_ENABLE_UTF8PROC=OFF -DLEPTRIS_ENABLE_ICONV=OFF
-          cmake --build /tmp/leptris-build -j"$(nproc)"
-          LIBLEPTRIS=$(find /tmp/leptris-build -name 'libleptris.so*' -type f | head -1)
-          echo "LEPTRIS_LIB_PATH=${LIBLEPTRIS}" >> "$GITHUB_ENV"
-        shell: bash
-
       - name: Create environment config
         run: |
           mkdir -p config/environments

data/.github/workflows/rake.yml

@@ -1,26 +1,23 @@
-# Auto-generated by Cimas: Do not edit it manually!
-# See https://github.com/metanorma/cimas
 name: rake
 
 on:
   push:
-    branches: [ master, main ]
+    branches: [ main ]
     tags: [ v* ]
-    paths-ignore:
-      - '.github/workflows/windows-debug.yml'
   pull_request:
-    paths-ignore:
-      - '.github/workflows/windows-debug.yml'
+
+permissions:
+  contents: read
 
 jobs:
-  rake:
-    uses: metanorma/ci/.github/workflows/generic-rake.yml@main
-    with:
-      before-setup-ruby: |
-        vcpkg install libxml2:x64-windows 2>&1 || true
-        vcpkg integrate install 2>&1 || true
-        mkdir -p .bundle 2>&1 || true
-        echo "---" > .bundle/config 2>&1 || true
-        echo 'BUNDLE_BUILD__LIBXML___RUBY: "--with-xml2-dir=C:/vcpkg/installed/x64-windows --with-xml2-include=C:/vcpkg/installed/x64-windows/include/libxml2 --with-xml2-lib=C:/vcpkg/installed/x64-windows/lib"' >> .bundle/config 2>&1 || true
-    secrets:
-      pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}
+  rspec:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+          bundler-cache: true
+
+      - run: bundle exec rspec --format progress

data/Gemfile

@@ -12,7 +12,7 @@ unless Gem.win_platform? && RUBY_PLATFORM.include?('aarch64')
   gem 'libxml-ruby'
 end
 
-gem 'leptris', '1.9.0'  # pin to latest git tag; newer gems expect APIs not in v1.9.0 source builds    # Needs the libleptris shared library (LEPTRIS_LIB_PATH or system install)
+gem 'leptris', '1.9.35'  # precompiled platform gems bundle libleptris (darwin/linux/mingw)
 gem 'benchmark'  # Removed from stdlib in Ruby 4.0
 gem 'base64'  # Required for Ruby 3.4+
 gem 'lutaml-model', '~> 0.7'

data/config/benchmarks/full-json.yml

@@ -17,8 +17,8 @@ iterations:
   large: 1
 
 operations:
-- parse
-- generate
+- parsing
+- generation
 - streaming
 - memory
 

data/config/benchmarks/full-toml.yml

@@ -17,8 +17,8 @@ iterations:
   large: 1
 
 operations:
-- parse
-- generate
+- parsing
+- generation
 - streaming
 - memory
 

data/config/benchmarks/full-xml.yml

@@ -17,8 +17,8 @@ iterations:
   large: 1
 
 operations:
-- parse
-- generate
+- parsing
+- generation
 - streaming
 - xpath
 - memory

data/config/benchmarks/full-yaml.yml

@@ -17,8 +17,8 @@ iterations:
   large: 1
 
 operations:
-- parse
-- generate
+- parsing
+- generation
 - streaming
 - memory
 

data/config/benchmarks/full.yml

@@ -25,8 +25,8 @@ iterations:
   large: 1
 
 operations:
-- parse
-- generate
+- parsing
+- generation
 - streaming
 - xpath
 - memory

data/config/benchmarks/short.yml

@@ -19,8 +19,8 @@ iterations:
   large: 1
 
 operations:
-- parse
-- generate
+- parsing
+- generation
 - streaming
 
 warmup: 2

data/lib/serialbench/benchmark_runner.rb

@@ -21,6 +21,7 @@ module Serialbench
       @serializers = Serializers.available
       @test_data = {}
       @results = []
+      validate_operations!
       load_test_data
     end
 
@@ -45,11 +46,13 @@ module Serialbench
       puts "Test data sizes: #{@test_data.keys.join(', ')}"
       puts
 
+      selected = @benchmark_config.operations
+      selected = OPERATIONS.keys + ['memory'] if selected.nil? || selected.empty?
       results = {}
       OPERATIONS.each do |name, handler|
-        results[name.to_sym] = run_benchmark_type(name, name, &handler)
+        results[name.to_sym] = selected.include?(name) ? run_benchmark_type(name, name, &handler) : []
       end
-      results[:memory] = run_memory_benchmarks
+      results[:memory] = selected.include?('memory') ? run_memory_benchmarks : []
 
       Models::BenchmarkResult.new(
         serializers: Serializers.information,
@@ -145,313 +148,35 @@ module Serialbench
 
       case type_name
       when 'generation'
-        serializers.select(&:supports_generation?)
+        serializers.select { |s| s.supports?(:generate) }
       when 'streaming'
-        serializers.select(&:supports_streaming?)
+        serializers.select { |s| s.supports?(:sax) || s.supports?(:streaming) }
+      when 'xpath'
+        serializers.select { |s| s.supports?(:xpath) }
       else
         serializers
       end
     end
 
     def get_iterations_for_size(size)
-      @benchmark_config.iterations.send(size.to_s)
-    end
-
-    def load_test_data
-      # Determine which data sizes to load based on configuration
-      data_sizes = @benchmark_config.data_sizes
-
-      # Initialize test data structure
-      @test_data = {}
-      data_sizes.each { |size| @test_data[size] = {} }
-
-      # Generate data for each format and size
-      @benchmark_config.formats.each do |format|
-        data_sizes.each do |size|
-          @test_data[size][format] = generate_test_data(format, size)
-        end
-      end
-
-      # Try to load real test files if they exist
-      data_sizes.each do |size|
-        @benchmark_config.formats.each do |format|
-          file_path = "test_data/#{size}.#{format}"
-          @test_data[size][format] = File.read(file_path) if File.exist?(file_path)
-        end
+      iterations = @benchmark_config.iterations
+      case size.to_s
+      when 'small' then iterations.small
+      when 'medium' then iterations.medium
+      when 'large' then iterations.large
+      else raise ArgumentError, "no iteration count configured for #{size}"
       end
     end
 
-    def generate_test_data(format, size)
-      method_name = "generate_#{size}_#{format}"
-      send(method_name)
-    end
-
-    # Shared data structure generators
-    def small_test_data_structure
-      {
-        config: {
-          database: {
-            host: 'localhost',
-            port: 5432,
-            name: 'myapp',
-            user: 'admin',
-            password: 'secret'
-          },
-          cache: {
-            enabled: true,
-            ttl: 3600
-          }
-        }
-      }
-    end
-
-    def medium_test_data_structure
-      {
-        users: (1..1000).map do |i|
-          {
-            id: i,
-            name: "User #{i}",
-            email: "user#{i}@example.com",
-            created_at: "2023-01-#{(i % 28) + 1}T10:00:00Z",
-            profile: {
-              age: 20 + (i % 50),
-              city: "City #{i % 100}",
-              preferences: {
-                theme: i.even? ? 'dark' : 'light',
-                notifications: (i % 3).zero?
-              }
-            }
-          }
-        end
-      }
-    end
-
-    def large_test_data_structure
-      {
-        dataset: {
-          header: {
-            created: '2023-01-01T00:00:00Z',
-            count: 10_000,
-            format: 'data'
-          },
-          records: (1..10_000).map do |i|
-            {
-              id: i,
-              timestamp: "2023-01-01T#{format('%02d', i % 24)}:#{format('%02d', i % 60)}:#{format('%02d', i % 60)}Z",
-              data: {
-                field1: "Value #{i}",
-                field2: i * 2,
-                field3: (i % 100).zero? ? 'special' : 'normal',
-                nested: [
-                  "Item #{i}-1",
-                  "Item #{i}-2",
-                  "Item #{i}-3"
-                ]
-              },
-              metadata: {
-                source: 'generator',
-                version: '1.0',
-                checksum: i.to_s(16)
-              }
-            }
-          end
-        }
-      }
-    end
-
-    # XML test data generators
-    def generate_small_xml
-      data = small_test_data_structure
-      <<~XML
-        <?xml version="1.0" encoding="UTF-8"?>
-        <config>
-          <database>
-            <host>#{data[:config][:database][:host]}</host>
-            <port>#{data[:config][:database][:port]}</port>
-            <name>#{data[:config][:database][:name]}</name>
-            <user>#{data[:config][:database][:user]}</user>
-            <password>#{data[:config][:database][:password]}</password>
-          </database>
-          <cache>
-            <enabled>#{data[:config][:cache][:enabled]}</enabled>
-            <ttl>#{data[:config][:cache][:ttl]}</ttl>
-          </cache>
-        </config>
-      XML
-    end
-
-    def generate_medium_xml
-      data = medium_test_data_structure
-      users = data[:users].map do |user|
-        <<~USER
-          <user id="#{user[:id]}">
-            <name>#{user[:name]}</name>
-            <email>#{user[:email]}</email>
-            <created_at>#{user[:created_at]}</created_at>
-            <profile>
-              <age>#{user[:profile][:age]}</age>
-              <city>#{user[:profile][:city]}</city>
-              <preferences>
-                <theme>#{user[:profile][:preferences][:theme]}</theme>
-                <notifications>#{user[:profile][:preferences][:notifications]}</notifications>
-              </preferences>
-            </profile>
-          </user>
-        USER
-      end.join
-
-      <<~XML
-        <?xml version="1.0" encoding="UTF-8"?>
-        <users>
-          #{users}
-        </users>
-      XML
-    end
-
-    def generate_large_xml
-      data = large_test_data_structure
-      records = data[:dataset][:records].map do |record|
-        nested_items = record[:data][:nested].map { |item| "    <item>#{item}</item>" }.join("\n")
-        <<~RECORD
-            <record id="#{record[:id]}">
-              <timestamp>#{record[:timestamp]}</timestamp>
-              <data>
-                <field1>#{record[:data][:field1]}</field1>
-                <field2>#{record[:data][:field2]}</field2>
-                <field3>#{record[:data][:field3]}</field3>
-                <nested>
-          #{nested_items}
-                </nested>
-              </data>
-              <metadata>
-                <source>#{record[:metadata][:source]}</source>
-                <version>#{record[:metadata][:version]}</version>
-                <checksum>#{record[:metadata][:checksum]}</checksum>
-              </metadata>
-            </record>
-        RECORD
-      end.join
-
-      <<~XML
-        <?xml version="1.0" encoding="UTF-8"?>
-        <dataset>
-          <header>
-            <created>#{data[:dataset][:header][:created]}</created>
-            <count>#{data[:dataset][:header][:count]}</count>
-            <format>xml</format>
-          </header>
-          <records>
-            #{records}
-          </records>
-        </dataset>
-      XML
-    end
-
-    # JSON test data generators
-    def generate_small_json
-      JSON.generate(small_test_data_structure)
-    end
-
-    def generate_medium_json
-      JSON.generate(medium_test_data_structure)
-    end
-
-    def generate_large_json
-      data = large_test_data_structure
-      data[:dataset][:header][:format] = 'json'
-      JSON.generate(data)
-    end
-
-    # YAML test data generators
-    def generate_small_yaml
-      small_test_data_structure.to_yaml
-    end
-
-    def generate_medium_yaml
-      medium_test_data_structure.to_yaml
-    end
+    def validate_operations!
+      unknown = (@benchmark_config.operations || []) - (OPERATIONS.keys + ['memory'])
+      return if unknown.empty?
 
-    def generate_large_yaml
-      data = large_test_data_structure
-      data[:dataset][:header][:format] = 'yaml'
-      data.to_yaml
+      raise ArgumentError, "unknown operations #{unknown.inspect}; expected: #{OPERATIONS.keys.join(', ')}, memory"
     end
 
-    # TOML test data generators
-    def generate_small_toml
-      data = small_test_data_structure
-      <<~TOML
-        [config]
-
-        [config.database]
-        host = "#{data[:config][:database][:host]}"
-        port = #{data[:config][:database][:port]}
-        name = "#{data[:config][:database][:name]}"
-        user = "#{data[:config][:database][:user]}"
-        password = "#{data[:config][:database][:password]}"
-
-        [config.cache]
-        enabled = #{data[:config][:cache][:enabled]}
-        ttl = #{data[:config][:cache][:ttl]}
-      TOML
-    end
-
-    def generate_medium_toml
-      data = medium_test_data_structure
-      # Use smaller dataset for TOML due to verbosity
-      users = data[:users].first(100)
-      users.map do |user|
-        <<~USER
-          [[users]]
-          id = #{user[:id]}
-          name = "#{user[:name]}"
-          email = "#{user[:email]}"
-          created_at = "#{user[:created_at]}"
-
-          [users.profile]
-          age = #{user[:profile][:age]}
-          city = "#{user[:profile][:city]}"
-
-          [users.profile.preferences]
-          theme = "#{user[:profile][:preferences][:theme]}"
-          notifications = #{user[:profile][:preferences][:notifications]}
-        USER
-      end.join("\n")
-    end
-
-    def generate_large_toml
-      data = large_test_data_structure
-      # Use smaller dataset for TOML due to verbosity
-      records = data[:dataset][:records].first(1000)
-      records_toml = records.map do |record|
-        <<~RECORD
-          [[dataset.records]]
-          id = #{record[:id]}
-          timestamp = "#{record[:timestamp]}"
-
-          [dataset.records.data]
-          field1 = "#{record[:data][:field1]}"
-          field2 = #{record[:data][:field2]}
-          field3 = "#{record[:data][:field3]}"
-          nested = #{record[:data][:nested].inspect}
-
-          [dataset.records.metadata]
-          source = "#{record[:metadata][:source]}"
-          version = "#{record[:metadata][:version]}"
-          checksum = "#{record[:metadata][:checksum]}"
-        RECORD
-      end.join("\n")
-
-      <<~TOML
-        [dataset]
-
-        [dataset.header]
-        created = "#{data[:dataset][:header][:created]}"
-        count = #{records.length}
-        format = "toml"
-
-        #{records_toml}
-      TOML
+    def load_test_data
+      @test_data = TestData.load(@benchmark_config)
     end
   end
 end

data/lib/serialbench/cli/benchmark_cli.rb

@@ -284,7 +284,6 @@ module Serialbench
 
         say '✅ Local benchmark completed successfully!', :green
         say "Results saved to: #{result_dir}", :cyan
-        say "Generate site: serialbench benchmark build-site #{result_dir}", :white
       rescue StandardError => e
         say "❌ Local benchmark failed: #{e.message}", :red
         say "Details: #{e.backtrace.first(3).join("\n")}", :white if options[:verbose]
@@ -308,7 +307,6 @@ module Serialbench
 
         say '✅ Docker benchmark completed successfully!', :green
         say "Results saved to: #{result_dir}", :cyan
-        say "Generate site: serialbench benchmark build-site #{result_dir}", :white
       rescue StandardError => e
         say "❌ Docker benchmark failed: #{e.message}", :red
         say "Details: #{e.backtrace.first(3).join("\n")}", :white if options[:verbose]
@@ -357,92 +355,21 @@ module Serialbench
         exit 1
       end
 
-      # def execute_local_benchmark(environment, config, benchmark_config_path)
-      #   say '🏠 Executing local benchmark', :green
-
-      #   # Create benchmark runner with config
-      #   runner_options = {
-      #     formats: (config['formats'] || %w[xml json yaml toml]).map(&:to_sym),
-      #     iterations: config['iterations'] || 10,
-      #     warmup: config['warmup'] || 3,
-      #     config: config
-      #   }
-
-      #   runner = Serialbench::BenchmarkRunner.new(**runner_options)
-
-      #   # Run benchmarks
-      #   say "Running benchmarks with #{runner_options[:iterations]} iterations...", :white
-      #   results = runner.run_all_benchmarks
-
-      #   # Create platform-specific directory name using environment's ruby_build_tag
-      #   require_relative '../models/platform'
-      #   platform = Serialbench::Models::Platform.current_local
-      #   platform_string = "local-#{platform.os}-#{platform.arch}-ruby-#{environment['ruby_build_tag']}"
-
-      #   # Create results directory
-      #   result_dir = "results/runs/#{environment['name']}"
-      #   FileUtils.mkdir_p(result_dir)
-
-      #   # Save results to single YAML file with platform and metadata merged in
-      #   results_file = File.join(result_dir, 'results.yaml')
-      #   full_results = {
-      #     'platform' => {
-      #       'platform_string' => platform_string,
-      #       'os' => platform.os,
-      #       'arch' => platform.arch
-      #     },
-      #     'metadata' => {
-      #       'environment_name' => environment['name'],
-      #       'benchmark_config' => benchmark_config_path,
-      #       'created_at' => Time.now.iso8601,
-      #       'tags' => ['local', platform.os, platform.arch, "ruby-#{environment['ruby_build_tag']}"]
-      #     },
-      #     'environment' => {
-      #       'name' => environment['name'],
-      #       'type' => environment.kind,
-      #       'ruby_build_tag' => environment['ruby_build_tag'],
-      #       'created_at' => Time.now.iso8601
-      #     },
-      #     'config' => {
-      #       'benchmark_config' => benchmark_config_path,
-      #       'formats' => config['formats'],
-      #       'iterations' => config['iterations'],
-      #       'data_sizes' => config['data_sizes']
-      #     },
-      #     'results' => results
-      #   }
-
-      #   File.write(results_file, full_results.to_yaml)
-
-      #   say '✅ Local benchmark completed successfully!', :green
-      #   say "Results saved to: #{result_dir}", :cyan
-      #   say "Generate site: serialbench benchmark build-site #{result_dir}", :white
-      # rescue StandardError => e
-      #   say "❌ Local benchmark failed: #{e.message}", :red
-      #   say "Details: #{e.backtrace.first(3).join("\n")}", :white if options[:verbose]
-      #   raise e
-      # end
-
-      def execute_asdf_benchmark(environment, _config, benchmark_config_path)
+      def execute_asdf_benchmark(environment, config, benchmark_config_path)
         say '🔧 Executing ASDF benchmark', :green
 
-        # Use the ASDF runner to execute the benchmark
-        require_relative '../asdf_runner'
+        require_relative '../runners/asdf_runner'
 
-        # Create a config object that AsdfRunner expects
-        asdf_config = environment.merge({
-                                          'benchmark_config' => benchmark_config_path
-                                        })
+        environment_config_path = "config/environments/#{environment.name}.yml"
+        runner = Runners::AsdfRunner.new(environment, environment_config_path)
 
-        runner = Serialbench::AsdfRunner.new(asdf_config)
+        result_dir = "results/runs/#{environment.name}-results"
+        FileUtils.mkdir_p(result_dir)
 
-        say "Installing Ruby #{environment['ruby_build_tag']} via ASDF...", :white
-        runner.prepare
-        runner.benchmark
+        runner.run_benchmark(config, benchmark_config_path, result_dir)
 
         say '✅ ASDF benchmark completed successfully!', :green
-        say "Results saved to: results/runs/#{environment['name']}", :cyan
-        say "Generate site: serialbench benchmark build-site results/runs/#{environment['name']}", :white
+        say "Results saved to: #{result_dir}", :cyan
       rescue StandardError => e
         say "❌ ASDF benchmark failed: #{e.message}", :red
         say "Details: #{e.backtrace.first(3).join("\n")}", :white if options[:verbose]

data/lib/serialbench/cli.rb

@@ -6,7 +6,6 @@ require_relative 'cli/environment_cli'
 require_relative 'cli/benchmark_cli'
 require_relative 'cli/ruby_build_cli'
 require_relative 'cli/validate_cli'
-require_relative 'cli/resultset_cli'
 
 module Serialbench
   # Main CLI entry point for the new object-oriented command structure
@@ -58,10 +57,6 @@ module Serialbench
             serialbench benchmark create my-benchmark
             serialbench benchmark execute my-benchmark.yml
 
-            # Create a result set for comparison
-
-            # Generate static sites
-            serialbench benchmark build-site results/my-benchmark
 
           For detailed help on any command, use:
             serialbench COMMAND help

data/lib/serialbench/models/benchmark_config.rb

@@ -51,7 +51,7 @@ module Serialbench
       attribute :formats, :string, collection: true, values: %w[xml json yaml toml]
       attribute :iterations, BenchmarkIteration
       attribute :warmup, :integer, default: -> { 1 }
-      attribute :operations, :string, collection: true, values: %w[parse generate memory streaming]
+      attribute :operations, :string, collection: true, values: %w[parsing generation xpath streaming memory]
 
       key_value do
         map 'name', to: :name