serialbench 0.1.1 → 0.1.3

data/Rakefile

@@ -5,59 +5,4 @@ require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-desc 'Run benchmarks'
-task :benchmark do
-  require_relative 'lib/serialbench'
-
-  puts 'Running XML benchmarks...'
-  results = Serialbench.run_benchmarks
-
-  puts "\nGenerating reports..."
-  report_files = Serialbench.generate_reports(results)
-
-  puts "\nBenchmark complete!"
-  puts 'Reports generated:'
-  puts "  HTML: #{report_files[:html]}"
-  puts "  AsciiDoc: #{report_files[:asciidoc]}"
-  puts "  Charts: #{report_files[:charts].length} SVG files"
-end
-
-desc 'Install all XML library dependencies'
-task :install_deps do
-  gems = %w[ox nokogiri libxml-ruby oga memory_profiler]
-
-  puts 'Installing XML library dependencies...'
-  gems.each do |gem_name|
-    puts "Installing #{gem_name}..."
-    system("gem install #{gem_name}")
-  rescue StandardError => e
-    puts "Warning: Failed to install #{gem_name}: #{e.message}"
-  end
-  puts 'Done!'
-end
-
-desc 'Check which XML libraries are available'
-task :check_libs do
-  require_relative 'lib/serialbench'
-
-  runner = Serialbench::BenchmarkRunner.new
-
-  puts 'XML Library Availability Check'
-  puts '=' * 40
-
-  runner.parsers.each do |parser|
-    status = parser.available? ? '✓ Available' : '✗ Not available'
-    version = parser.available? ? " (#{parser.version})" : ''
-    puts "#{parser.name.ljust(15)} #{status}#{version}"
-  end
-
-  puts "\nMemory profiler: #{Serialbench::MemoryProfiler.available? ? '✓ Available' : '✗ Not available'}"
-end
-
-desc 'Clean generated files'
-task :clean do
-  FileUtils.rm_rf('results')
-  puts 'Cleaned generated files'
-end
-
 task default: :spec

data/config/benchmarks/full.yml

@@ -0,0 +1,29 @@
+# Configuration for comprehensive benchmarks - Full testing with all data sizes
+# Used by Docker script for complete performance analysis
+
+# Run name for identification
+name: full-benchmark
+
+data_sizes:
+- small
+- medium
+- large
+
+formats:
+- xml
+- json
+- yaml
+- toml
+
+iterations:
+  small: 20
+  medium: 5
+  large: 2
+
+operations:
+- parse
+- generate
+- streaming
+- memory
+
+warmup: 3

data/config/benchmarks/short.yml

@@ -0,0 +1,26 @@
+# Configuration for short benchmarks - Minimal memory usage for CI runners
+# Designed for Windows and Ubuntu runners with limited memory
+
+# Run name for identification
+name: short-benchmark
+
+data_sizes:
+- small
+
+formats:
+- xml
+- json
+- yaml
+- toml
+
+iterations:
+  small: 5
+  medium: 2
+  large: 1
+
+operations:
+- parse
+- generate
+- streaming
+
+warmup: 2

data/config/environments/asdf-ruby-3.2.yml

@@ -0,0 +1,8 @@
+---
+name: ruby-324-asdf
+kind: asdf
+created_at: '2025-06-12T22:54:43+08:00'
+ruby_build_tag: 3.2.4
+description: ASDF environment
+asdf:
+  auto_install: true

data/config/environments/asdf-ruby-3.3.yml

@@ -0,0 +1,8 @@
+---
+name: ruby-332-asdf
+kind: asdf
+created_at: '2025-06-12T22:53:24+08:00'
+ruby_build_tag: 3.3.2
+description: ASDF environment
+asdf:
+  auto_install: true

data/config/environments/docker-ruby-3.0.yml

@@ -0,0 +1,9 @@
+---
+name: docker-ruby-3.0
+kind: docker
+created_at: '2025-06-13T15:18:43+08:00'
+ruby_build_tag: "3.0.7"
+description: Docker environment for Ruby 3.0 benchmarks
+docker:
+  image: 'ruby:3.0-slim'
+  dockerfile: '../../docker/Dockerfile.ubuntu'

data/config/environments/docker-ruby-3.1.yml

@@ -0,0 +1,9 @@
+---
+name: docker-ruby-3.1
+kind: docker
+created_at: '2025-06-13T15:18:43+08:00'
+ruby_build_tag: "3.1.6"
+description: Docker environment for Ruby 3.1 benchmarks
+docker:
+  image: 'ruby:3.1-slim'
+  dockerfile: '../../docker/Dockerfile.ubuntu'

data/config/environments/docker-ruby-3.2.yml

@@ -0,0 +1,9 @@
+---
+name: docker-ruby-3.2
+kind: docker
+created_at: '2025-06-13T15:18:43+08:00'
+ruby_build_tag: "3.2.4"
+description: Docker environment for Ruby 3.2 benchmarks
+docker:
+  image: 'ruby:3.2-slim'
+  dockerfile: '../../docker/Dockerfile.ubuntu'

data/config/environments/docker-ruby-3.3.yml

@@ -0,0 +1,9 @@
+---
+name: docker-ruby-3.3
+kind: docker
+created_at: '2025-06-13T15:18:43+08:00'
+ruby_build_tag: "3.3.7"
+description: Docker environment for Ruby 3.3 benchmarks
+docker:
+  image: 'ruby:3.3-slim'
+  dockerfile: '../../docker/Dockerfile.ubuntu'

data/config/environments/docker-ruby-3.4.yml

@@ -0,0 +1,9 @@
+---
+name: docker-ruby-3.4
+kind: docker
+created_at: '2025-06-13T15:18:43+08:00'
+ruby_build_tag: "3.4.1"
+description: Docker environment for Ruby 3.4 benchmarks
+docker:
+  image: 'ruby:3.4-slim'
+  dockerfile: '../../docker/Dockerfile.ubuntu'

data/data/schemas/result.yml

@@ -0,0 +1,29 @@
+---
+type: object
+title: Serialbench Result
+description: Schema for serialbench benchmark result files (basic structure validation)
+properties:
+  platform:
+    type: object
+    properties:
+      platform_string:
+        type: string
+      os:
+        type: string
+      arch:
+        type: string
+      ruby_version:
+        type: string
+  metadata:
+    type: object
+  environment_config:
+    type: object
+    properties:
+      name:
+        type: string
+      kind:
+        type: string
+  benchmark_config:
+    type: object
+  benchmark_result:
+    type: object

data/docker/Dockerfile.alpine

@@ -0,0 +1,33 @@
+# Alpine-based Ruby benchmark environment using official Ruby Alpine images
+ARG BASE_IMAGE=ruby:3.4-alpine
+FROM ${BASE_IMAGE}
+
+# Install system dependencies for XML libraries and build tools
+RUN apk add --no-cache \
+    build-base \
+    libxml2-dev \
+    libxslt-dev \
+    yaml-dev \
+    zlib-dev \
+    openssl-dev \
+    linux-headers \
+    git
+
+# Set working directory
+WORKDIR /app
+
+# Copy the entire application (gemspec needs full context)
+COPY . .
+
+# Update bundler and configure bundle for cross-platform compatibility
+RUN gem install bundler:2.5.22 && \
+    bundle config set --local deployment 'false' && \
+    bundle config set --local path '/usr/local/bundle' && \
+    bundle config set --local force_ruby_platform true && \
+    bundle install --jobs 4 --retry 3
+
+# Create results directory
+RUN mkdir -p /app/results
+
+# Default command runs benchmarks
+ENTRYPOINT ["bundle", "exec", "serialbench"]

data/docker/{Dockerfile.benchmark → Dockerfile.ubuntu}

@@ -1,14 +1,15 @@
 # Multi-stage Dockerfile for Ruby serialization benchmarks
 # Supports multiple Ruby versions for comprehensive testing
 
-ARG RUBY_VERSION=3.3
-FROM ruby:${RUBY_VERSION}
+ARG BASE_IMAGE=ruby:3.4-slim
+FROM ${BASE_IMAGE}
 
 # Install system dependencies for XML libraries
 RUN apt-get update && apt-get install -y \
     libxml2-dev \
     libxslt1-dev \
     build-essential \
+    git \
     && rm -rf /var/lib/apt/lists/*
 
 # Set working directory
@@ -28,4 +29,4 @@ RUN gem install bundler:2.5.22 && \
 RUN mkdir -p /app/results
 
 # Default command runs parsing and generation benchmarks (memory benchmarks disabled due to hanging in Docker)
-CMD ["bundle", "exec", "ruby", "exe/serialbench", "benchmark", "--formats", "xml", "json", "yaml", "toml", "--iterations", "1", "--warmup", "0", "--parsing_only"]
+ENTRYPOINT ["bundle", "exec", "serialbench"]

data/docker/README.md

@@ -4,7 +4,7 @@ This directory contains Docker infrastructure for running Serialbench across mul
 
 ## Files
 
-- `Dockerfile.benchmark` - Multi-stage Dockerfile for building benchmark environments
+- `Dockerfile.ubuntu` - Multi-stage Dockerfile for building benchmark environments
 - `run-benchmarks.sh` - Automated script for running benchmarks across multiple Ruby versions
 
 ## Quick Start
@@ -77,7 +77,7 @@ docker-results/
 docker build \
   --build-arg RUBY_VERSION=3.3 \
   -t serialbench:ruby-3.3 \
-  -f docker/Dockerfile.benchmark \
+  -f docker/Dockerfile.ubuntu \
   .
 ```
 

data/docs/PLATFORM_VALIDATION_FIX.md

@@ -0,0 +1,79 @@
+# Platform Validation Fix
+
+## Problem
+
+GitHub Actions workflow run [18624230866](https://github.com/metanorma/serialbench/actions/runs/18624230866/job/53099708431) failed with the error:
+
+```
+Error adding run to resultset: undefined method `platform_string' for nil
+```
+
+This occurred when trying to add benchmark results from `artifacts/benchmark-results-macos-latest-ruby-3.2` to the weekly resultset.
+
+## Root Cause
+
+The error occurred in `lib/serialbench/models/result_set.rb` line 56, where the code attempted to access `result.platform.platform_string` to check for duplicate results. When `result.platform` is `nil`, this causes a NoMethodError.
+
+The underlying issue is that some result files were missing the `platform` section in their YAML data, causing `Result.load()` to return a Result object with a nil platform.
+
+## Solution Implemented
+
+Added validation in the `ResultSet#add_result` method to check for required fields before attempting to access their properties:
+
+```ruby
+# Validate that the result has required fields
+raise ArgumentError, "Result from #{result_path} is missing platform information" if result.platform.nil?
+raise ArgumentError, "Result from #{result_path} is missing environment_config" if result.environment_config.nil?
+raise ArgumentError, "Result from #{result_path} is missing benchmark_config" if result.benchmark_config.nil?
+```
+
+This provides clear, actionable error messages that indicate:
+1. Which result file has the problem
+2. Which specific field is missing
+
+## Testing
+
+Created comprehensive RSpec tests in `spec/serialbench/models/result_set_spec.rb` that verify:
+- Missing platform information raises appropriate error
+- Missing environment_config raises appropriate error
+- Missing benchmark_config raises appropriate error
+- Complete results are added successfully
+
+All tests pass.
+
+## Next Steps
+
+While this fix improves error reporting, the underlying issue of why platform data is missing needs investigation:
+
+1. **Investigate Result Generation**: Check why some results are being created without platform data
+   - Review `LocalRunner#run_benchmark` in `lib/serialbench/runners/local_runner.rb`
+   - Review how platform data is serialized in `Platform.to_yaml`
+   - Check if there's a Lutaml::Model serialization issue
+
+2. **GitHub Actions Debugging**:
+   - Add debugging output to show the contents of result files before adding them to resultsets
+   - Verify that all benchmark runs are creating complete result files with platform data
+
+3. **Platform Serialization**: Ensure `Platform` model properly serializes/deserializes:
+   - All attributes are included in YAML output
+   - Default values are properly set
+   - No Lutaml::Model configuration issues
+
+## Commit
+
+```
+fix(resultset): add validation for missing platform data in results
+
+Previously, when adding a result to a resultset, if the result's YAML file
+was missing platform information (or environment_config/benchmark_config),
+the code would fail with 'undefined method platform_string for nil' when
+trying to check for duplicates.
+
+This commit adds proper validation to check for nil values before accessing
+their properties, providing clear error messages that indicate which field
+is missing and which result path has the issue.
+
+Also adds comprehensive RSpec tests to verify the validation works correctly
+for all three required fields: platform, environment_config, and benchmark_config.
+
+Fixes the error seen in GitHub Actions workflow run 18624230866.

data/docs/SYCK_YAML_FIX.md

@@ -0,0 +1,91 @@
+# Syck YAML Constant Fix
+
+## Problem Summary
+
+GitHub Actions workflow failed with cryptic error:
+```
+Error adding run to resultset: undefined method `platform_string' for nil
+```
+
+Root cause: Benchmark results.yaml files were only 2 bytes containing `{}` instead of the expected ~10KB files with complete benchmark data.
+
+## Root Cause Analysis
+
+After 4 rounds of progressive fixes, we discovered the Syck gem was overriding the YAML constant, causing Lutaml::Model's `to_yaml` method to produce empty output.
+
+### Fix Timeline
+
+1. **Fix #1 (Commit 53f1799)**: Corrected `memory_usage` → `memory` attribute name
+   - Result: Still produced empty `{}` files
+
+2. **Fix #2 (Commit fcd23f2)**: Added key_value blocks to BenchmarkResult models
+   - Result: Still produced empty `{}` files
+
+3. **Fix #3 (Commit d8fb78b)**: Added key_value blocks to all remaining models
+   - Result: Still produced empty `{}` files
+
+4. **Fix #4 (Commit 69a2bd7)**: **Restored YAML constant to Psych in LocalRunner**
+   - Result: ✅ **SUCCESS! 10KB files with complete data**
+
+## The Critical Fix
+
+Added to `lib/serialbench/runners/local_runner.rb`:
+
+```ruby
+# Restore YAML to use Psych for output, otherwise lutaml-model's to_yaml
+# will have no output (Syck gem overrides YAML constant)
+Object.const_set(:YAML, Psych)
+
+results_file = File.join(result_dir, 'results.yaml')
+results_model.to_file(results_file)
+```
+
+## Why This Happened
+
+- The `_docker_execute` method in `benchmark_cli.rb` already had this fix (lines 151-154)
+- Local testing with `serialbench benchmark _docker_execute` worked fine
+- GHA with `serialbench environment execute` (uses LocalRunner) failed
+- The Syck gem overrides the YAML constant during benchmark execution
+- Without restoring it to Psych, Lutaml::Model serialization produces empty output
+
+## Verification
+
+From successful GHA run #51:
+
+```
+File size: 10258 bytes
+✅ File size OK: 10258 bytes
+✅ YAML syntax valid
+
+---
+platform:
+  platform_string: local-3.4.7
+  kind: local
+  os: macos
+  arch: arm64
+  ruby_build_tag: 3.4.7
+metadata:
+  benchmark_config_path: config/benchmarks/short.yml
+  environment_config_path: config/environments/ci-ruby-3.4.yml
+  tags:
+  - local
+  - macos
+  - arm64
+  - ruby-3.4
+[...]
+```
+
+## Impact
+
+This fix ensures:
+1. Benchmark results are properly serialized in all execution contexts
+2. ResultSet aggregation works correctly
+3. No more cryptic nil errors
+4. Complete benchmark data flows through the entire pipeline
+
+## Lessons Learned
+
+1. Always check for gem interference with global constants
+2. The Syck gem is a known source of YAML constant conflicts
+3. Test in the same execution context as production (LocalRunner vs DockerRunner behavior differs)
+4. Even with proper model definitions, external factors can break serialization