junit_timing_splitter 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c22c4ca7f4c88bf202a2210e0f0e0739a3526c253e4936c96da3eef4128cab86
+  data.tar.gz: f162822a717e86cefa18d4f6ececc44bb2db61d7d02d7490c089f795f35eb7af
+SHA512:
+  metadata.gz: 69387862b8500f9ee3a1a4b5e944b9f2902c89d93c63125c8eaae38994ba8527fb10327df1888a4ea8c9d51d2d948ea0a2afca1f58bcbd3867a22aad097ec624
+  data.tar.gz: 68e59f1bf1c3039f4a6fdd19d67b2a5b4a970262035bccb38a15f639345ca4ef7531a6db401760db5bd8a37122b429655ad9f44f0e1282d075de75c527d073f0

data/bin/junit_timing_splitter

@@ -0,0 +1,5 @@
+#!/usr/bin/env
+
+require 'junit_timing_splitter'
+
+JunitTimingSplitter::Cli.start(ARGV)

data/lib/junit_timing_splitter/bucket.rb

@@ -0,0 +1,21 @@
+module JunitTimingSplitter
+  class Bucket
+    attr_accessor :files, :total_time
+
+    def initialize(files: [], total_time: 0.0)
+      @files = files
+      @total_time = total_time
+    end
+
+    def to_h
+      {
+        files: @files,
+        total_time: @total_time
+      }
+    end
+
+    def to_s
+      files.join(' ')
+    end
+  end
+end

data/lib/junit_timing_splitter/cli.rb

@@ -0,0 +1,84 @@
+# The CLI to split testcases into n buckets and read a specific bucket
+module JunitTimingSplitter
+  class Cli < Thor
+    desc 'split', 'Split test files by timing'
+    option :files, required: true, aliases: '-f', desc: 'Glob path to scan for test files'
+    option :buckets, required: true, aliases: '-b', type: :numeric, desc: 'Number of buckets'
+    option :schema, required: true, aliases: '-o', desc: 'Output JSON file that contains information for each bucket'
+    def split
+      parsed_timings = JunitTimingSplitter::Parser.new(options[:files]).execute
+      buckets = JunitTimingSplitter::Splitter.new(parsed_timings, options[:buckets]).execute
+      buckets_as_hashes = buckets.map(&:to_h)
+      FileUtils.mkdir_p(File.dirname(options[:schema]))
+      File.write(options[:schema], JSON.pretty_generate(buckets_as_hashes))
+      puts "Buckets written to #{options[:schema]}"
+    end
+
+    desc 'show', 'Show test files of a specific bucket from JSON file'
+    option :schema, required: true, aliases: '-s', desc: 'Specific Generated JSON file from split step'
+    option :bucket, required: true, aliases: '-i', type: :numeric, desc: 'Bucket number to read'
+    def show
+      begin
+        schema = JunitTimingSplitter::Schema.new(options[:schema])
+        files = schema.files_for_bucket(options[:bucket].to_i)
+
+        if files.any?
+          puts files.join(' ')
+        else
+          puts 'Bucket not found'
+          exit(1)
+        end
+      rescue IOError => e
+        puts e.message
+        exit(1)
+      end
+    end
+
+    desc 'scan', 'Scan folder or glob path for missing test files'
+    option :schema, required: true, aliases: '-s', desc: 'Specific Generated JSON file from split step'
+    option :files, required: true, aliases: '-f', desc: 'Glob path to scan for test files'
+    def scan
+      begin
+        schema = JunitTimingSplitter::Schema.new(options[:schema])
+        missing_files = schema.scan_missing_files(options[:files])
+
+        if missing_files.empty?
+          puts 'No missing test files detected.'
+        else
+          puts 'Missing test files:'
+          missing_files.each { |file| puts file }
+        end
+      rescue IOError => e
+        puts e.message
+        exit(1)
+      end
+    end
+
+    desc 'merge', 'Merge missing test files into buckets'
+    option :schema, required: true, aliases: '-s', desc: 'Specific Generated JSON file from split step'
+    option :files, required: true, aliases: '-f', desc: 'Glob path to scan for test files'
+    def merge
+      begin
+        schema = JunitTimingSplitter::Schema.new(options[:schema])
+        missing_files = schema.scan_missing_files(options[:files])
+
+        if missing_files.empty?
+          puts 'No missing test files to merge.'
+        else
+          splitter = JunitTimingSplitter::Splitter.new([], schema.buckets.size)
+          buckets = splitter.merge_missing_files(missing_files)
+          buckets_as_hashes = buckets.map(&:to_h)
+          File.write(options[:schema], JSON.pretty_generate(buckets_as_hashes))
+          puts "Missing files merged into buckets and written to #{options[:schema]}"
+        end
+      rescue IOError => e
+        puts e.message
+        exit(1)
+      end
+    end
+
+    def self.exit_on_failure?
+      true
+    end
+  end
+end

data/lib/junit_timing_splitter/parsed_timing.rb

@@ -0,0 +1,11 @@
+module JunitTimingSplitter
+  class ParsedTiming
+    attr_accessor :file, :total_time, :total_testcases
+
+    def initialize(file:, total_time:, total_testcases: 1)
+      @file = file
+      @total_time = total_time
+      @total_testcases = total_testcases
+    end
+  end
+end

data/lib/junit_timing_splitter/parser.rb

@@ -0,0 +1,39 @@
+module JunitTimingSplitter
+  class Parser
+    attr_reader :file_paths, :files, :parsed_timings
+
+    def initialize(file_paths)
+      @file_paths = file_paths
+      @files = Dir.glob(file_paths)
+      @parsed_timings = []
+
+      puts "Detected #{files.size} files"
+      @files.each { |file| puts "Detected file: #{file}" }
+    end
+
+    # Parse multiple rspec-results.xml files into a list of files and their execution times
+    def execute
+      files.each do |file_path|
+        File.open(file_path) do |file|
+          doc = Nokogiri::XML(file)
+
+          doc.xpath('//testcase').each do |testcase|
+            file = testcase['file']
+            time = testcase['time'].to_f
+            next if file.nil? || time.nil?
+
+            existing = @parsed_timings.find { |pd| pd.file == file }
+            if existing
+              existing.total_time += time
+              existing.total_testcases += 1
+            else
+              @parsed_timings << ParsedTiming.new(file: file, total_time: time)
+            end
+          end
+        end
+      end
+
+      @parsed_timings
+    end
+  end
+end

data/lib/junit_timing_splitter/schema.rb

@@ -0,0 +1,37 @@
+module JunitTimingSplitter
+  class Schema
+    attr_reader :path, :buckets
+
+    def initialize(path)
+      @path = path
+      validate_file
+      @buckets = JSON.parse(File.read(path))
+    end
+
+    # Retrieve files from a specific bucket
+    def files_for_bucket(bucket_number)
+      bucket = buckets[bucket_number]
+      bucket ? bucket['files'] : []
+    end
+
+    # Retrieve all parsed files across all buckets
+    def all_parsed_files
+      buckets.flat_map { |bucket| bucket['files'] }
+    end
+
+    # Scan for missing files in a specified glob path
+    def scan_missing_files(glob_path)
+      parsed_files = all_parsed_files.map { |file| File.expand_path(file) }
+      all_files = Dir.glob(glob_path).map { |file| File.expand_path(file) }
+      all_files - parsed_files
+    end
+
+    private
+
+    def validate_file
+      unless File.exist?(path)
+        raise IOError, "Schema file not found: #{path}"
+      end
+    end
+  end
+end

data/lib/junit_timing_splitter/splitter.rb

@@ -0,0 +1,59 @@
+module JunitTimingSplitter
+  class Splitter
+    attr_reader :parsed_timings, :total_splits, :buckets
+
+    def initialize(parsed_timings, total_splits)
+      @parsed_timings = parsed_timings
+      @total_splits = total_splits
+      @buckets = Array.new(total_splits) { Bucket.new }
+    end
+
+    # Split the parsed timings into buckets based on total_splits
+    def execute
+      # Sort by time descending
+      sorted_timings = parsed_timings.sort_by { |parsed_timing| -parsed_timing.total_time }
+
+      # Initialize buckets
+      @buckets = Array.new(total_splits) { Bucket.new }
+
+      # Greedily distribute files to minimize total time imbalance
+      sorted_timings.each do |timing|
+        min_bucket = @buckets.min_by { |bucket| bucket.total_time }
+        min_bucket.files << timing.file
+        min_bucket.total_time += timing.total_time
+      end
+
+      @buckets
+    end
+
+    def merge_missing_files(missing_files)
+      missing_files.each_with_index do |file, index|
+        bucket = @buckets[index % total_splits]
+        bucket.files << file
+        # Assuming a default time for missing files, e.g., 1.0
+        bucket.total_time += 1.0
+      end
+      @buckets
+    end
+
+    # Command to display a specific split, start with index 0
+    def inspect(split_number: nil)
+      unless split_number
+        buckets.each_with_index do |bucket, index|
+          puts "[BUCKET #{index} - #{bucket.total_time.round(2)}s] #{bucket.files.join(', ')}"
+        end
+
+        return
+      end
+
+      # split_number exists
+      if split_number > total_splits - 1
+        puts "Invalid split number. Total splits: #{total_splits}"
+        return
+      end
+
+      specific_bucket = buckets[split_number]
+      puts "[BUCKET #{split_number} - #{specific_bucket.total_time.round(2)}s] #{specific_bucket.files.join(', ')}"
+    end
+  end
+end

data/lib/junit_timing_splitter.rb

@@ -0,0 +1,13 @@
+require 'nokogiri'
+require "zeitwerk"
+require 'thor'
+require 'json'
+
+# Example Usage:
+# To analyze all XML files:
+# parsed_timings = JunitTimingSplitter::Parser.new('results_*.xml').execute
+# buckets = JunitTimingSplitter::Split.new(parsed_timings, 5).execute
+module JunitTimingSplitter
+  loader = Zeitwerk::Loader.for_gem
+  loader.setup # ready!
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: junit_timing_splitter
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Kim Yu Ng
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-11-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.15'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.15'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: A tool to optimize parallel test execution by analyzing JUnit XML results
+  and distributing test files across buckets.
+email: kimyu92@gmail.com
+executables:
+- junit_timing_splitter
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/junit_timing_splitter
+- lib/junit_timing_splitter.rb
+- lib/junit_timing_splitter/bucket.rb
+- lib/junit_timing_splitter/cli.rb
+- lib/junit_timing_splitter/parsed_timing.rb
+- lib/junit_timing_splitter/parser.rb
+- lib/junit_timing_splitter/schema.rb
+- lib/junit_timing_splitter/splitter.rb
+homepage: https://github.com/kimyu92/junit_timing_splitter
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: '4.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Split test files into evenly distributed buckets based on execution time
+test_files: []