xcknife 0.6.6 → 0.13.0

data/example/xcknife-exemplar.json-stream

@@ -1,15 +1,15 @@
-{"bundleName":"iPadTestTarget.xctest","deviceName":"iPad Air","event":"begin-ocunit","targetName":"iPadTestTarget","testType":"application-test","sdkName":"iphonesimulator9.2"}
+{"bundleName":"iPadTestTarget.xctest","deviceName":"iPad Air","event":"begin-ocunit","targetName":"iPadTestTarget","testType":"application-test","sdkName":"iphonesimulator13.6"}
 {"test":"1","className":"iPadTestClassFour","event":"end-test","totalDuration":"0"}
 {"test":"1","className":"iPadTestClassOne","event":"end-test","totalDuration":"0"}
 {"test":"1","className":"iPadTestClassThree","event":"end-test","totalDuration":"0"}
 {"test":"1","className":"iPadTestClassTwo","event":"end-test","totalDuration":"0"}
-{"bundleName":"iPhoneTestTarget.xctest","deviceName":"iPad Air","event":"begin-ocunit","targetName":"iPhoneTestTarget","testType":"application-test","sdkName":"iphonesimulator9.2"}
+{"bundleName":"iPhoneTestTarget.xctest","deviceName":"iPad Air","event":"begin-ocunit","targetName":"iPhoneTestTarget","testType":"application-test","sdkName":"iphonesimulator13.6"}
 {"test":"1","className":"iPhoneTestClassAlpha","event":"end-test","totalDuration":"0"}
 {"test":"1","className":"iPhoneTestClassBeta","event":"end-test","totalDuration":"0"}
 {"test":"1","className":"iPhoneTestClassDelta","event":"end-test","totalDuration":"0"}
 {"test":"1","className":"iPhoneTestClassGama","event":"end-test","totalDuration":"0"}
 {"test":"1","className":"iPhoneTestClassOmega","event":"end-test","totalDuration":"0"}
-{"bundleName":"CommonTestTarget.xctest","deviceName":"iPad Air","event":"begin-ocunit","targetName":"CommonTestTarget","testType":"logic-test","sdkName":"iphonesimulator9.2"}
+{"bundleName":"CommonTestTarget.xctest","deviceName":"iPad Air","event":"begin-ocunit","targetName":"CommonTestTarget","testType":"logic-test","sdkName":"iphonesimulator13.6"}
 {"test":"1","className":"CommonTestClass","event":"end-test","totalDuration":"0"}
 {"test":"1","className":"CommonTestClass","event":"end-test","totalDuration":"0"}
 {"test":"1","className":"CommonTestClass","event":"end-test","totalDuration":"0"}

data/lib/xcknife.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'xcknife/events_analyzer'
 require 'xcknife/stream_parser'
 require 'xcknife/xctool_cmd_helper'
@@ -7,5 +9,5 @@ require 'xcknife/exceptions'
 require 'xcknife/xcscheme_analyzer'
 
 module XCKnife
-  VERSION = '0.6.6'
+  VERSION = '0.13.0'
 end

data/lib/xcknife/events_analyzer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'xcknife/json_stream_parser_helper'
 require 'set'
 
@@ -8,6 +10,7 @@ module XCKnife
 
     def self.for(events, relevant_partitions)
       return NullEventsAnalyzer.new if events.nil?
+
       new(events, relevant_partitions)
     end
 
@@ -17,19 +20,21 @@ module XCKnife
       @target_class_map = analyze_events(events)
     end
 
-    def has_test_target?(target)
-      target_class_map.has_key?(target)
+    def test_target?(target)
+      target_class_map.key?(target)
     end
 
-    def has_test_class?(target, clazz)
-      has_test_target?(target) and target_class_map[target].include?(clazz)
+    def test_class?(target, clazz)
+      test_target?(target) and target_class_map[target].include?(clazz)
     end
 
     private
+
     def analyze_events(events)
       ret = Hash.new { |h, key| h[key] = Set.new }
       each_test_event(events) do |target_name, result|
         next unless @relevant_partitions.include?(target_name)
+
         @total_tests += 1
         ret[target_name] << result.className
       end
@@ -40,11 +45,11 @@ module XCKnife
   # Null object for EventsAnalyzer
   # @ref https://en.wikipedia.org/wiki/Null_Object_pattern
   class NullEventsAnalyzer
-    def has_test_target?(target)
+    def test_target?(_target)
       true
     end
 
-    def has_test_class?(target, clazz)
+    def test_class?(_target, _clazz)
       true
     end
 

data/lib/xcknife/exceptions.rb

@@ -1,6 +1,10 @@
+# frozen_string_literal: true
+
 module XCKnife
   # Base error class for xcknife
-  XCKnifeError = Class.new(StandardError)
+  XCKnifeError = Class.new(RuntimeError)
+
+  TestDumpError = Class.new(XCKnifeError)
 
   StreamParsingError = Class.new(XCKnifeError)
-end
+end

data/lib/xcknife/json_stream_parser_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module XCKnife
   module JsonStreamParserHelper
     extend self
@@ -6,18 +8,16 @@ module XCKnife
     def each_test_event(events, &block)
       current_target = nil
       events.each do |result|
-        current_target = result.targetName if result.event == "begin-ocunit"
-        if result.test and result.event == "end-test"
-          raise XCKnife::StreamParsingError, "No test target defined" if current_target.nil?
-          block.call(current_target, normalize_result(result))
-        end
+        current_target = result.targetName if result.event == 'begin-ocunit'
+        next unless result.test && (result.event == 'end-test')
+        raise XCKnife::StreamParsingError, 'No test target defined' if current_target.nil?
+
+        block.call(current_target, normalize_result(result))
       end
     end
 
     def normalize_result(result)
-      if result.totalDuration.is_a?(String)
-        result.totalDuration = result.totalDuration.to_f
-      end
+      result.totalDuration = result.totalDuration.to_f if result.totalDuration.is_a?(String)
       result
     end
   end

data/lib/xcknife/runner.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'optparse'
 
 module XCKnife
@@ -33,11 +35,11 @@ module XCKnife
     end
 
     private
+
     def gen_abbreviated_output(result)
       result.test_maps.map { |partition_set| only_arguments_for_a_partition_set(partition_set, output_type) }
     end
 
-
     def output_type
       @xcodebuild_output ? :xcodebuild : :xctool
     end
@@ -47,7 +49,7 @@ module XCKnife
         metadata: {
           worker_count: @worker_count,
           partition_set_count: result.test_maps.size,
-          total_time_in_ms: result.total_test_time,
+          total_time_in_ms: result.total_test_time
         }.merge(result.stats.to_h),
         partition_set_data: partition_sets_data(result)
       }
@@ -81,18 +83,15 @@ module XCKnife
     def write_output(data)
       json = JSON.pretty_generate(data)
       return puts json if @output_file_name.nil?
-      File.open(@output_file_name, "w") { |f| f.puts(json) }
+
+      File.open(@output_file_name, 'w') { |f| f.puts(json) }
       puts "Wrote file to: #{@output_file_name}"
     end
 
     def parse_arguments(args)
       positional_arguments = parse_options(args)
-      if positional_arguments.size < required_arguments.size
-        warn_and_exit("You must specify *all* required arguments: #{required_arguments.join(", ")}")
-      end
-      if @partitions.empty?
-        warn_and_exit("At least one target partition set must be provided with -p flag")
-      end
+      warn_and_exit("You must specify *all* required arguments: #{required_arguments.join(', ')}") if positional_arguments.size < required_arguments.size
+      warn_and_exit('At least one target partition set must be provided with -p flag') if @partitions.empty?
       worker_count, @historical_timings_file, @current_tests_file = positional_arguments
       @worker_count = Integer(worker_count)
     end
@@ -101,24 +100,24 @@ module XCKnife
       build_parser
       begin
         parser.parse(args)
-      rescue OptionParser::ParseError => error
-        warn_and_exit(error)
+      rescue OptionParser::ParseError => e
+        warn_and_exit(e)
       end
     end
 
     def build_parser
       @parser = OptionParser.new do |opts|
         opts.banner += " #{arguments_banner}"
-        opts.on("-p", "--partition TARGETS",
-          "Comma separated list of targets. Can be used multiple times.") do |v|
+        opts.on('-p', '--partition TARGETS',
+                'Comma separated list of targets. Can be used multiple times.') do |v|
           @partition_names << v
-          @partitions << v.split(",")
+          @partitions << v.split(',')
         end
-        opts.on("-o", "--output FILENAME", "Output file. Defaults to STDOUT") { |v| @output_file_name = v }
-        opts.on("-a", "--abbrev", "Results are abbreviated") { |v| @abbreviated_output = v }
-        opts.on("-x", "--xcodebuild-output", "Output is formatted for xcodebuild") { |v| @xcodebuild_output = v }
+        opts.on('-o', '--output FILENAME', 'Output file. Defaults to STDOUT') { |v| @output_file_name = v }
+        opts.on('-a', '--abbrev', 'Results are abbreviated') { |v| @abbreviated_output = v }
+        opts.on('-x', '--xcodebuild-output', 'Output is formatted for xcodebuild') { |v| @xcodebuild_output = v }
 
-        opts.on_tail("-h", "--help", "Show this message") do
+        opts.on_tail('-h', '--help', 'Show this message') do
           puts opts
           exit
         end
@@ -135,7 +134,7 @@ module XCKnife
 
     def arguments_banner
       optional_args = optional_arguments.map { |a| "[#{a}]" }
-      (required_arguments + optional_args).join(" ")
+      (required_arguments + optional_args).join(' ')
     end
 
     def warn_and_exit(msg)

data/lib/xcknife/stream_parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'xcknife/json_stream_parser_helper'
 require 'json'
 require 'set'
@@ -10,21 +12,31 @@ module XCKnife
 
     attr_reader :number_of_shards, :test_partitions, :stats, :relevant_partitions
 
-    def initialize(number_of_shards, test_partitions)
+    def initialize(number_of_shards, test_partitions, options_for_metapartition: Array.new(test_partitions.size, {}), allow_fewer_shards: false, on_extrapolation: nil)
       @number_of_shards = number_of_shards
       @test_partitions = test_partitions.map(&:to_set)
       @relevant_partitions = test_partitions.flatten.to_set
       @stats = ResultStats.new
-      ResultStats.members.each { |k| @stats[k] = 0}
+      @options_for_metapartition = options_for_metapartition.map { |o| Options::DEFAULT.merge(o) }
+      @allow_fewer_shards = allow_fewer_shards
+      @on_extrapolation = on_extrapolation
+      ResultStats.members.each { |k| @stats[k] = 0 }
     end
 
-    PartitionWithMachines = Struct.new :test_time_map, :number_of_shards, :partition_time, :max_shard_count
+    PartitionWithMachines = Struct.new :test_time_map, :number_of_shards, :partition_time, :max_shard_count, :options
     MachineAssignment = Struct.new :test_time_map, :total_time
     ResultStats = Struct.new :historical_total_tests, :current_total_tests, :class_extrapolations, :target_extrapolations
+    Options = Struct.new :max_shard_count, :split_bundles_across_machines, :allow_fewer_shards do
+      def merge(hash)
+        self.class.new(*to_h.merge(hash).values_at(*members))
+      end
+    end
+    Options::DEFAULT = Options.new(nil, true, false).freeze
 
     class PartitionResult
       TimeImbalances = Struct.new :partition_set, :partitions
       attr_reader :stats, :test_maps, :test_times, :total_test_time, :test_time_imbalances, :test_time_for_partitions
+
       extend Forwardable
       delegate ResultStats.members => :@stats
 
@@ -39,10 +51,11 @@ module XCKnife
       end
 
       private
+
       # Yields the imbalances ratios of the partition sets, and the internal imbalance ratio of the respective partitions
       def compute_test_time_imbalances
         times = test_times
-        average_partition_size = times.map { |l| l.inject(:+).to_f / l.size}
+        average_partition_size = times.map { |l| l.inject(:+).to_f / l.size }
         ideal_partition_set_avg = average_partition_size.inject(:+) / @partition_sets.size
         partition_set_imbalance = average_partition_size.map { |avg| avg / ideal_partition_set_avg }
 
@@ -75,18 +88,20 @@ module XCKnife
 
     def compute_shards_for_partitions(test_time_for_partitions)
       PartitionResult.new(@stats, split_machines_proportionally(test_time_for_partitions).map do |partition|
-        compute_single_shards(partition.number_of_shards, partition.test_time_map)
+        compute_single_shards(partition.number_of_shards, partition.test_time_map, options: partition.options)
       end, test_time_for_partitions)
     end
 
     def test_time_for_partitions(historical_events, current_events = nil)
-      analyzer =  EventsAnalyzer.for(current_events, relevant_partitions)
+      analyzer = EventsAnalyzer.for(current_events, relevant_partitions)
       @stats[:current_total_tests] = analyzer.total_tests
       times_for_target_class = Hash.new { |h, current_target| h[current_target] = Hash.new(0) }
       each_test_event(historical_events) do |target_name, result|
         next unless relevant_partitions.include?(target_name)
+
         inc_stat :historical_total_tests
-        next unless analyzer.has_test_class?(target_name, result.className)
+        next unless analyzer.test_class?(target_name, result.className)
+
         times_for_target_class[target_name][result.className] += (result.totalDuration * 1000).ceil
       end
 
@@ -94,43 +109,47 @@ module XCKnife
       hash_partitions(times_for_target_class)
     end
 
+    # rubocop:disable Metrics/CyclomaticComplexity
     def split_machines_proportionally(partitions)
       total = 0
       partitions.each do |test_time_map|
-        each_duration(test_time_map) { |duration_in_milliseconds| total += duration_in_milliseconds}
+        each_duration(test_time_map) { |duration_in_milliseconds| total += duration_in_milliseconds }
       end
 
       used_shards = 0
       assignable_shards = number_of_shards - partitions.size
-      partition_with_machines_list = partitions.map do |test_time_map|
+      partition_with_machines_list = partitions.each_with_index.map do |test_time_map, metapartition|
+        options = @options_for_metapartition[metapartition]
         partition_time = 0
-        max_shard_count = test_time_map.values.map(&:size).inject(:+) || 1
-        each_duration(test_time_map) { |duration_in_milliseconds| partition_time += duration_in_milliseconds}
+        max_shard_count = test_time_map.each_value.map(&:size).reduce(&:+) || 1
+        max_shard_count = [max_shard_count, options.max_shard_count].min if options.max_shard_count
+        each_duration(test_time_map) { |duration_in_milliseconds| partition_time += duration_in_milliseconds }
         n = [1 + (assignable_shards * partition_time.to_f / total).floor, max_shard_count].min
         used_shards += n
-        PartitionWithMachines.new(test_time_map, n, partition_time, max_shard_count)
+        PartitionWithMachines.new(test_time_map, n, partition_time, max_shard_count, options)
       end
 
-      fifo_with_machines_who_can_use_more_shards = partition_with_machines_list.select { |x| x.number_of_shards < x.max_shard_count}.sort_by(&:partition_time)
-      while (number_of_shards - used_shards) > 0
+      fifo_with_machines_who_can_use_more_shards = partition_with_machines_list.select { |x| x.number_of_shards < x.max_shard_count }.sort_by(&:partition_time)
+      while number_of_shards > used_shards
         if fifo_with_machines_who_can_use_more_shards.empty?
+          break if @allow_fewer_shards
+
           raise XCKnife::XCKnifeError, "There are #{number_of_shards - used_shards} extra machines"
         end
         machine = fifo_with_machines_who_can_use_more_shards.pop
         machine.number_of_shards += 1
         used_shards += 1
-        if machine.number_of_shards < machine.max_shard_count
-          fifo_with_machines_who_can_use_more_shards.unshift(machine)
-        end
+        fifo_with_machines_who_can_use_more_shards.unshift(machine) if machine.number_of_shards < machine.max_shard_count
       end
       partition_with_machines_list
     end
 
     # Computes a 2-aproximation to the optimal partition_time, which is an instance of the Open shop scheduling problem (which is NP-hard)
     # see: https://en.wikipedia.org/wiki/Open-shop_scheduling
-    def compute_single_shards(number_of_shards, test_time_map)
-      raise XCKnife::XCKnifeError, "There are not enough workers provided" if number_of_shards <= 0
-      raise XCKnife::XCKnifeError, "Cannot shard an empty partition_time" if test_time_map.empty?
+    def compute_single_shards(number_of_shards, test_time_map, options: Options::DEFAULT)
+      raise XCKnife::XCKnifeError, 'There are not enough workers provided' if number_of_shards <= 0
+      raise XCKnife::XCKnifeError, 'Cannot shard an empty partition_time' if test_time_map.empty?
+
       assignements = Array.new(number_of_shards) { MachineAssignment.new(Hash.new { |k, v| k[v] = [] }, 0) }
 
       list_of_test_target_class_times = []
@@ -140,36 +159,61 @@ module XCKnife
         end
       end
 
-      list_of_test_target_class_times.sort_by! { |test_target, class_name, duration_in_milliseconds| -duration_in_milliseconds }
-      list_of_test_target_class_times.each do |test_target, class_name, duration_in_milliseconds|
+      # This might seem like an uncessary level of indirection, but it allows us to keep
+      # logic consistent regardless of the `split_bundles_across_machines` option
+      group = list_of_test_target_class_times.group_by do |test_target, class_name, _duration_in_milliseconds|
+        options.split_bundles_across_machines ? [test_target, class_name] : test_target
+      end
+
+      list_of_test_target_classes_times = group.map do |(test_target, _), classes|
+        [
+          test_target,
+          classes.map { |_test_target, class_name, _duration_in_milliseconds| class_name },
+          classes.reduce(0) { |total_duration, (_test_target, _class_name, duration_in_milliseconds)| total_duration + duration_in_milliseconds }
+        ]
+      end
+
+      list_of_test_target_classes_times.sort_by! { |_test_target, _class_names, duration_in_milliseconds| -duration_in_milliseconds }
+      list_of_test_target_classes_times.each do |test_target, class_names, duration_in_milliseconds|
         assignemnt = assignements.min_by(&:total_time)
-        assignemnt.test_time_map[test_target] << class_name
+        assignemnt.test_time_map[test_target].concat class_names
         assignemnt.total_time += duration_in_milliseconds
       end
-      raise XCKnife::XCKnifeError, "Too many shards" if assignements.any? { |a| a.test_time_map.empty? }
+
+      if (empty_test_map_assignments = assignements.select { |a| a.test_time_map.empty? }) && !empty_test_map_assignments.empty? && !options.allow_fewer_shards
+        test_grouping = options.split_bundles_across_machines ? 'classes' : 'targets'
+        raise XCKnife::XCKnifeError, "Too many shards -- #{empty_test_map_assignments.size} of #{number_of_shards} assignments are empty," \
+                                     " because there are not enough test #{test_grouping} for that many shards."
+      end
+      assignements.reject! { |a| a.test_time_map.empty? }
+
       assignements
     end
+    # rubocop:enable Metrics/CyclomaticComplexity
 
     def parse_json_stream_file(filename)
       return nil if filename.nil?
-      return [] unless File.exists?(filename)
+      return [] unless File.exist?(filename)
+
       lines = IO.readlines(filename)
-      lines.lazy.map { |line| OpenStruct.new(JSON.load(line)) }
+      lines.lazy.map { |line| OpenStruct.new(JSON.parse(line)) }
     end
 
     private
+
     def inc_stat(name)
       @stats[name] += 1
     end
 
-    def each_duration(test_time_map, &block)
-      test_time_map.each do |test_target, class_times|
-        class_times.each do |class_name, duration_in_milliseconds|
+    def each_duration(test_time_map)
+      test_time_map.each do |_test_target, class_times|
+        class_times.each do |_class_name, duration_in_milliseconds|
           yield(duration_in_milliseconds)
         end
       end
     end
 
+    # rubocop:disable Metrics/CyclomaticComplexity
     def extrapolate_times_for_current_events(analyzer, times_for_target_class)
       median_map = {}
       times_for_target_class.each do |test_target, class_times|
@@ -179,17 +223,20 @@ module XCKnife
       all_times_for_all_classes = times_for_target_class.values.flat_map(&:values)
       median_of_targets = median(all_times_for_all_classes)
       analyzer.target_class_map.each do |test_target, class_set|
-        if times_for_target_class.has_key?(test_target)
+        if times_for_target_class.key?(test_target)
           class_set.each do |clazz|
-            unless times_for_target_class[test_target].has_key?(clazz)
-              inc_stat :class_extrapolations
-              times_for_target_class[test_target][clazz] = median_map[test_target]
-            end
+            next if times_for_target_class[test_target].key?(clazz)
+
+            inc_stat :class_extrapolations
+            @on_extrapolation&.call(test_target: test_target, test_class: clazz)
+            times_for_target_class[test_target][clazz] = median_map[test_target]
           end
         else
           inc_stat :target_extrapolations
+          @on_extrapolation&.call(test_target: test_target, test_class: nil)
           class_set.each do |clazz|
             inc_stat :class_extrapolations
+            @on_extrapolation&.call(test_target: test_target, test_class: clazz)
             times_for_target_class[test_target][clazz] = extrapolated_duration(median_of_targets, class_set)
           end
         end
@@ -199,6 +246,7 @@ module XCKnife
     DEFAULT_EXTRAPOLATED_DURATION = 1000
     def extrapolated_duration(median_of_targets, class_set)
       return DEFAULT_EXTRAPOLATED_DURATION if median_of_targets.nil?
+
       median_of_targets / class_set.size
     end
 
@@ -214,10 +262,9 @@ module XCKnife
         end
       end
       ret.each_with_index do |partition, index|
-        if partition.empty?
-          raise XCKnife::XCKnifeError, "The following partition has no tests: #{test_partitions[index].to_a.inspect}"
-        end
+        raise XCKnife::XCKnifeError, "The following partition has no tests: #{test_partitions[index].to_a.inspect}" if partition.empty?
       end
     end
   end
+  # rubocop:enable Metrics/CyclomaticComplexity
 end