xcknife 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,18 @@
1
1
  require 'pp'
2
2
  module XCKnife
3
3
  module XCToolCmdHelper
4
+ def only_arguments_for_a_partition_set(output_type, partition_set)
5
+ method = "#{output_type}_only_arguments_for_a_partition_set"
6
+ raise "Unknown output_type: #{output_type}" unless respond_to?(method)
7
+ __send__(method, partition_set)
8
+ end
9
+
10
+ def only_arguments(output_type, partition)
11
+ method = "#{output_type}_only_arguments"
12
+ raise "Unknown output_type: #{output_type}" unless respond_to?(method)
13
+ __send__(method, partition)
14
+ end
15
+
4
16
  def xctool_only_arguments(single_partition)
5
17
  single_partition.flat_map do |test_target, classes|
6
18
  ['-only', "#{test_target}:#{classes.sort.join(',')}"]
@@ -10,5 +22,19 @@ module XCKnife
10
22
  def xctool_only_arguments_for_a_partition_set(partition_set)
11
23
  partition_set.map { |partition| xctool_only_arguments(partition) }
12
24
  end
25
+
26
+ # only-testing is available since Xcode 8
27
+ def xcodebuild_only_arguments(single_partition)
28
+ single_partition.flat_map do |test_target, classes|
29
+ classes.sort.map do |clazz|
30
+ "-only-testing:#{test_target}/#{clazz}"
31
+ end
32
+
33
+ end
34
+ end
35
+
36
+ def xcodebuild_only_arguments_for_a_partition_set(partition_set)
37
+ partition_set.map { |partition| xctool_only_arguments(partition) }
38
+ end
13
39
  end
14
40
  end
data/xcknife.gemspec CHANGED
@@ -20,9 +20,9 @@ Gem::Specification.new do |s|
20
20
  # Only allow gem to be pushed to https://rubygems.org
21
21
  s.metadata["allowed_push_host"] = 'https://rubygems.org'
22
22
 
23
- s.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
23
+ s.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR).reject { |f| f =~ /^spec/} + ["TestDumper/TestDumper.dylib"]
24
24
  s.bindir = 'bin'
25
- s.executables = ['xcknife', 'xcknife-min']
25
+ s.executables = ['xcknife', 'xcknife-min', 'xcknife-test-dumper']
26
26
  s.require_paths = ['lib']
27
27
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
28
28
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: xcknife
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.0
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Ribeiro
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-07 00:00:00.000000000 Z
11
+ date: 2017-01-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -33,19 +33,31 @@ email:
33
33
  executables:
34
34
  - xcknife
35
35
  - xcknife-min
36
+ - xcknife-test-dumper
36
37
  extensions: []
37
38
  extra_rdoc_files: []
38
39
  files:
39
40
  - ".gitignore"
41
+ - ".gitmodules"
40
42
  - ".rspec"
41
43
  - ".ruby-version"
44
+ - ".travis.yml"
42
45
  - CONTRIBUTING.md
43
46
  - Gemfile
44
47
  - LICENSE
45
48
  - README.md
46
49
  - Rakefile
50
+ - TestDumper/README.md
51
+ - TestDumper/TestDumper.dylib
52
+ - TestDumper/TestDumper.xcodeproj/project.pbxproj
53
+ - TestDumper/TestDumper.xcodeproj/xcshareddata/xcschemes/TestDumper.xcscheme
54
+ - TestDumper/TestDumper/Info.plist
55
+ - TestDumper/TestDumper/Initialize.m
56
+ - TestDumper/TestDumper/TestDumper.h
57
+ - TestDumper/build.sh
47
58
  - bin/xcknife
48
59
  - bin/xcknife-min
60
+ - bin/xcknife-test-dumper
49
61
  - example/README.md
50
62
  - example/run_example.rb
51
63
  - example/xcknife-exemplar-historical-data.json-stream
@@ -56,10 +68,8 @@ files:
56
68
  - lib/xcknife/json_stream_parser_helper.rb
57
69
  - lib/xcknife/runner.rb
58
70
  - lib/xcknife/stream_parser.rb
71
+ - lib/xcknife/test_dumper.rb
59
72
  - lib/xcknife/xctool_cmd_helper.rb
60
- - spec/spec_helper.rb
61
- - spec/xcknife_spec.rb
62
- - spec/xctool_cmd_helper_spec.rb
63
73
  - xcknife.gemspec
64
74
  homepage: https://github.com/square/xcknife
65
75
  licenses:
@@ -86,7 +96,4 @@ rubygems_version: 2.4.5.1
86
96
  signing_key:
87
97
  specification_version: 4
88
98
  summary: Simple tool for optimizing XCTest runs across machines
89
- test_files:
90
- - spec/spec_helper.rb
91
- - spec/xcknife_spec.rb
92
- - spec/xctool_cmd_helper_spec.rb
99
+ test_files: []
data/spec/spec_helper.rb DELETED
@@ -1,4 +0,0 @@
1
- $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
2
- require 'xcknife'
3
- require 'set'
4
- require 'pp'
data/spec/xcknife_spec.rb DELETED
@@ -1,344 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe XCKnife::StreamParser do
4
- context 'test_time_for_partitions' do
5
- subject { XCKnife::StreamParser.new(2, [["TestTarget1"], ["TestTarget2"]]) }
6
-
7
- it 'decide how many shards each partition set needs' do
8
- stream = [xctool_target_event("TestTarget1"),
9
- xctool_test_event("ClassTest1", "test1"),
10
- xctool_target_event("TestTarget2"),
11
- xctool_test_event("ClassTest2", "test1")
12
- ]
13
- result = subject.test_time_for_partitions(stream)
14
- expect(result).to eq([{ "TestTarget1" => { "ClassTest1" => 1000 } },
15
- { "TestTarget2" => { "ClassTest2" => 1000 } }])
16
- end
17
-
18
- it 'aggretates the times at the class level' do
19
- stream_parser = XCKnife::StreamParser.new(2, [["TestTarget1"]])
20
- stream = [xctool_target_event("TestTarget1"),
21
- xctool_test_event("ClassTest1", "test1", 1.0),
22
- xctool_test_event("ClassTest1", "test2", 2.0)
23
- ]
24
- result = stream_parser.test_time_for_partitions(stream)
25
- expect(result).to eq([{ "TestTarget1" => { "ClassTest1" => 3000 } }])
26
- end
27
-
28
- it 'works with multiple partitions' do
29
- stream_parser = XCKnife::StreamParser.new(2, [["TestTarget1"], ["TestTarget2"], ["TestTarget3"]])
30
-
31
- stream = [xctool_target_event("TestTarget1"),
32
- xctool_test_event("Class1", "test1"),
33
- xctool_target_event("TestTarget2"),
34
- xctool_test_event("Class2", "test1"),
35
- xctool_target_event("TestTarget3"),
36
- xctool_test_event("Class3", "test1"),
37
- ]
38
- result = stream_parser.test_time_for_partitions(stream)
39
- expect(result).to eq([{ "TestTarget1" => { "Class1" => 1000 } },
40
- { "TestTarget2" => { "Class2" => 1000 } },
41
- { "TestTarget3" => { "Class3" => 1000 } }])
42
- end
43
-
44
- it 'allows the same target to be listed on multiple partitions' do
45
- stream_parser = XCKnife::StreamParser.new(2, [["TestTarget1"], ["TestTarget2", "TestTarget1"]])
46
- stream = [xctool_target_event("TestTarget1"),
47
- xctool_test_event("ClassTest1", "test1"),
48
- xctool_target_event("TestTarget2"),
49
- xctool_test_event("ClassTest2", "test1"),
50
- ]
51
- result = stream_parser.test_time_for_partitions(stream)
52
- expect(result).to eq([{ "TestTarget1" => { "ClassTest1" => 1000 } },
53
- { "TestTarget2" => { "ClassTest2" => 1000 },
54
- "TestTarget1" => { "ClassTest1" => 1000 }
55
- }])
56
- end
57
-
58
- it 'raises error when an empty partition is specified' do
59
- stream_parser = XCKnife::StreamParser.new(1, [["TestTarget1"]])
60
- expect { stream_parser.test_time_for_partitions([]) }.to raise_error(XCKnife::XCKnifeError, 'The following partition has no tests: ["TestTarget1"]')
61
- end
62
- end
63
-
64
- context 'provided historical events' do
65
- subject { XCKnife::StreamParser.new(2, [["TestTarget1", "TestTarget2", "TestTarget3", "NewTestTarget1"]]) }
66
-
67
- it 'ignores test targets not present on current events' do
68
- historical_events = [xctool_target_event("TestTarget1"),
69
- xctool_test_event("ClassTest1", "test1"),
70
- xctool_test_event("ClassTest1", "test2"),
71
- xctool_target_event("TestTarget2"),
72
- xctool_test_event("ClassTest2", "test1"),
73
- xctool_test_event("ClassTest2", "test2")
74
- ]
75
- current_events = [
76
- xctool_target_event("TestTarget1"),
77
- xctool_test_event("ClassTest1", "test1")
78
- ]
79
- result = subject.test_time_for_partitions(historical_events, current_events)
80
- expect(result).to eq([{ "TestTarget1" => { "ClassTest1" => 2000 } }])
81
- expect(subject.stats.to_h).to eq({historical_total_tests: 4, current_total_tests: 1, class_extrapolations: 0, target_extrapolations: 0})
82
- end
83
-
84
-
85
- it 'ignores test classes not present on current events' do
86
- historical_events = [xctool_target_event("TestTarget1"),
87
- xctool_test_event("ClassTest1", "test1"),
88
- xctool_test_event("ClassTest1", "test2"),
89
- xctool_test_event("ClassTest2", "test1"),
90
- xctool_test_event("ClassTest2", "test2")
91
- ]
92
- current_events = [
93
- xctool_target_event("TestTarget1"),
94
- xctool_test_event("ClassTest1", "test1")
95
- ]
96
- result = subject.test_time_for_partitions(historical_events, current_events)
97
- expect(result).to eq([{ "TestTarget1" => { "ClassTest1" => 2000 } }])
98
- expect(subject.stats.to_h).to eq({historical_total_tests: 4, current_total_tests: 1, class_extrapolations: 0, target_extrapolations: 0})
99
- end
100
-
101
- it 'extrapolates for new test targets' do
102
- historical_events = [
103
- xctool_target_event("TestTarget1"),
104
- xctool_test_event("ClassTest1", "test1")
105
- ]
106
- current_events = [
107
- xctool_target_event("TestTarget1"),
108
- xctool_test_event("ClassTest1", "test1"),
109
- xctool_target_event("NewTestTargetButNotRelevant"),
110
- xctool_test_event("ClassTest10", "test1")
111
- ]
112
- result = subject.test_time_for_partitions(historical_events, current_events)
113
- expect(result.to_set).to eq([{
114
- "TestTarget1" => { "ClassTest1" => 1000 }
115
- }
116
- ].to_set)
117
- expect(subject.stats.to_h).to eq({historical_total_tests: 1, current_total_tests: 1, class_extrapolations: 0, target_extrapolations: 0})
118
- end
119
-
120
- it 'extrapolates for new test classes' do
121
- historical_events = [
122
- xctool_target_event("TestTarget1"),
123
- xctool_test_event("ClassTest1", "test1", 1.0),
124
- xctool_test_event("ClassTest2", "test2", 5.0),
125
- xctool_test_event("ClassTest3", "test3", 10000.0)
126
- ]
127
- current_events = historical_events + [
128
- xctool_target_event("TestTarget1"),
129
- xctool_test_event("ClassTest2", "test2"),
130
- xctool_test_event("ClassTestNew", "test1")
131
- ]
132
- result = subject.test_time_for_partitions(historical_events, current_events)
133
- median = 5000
134
- expect(result).to eq([{
135
- "TestTarget1" =>
136
- {
137
- "ClassTest1" => 1000,
138
- "ClassTest2" => 5000,
139
- "ClassTest3" => 10000000,
140
- "ClassTestNew" => median
141
- },
142
- }])
143
- expect(subject.stats.to_h).to eq({historical_total_tests: 3, current_total_tests: 5, class_extrapolations: 1, target_extrapolations: 0})
144
- end
145
-
146
- it "ignores test classes that don't belong to relevant targets" do
147
- historical_events = [
148
- xctool_target_event("TestTarget1"),
149
- xctool_test_event("ClassTest1", "test1", 1.0),
150
- xctool_test_event("ClassTest2", "test2", 5.0),
151
- xctool_test_event("ClassTest3", "test3", 10000.0)
152
- ]
153
- current_events = historical_events + [
154
- xctool_target_event("TestTarget1"),
155
- xctool_test_event("ClassTest2", "test2"),
156
- xctool_test_event("ClassTestNew", "test1")
157
- ]
158
- result = subject.test_time_for_partitions(historical_events, current_events)
159
- median = 5000
160
- expect(result).to eq([{
161
- "TestTarget1" =>
162
- {
163
- "ClassTest1" => 1000,
164
- "ClassTest2" => 5000,
165
- "ClassTest3" => 10000000,
166
- "ClassTestNew" => median
167
- },
168
- }])
169
- expect(subject.stats.to_h).to eq({historical_total_tests: 3, current_total_tests: 5, class_extrapolations: 1, target_extrapolations: 0})
170
- end
171
- end
172
-
173
- context 'provided an empty set of applicable historical events' do
174
- subject { XCKnife::StreamParser.new(2, [["TestTarget1", "TestTarget2", "TestTarget3", "NewTestTarget1"]]) }
175
-
176
- let(:empty_historical_events) { [] }
177
- let(:default_extrapolated_duration) { XCKnife::StreamParser::DEFAULT_EXTRAPOLATED_DURATION }
178
-
179
- it 'extrapolates the test target duration and classes get extrapolated' do
180
- current_events = [
181
- xctool_target_event("TestTarget1"),
182
- xctool_test_event("ClassTest1", "test1")
183
- ]
184
- result = subject.test_time_for_partitions(empty_historical_events, current_events)
185
- expect(result).to eq([{ "TestTarget1" => { "ClassTest1" => default_extrapolated_duration } }])
186
- end
187
-
188
- it 'extrapolates the test target to different classes' do
189
- effectively_empty_historical_events = [
190
- xctool_target_event("TestTarget2"),
191
- xctool_test_event("IgnoredClass", "ignoredTest"),
192
- ]
193
- current_events = [
194
- xctool_target_event("TestTarget1"),
195
- xctool_test_event("ClassTest1", "test1"),
196
- xctool_test_event("ClassTest2", "test2")
197
- ]
198
- result = subject.test_time_for_partitions(effectively_empty_historical_events, current_events)
199
- duration = default_extrapolated_duration
200
- expect(result).to eq([{ "TestTarget1" => { "ClassTest1" => duration, "ClassTest2" => duration } }])
201
- end
202
-
203
- it "can handle multiple test targets and test classes" do
204
- current_events = [
205
- xctool_target_event("TestTarget1"),
206
- xctool_test_event("ClassTest11", "test1"),
207
- xctool_test_event("ClassTest12", "test1"),
208
- xctool_test_event("ClassTest13", "test1"),
209
- xctool_target_event("TestTarget2"),
210
- xctool_test_event("ClassTest21", "test1"),
211
- xctool_test_event("ClassTest22", "test1"),
212
- xctool_test_event("ClassTest23", "test1"),
213
- xctool_target_event("TestTarget3"),
214
- xctool_test_event("ClassTest31", "test1"),
215
- xctool_test_event("ClassTest32", "test1"),
216
- xctool_test_event("ClassTest33", "test1"),
217
- ]
218
- result = subject.test_time_for_partitions(empty_historical_events, current_events)
219
- duration = default_extrapolated_duration
220
- expect(result).to eq(
221
- [
222
- {
223
- "TestTarget1" => { "ClassTest11" => duration, "ClassTest12" => duration, "ClassTest13" => duration },
224
- "TestTarget2" => { "ClassTest21" => duration, "ClassTest22" => duration, "ClassTest23" => duration },
225
- "TestTarget3" => { "ClassTest31" => duration, "ClassTest32" => duration, "ClassTest33" => duration } }]
226
- )
227
- end
228
- end
229
-
230
- it "can split_machines_proportionally" do
231
- stream_parser = XCKnife::StreamParser.new(5, [["TargetOnPartition1"], ["TargetOnPartition2"]])
232
- result = stream_parser.split_machines_proportionally([
233
- { "TargetOnPartition1" => { "TestClass1" => 500, "TestClass2" => 500 } },
234
- { "TargetOnPartition2" => { "TestClass3" => 1000, "TestClass4" => 1000, "TestClass5" => 1000, "TestClass6" => 1000 } }])
235
- expect(result.map(&:number_of_shards)).to eq([1, 4])
236
- end
237
-
238
- it "can split_machines_proportionally even when in the presence of large imbalances" do
239
- stream_parser = XCKnife::StreamParser.new(5, [["TargetOnPartition1"], ["TargetOnPartition2"], ["TargetOnPartition3"]])
240
- result = stream_parser.split_machines_proportionally([{ "TargetOnPartition1" => { "TestClass1" => 1 } },
241
- { "TargetOnPartition2" => { "TestClass2" => 1} },
242
- { "TargetOnPartition3" => { "TestClass3" => 1000, "TestClass4" => 1000, "TestClass5" => 1000} }])
243
- expect(result.map(&:number_of_shards)).to eq([1, 1, 3])
244
- end
245
-
246
-
247
- it "should never let partition_sets have less than 1 machine alocated to them" do
248
- stream_parser = XCKnife::StreamParser.new(3, [["TestTarget1"], ["TestTarget2"]])
249
- result = stream_parser.split_machines_proportionally([{ "TargetOnPartition1" => { "TestClass1" => 1 } },
250
- { "TargetOnPartition2" => { "TestClass2" => 2000, "TestClass3" => 2000 } }])
251
- expect(result.map(&:number_of_shards)).to eq([1, 2])
252
- end
253
-
254
-
255
- context 'test_time_for_partitions' do
256
- it "partitions the test classes accross the number of machines" do
257
- stream_parser = XCKnife::StreamParser.new(2, [["Test Target"]])
258
- partition = { "Test Target" => { "Class1" => 1000, "Class2" => 1000, "Class3" => 2000 } }
259
- shards = stream_parser.compute_single_shards(2, partition).map(&:test_time_map)
260
- expect(shards.size).to eq 2
261
- first_shard, second_shard = shards.sort_by { |map| map.values.flatten.size }
262
- expect(first_shard.keys).to eq(["Test Target"])
263
- expect(first_shard.values).to eq([["Class3"]])
264
-
265
- expect(second_shard.keys).to eq(["Test Target"])
266
- expect(second_shard.values.map(&:to_set)).to eq([["Class1", "Class2"].to_set])
267
- end
268
-
269
- it "partitions the test, across targets" do
270
- stream_parser = XCKnife::StreamParser.new(2, [["Test Target1", "Test Target2", "Test Target3"]])
271
- partition = { "Test Target1" => { "Class1" => 1000 },
272
- "Test Target2" => { "Class2" => 1000 },
273
- "Test Target3" => { "Class3" => 2000 } }
274
- shards = stream_parser.compute_single_shards(2, partition).map(&:test_time_map)
275
- expect(shards.size).to eq 2
276
- first_shard, second_shard = shards.sort_by { |map| map.values.flatten.size }
277
- expect(first_shard.keys).to eq(["Test Target3"])
278
- expect(first_shard.values).to eq([["Class3"]])
279
-
280
- expect(second_shard.keys.to_set).to eq(["Test Target1", "Test Target2"].to_set)
281
- expect(second_shard.values.to_set).to eq([["Class1"], ["Class2"]].to_set)
282
- end
283
-
284
- it "raises an error if there are too many shards" do
285
- too_many_machines = 2
286
- stream_parser = XCKnife::StreamParser.new(too_many_machines, [["Test Target1"]])
287
- partition = { "Test Target1" => { "Class1" => 1000 } }
288
- expect { stream_parser.compute_single_shards(too_many_machines, partition) }.
289
- to raise_error(XCKnife::XCKnifeError, "Too many shards")
290
- end
291
- end
292
-
293
- it "can compute test for all partitions" do
294
- stream_parser = XCKnife::StreamParser.new(3, [["TargetOnPartition1"], ["TargetOnPartition2"]])
295
- result = stream_parser.compute_shards_for_partitions([{ "TargetOnPartition1" => { "TestClass1" => 1000 } },
296
- { "TargetOnPartition2" => { "TestClass2" => 4000, "TestClass3" => 4000 } }])
297
- expect(result.test_maps).to eq([[{ "TargetOnPartition1" => ["TestClass1"] }],
298
- [{ "TargetOnPartition2" => ["TestClass2"] },
299
- { "TargetOnPartition2" => ["TestClass3"] }]])
300
- expect(result.test_times).to eq [[1000], [4000, 4000]]
301
- expect(result.total_test_time).to eq 9000
302
- expect(result.test_time_imbalances.to_h).to eq({
303
- partition_set: [0.4, 1.6],
304
- partitions: [[1.0], [1.0, 1.0]]
305
- })
306
- end
307
-
308
- it "can compute for only one partition set" do
309
- stream_parser = XCKnife::StreamParser.new(1, [["TargetOnPartition1"]])
310
- historical_events = [xctool_target_event("TargetOnPartition1"),
311
- xctool_test_event("ClassTest1", "test1"),
312
- ]
313
- result = stream_parser.compute_shards_for_events(historical_events)
314
- expect(result.test_maps).to eq([[{ "TargetOnPartition1" => ["ClassTest1"] }]])
315
- expect(result.test_times).to eq [[1000]]
316
- expect(result.total_test_time).to eq 1000
317
- expect(result.stats.to_h).to eq({historical_total_tests: 1, current_total_tests: 0, class_extrapolations: 0, target_extrapolations: 0})
318
- expect(result.test_time_imbalances.to_h).to eq({
319
- partition_set: [1.0],
320
- partitions: [[1.0]]
321
- })
322
- end
323
-
324
- def xctool_test_event(class_name, method_name, duration = 1.0)
325
- OpenStruct.new({ result: "success",
326
- exceptions: [],
327
- test: "-[#{class_name} #{method_name}]",
328
- className: class_name,
329
- event: "end-test",
330
- methodName: method_name,
331
- succeeded: true,
332
- output: "",
333
- totalDuration: duration,
334
- timestamp: 0
335
- })
336
- end
337
-
338
- def xctool_target_event(target_name)
339
- OpenStruct.new({ result: "success",
340
- event: "begin-ocunit",
341
- targetName: target_name
342
- })
343
- end
344
- end