illuminator 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/gem/README.md +37 -0
  3. data/gem/bin/illuminatorTestRunner.rb +22 -0
  4. data/gem/lib/illuminator.rb +171 -0
  5. data/gem/lib/illuminator/argument-parsing.rb +299 -0
  6. data/gem/lib/illuminator/automation-builder.rb +39 -0
  7. data/gem/lib/illuminator/automation-runner.rb +589 -0
  8. data/gem/lib/illuminator/build-artifacts.rb +118 -0
  9. data/gem/lib/illuminator/device-installer.rb +45 -0
  10. data/gem/lib/illuminator/host-utils.rb +42 -0
  11. data/gem/lib/illuminator/instruments-runner.rb +301 -0
  12. data/gem/lib/illuminator/javascript-runner.rb +98 -0
  13. data/gem/lib/illuminator/listeners/console-logger.rb +32 -0
  14. data/gem/lib/illuminator/listeners/full-output.rb +13 -0
  15. data/gem/lib/illuminator/listeners/instruments-listener.rb +22 -0
  16. data/gem/lib/illuminator/listeners/intermittent-failure-detector.rb +49 -0
  17. data/gem/lib/illuminator/listeners/pretty-output.rb +26 -0
  18. data/gem/lib/illuminator/listeners/saltinel-agent.rb +66 -0
  19. data/gem/lib/illuminator/listeners/saltinel-listener.rb +26 -0
  20. data/gem/lib/illuminator/listeners/start-detector.rb +52 -0
  21. data/gem/lib/illuminator/listeners/stop-detector.rb +46 -0
  22. data/gem/lib/illuminator/listeners/test-listener.rb +58 -0
  23. data/gem/lib/illuminator/listeners/trace-error-detector.rb +38 -0
  24. data/gem/lib/illuminator/options.rb +96 -0
  25. data/gem/lib/illuminator/resources/IlluminatorGeneratedEnvironment.erb +13 -0
  26. data/gem/lib/illuminator/resources/IlluminatorGeneratedRunnerForInstruments.erb +19 -0
  27. data/gem/lib/illuminator/test-definitions.rb +23 -0
  28. data/gem/lib/illuminator/test-suite.rb +155 -0
  29. data/gem/lib/illuminator/version.rb +3 -0
  30. data/gem/lib/illuminator/xcode-builder.rb +144 -0
  31. data/gem/lib/illuminator/xcode-utils.rb +219 -0
  32. data/gem/resources/BuildConfiguration.xcconfig +10 -0
  33. data/gem/resources/js/AppMap.js +767 -0
  34. data/gem/resources/js/Automator.js +1132 -0
  35. data/gem/resources/js/Base64.js +142 -0
  36. data/gem/resources/js/Bridge.js +102 -0
  37. data/gem/resources/js/Config.js +92 -0
  38. data/gem/resources/js/Extensions.js +2025 -0
  39. data/gem/resources/js/Illuminator.js +228 -0
  40. data/gem/resources/js/Preferences.js +24 -0
  41. data/gem/resources/scripts/UIAutomationBridge.rb +248 -0
  42. data/gem/resources/scripts/common.applescript +25 -0
  43. data/gem/resources/scripts/diff_png.sh +61 -0
  44. data/gem/resources/scripts/kill_all_sim_processes.sh +17 -0
  45. data/gem/resources/scripts/plist_to_json.sh +40 -0
  46. data/gem/resources/scripts/set_hardware_keyboard.applescript +0 -0
  47. metadata +225 -0
@@ -0,0 +1,118 @@
1
+ require 'singleton'
2
+ require 'fileutils'
3
+
4
+ # Convenience functions for command-line actions done in Xcode
5
+ module Illuminator
6
+ class BuildArtifacts
7
+ include Singleton
8
+
9
+ def initialize
10
+ @_root = nil
11
+ @artifacts_have_been_created = false
12
+ end
13
+
14
+ def set_root(dir_raw)
15
+ dir = Illuminator::HostUtils.realpath(dir_raw)
16
+ if @_root != dir and @artifacts_have_been_created
17
+ puts "Warning: changing BuildArtifacts root to '#{dir}' after creating artifacts in '#{@_root}'".red
18
+ end
19
+ @_root = dir
20
+ end
21
+
22
+ def _setup_and_use(dir, skip_setup)
23
+ raise TypeError, "The buildArtifact root directory is nil; perhaps it was not set" if @_root.nil?
24
+ unless skip_setup or File.directory?(dir)
25
+ FileUtils.mkdir_p dir
26
+ @artifacts_have_been_created = true
27
+ end
28
+ dir
29
+ end
30
+
31
+ ################## Directories
32
+
33
+ def root(skip_setup = false)
34
+ _setup_and_use @_root, skip_setup
35
+ end
36
+
37
+ def xcode(skip_setup = false)
38
+ _setup_and_use "#{@_root}/xcode", skip_setup
39
+ end
40
+
41
+ def derived_data(skip_setup = false)
42
+ _setup_and_use "#{@_root}/xcodeDerivedData", skip_setup
43
+ end
44
+
45
+ def instruments(skip_setup = false)
46
+ _setup_and_use "#{@_root}/instruments", skip_setup
47
+ end
48
+
49
+ def crash_reports(skip_setup = false)
50
+ _setup_and_use "#{@_root}/crashReports", skip_setup
51
+ end
52
+
53
+ def object_files(skip_setup = false)
54
+ _setup_and_use "#{@_root}/objectFiles", skip_setup
55
+ end
56
+
57
+ def console(skip_setup = false)
58
+ _setup_and_use "#{@_root}/console", skip_setup
59
+ end
60
+
61
+ def ui_automation(skip_setup = false)
62
+ _setup_and_use "#{@_root}/UIAutomation-outputs", skip_setup
63
+ end
64
+
65
+ def state(skip_setup = false)
66
+ _setup_and_use "#{@_root}/Illuminator-state", skip_setup
67
+ end
68
+
69
+
70
+ ################## FILES
71
+
72
+ def app_location(app_name = nil)
73
+ app_output_directory = xcode
74
+ if app_name.nil?
75
+ # assume that only one app exists and use that
76
+ return Dir["#{app_output_directory}/*.app"][0]
77
+ else
78
+ return "#{app_output_directory}/#{app_name}.app"
79
+ end
80
+ end
81
+
82
+ def xcpretty_report_file(skip_setup = false)
83
+ _setup_and_use "#{@_root}/xcpretty", skip_setup
84
+ "#{@_root}/xcpretty/report.xml"
85
+ end
86
+
87
+ def coverage_report_file(skip_setup = false)
88
+ _setup_and_use @_root, skip_setup
89
+ "#{@_root}/coverage.xml"
90
+ end
91
+
92
+ def illuminator_js_runner(skip_setup = false)
93
+ _setup_and_use @_root, skip_setup
94
+ "#{@_root}/IlluminatorGeneratedRunnerForInstruments.js"
95
+ end
96
+
97
+ def illuminator_js_environment(skip_setup = false)
98
+ _setup_and_use @_root, skip_setup
99
+ "#{@_root}/IlluminatorGeneratedEnvironment.js"
100
+ end
101
+
102
+ def illuminator_config_file(skip_setup = false)
103
+ _setup_and_use @_root, skip_setup
104
+ "#{@_root}/IlluminatorGeneratedConfig.json"
105
+ end
106
+
107
+ def junit_report_file(skip_setup = false)
108
+ _setup_and_use @_root, skip_setup
109
+ "#{@_root}/IlluminatorJUnitReport.xml"
110
+ end
111
+
112
+ def illuminator_rerun_failed_tests_settings(skip_setup = false)
113
+ _setup_and_use @_root, skip_setup
114
+ "#{@_root}/IlluminatorRerunFailedTestsSettings.json"
115
+ end
116
+
117
+ end
118
+ end
@@ -0,0 +1,45 @@
1
+ require 'singleton'
2
+
3
+ require_relative './host-utils'
4
+
5
+ # Wrapper for possible methods of installing an app on physical hardware
6
+ class DeviceInstaller
7
+ include Singleton
8
+
9
+ def initialize
10
+ installers = ['ios-deploy']
11
+
12
+ @installed_installers = {}
13
+ installers.each { |exe| @installed_installers[exe] = Illuminator::HostUtils.which(exe) }
14
+ end
15
+
16
+
17
+ def _install_using_ios_deploy(app_location, hardware_id)
18
+ # TODO: actually watch the output of this command
19
+ cmd = "#{@installed_installers['ios-deploy']} -b '#{app_location}' -i #{hardware_id} -r -n"
20
+ puts cmd.green
21
+ puts `#{cmd}`
22
+ end
23
+
24
+
25
+ def install_on_device(app_location, hardware_id, specific_method = nil)
26
+ # if nothing is specified, just get the first one that exists
27
+ if specific_method.nil?
28
+ @installed_installers.each do |name, path|
29
+ specific_method = path
30
+ break unless specific_method.nil?
31
+ end
32
+ end
33
+
34
+ # run the appropriate helper for doing this
35
+ puts "Installing #{app_location} on device #{hardware_id} using #{specific_method}"
36
+ case specific_method
37
+ when /ios-deploy$/
38
+ _install_using_ios_deploy(app_location, hardware_id)
39
+ else
40
+ puts "None of the following utilities for app installation appear to be installed: #{@installed_installers.keys.to_s}".red
41
+ raise NotImplementedError, "No app installation available with name " + specific_method.to_s
42
+ end
43
+ end
44
+
45
+ end
@@ -0,0 +1,42 @@
1
+ require 'pathname'
2
+
3
+ module Illuminator
4
+ class HostUtils
5
+
6
+ # Cross-platform way of finding an executable in the $PATH.
7
+ # based on http://stackoverflow.com/a/5471032/2063546
8
+ #
9
+ # which('ruby') #=> /usr/bin/ruby
10
+ def self.which program
11
+ exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
12
+ ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
13
+ exts.each do |ext|
14
+ exe = File.join(path, "#{program}#{ext}")
15
+ return exe if File.executable?(exe) unless File.directory?(exe)
16
+ end
17
+ end
18
+ return nil
19
+ end
20
+
21
+ def self.save_json(hash_object, path)
22
+ f = File.open(path, 'w')
23
+ f << JSON.pretty_generate(hash_object)
24
+ f.close
25
+ end
26
+
27
+ # try to simplify the path, if it exists
28
+ def self.realpath(path)
29
+ epath = File.expand_path path
30
+
31
+ # use expanded path if regular one fails
32
+ path = epath unless (File.exists? path) or (File.directory? path)
33
+
34
+ # use given path if it doesn't exist (can't take a real path of a nonexistent path)
35
+ return path unless (File.exists? path) or (File.directory? path)
36
+
37
+ Pathname.new(path).realpath.to_s
38
+ end
39
+
40
+ end
41
+
42
+ end
@@ -0,0 +1,301 @@
1
+ require 'fileutils'
2
+ require 'colorize'
3
+ require 'pty'
4
+
5
+ require_relative './xcode-utils'
6
+ require_relative './build-artifacts'
7
+ require_relative 'listeners/start-detector'
8
+ require_relative 'listeners/stop-detector'
9
+ require_relative 'listeners/intermittent-failure-detector'
10
+ require_relative 'listeners/trace-error-detector'
11
+
12
+ ####################################################################################################
13
+ # status
14
+ ####################################################################################################
15
+
16
+ class ParsedInstrumentsMessage
17
+
18
+ attr_accessor :message
19
+ attr_accessor :full_line
20
+ attr_accessor :status
21
+ attr_accessor :date
22
+ attr_accessor :time
23
+ attr_accessor :tz
24
+
25
+ # parse lines in the form: 2014-10-20 20:43:41 +0000 Default: BLAH BLAH BLAH ACTUAL MESSAGE
26
+ def self.from_line (line)
27
+ parsed = line.match(/^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) ([+-]\d{4}) ([^:]+): (.*)$/).to_a
28
+ _, date_string, time_string, tz_string, status_string, msg_string = parsed
29
+
30
+ message = ParsedInstrumentsMessage.new
31
+ message.full_line = line
32
+ message.message = msg_string
33
+ message.status = parse_status(status_string)
34
+ message.date = date_string
35
+ message.time = time_string
36
+ message.tz = tz_string
37
+
38
+ message
39
+ end
40
+
41
+ def self.parse_status(status)
42
+ case status
43
+ when /start/i then :start
44
+ when /stopped/i then :stopped
45
+ when /pass/i then :pass
46
+ when /fail/i then :fail
47
+ when /error/i then :error
48
+ when /warning/i then :warning
49
+ when /issue/i then :issue
50
+ when /default/i then :default
51
+ when /debug/i then :debug
52
+ else :unknown
53
+ end
54
+ end
55
+
56
+ end
57
+
58
+
59
+ class InstrumentsExitStatus
60
+ attr_accessor :normal # whether the command finished under normal circumstances (eventually)
61
+ attr_accessor :fatal_error # whether the command finished in a way that restarting won't fix
62
+ attr_accessor :fatal_reason # a message about why the command can't be restarted
63
+ end
64
+
65
+
66
+ ####################################################################################################
67
+ # command builder
68
+ ####################################################################################################
69
+
70
+ class InstrumentsRunner
71
+ include StartDetectorEventSink
72
+ include StopDetectorEventSink
73
+ include IntermittentFailureDetectorEventSink
74
+ include TraceErrorDetectorEventSink
75
+
76
+ attr_accessor :app_location # the app to run
77
+ attr_accessor :hardware_id # hardware id specifier
78
+ attr_accessor :sim_device # sim device id specifier
79
+ attr_accessor :sim_language # sim language (unsupported currently)
80
+ attr_accessor :attempts # number of times to try running instruments before giving up
81
+ attr_accessor :startup_timeout # amount of time to wait for initial instruments output
82
+ attr_accessor :max_silence # if instruments goes silent for this long, we kill it.
83
+
84
+ attr_reader :started
85
+
86
+ def initialize
87
+ @listeners = Hash.new
88
+ @attempts = 5
89
+ @startup_timeout = 30
90
+ end
91
+
92
+ def add_listener (name, listener)
93
+ @listeners[name] = listener
94
+ end
95
+
96
+ def cleanup
97
+ dirs_to_remove = []
98
+ build_artifact_keys = [:instruments]
99
+ # get the directories without creating them (the 'true' arg), add them to our list
100
+ build_artifact_keys.each do |key|
101
+ dir = Illuminator::BuildArtifacts.instance.method(key).call(true)
102
+ dirs_to_remove << dir
103
+ end
104
+
105
+ # remove directories in the list
106
+ dirs_to_remove.each do |d|
107
+ dir = Illuminator::HostUtils.realpath d
108
+ puts "InstrumentsRunner cleanup: removing #{dir}"
109
+ FileUtils.rmtree dir
110
+ end
111
+ end
112
+
113
+ def start_detector_triggered
114
+ @fully_started = true
115
+ end
116
+
117
+ def stop_detector_triggered(fatal, message)
118
+ if !fatal
119
+ @should_abort = true
120
+ else
121
+ @fatal_error = true
122
+ @fatal_reason = message
123
+ end
124
+ end
125
+
126
+ def intermittent_failure_detector_triggered message
127
+ @fully_started = true
128
+ force_stop("Detected an intermittent failure condition - #{message}")
129
+ end
130
+
131
+ def trace_error_detector_triggered(fatal, message)
132
+ puts "Detected a trace error - #{message}".yellow
133
+ if fatal
134
+ @fatal_error = true
135
+ @fatal_reason = message
136
+ else
137
+ @should_reset_everything = true
138
+ end
139
+ end
140
+
141
+ def force_stop why
142
+ puts "\n #{why}".red
143
+ @should_abort = true
144
+ end
145
+
146
+
147
+ def add_necessary_instruments_listeners(saltinel)
148
+ # add saltinel listener
149
+ start_detector = StartDetector.new(saltinel)
150
+ start_detector.event_sink = self
151
+ add_listener("start_detector", start_detector)
152
+
153
+ # add trace error listener
154
+ trace_detector = TraceErrorDetector.new
155
+ trace_detector.event_sink = self
156
+ add_listener("trace_error_detector", trace_detector)
157
+
158
+ # add fatal error listener
159
+ stop_detector = StopDetector.new
160
+ stop_detector.event_sink = self
161
+ add_listener("stop_detector", stop_detector)
162
+ end
163
+
164
+
165
+ # Build the proper command and run it
166
+ def run_once saltinel
167
+ report_path = Illuminator::BuildArtifacts.instance.instruments
168
+
169
+ add_necessary_instruments_listeners(saltinel)
170
+
171
+ global_js_file = Illuminator::BuildArtifacts.instance.illuminator_js_runner
172
+ xcode_path = Illuminator::XcodeUtils.instance.get_xcode_path
173
+ template_path = Illuminator::XcodeUtils.instance.get_instruments_template_path
174
+
175
+ command = "env DEVELOPER_DIR='#{xcode_path}' /usr/bin/instruments"
176
+ if !@hardware_id.nil?
177
+ command << " -w '" + @hardware_id + "'"
178
+ elsif !@sim_device.nil?
179
+ command << " -w '" + @sim_device + "'"
180
+ end
181
+
182
+ command << " -t '#{template_path}' "
183
+ command << "'#{@app_location}'"
184
+ command << " -e UIASCRIPT '#{global_js_file}'"
185
+ command << " -e UIARESULTSPATH '#{report_path}'"
186
+
187
+ command << " #{@sim_language}" if @sim_language
188
+
189
+ directory = Dir.pwd
190
+ ret = nil
191
+ # change directories and successfully change back
192
+ begin
193
+ Dir.chdir(report_path)
194
+ ret = run_instruments_command command
195
+ ensure
196
+ Dir.chdir(directory)
197
+ end
198
+ return ret
199
+ end
200
+
201
+
202
+ # kill the instruments child process
203
+ def kill_instruments(r, w, pid)
204
+ puts "killing Instruments (pid #{pid})...".red
205
+ begin
206
+ Process.kill(9, pid)
207
+ w.close
208
+ r.close
209
+ Process.wait(pid)
210
+ rescue PTY::ChildExited
211
+ end
212
+ end
213
+
214
+
215
+ # Run an instruments command until it looks like it started successfully or failed non-intermittently
216
+ def run_instruments_command (command)
217
+ @fully_started = false
218
+ @should_abort = false
219
+ @fatal_error = false
220
+ @fatal_reason = nil
221
+ puts command.green
222
+ remaining_attempts = @attempts
223
+
224
+ # launch & re-launch instruments until it triggers the StartDetector
225
+ while (not @fatal_error) && (not @fully_started) && remaining_attempts > 0 do
226
+ @should_reset_everything = false
227
+ successful_run = true
228
+ remaining_attempts = remaining_attempts - 1
229
+
230
+ Illuminator::XcodeUtils.kill_all_instruments_processes # because sometimes they stick around and add up
231
+
232
+ puts "\nRelaunching instruments. #{remaining_attempts} retries left".red unless (remaining_attempts + 1) == @attempts
233
+
234
+ # spawn process and catch unexpected exits
235
+ begin
236
+ PTY.spawn(*command) do |r, w, pid|
237
+ silence_duration = 0
238
+
239
+ done_reading_output = false
240
+ # select on the output and send it to the listeners
241
+ while not done_reading_output do
242
+ if @should_abort || @fatal_error
243
+ successful_run = false
244
+ done_reading_output = true
245
+ kill_instruments(r, w, pid)
246
+
247
+ elsif @should_reset_everything
248
+ successful_run = false
249
+ done_reading_output = true
250
+ kill_instruments(r, w, pid)
251
+ unless @sim_device.nil?
252
+ Illuminator::XcodeUtils.kill_all_simulator_processes @sim_device
253
+ Illuminator::XcodeUtils.instance.reset_simulator @sim_device
254
+ end
255
+
256
+ elsif IO.select([r], nil, nil, @startup_timeout) then
257
+ line = r.readline.rstrip
258
+ @listeners.each { |_, listener| listener.receive(ParsedInstrumentsMessage.from_line(line)) }
259
+ elsif not @fully_started
260
+ successful_run = false
261
+ done_reading_output = true
262
+ puts "\n Timeout #{@startup_timeout} reached without any output - ".red
263
+ kill_instruments(r, w, pid)
264
+ puts "killing simulator processes...".red
265
+ Illuminator::XcodeUtils.kill_all_simulator_processes @sim_device
266
+ # TODO: might be necessary to delete any app crashes at this point
267
+ else
268
+ # We failed to get output for @startupTimeout, but that's probably OK since we've successfully started
269
+ # But we still enforce a maximum time spent without output
270
+
271
+ silence_duration += @startup_timeout # (the amount of time we waited for a select)
272
+ puts "Instruments seems to have started but has not produced output in #{@startup_timeout} seconds".yellow
273
+ if @max_silence < silence_duration
274
+ @should_abort = true
275
+ end
276
+ end
277
+ end
278
+ end
279
+
280
+ rescue EOFError
281
+ # normal termination
282
+ rescue Errno::ECHILD, Errno::EIO, PTY::ChildExited
283
+ STDERR.puts 'Instruments exited unexpectedly'
284
+ if @fully_started
285
+ successful_run = false
286
+ done_reading_output = true
287
+ end
288
+ ensure
289
+ @listeners.each { |_, listener| listener.on_automation_finished }
290
+ end
291
+ end
292
+
293
+ exit_status = InstrumentsExitStatus.new
294
+ exit_status.normal = successful_run
295
+ exit_status.fatal_error = @fatal_error
296
+ exit_status.fatal_reason = @fatal_reason
297
+
298
+ return exit_status
299
+ end
300
+
301
+ end