illuminator 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/gem/README.md +37 -0
  3. data/gem/bin/illuminatorTestRunner.rb +22 -0
  4. data/gem/lib/illuminator.rb +171 -0
  5. data/gem/lib/illuminator/argument-parsing.rb +299 -0
  6. data/gem/lib/illuminator/automation-builder.rb +39 -0
  7. data/gem/lib/illuminator/automation-runner.rb +589 -0
  8. data/gem/lib/illuminator/build-artifacts.rb +118 -0
  9. data/gem/lib/illuminator/device-installer.rb +45 -0
  10. data/gem/lib/illuminator/host-utils.rb +42 -0
  11. data/gem/lib/illuminator/instruments-runner.rb +301 -0
  12. data/gem/lib/illuminator/javascript-runner.rb +98 -0
  13. data/gem/lib/illuminator/listeners/console-logger.rb +32 -0
  14. data/gem/lib/illuminator/listeners/full-output.rb +13 -0
  15. data/gem/lib/illuminator/listeners/instruments-listener.rb +22 -0
  16. data/gem/lib/illuminator/listeners/intermittent-failure-detector.rb +49 -0
  17. data/gem/lib/illuminator/listeners/pretty-output.rb +26 -0
  18. data/gem/lib/illuminator/listeners/saltinel-agent.rb +66 -0
  19. data/gem/lib/illuminator/listeners/saltinel-listener.rb +26 -0
  20. data/gem/lib/illuminator/listeners/start-detector.rb +52 -0
  21. data/gem/lib/illuminator/listeners/stop-detector.rb +46 -0
  22. data/gem/lib/illuminator/listeners/test-listener.rb +58 -0
  23. data/gem/lib/illuminator/listeners/trace-error-detector.rb +38 -0
  24. data/gem/lib/illuminator/options.rb +96 -0
  25. data/gem/lib/illuminator/resources/IlluminatorGeneratedEnvironment.erb +13 -0
  26. data/gem/lib/illuminator/resources/IlluminatorGeneratedRunnerForInstruments.erb +19 -0
  27. data/gem/lib/illuminator/test-definitions.rb +23 -0
  28. data/gem/lib/illuminator/test-suite.rb +155 -0
  29. data/gem/lib/illuminator/version.rb +3 -0
  30. data/gem/lib/illuminator/xcode-builder.rb +144 -0
  31. data/gem/lib/illuminator/xcode-utils.rb +219 -0
  32. data/gem/resources/BuildConfiguration.xcconfig +10 -0
  33. data/gem/resources/js/AppMap.js +767 -0
  34. data/gem/resources/js/Automator.js +1132 -0
  35. data/gem/resources/js/Base64.js +142 -0
  36. data/gem/resources/js/Bridge.js +102 -0
  37. data/gem/resources/js/Config.js +92 -0
  38. data/gem/resources/js/Extensions.js +2025 -0
  39. data/gem/resources/js/Illuminator.js +228 -0
  40. data/gem/resources/js/Preferences.js +24 -0
  41. data/gem/resources/scripts/UIAutomationBridge.rb +248 -0
  42. data/gem/resources/scripts/common.applescript +25 -0
  43. data/gem/resources/scripts/diff_png.sh +61 -0
  44. data/gem/resources/scripts/kill_all_sim_processes.sh +17 -0
  45. data/gem/resources/scripts/plist_to_json.sh +40 -0
  46. data/gem/resources/scripts/set_hardware_keyboard.applescript +0 -0
  47. metadata +225 -0
@@ -0,0 +1,98 @@
1
+ require 'erb'
2
+ require 'pathname'
3
+ require 'json'
4
+ require 'socket'
5
+ require 'digest/sha1'
6
+
7
+ require_relative './build-artifacts'
8
+ require_relative './host-utils'
9
+ require_relative './xcode-utils'
10
+
11
+ # Class to handle all configuration relating to the javascript environment
12
+ # "runner" is a bit of a misnomer (this runs as part of instruments) but without this code, nothing runs
13
+ class JavascriptRunner
14
+
15
+ attr_reader :saltinel # the salted sentinel
16
+ attr_accessor :target_device_id
17
+ attr_accessor :is_hardware
18
+ attr_accessor :entry_point
19
+ attr_accessor :test_path
20
+ attr_accessor :implementation
21
+ attr_accessor :sim_device
22
+ attr_accessor :sim_version
23
+ attr_accessor :random_seed
24
+ attr_accessor :tags_any
25
+ attr_accessor :tags_all
26
+ attr_accessor :tags_none
27
+ attr_accessor :scenario_list
28
+ attr_accessor :scenario_number_offset # for consistent numbering after restarts
29
+ attr_accessor :app_specific_config
30
+
31
+ def initialize
32
+ @tags_any = Array.new(0)
33
+ @tags_all = Array.new(0)
34
+ @tags_none = Array.new(0)
35
+ @scenario_list = nil
36
+ end
37
+
38
+
39
+ def assemble_config
40
+ @full_config = {}
41
+
42
+ # a mapping of the full config key name to the local value that corresponds to it
43
+ key_defs = {
44
+ 'saltinel' => @saltinel,
45
+ 'entryPoint' => @entry_point,
46
+ 'implementation' => @implementation,
47
+ 'automatorDesiredSimDevice' => @sim_device,
48
+ 'automatorDesiredSimVersion' => @sim_version,
49
+ 'targetDeviceID' => @target_device_id,
50
+ 'isHardware' => @is_hardware,
51
+ 'xcodePath' => Illuminator::XcodeUtils.instance.get_xcode_path,
52
+ 'automatorSequenceRandomSeed' => @random_seed,
53
+ 'automatorTagsAny' => @tags_any,
54
+ 'automatorTagsAll' => @tags_all,
55
+ 'automatorTagsNone' => @tags_none,
56
+ 'automatorScenarioNames' => @scenario_list,
57
+ 'automatorScenarioOffset' => @scenario_number_offset,
58
+ 'customConfig' => (@app_specific_config.is_a? Hash) ? @app_specific_config : @app_specific_config.to_h,
59
+ }
60
+
61
+ key_defs.each do |key, value|
62
+ @full_config[key] = value unless value.nil?
63
+ end
64
+
65
+ end
66
+
67
+
68
+ def write_configuration()
69
+ # instance variables required for render_template
70
+ @saltinel = Digest::SHA1.hexdigest (Time.now.to_i.to_s + Socket.gethostname)
71
+ @illuminator_root = Illuminator::HostUtils.realpath(File.join(File.dirname(__FILE__), "../../resources/js/"))
72
+ @illuminator_scripts = Illuminator::HostUtils.realpath(File.join(File.dirname(__FILE__), "../../resources/scripts/"))
73
+ @artifacts_root = Illuminator::BuildArtifacts.instance.root
74
+ @illuminator_instruments_root = Illuminator::BuildArtifacts.instance.instruments
75
+ @environment_file = Illuminator::BuildArtifacts.instance.illuminator_js_environment
76
+
77
+ # prepare @full_config
78
+ assemble_config
79
+
80
+ render_template('/resources/IlluminatorGeneratedRunnerForInstruments.erb',
81
+ Illuminator::BuildArtifacts.instance.illuminator_js_runner)
82
+ render_template('/resources/IlluminatorGeneratedEnvironment.erb',
83
+ Illuminator::BuildArtifacts.instance.illuminator_js_environment)
84
+
85
+ Illuminator::HostUtils.save_json(@full_config, Illuminator::BuildArtifacts.instance.illuminator_config_file)
86
+ end
87
+
88
+
89
+ def render_template source_file, destination_file
90
+ contents = File.open(File.dirname(__FILE__) + source_file, 'r') { |f| f.read }
91
+
92
+ renderer = ERB.new(contents)
93
+ new_file = File.open(destination_file, 'w')
94
+ new_file.write(renderer.result(binding))
95
+ new_file.close
96
+ end
97
+
98
+ end
@@ -0,0 +1,32 @@
1
+
2
+ require 'logger'
3
+ require_relative '../build-artifacts'
4
+ require_relative './instruments-listener'
5
+
6
+ class ConsoleLogger < InstrumentsListener
7
+
8
+ def initialize
9
+ @run_number = 0
10
+ @logger = nil
11
+ end
12
+
13
+ def prepare_logger
14
+ return unless @logger.nil?
15
+ @run_number += 1
16
+ filename = File.join(Illuminator::BuildArtifacts.instance.console, "instruments#{@run_number.to_s.rjust(3, "0")}.log")
17
+ FileUtils.rmtree filename
18
+ @logger = Logger.new(filename)
19
+ end
20
+
21
+ def receive (message)
22
+ prepare_logger
23
+ @logger << message.full_line
24
+ @logger << "\n"
25
+ end
26
+
27
+ def on_automation_finished
28
+ @logger.close unless @logger.nil?
29
+ @logger = nil
30
+ end
31
+
32
+ end
@@ -0,0 +1,13 @@
1
+
2
+ require_relative './instruments-listener'
3
+
4
+ class FullOutput < InstrumentsListener
5
+
6
+ def receive (message)
7
+ puts message.full_line
8
+ end
9
+
10
+ def on_automation_finished
11
+ end
12
+
13
+ end
@@ -0,0 +1,22 @@
1
+
2
+ class InstrumentsListener
3
+
4
+ def receive (message)
5
+ puts message.full_line
6
+ puts " date: #{message.date}"
7
+ puts " time: #{message.time}"
8
+ puts " tz: #{message.tz}"
9
+ puts " status: #{message.status}"
10
+ puts " message: #{message.message}"
11
+ puts " --- If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
12
+ puts
13
+ end
14
+
15
+ def on_automation_finished
16
+ puts " ==="
17
+ puts " === If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
18
+ puts " ==="
19
+ puts
20
+ end
21
+
22
+ end
@@ -0,0 +1,49 @@
1
+
2
+ require_relative './saltinel-listener'
3
+
4
+ module IntermittentFailureDetectorEventSink
5
+
6
+ def intermittent_failure_detector_triggered message
7
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
8
+ end
9
+
10
+ end
11
+
12
+ # IntermittentFailureDetector monitors the logs for things that indicate a transient failure to start instruments
13
+ # - UIATargetHasGoneAWOLException
14
+ # - etc
15
+ class IntermittentFailureDetector < SaltinelListener
16
+
17
+ attr_accessor :event_sink
18
+
19
+ def on_init
20
+ @already_started = false
21
+ end
22
+
23
+ def trigger message
24
+ @event_sink.intermittent_failure_detector_triggered message
25
+ end
26
+
27
+ def receive message
28
+ super # run the SaltinelListener processor
29
+
30
+ # error cases that should trigger a restart
31
+ p = /Automation Instrument ran into an exception while trying to run the script. UIATargetHasGoneAWOLException/
32
+ if p =~ message.full_line
33
+ trigger "UIATargetHasGoneAWOLException"
34
+ end
35
+
36
+ p = /Fail: Could not start script, target application is not frontmost./
37
+ if p =~ message.full_line
38
+ trigger "Could not start script, target application is not frontmost"
39
+ end
40
+ end
41
+
42
+ def on_saltinel inner_message
43
+ # no cases yet
44
+ end
45
+
46
+ def on_automation_finished
47
+ end
48
+
49
+ end
@@ -0,0 +1,26 @@
1
+
2
+ require_relative './instruments-listener'
3
+
4
+ class PrettyOutput < InstrumentsListener
5
+
6
+ def receive (message)
7
+ case message.status
8
+ when :start
9
+ puts "Start: #{message.message}".green
10
+ when :pass
11
+ puts "Pass: #{message.message}".green
12
+ puts
13
+ when :fail, :error, :issue
14
+ puts "Fail: #{message.message}".red
15
+ when :warning
16
+ puts "Warn: #{message.message}".yellow
17
+ when :default
18
+ puts " #{message.message}"
19
+ end
20
+
21
+ end
22
+
23
+ def on_automation_finished
24
+ end
25
+
26
+ end
@@ -0,0 +1,66 @@
1
+
2
+ require_relative './saltinel-listener'
3
+
4
+ module SaltinelAgentEventSink
5
+
6
+ def saltinel_agent_got_scenario_list jsonPath
7
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
8
+ end
9
+
10
+ def saltinel_agent_got_scenario_definitions jsonPath
11
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
12
+ end
13
+
14
+ def saltinel_agent_got_stacktrace_hint
15
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
16
+ end
17
+
18
+ def saltinel_agent_got_restart_request
19
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
20
+ end
21
+
22
+ end
23
+
24
+ # Saltinel Agent handles all known saltinel messages and runs callbacks
25
+ class SaltinelAgent < SaltinelListener
26
+
27
+ attr_accessor :event_sink
28
+
29
+ def on_init
30
+ @recognizers = {
31
+ "recognize_test_list" => /Saved intended test list to: (.*)/,
32
+ "recognize_test_defs" => /Saved scenario definitions to: (.*)/,
33
+ "recognize_stacktrace" => /Stack trace follows:/,
34
+ "recognize_restart_request" => /Request instruments restart/,
35
+ }
36
+ end
37
+
38
+ def recognize_test_list regexResult
39
+ # assume developer has set event_sink already
40
+ @event_sink.saltinel_agent_got_scenario_list(regexResult.to_a[1])
41
+ end
42
+
43
+ def recognize_test_defs regexResult
44
+ # assume developer has set event_sink already
45
+ @event_sink.saltinel_agent_got_scenario_definitions(regexResult.to_a[1])
46
+ end
47
+
48
+ def recognize_stacktrace _
49
+ @event_sink.saltinel_agent_got_stacktrace_hint
50
+ end
51
+
52
+ def recognize_restart_request _
53
+ @event_sink.saltinel_agent_got_restart_request
54
+ end
55
+
56
+ def on_saltinel inner_message
57
+ @recognizers.each do |fn, regex|
58
+ result = regex.match(inner_message)
59
+ send(fn, result) unless result.nil?
60
+ end
61
+ end
62
+
63
+ def on_automation_finished
64
+ end
65
+
66
+ end
@@ -0,0 +1,26 @@
1
+
2
+ require_relative './instruments-listener'
3
+
4
+ # A listener class that just looks for saltinel messages and parses them,
5
+ # sending them off to a subclassable handler
6
+ class SaltinelListener < InstrumentsListener
7
+
8
+ def initialize saltinel
9
+ @saltinel = saltinel
10
+ on_init
11
+ end
12
+
13
+ def on_init
14
+ end
15
+
16
+ def receive (message)
17
+ unsalted = /^#{@saltinel} (.*) #{@saltinel}/.match(message.message)
18
+ on_saltinel(unsalted.to_a[1]) unless unsalted.nil?
19
+ end
20
+
21
+ def on_saltinel inner_message
22
+ puts "CAUGHT SALTINEL MESSAGE in #{self.class.name}.on_saltinel".red
23
+ puts inner_message.green
24
+ end
25
+
26
+ end
@@ -0,0 +1,52 @@
1
+
2
+ require_relative './saltinel-listener'
3
+
4
+ module StartDetectorEventSink
5
+
6
+ def start_detector_triggered
7
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
8
+ end
9
+
10
+ end
11
+
12
+ # StartDetector monitors the logs for things that indicate a successful startup of instruments
13
+ # - the saltinel for the intended test list
14
+ # - a javascript error
15
+ # - etc
16
+ class StartDetector < SaltinelListener
17
+
18
+ attr_accessor :event_sink
19
+
20
+ def on_init
21
+ @already_started = false
22
+ end
23
+
24
+ def trigger
25
+ # assume developer has set event_sink already
26
+ @event_sink.start_detector_triggered unless @already_started
27
+ @already_started = true
28
+ end
29
+
30
+ def receive message
31
+ super # run the SaltinelListener processor
32
+
33
+ # error cases that indicate successful start but involve errors that won't be fixed by a restart
34
+ trigger if :error == message.status and /Script threw an uncaught JavaScript error:/ =~ message.message
35
+
36
+ # Instruments usage error generally means we can't recover... unless it's a device booting issue
37
+ if /^Instruments Usage Error :/ =~ message.full_line
38
+ unless /^Instruments Usage Error : Timed out waiting for device to boot:/ =~ message.full_line
39
+ trigger
40
+ end
41
+ end
42
+ end
43
+
44
+ def on_saltinel inner_message
45
+ trigger if /Saved intended test list to/ =~ inner_message
46
+ trigger if /Successful launch/ =~ inner_message
47
+ end
48
+
49
+ def on_automation_finished
50
+ end
51
+
52
+ end
@@ -0,0 +1,46 @@
1
+
2
+ require_relative 'instruments-listener'
3
+
4
+ module StopDetectorEventSink
5
+
6
+ # fatal means that restarting instruments won't fix it
7
+ def stop_detector_triggered(fatal, message)
8
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
9
+ end
10
+
11
+ end
12
+
13
+ # StopDetector monitors the logs for things that indicate that instruments should be stopped (user errors)
14
+ # - a message that automation has stopped
15
+ # - a message that the user needs to enter credentials
16
+ # - etc
17
+ class StopDetector < InstrumentsListener
18
+
19
+ attr_accessor :event_sink
20
+
21
+ def trigger(fatal, message)
22
+ # assume developer has set event_sink already
23
+ @event_sink.stop_detector_triggered(fatal, message)
24
+ end
25
+
26
+ def receive message
27
+ # error cases that indicate successful stop but involve errors that won't be fixed by a restart
28
+
29
+ # like if instruments says it stopped
30
+ trigger(false, message.message) if :stopped == message.status
31
+
32
+ # or if instruments prompts for username/password
33
+ p1 = "instruments: Instruments wants permission to analyze other processes."
34
+ p2 = "Please enter an administrator username and password to allow this."
35
+
36
+ p = /#{p1} #{p2}/
37
+ if p =~ message.full_line
38
+ trigger(true, "Instruments needs permission to analyze other processes. Please run instruments manually to permit this.")
39
+ end
40
+
41
+ end
42
+
43
+ def on_automation_finished
44
+ end
45
+
46
+ end
@@ -0,0 +1,58 @@
1
+ require 'date'
2
+
3
+ require_relative './instruments-listener'
4
+
5
+ module TestListenerEventSink
6
+
7
+ def test_listener_got_test_start name
8
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
9
+ end
10
+
11
+ def test_listener_got_test_pass name
12
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
13
+ end
14
+
15
+ def test_listener_got_test_fail message
16
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
17
+ end
18
+
19
+ def test_listener_got_test_error message
20
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
21
+ end
22
+
23
+
24
+ def test_listener_got_line(status, message)
25
+ puts " +++ If you're seeing this, #{self.class.name}.#{__method__} was not overridden"
26
+ end
27
+
28
+ end
29
+
30
+
31
+ class TestListener < InstrumentsListener
32
+
33
+ attr_accessor :event_sink
34
+
35
+ def receive(message)
36
+ # assume developer has set event_sink already
37
+
38
+ # signal test starts before general logging
39
+ @event_sink.test_listener_got_test_start message.message if message.status == :start
40
+
41
+ # log all lines
42
+ if message.status == :unknown
43
+ @event_sink.test_listener_got_line nil, message.full_line
44
+ else
45
+ @event_sink.test_listener_got_line message.status, message.message
46
+ end
47
+
48
+ # signal test ends after logs
49
+ @event_sink.test_listener_got_test_pass message.message if message.status == :pass
50
+ @event_sink.test_listener_got_test_fail message.message if message.status == :fail
51
+ @event_sink.test_listener_got_test_error message.message if message.status == :error
52
+
53
+ end
54
+
55
+ def on_automation_finished
56
+ end
57
+
58
+ end