flaky 0.0.22 → 0.0.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5a66676fa3248d3b07816db77305c598ca448155
-  data.tar.gz: b02de59738db71d7737d4bdcb03286935965a964
+  metadata.gz: 3d47a6d81b834bcde7d9bf32225bb93a07961a0f
+  data.tar.gz: e17bdee7458eb5cd3f8b3610ca2ef33a80a82a12
 SHA512:
-  metadata.gz: d9f8b2a16dbe1eb018a1e201880a07c46182938b17ff8582664f39fcc4e4189037fa434e78b80e664e3be4311867852bcb99f52446ff1865ba3f7ab9f3ec8242
-  data.tar.gz: 48293b05c801107679a43639a022a8e9c20e2d270c1e9bbf92712b7d24fb4955094f397222cc8deadf0f8944e385184dcd32114c9ae3dd2c21b062076d1f7f5a
+  metadata.gz: 9a11390a90e848bb18de2adfd0154af6546b61527f341cbeef135ec198251ec745b9ea1be5eef122a16e48c1610bed10c96b3bfbe00cf4bd75afd51c27684b13
+  data.tar.gz: 82e6234ef0789952a1eab8438a9d836f23ec07a0956cae9083812b50533df4cf52a395dd3ff948ba0aeadb89a600391a114b781105bb6ad8bce8309bd75a1af1

data/lib/flaky.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 require 'fileutils' # system requires
 require 'open3'
-require 'timeout' # to timeout long running color runs
+require 'timeout'
 
 require 'rubygems' # gem requires
 require 'chronic_duration'
@@ -9,8 +9,8 @@ require 'posix/spawn' # http://rubygems.org/gems/posix-spawn
 require 'digest/md5'
 
 module Flaky
-  VERSION = '0.0.22' unless defined? ::Flaky::VERSION
-  DATE = '2014-01-03' unless defined? ::Flaky::DATE
+  VERSION = '0.0.23' unless defined? ::Flaky::VERSION
+  DATE = '2014-01-08' unless defined? ::Flaky::DATE
 
   class << self; attr_accessor :no_video; end
   self.no_video = false; # set default value

data/lib/flaky/appium.rb

@@ -5,7 +5,8 @@ module Flaky
     attr_reader :pid, :in, :out, :err
 
     def initialize cmd
-      @pid, @in, @out, @err = POSIX::Spawn::popen4 cmd
+      # redirect err to child's out
+      @pid, @in, @out, @err = POSIX::Spawn::popen4 cmd, {:err => [:child, :out]}
       @in.close
     end
 
@@ -68,7 +69,7 @@ module Flaky
 
     def start
       self.stop # stop existing process
-      @log = "/tmp/flaky/tmp_log"
+      @log = "/tmp/flaky/appium_tmp_log.txt"
       File.delete(@log) if File.exists? @log
       if @android
         @droid.reset
@@ -133,6 +134,9 @@ module Flaky
       end
     end
 
+    # if this is defined using self, then instance methods must refer using
+    # self.class.end_all_nodes
+    # instead self.end_all_nodes is cleaner.
     # https://github.com/rtomayko/posix-spawn#posixspawn-as-a-mixin
     def end_all_nodes
       self.class.kill_all 'node'

data/lib/flaky/run.rb

@@ -17,7 +17,7 @@ module Flaky
   class LogArtifact
     def initialize opts={}
       @result_dir = opts.fetch :result_dir, ''
-      @pass_str   = opts.fetch :pass_str, ''
+      @pass_str = opts.fetch :pass_str, ''
       @test_name = opts.fetch :test_name, ''
     end
 
@@ -64,8 +64,10 @@ module Flaky
         runs = stats[:runs]
         pass = stats[:pass]
         fail = stats[:fail]
+        timedout = ''
+        timedout = '-- TIMED OUT' if stats[:timedout] == true
         line = "#{name}, runs: #{runs}, pass: #{pass}," +
-            " fail: #{fail}\n"
+            " fail: #{fail} #{timedout}\n"
         if fail > 0 && pass <= 0
           failure_name_only += "#{File.basename(name)}\n"
           failure += line
@@ -76,7 +78,8 @@ module Flaky
         end
       end
 
-      out = "#{total_success + total_failure} Tests\n\n"
+      total_tests = total_success + total_failure
+      out = "#{total_tests} Tests\n\n"
       out += "Failure (#{total_failure}):\n#{failure}\n" unless failure.empty?
       out += "Success (#{total_success}):\n#{success}" unless success.empty?
 
@@ -87,6 +90,12 @@ module Flaky
       time_format = '%b %d %l:%M %P'
       time_format2 = '%l:%M %P'
       out += "#{@start_time.strftime(time_format)} - #{time_now.strftime(time_format2)}"
+
+      month_day_year = Time.now.strftime '%-m/%-d/%Y'
+      success_percent = (total_success.to_f/total_tests.to_f*100).round(2)
+      success_percent = 100 if total_failure <= 0
+      google_docs_line = [month_day_year, total_tests, total_failure, total_success, success_percent].join("\t")
+      out += "\n#{google_docs_line}"
       out += "\n--\n"
 
       if save_file
@@ -103,29 +112,81 @@ module Flaky
       puts out
     end
 
+    def process_exists pid
+      begin
+        Process.waitpid(pid, Process::WNOHANG)
+        true
+      rescue
+        false
+      end
+    end
+
     def _execute run_cmd, test_name, runs, appium, sauce
       # must capture exit code or log is an array.
-      log, exit_code = Open3.capture2e run_cmd
-
       result = /\d+ runs, \d+ assertions, \d+ failures, \d+ errors, \d+ skips/
       success = /0 failures, 0 errors, 0 skips/
       passed = true
 
-      found_results = log.scan result
-      # all result instances must match success
-      found_results.each do |result|
-        # runs must be >= 1. 0 runs mean no tests were run.
-        r_count = result.match /(\d+) runs/
-        runs_not_zero = r_count && r_count[1] && r_count[1].to_i > 0 ? true : false
-
-        unless result.match(success) && runs_not_zero
-          passed = false
-          break
+      exit_code = -1
+      timedout = false
+      rake_pid = -1
+      # we need the minitest log in memory to scan for results.
+      log = ''
+      tmp_ruby_log = '/tmp/flaky/ruby_log_tmp.txt'
+      File.delete(tmp_ruby_log) if File.exists? tmp_ruby_log
+      begin
+        ten_minutes = 10 * 60
+        timeout ten_minutes do
+          rake = Flaky::Cmd.new run_cmd
+          rake_pid = rake.pid
+
+          while process_exists rake_pid
+            sleep 0.5
+            begin
+              # readpartial throws end of file reached error
+              log += rake.out.readpartial 999_999
+
+              File.open(tmp_ruby_log, 'a') do |f|
+                f.write log
+              end
+            rescue
+              break
+            end
+          end
+        end
+      rescue Exception => e
+        timedout = true
+        passed = false
+        # after_run in run.rb is triggered by sigint
+        Process.kill :SIGINT, rake_pid
+
+        begin
+          two_minutes = 2 * 60
+          timeout two_minutes do
+            Process::waitpid rake_pid
+          end
+        rescue # if the process still isn't done after sigint, use sigkill
+          Process.kill :SIGKILL, rake_pid
         end
       end
 
-      # no results found.
-      passed = false if found_results.length <= 0
+      unless timedout
+        found_results = log.scan result
+        # all result instances must match success
+        found_results.each do |result|
+          # runs must be >= 1. 0 runs mean no tests were run.
+          r_count = result.match /(\d+) runs/
+          runs_not_zero = r_count && r_count[1] && r_count[1].to_i > 0 ? true : false
+
+          unless result.match(success) && runs_not_zero
+            passed = false
+            break
+          end
+        end
+
+        # no results found.
+        passed = false if found_results.length <= 0
+      end
       pass_str = passed ? 'pass' : 'fail'
       test = @tests[test_name]
       # save log
@@ -135,6 +196,7 @@ module Flaky
       else
         fail = test[:fail] += 1
         postfix = "fail_#{fail}"
+        test[:timedout] = true if timedout
       end
 
       postfix = "#{runs}_#{test_name}_" + postfix
@@ -143,18 +205,13 @@ module Flaky
       log_file = LogArtifact.new result_dir: result_dir, pass_str: pass_str, test_name: test_name
 
       # File.open 'w' will not create folders. Use mkdir_p before.
-      test_file_path = log_file.name("#{postfix}.html")
+      test_file_path = log_file.name("#{postfix}.txt")
       FileUtils.mkdir_p File.dirname(test_file_path)
       # html Ruby test log
       File.open(test_file_path, 'w') do |f|
         f.write log
       end
 
-      # TODO: Get iOS simulator system log from appium
-      # File.open(log_file.name("#{postfix}.server.log.txt"), 'w') do |f|
-      #  f.write appium.tail.out.readpartial(999_999_999)
-      # end
-
       unless sauce
         movie_path = log_file.name("#{postfix}.mov")
         FileUtils.mkdir_p File.dirname(movie_path)
@@ -168,6 +225,13 @@ module Flaky
           File.delete movie_src if File.exists? movie_src
         end
 
+        # save .TIMED_OUT.txt in timeout fails
+        if timedout
+          timeout_path = log_file.name("#{postfix}.TIMED_OUT.txt")
+          FileUtils.mkdir_p File.dirname(timeout_path)
+          File.open(timeout_path, 'w') {}
+        end
+
         src_system_log = '/tmp/flaky_logs.txt'
         if File.exists? src_system_log
           # postfix is required! or the log will be saved to an incorrect path
@@ -178,7 +242,7 @@ module Flaky
         end
 
         # appium server log
-        appium_server_path = log_file.name("#{postfix}.appium.html")
+        appium_server_path = log_file.name("#{postfix}.appium.txt")
         FileUtils.mkdir_p File.dirname(appium_server_path)
 
         tmp_file = appium.flush_buffer
@@ -186,6 +250,8 @@ module Flaky
           FileUtils.copy_file tmp_file, appium_server_path
         end
         File.delete tmp_file if File.exists? tmp_file
+        # also delete the temp ruby log
+        File.delete tmp_ruby_log if File.exists? tmp_ruby_log
       end
 
       passed
@@ -215,13 +281,13 @@ module Flaky
       # local appium is not required when running on Sauce
       raise 'must pass :appium' unless appium || sauce
 
-      test = @tests[test_name] ||= {runs: 0, pass: 0, fail: 0}
+      test = @tests[test_name] ||= {runs: 0, pass: 0, fail: 0, timedout: false}
       runs = test[:runs] += 1
 
       passed = _execute run_cmd, test_name, runs, appium, sauce
       unless sauce
         print cyan("\n #{test_name} ") if @last_test.nil? ||
-          @last_test != test_name
+            @last_test != test_name
 
         print passed ? green(' ✓') : red(' ✖')
       else

data/readme.md

@@ -83,7 +83,7 @@ Run `flake auth` to automatically dismiss security dialogs.
       save_logs
       puts "Ending pid: #{flaky_screen_recording_pid}"
       Flaky.screen_recording_stop flaky_screen_recording_pid # save video
-      $driver.x
+      ignore { wait(10) { $driver.x } }
     end
   end
 ```

data/release_notes.md

@@ -1,3 +1,15 @@
+#### v0.0.22 2014-01-03
+
+- [e60803e](https://github.com/appium/flaky/commit/e60803e2e240617f7d7500c0cfc46e919033235d) Release 0.0.22
+- [a2783c2](https://github.com/appium/flaky/commit/a2783c2ebb7f1c53636ad8e946371443408f1327) Update screen recording
+- [9f6809f](https://github.com/appium/flaky/commit/9f6809ff8ef46d3d8c690ae208a98431a951164b) Ensure unresponsive screen-recording doesn't stop test execution
+- [ca63614](https://github.com/appium/flaky/commit/ca636144dbb2871c86773711fadc6e176ed239e9) Fix video in run modes
+- [d00e5d6](https://github.com/appium/flaky/commit/d00e5d6f3f73f3ba0f3336ae7666587ad1b264fb) Pass no_video boolean to rake
+- [91ec551](https://github.com/appium/flaky/commit/91ec551284736f76b137b18b028f4211d9b46287) Update readme.md
+- [2d6b276](https://github.com/appium/flaky/commit/2d6b2766a110f187d903f77bdd73999d3930e845) Add --no-video flag
+- [e9312c1](https://github.com/appium/flaky/commit/e9312c1f2a9a6c729658ddb6fc826a169a86be0e) Add sample Ruby integration
+
+
 #### v0.0.21 2014-01-02
 
 - [10babb7](https://github.com/appium/flaky/commit/10babb7dcee2e811c6fb6a78d5dcf2c6fffba2f2) Release 0.0.21

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: flaky
 version: !ruby/object:Gem::Version
-  version: 0.0.22
+  version: 0.0.23
 platform: ruby
 authors:
 - code@bootstraponline.com
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-03 00:00:00.000000000 Z
+date: 2014-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chronic_duration