flaky 0.0.22 → 0.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/flaky.rb +3 -3
- data/lib/flaky/appium.rb +6 -2
- data/lib/flaky/run.rb +92 -26
- data/readme.md +1 -1
- data/release_notes.md +12 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3d47a6d81b834bcde7d9bf32225bb93a07961a0f
|
4
|
+
data.tar.gz: e17bdee7458eb5cd3f8b3610ca2ef33a80a82a12
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9a11390a90e848bb18de2adfd0154af6546b61527f341cbeef135ec198251ec745b9ea1be5eef122a16e48c1610bed10c96b3bfbe00cf4bd75afd51c27684b13
|
7
|
+
data.tar.gz: 82e6234ef0789952a1eab8438a9d836f23ec07a0956cae9083812b50533df4cf52a395dd3ff948ba0aeadb89a600391a114b781105bb6ad8bce8309bd75a1af1
|
data/lib/flaky.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'fileutils' # system requires
|
3
3
|
require 'open3'
|
4
|
-
require 'timeout'
|
4
|
+
require 'timeout'
|
5
5
|
|
6
6
|
require 'rubygems' # gem requires
|
7
7
|
require 'chronic_duration'
|
@@ -9,8 +9,8 @@ require 'posix/spawn' # http://rubygems.org/gems/posix-spawn
|
|
9
9
|
require 'digest/md5'
|
10
10
|
|
11
11
|
module Flaky
|
12
|
-
VERSION = '0.0.
|
13
|
-
DATE = '2014-01-
|
12
|
+
VERSION = '0.0.23' unless defined? ::Flaky::VERSION
|
13
|
+
DATE = '2014-01-08' unless defined? ::Flaky::DATE
|
14
14
|
|
15
15
|
class << self; attr_accessor :no_video; end
|
16
16
|
self.no_video = false; # set default value
|
data/lib/flaky/appium.rb
CHANGED
@@ -5,7 +5,8 @@ module Flaky
|
|
5
5
|
attr_reader :pid, :in, :out, :err
|
6
6
|
|
7
7
|
def initialize cmd
|
8
|
-
|
8
|
+
# redirect err to child's out
|
9
|
+
@pid, @in, @out, @err = POSIX::Spawn::popen4 cmd, {:err => [:child, :out]}
|
9
10
|
@in.close
|
10
11
|
end
|
11
12
|
|
@@ -68,7 +69,7 @@ module Flaky
|
|
68
69
|
|
69
70
|
def start
|
70
71
|
self.stop # stop existing process
|
71
|
-
@log = "/tmp/flaky/
|
72
|
+
@log = "/tmp/flaky/appium_tmp_log.txt"
|
72
73
|
File.delete(@log) if File.exists? @log
|
73
74
|
if @android
|
74
75
|
@droid.reset
|
@@ -133,6 +134,9 @@ module Flaky
|
|
133
134
|
end
|
134
135
|
end
|
135
136
|
|
137
|
+
# if this is defined using self, then instance methods must refer using
|
138
|
+
# self.class.end_all_nodes
|
139
|
+
# instead self.end_all_nodes is cleaner.
|
136
140
|
# https://github.com/rtomayko/posix-spawn#posixspawn-as-a-mixin
|
137
141
|
def end_all_nodes
|
138
142
|
self.class.kill_all 'node'
|
data/lib/flaky/run.rb
CHANGED
@@ -17,7 +17,7 @@ module Flaky
|
|
17
17
|
class LogArtifact
|
18
18
|
def initialize opts={}
|
19
19
|
@result_dir = opts.fetch :result_dir, ''
|
20
|
-
@pass_str
|
20
|
+
@pass_str = opts.fetch :pass_str, ''
|
21
21
|
@test_name = opts.fetch :test_name, ''
|
22
22
|
end
|
23
23
|
|
@@ -64,8 +64,10 @@ module Flaky
|
|
64
64
|
runs = stats[:runs]
|
65
65
|
pass = stats[:pass]
|
66
66
|
fail = stats[:fail]
|
67
|
+
timedout = ''
|
68
|
+
timedout = '-- TIMED OUT' if stats[:timedout] == true
|
67
69
|
line = "#{name}, runs: #{runs}, pass: #{pass}," +
|
68
|
-
" fail: #{fail}\n"
|
70
|
+
" fail: #{fail} #{timedout}\n"
|
69
71
|
if fail > 0 && pass <= 0
|
70
72
|
failure_name_only += "#{File.basename(name)}\n"
|
71
73
|
failure += line
|
@@ -76,7 +78,8 @@ module Flaky
|
|
76
78
|
end
|
77
79
|
end
|
78
80
|
|
79
|
-
|
81
|
+
total_tests = total_success + total_failure
|
82
|
+
out = "#{total_tests} Tests\n\n"
|
80
83
|
out += "Failure (#{total_failure}):\n#{failure}\n" unless failure.empty?
|
81
84
|
out += "Success (#{total_success}):\n#{success}" unless success.empty?
|
82
85
|
|
@@ -87,6 +90,12 @@ module Flaky
|
|
87
90
|
time_format = '%b %d %l:%M %P'
|
88
91
|
time_format2 = '%l:%M %P'
|
89
92
|
out += "#{@start_time.strftime(time_format)} - #{time_now.strftime(time_format2)}"
|
93
|
+
|
94
|
+
month_day_year = Time.now.strftime '%-m/%-d/%Y'
|
95
|
+
success_percent = (total_success.to_f/total_tests.to_f*100).round(2)
|
96
|
+
success_percent = 100 if total_failure <= 0
|
97
|
+
google_docs_line = [month_day_year, total_tests, total_failure, total_success, success_percent].join("\t")
|
98
|
+
out += "\n#{google_docs_line}"
|
90
99
|
out += "\n--\n"
|
91
100
|
|
92
101
|
if save_file
|
@@ -103,29 +112,81 @@ module Flaky
|
|
103
112
|
puts out
|
104
113
|
end
|
105
114
|
|
115
|
+
def process_exists pid
|
116
|
+
begin
|
117
|
+
Process.waitpid(pid, Process::WNOHANG)
|
118
|
+
true
|
119
|
+
rescue
|
120
|
+
false
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
106
124
|
def _execute run_cmd, test_name, runs, appium, sauce
|
107
125
|
# must capture exit code or log is an array.
|
108
|
-
log, exit_code = Open3.capture2e run_cmd
|
109
|
-
|
110
126
|
result = /\d+ runs, \d+ assertions, \d+ failures, \d+ errors, \d+ skips/
|
111
127
|
success = /0 failures, 0 errors, 0 skips/
|
112
128
|
passed = true
|
113
129
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
130
|
+
exit_code = -1
|
131
|
+
timedout = false
|
132
|
+
rake_pid = -1
|
133
|
+
# we need the minitest log in memory to scan for results.
|
134
|
+
log = ''
|
135
|
+
tmp_ruby_log = '/tmp/flaky/ruby_log_tmp.txt'
|
136
|
+
File.delete(tmp_ruby_log) if File.exists? tmp_ruby_log
|
137
|
+
begin
|
138
|
+
ten_minutes = 10 * 60
|
139
|
+
timeout ten_minutes do
|
140
|
+
rake = Flaky::Cmd.new run_cmd
|
141
|
+
rake_pid = rake.pid
|
142
|
+
|
143
|
+
while process_exists rake_pid
|
144
|
+
sleep 0.5
|
145
|
+
begin
|
146
|
+
# readpartial throws end of file reached error
|
147
|
+
log += rake.out.readpartial 999_999
|
148
|
+
|
149
|
+
File.open(tmp_ruby_log, 'a') do |f|
|
150
|
+
f.write log
|
151
|
+
end
|
152
|
+
rescue
|
153
|
+
break
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
rescue Exception => e
|
158
|
+
timedout = true
|
159
|
+
passed = false
|
160
|
+
# after_run in run.rb is triggered by sigint
|
161
|
+
Process.kill :SIGINT, rake_pid
|
162
|
+
|
163
|
+
begin
|
164
|
+
two_minutes = 2 * 60
|
165
|
+
timeout two_minutes do
|
166
|
+
Process::waitpid rake_pid
|
167
|
+
end
|
168
|
+
rescue # if the process still isn't done after sigint, use sigkill
|
169
|
+
Process.kill :SIGKILL, rake_pid
|
124
170
|
end
|
125
171
|
end
|
126
172
|
|
127
|
-
|
128
|
-
|
173
|
+
unless timedout
|
174
|
+
found_results = log.scan result
|
175
|
+
# all result instances must match success
|
176
|
+
found_results.each do |result|
|
177
|
+
# runs must be >= 1. 0 runs mean no tests were run.
|
178
|
+
r_count = result.match /(\d+) runs/
|
179
|
+
runs_not_zero = r_count && r_count[1] && r_count[1].to_i > 0 ? true : false
|
180
|
+
|
181
|
+
unless result.match(success) && runs_not_zero
|
182
|
+
passed = false
|
183
|
+
break
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
# no results found.
|
188
|
+
passed = false if found_results.length <= 0
|
189
|
+
end
|
129
190
|
pass_str = passed ? 'pass' : 'fail'
|
130
191
|
test = @tests[test_name]
|
131
192
|
# save log
|
@@ -135,6 +196,7 @@ module Flaky
|
|
135
196
|
else
|
136
197
|
fail = test[:fail] += 1
|
137
198
|
postfix = "fail_#{fail}"
|
199
|
+
test[:timedout] = true if timedout
|
138
200
|
end
|
139
201
|
|
140
202
|
postfix = "#{runs}_#{test_name}_" + postfix
|
@@ -143,18 +205,13 @@ module Flaky
|
|
143
205
|
log_file = LogArtifact.new result_dir: result_dir, pass_str: pass_str, test_name: test_name
|
144
206
|
|
145
207
|
# File.open 'w' will not create folders. Use mkdir_p before.
|
146
|
-
test_file_path = log_file.name("#{postfix}.
|
208
|
+
test_file_path = log_file.name("#{postfix}.txt")
|
147
209
|
FileUtils.mkdir_p File.dirname(test_file_path)
|
148
210
|
# html Ruby test log
|
149
211
|
File.open(test_file_path, 'w') do |f|
|
150
212
|
f.write log
|
151
213
|
end
|
152
214
|
|
153
|
-
# TODO: Get iOS simulator system log from appium
|
154
|
-
# File.open(log_file.name("#{postfix}.server.log.txt"), 'w') do |f|
|
155
|
-
# f.write appium.tail.out.readpartial(999_999_999)
|
156
|
-
# end
|
157
|
-
|
158
215
|
unless sauce
|
159
216
|
movie_path = log_file.name("#{postfix}.mov")
|
160
217
|
FileUtils.mkdir_p File.dirname(movie_path)
|
@@ -168,6 +225,13 @@ module Flaky
|
|
168
225
|
File.delete movie_src if File.exists? movie_src
|
169
226
|
end
|
170
227
|
|
228
|
+
# save .TIMED_OUT.txt in timeout fails
|
229
|
+
if timedout
|
230
|
+
timeout_path = log_file.name("#{postfix}.TIMED_OUT.txt")
|
231
|
+
FileUtils.mkdir_p File.dirname(timeout_path)
|
232
|
+
File.open(timeout_path, 'w') {}
|
233
|
+
end
|
234
|
+
|
171
235
|
src_system_log = '/tmp/flaky_logs.txt'
|
172
236
|
if File.exists? src_system_log
|
173
237
|
# postfix is required! or the log will be saved to an incorrect path
|
@@ -178,7 +242,7 @@ module Flaky
|
|
178
242
|
end
|
179
243
|
|
180
244
|
# appium server log
|
181
|
-
appium_server_path = log_file.name("#{postfix}.appium.
|
245
|
+
appium_server_path = log_file.name("#{postfix}.appium.txt")
|
182
246
|
FileUtils.mkdir_p File.dirname(appium_server_path)
|
183
247
|
|
184
248
|
tmp_file = appium.flush_buffer
|
@@ -186,6 +250,8 @@ module Flaky
|
|
186
250
|
FileUtils.copy_file tmp_file, appium_server_path
|
187
251
|
end
|
188
252
|
File.delete tmp_file if File.exists? tmp_file
|
253
|
+
# also delete the temp ruby log
|
254
|
+
File.delete tmp_ruby_log if File.exists? tmp_ruby_log
|
189
255
|
end
|
190
256
|
|
191
257
|
passed
|
@@ -215,13 +281,13 @@ module Flaky
|
|
215
281
|
# local appium is not required when running on Sauce
|
216
282
|
raise 'must pass :appium' unless appium || sauce
|
217
283
|
|
218
|
-
test = @tests[test_name] ||= {runs: 0, pass: 0, fail: 0}
|
284
|
+
test = @tests[test_name] ||= {runs: 0, pass: 0, fail: 0, timedout: false}
|
219
285
|
runs = test[:runs] += 1
|
220
286
|
|
221
287
|
passed = _execute run_cmd, test_name, runs, appium, sauce
|
222
288
|
unless sauce
|
223
289
|
print cyan("\n #{test_name} ") if @last_test.nil? ||
|
224
|
-
|
290
|
+
@last_test != test_name
|
225
291
|
|
226
292
|
print passed ? green(' ✓') : red(' ✖')
|
227
293
|
else
|
data/readme.md
CHANGED
@@ -83,7 +83,7 @@ Run `flake auth` to automatically dismiss security dialogs.
|
|
83
83
|
save_logs
|
84
84
|
puts "Ending pid: #{flaky_screen_recording_pid}"
|
85
85
|
Flaky.screen_recording_stop flaky_screen_recording_pid # save video
|
86
|
-
$driver.x
|
86
|
+
ignore { wait(10) { $driver.x } }
|
87
87
|
end
|
88
88
|
end
|
89
89
|
```
|
data/release_notes.md
CHANGED
@@ -1,3 +1,15 @@
|
|
1
|
+
#### v0.0.22 2014-01-03
|
2
|
+
|
3
|
+
- [e60803e](https://github.com/appium/flaky/commit/e60803e2e240617f7d7500c0cfc46e919033235d) Release 0.0.22
|
4
|
+
- [a2783c2](https://github.com/appium/flaky/commit/a2783c2ebb7f1c53636ad8e946371443408f1327) Update screen recording
|
5
|
+
- [9f6809f](https://github.com/appium/flaky/commit/9f6809ff8ef46d3d8c690ae208a98431a951164b) Ensure unresponsive screen-recording doesn't stop test execution
|
6
|
+
- [ca63614](https://github.com/appium/flaky/commit/ca636144dbb2871c86773711fadc6e176ed239e9) Fix video in run modes
|
7
|
+
- [d00e5d6](https://github.com/appium/flaky/commit/d00e5d6f3f73f3ba0f3336ae7666587ad1b264fb) Pass no_video boolean to rake
|
8
|
+
- [91ec551](https://github.com/appium/flaky/commit/91ec551284736f76b137b18b028f4211d9b46287) Update readme.md
|
9
|
+
- [2d6b276](https://github.com/appium/flaky/commit/2d6b2766a110f187d903f77bdd73999d3930e845) Add --no-video flag
|
10
|
+
- [e9312c1](https://github.com/appium/flaky/commit/e9312c1f2a9a6c729658ddb6fc826a169a86be0e) Add sample Ruby integration
|
11
|
+
|
12
|
+
|
1
13
|
#### v0.0.21 2014-01-02
|
2
14
|
|
3
15
|
- [10babb7](https://github.com/appium/flaky/commit/10babb7dcee2e811c6fb6a78d5dcf2c6fffba2f2) Release 0.0.21
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flaky
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.23
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- code@bootstraponline.com
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-01-
|
11
|
+
date: 2014-01-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: chronic_duration
|