advance 0.4.5 → 0.4.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +2 -2
- data/lib/advance.rb +34 -23
- data/lib/advance/version.rb +1 -1
- metadata +6 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 36440c8a7f47c017df2840a92518a542a8b92d6b5cece7ae5edb9c8a6b8e7399
|
4
|
+
data.tar.gz: 88940f75013c5f390a4855a859dbc13175e6cc8dddeb2ee4b72ca28c3d43d029
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d025732524521d9f3b6ec95e26e3e0113bc083a472a875b2d5d83ec7ca6dfd243245089e30a12a8541b1da50527a38c5e1f3850142a82c3fcb19eff98552ef89
|
7
|
+
data.tar.gz: 9673c1369a07047d779634e37ba746f8f305cc2547f271c85e1c251aba06e231548408021f6c0410873f430968abe93133a0fa4ea6df8af3fa783c6fa85b0291
|
data/Gemfile.lock
CHANGED
data/lib/advance.rb
CHANGED
@@ -21,6 +21,18 @@ module Advance
|
|
21
21
|
WHITE="\e[1;37m"
|
22
22
|
YELLOW="\e[33m"
|
23
23
|
|
24
|
+
def env_is?(env_var_name, default)
|
25
|
+
case ENV[env_var_name]
|
26
|
+
when "true"; true
|
27
|
+
when "false"; false
|
28
|
+
when nil; default
|
29
|
+
else
|
30
|
+
puts "env variable #{env_var_name} should be 'true', 'false', or not present (defaults to '#{default}')"
|
31
|
+
puts "currently set to >#{ENV[env_var_name]}<"
|
32
|
+
false
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
24
36
|
def self.included(pipeline_module)
|
25
37
|
$pipeline = caller_locations.first.path
|
26
38
|
meta =
|
@@ -33,17 +45,6 @@ module Advance
|
|
33
45
|
$run_number = last_run_number + 1
|
34
46
|
$cores=`nproc`.to_i
|
35
47
|
puts "Multi steps will use #{$cores} cores"
|
36
|
-
|
37
|
-
$verbose_logging = case ENV["ADVANCE_VERBOSE_LOGGING"]
|
38
|
-
when "true"; true
|
39
|
-
when "false"; false
|
40
|
-
when nil
|
41
|
-
puts "For detailed logging of multi steps, rerun after 'export ADVANCE_VERBOSE_LOGGING=true'"
|
42
|
-
false
|
43
|
-
else
|
44
|
-
puts "env variable ADVANCE_VERBOSE_LOGGING should be 'true', 'false', or not present (defaults to 'false')"
|
45
|
-
puts "currently set to >#{ENV["ADVANCE_VERBOSE_LOGGING"]}<"
|
46
|
-
end
|
47
48
|
end
|
48
49
|
|
49
50
|
def update_meta(step_number, processing_mode, label, command, start_time, duration, file_count)
|
@@ -125,6 +126,11 @@ module Advance
|
|
125
126
|
end
|
126
127
|
previous_dir_path = previous_dir_path.gsub(/\.tgz$/, "")
|
127
128
|
start_time = Time.now
|
129
|
+
|
130
|
+
puts "#{GRAY}storage summary"
|
131
|
+
system('df -h .')
|
132
|
+
puts "---------#{RESET}"
|
133
|
+
|
128
134
|
send(processing_mode, command, previous_dir_path, dir_name)
|
129
135
|
file_count = count_files(dir_name)
|
130
136
|
duration = Time.now - start_time
|
@@ -132,7 +138,7 @@ module Advance
|
|
132
138
|
end
|
133
139
|
previous_dir_path = previous_dir_path.gsub(/\.tgz$/, "")
|
134
140
|
if File.basename(previous_dir_path) =~ /^step_/
|
135
|
-
if !File.exist?("#{previous_dir_path}.tgz")
|
141
|
+
if env_is?("ADVANCE_SAVE_HISTORY", true) && !File.exist?("#{previous_dir_path}.tgz")
|
136
142
|
do_command_wo_log "tar czf #{previous_dir_path}.tgz #{File.basename(previous_dir_path)}"
|
137
143
|
end
|
138
144
|
do_command_wo_log "rm -rf #{previous_dir_path}"
|
@@ -235,10 +241,11 @@ module Advance
|
|
235
241
|
command.gsub!("{input_file}", file_path)
|
236
242
|
command.gsub!("{file_name}", basename)
|
237
243
|
command.gsub!("{file_name_without_extension}", root_file_name)
|
238
|
-
puts "#{YELLOW}#{command}#{RESET} " if
|
244
|
+
puts "#{YELLOW}#{command}#{RESET} " if env_is?("ADVANCE_VERBOSE_LOGGING", false)
|
239
245
|
work_in_sub_dir(new_dir_name) do
|
240
|
-
do_command command,
|
246
|
+
do_command command, env_is?("ADVANCE_VERBOSE_LOGGING", false)
|
241
247
|
end
|
248
|
+
do_command_wo_log("rm #{file_path}", false) if !env_is?("ADVANCE_SAVE_HISTORY", true)
|
242
249
|
rescue
|
243
250
|
puts "%%%% error while processing >>#{file_path}<<"
|
244
251
|
raise
|
@@ -311,16 +318,20 @@ module Advance
|
|
311
318
|
puts "#{YELLOW}#{command}#{RESET} " if feedback
|
312
319
|
start_time = Time.now
|
313
320
|
stdout, stderr, status = Open3.capture3(command)
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
321
|
+
|
322
|
+
if env_is?("ADVANCE_LOG_FILES", true)
|
323
|
+
elapsed_time = Time.now - start_time
|
324
|
+
File.open("log", "w") do |f|
|
325
|
+
f.puts "%%% command: >#{command}<"
|
326
|
+
f.puts "%%% returned status: >#{status}<"
|
327
|
+
f.puts "%%% elapsed time: #{elapsed_time} seconds"
|
328
|
+
f.puts "%%% stdout:"
|
329
|
+
f.puts stdout
|
330
|
+
f.puts "%%% stderr:"
|
331
|
+
f.puts stderr
|
332
|
+
end
|
323
333
|
end
|
334
|
+
|
324
335
|
if !status.success?
|
325
336
|
raise "step #{$step} failed with #{status}\n#{stderr}"
|
326
337
|
end
|
data/lib/advance/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: advance
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- janemacfarlane
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-08-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: team_effort
|
@@ -106,7 +106,7 @@ homepage: https://github.com/doctorjane/advance
|
|
106
106
|
licenses:
|
107
107
|
- MIT
|
108
108
|
metadata: {}
|
109
|
-
post_install_message:
|
109
|
+
post_install_message:
|
110
110
|
rdoc_options: []
|
111
111
|
require_paths:
|
112
112
|
- lib
|
@@ -121,9 +121,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
121
|
- !ruby/object:Gem::Version
|
122
122
|
version: '0'
|
123
123
|
requirements: []
|
124
|
-
|
125
|
-
|
126
|
-
signing_key:
|
124
|
+
rubygems_version: 3.1.2
|
125
|
+
signing_key:
|
127
126
|
specification_version: 4
|
128
127
|
summary: A framework for building data transformation pipelines
|
129
128
|
test_files: []
|