advance 0.4.1 → 0.4.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/Gemfile.lock +3 -3
- data/advance.gemspec +1 -1
- data/bin/split_csv.rb +6 -4
- data/lib/advance.rb +17 -8
- data/lib/advance/version.rb +1 -1
- metadata +8 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ca35e055d27b22f73b7187a6f3fb2d1f33aa34038fa6460ade966edf48b69847
|
4
|
+
data.tar.gz: 2f5025dfcd45580a7b8b8c52e671de06d3c45de2e8e25d0769e3126426548dcd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4a7b9a3685f2e6fad1b73f730801b83f0873e844f5203733b2d45cc43950571fd43745c9a86efae35ae67b4c5f503fccae7ef11dd481187fcfff5805878df83f
|
7
|
+
data.tar.gz: ab02b802b9a533f48999122789d0d4e7526a4cd8da84e83d440acfbc6dc457d5be41089e1fca51b08d53e47edc38e57bffef61e7169e1427fe8f0382996cf949
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
advance (0.4.
|
4
|
+
advance (0.4.6)
|
5
5
|
team_effort
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
9
9
|
specs:
|
10
10
|
minitest (5.10.1)
|
11
|
-
rake (
|
11
|
+
rake (13.0.1)
|
12
12
|
team_effort (1.0.1)
|
13
13
|
|
14
14
|
PLATFORMS
|
@@ -18,7 +18,7 @@ DEPENDENCIES
|
|
18
18
|
advance!
|
19
19
|
bundler (~> 1.16)
|
20
20
|
minitest (~> 5.0)
|
21
|
-
rake (~>
|
21
|
+
rake (~> 13.0)
|
22
22
|
|
23
23
|
BUNDLED WITH
|
24
24
|
1.17.3
|
data/advance.gemspec
CHANGED
@@ -30,6 +30,6 @@ step become the input to the next step.
|
|
30
30
|
|
31
31
|
spec.add_runtime_dependency "team_effort"
|
32
32
|
spec.add_development_dependency "bundler", "~> 1.16"
|
33
|
-
spec.add_development_dependency "rake", "~>
|
33
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
34
34
|
spec.add_development_dependency "minitest", "~> 5.0"
|
35
35
|
end
|
data/bin/split_csv.rb
CHANGED
@@ -9,13 +9,15 @@ end
|
|
9
9
|
csv_file = ARGV[0]
|
10
10
|
lines = ARGV[1]
|
11
11
|
csv_file_name = File.basename(csv_file)
|
12
|
-
|
12
|
+
header_file_name = "#{csv_file_name}_header"
|
13
|
+
system "head -n 1 #{csv_file} > #{header_file_name}"
|
13
14
|
system "tail -n +2 #{csv_file} | split -l #{lines} -a 3 - #{csv_file_name}_"
|
14
|
-
files = Dir.entries(".").reject { |f| f =~ %r{^(\.\.?|log)$} }
|
15
|
+
files = Dir.entries(".").reject { |f| f.end_with?(header_file_name) || f =~ %r{^(\.\.?|log)$} }
|
15
16
|
TeamEffort.work(files, 1) do |file|
|
16
17
|
tmp_file = "tmp_#{file}"
|
17
|
-
do_cmd "cat #{
|
18
|
+
do_cmd "cat #{header_file_name} #{file} >> #{tmp_file}"
|
18
19
|
do_cmd "mv #{tmp_file} #{file}"
|
19
20
|
end
|
20
|
-
do_cmd "rm #{
|
21
|
+
do_cmd "rm #{header_file_name}"
|
22
|
+
do_cmd "rename 's/^(.*)\.csv_(.*)$/$1_$2.csv/' *csv*"
|
21
23
|
puts ""
|
data/lib/advance.rb
CHANGED
@@ -37,13 +37,21 @@ module Advance
|
|
37
37
|
$verbose_logging = case ENV["ADVANCE_VERBOSE_LOGGING"]
|
38
38
|
when "true"; true
|
39
39
|
when "false"; false
|
40
|
-
when nil
|
41
|
-
puts "For detailed logging of multi steps, rerun after 'export ADVANCE_VERBOSE_LOGGING=true'"
|
42
|
-
false
|
40
|
+
when nil; false
|
43
41
|
else
|
44
42
|
puts "env variable ADVANCE_VERBOSE_LOGGING should be 'true', 'false', or not present (defaults to 'false')"
|
45
43
|
puts "currently set to >#{ENV["ADVANCE_VERBOSE_LOGGING"]}<"
|
44
|
+
false
|
46
45
|
end
|
46
|
+
$save_history = case ENV["ADVANCE_SAVE_HISTORY"]
|
47
|
+
when "true"; true
|
48
|
+
when "false"; false
|
49
|
+
when nil; true
|
50
|
+
else
|
51
|
+
puts "env variable ADVANCE_SAVE_HISTORY should be 'true', 'false', or not present (defaults to 'true')"
|
52
|
+
puts "currently set to >#{ENV["ADVANCE_SAVE_HISTORY"]}<"
|
53
|
+
true
|
54
|
+
end
|
47
55
|
end
|
48
56
|
|
49
57
|
def update_meta(step_number, processing_mode, label, command, start_time, duration, file_count)
|
@@ -132,7 +140,7 @@ module Advance
|
|
132
140
|
end
|
133
141
|
previous_dir_path = previous_dir_path.gsub(/\.tgz$/, "")
|
134
142
|
if File.basename(previous_dir_path) =~ /^step_/
|
135
|
-
if !File.exist?("#{previous_dir_path}.tgz")
|
143
|
+
if $save_history && !File.exist?("#{previous_dir_path}.tgz")
|
136
144
|
do_command_wo_log "tar czf #{previous_dir_path}.tgz #{File.basename(previous_dir_path)}"
|
137
145
|
end
|
138
146
|
do_command_wo_log "rm -rf #{previous_dir_path}"
|
@@ -140,8 +148,7 @@ module Advance
|
|
140
148
|
end
|
141
149
|
|
142
150
|
def pipeline(pipeline_path)
|
143
|
-
|
144
|
-
load expanded_path
|
151
|
+
load pipeline_path
|
145
152
|
end
|
146
153
|
|
147
154
|
def count_files(dir)
|
@@ -220,7 +227,7 @@ module Advance
|
|
220
227
|
|
221
228
|
last_progress = ""
|
222
229
|
progress_proc = ->(index, max_index) do
|
223
|
-
latest_progress = sprintf("%3i
|
230
|
+
latest_progress = sprintf("%3i%%", index.to_f / max_index * 100)
|
224
231
|
puts latest_progress if last_progress != latest_progress
|
225
232
|
last_progress = latest_progress
|
226
233
|
end
|
@@ -309,7 +316,7 @@ module Advance
|
|
309
316
|
end
|
310
317
|
|
311
318
|
def do_command(command, feedback = true)
|
312
|
-
puts "#{YELLOW}#{command}#{RESET} "
|
319
|
+
puts "#{YELLOW}#{command}#{RESET} " if feedback
|
313
320
|
start_time = Time.now
|
314
321
|
stdout, stderr, status = Open3.capture3(command)
|
315
322
|
elapsed_time = Time.now - start_time
|
@@ -346,6 +353,8 @@ module Advance
|
|
346
353
|
end
|
347
354
|
|
348
355
|
def ensure_bin_on_path
|
356
|
+
$LOAD_PATH << File.expand_path(File.join(caller_locations.first.path, "../../lib"))
|
357
|
+
|
349
358
|
advance_bin_path = File.expand_path(File.join(File.dirname(__FILE__), "../bin"))
|
350
359
|
add_dir_to_path(advance_bin_path)
|
351
360
|
|
data/lib/advance/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: advance
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- janemacfarlane
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-06-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: team_effort
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '13.0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '13.0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: minitest
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -106,7 +106,7 @@ homepage: https://github.com/doctorjane/advance
|
|
106
106
|
licenses:
|
107
107
|
- MIT
|
108
108
|
metadata: {}
|
109
|
-
post_install_message:
|
109
|
+
post_install_message:
|
110
110
|
rdoc_options: []
|
111
111
|
require_paths:
|
112
112
|
- lib
|
@@ -121,9 +121,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
121
|
- !ruby/object:Gem::Version
|
122
122
|
version: '0'
|
123
123
|
requirements: []
|
124
|
-
|
125
|
-
|
126
|
-
signing_key:
|
124
|
+
rubygems_version: 3.0.6
|
125
|
+
signing_key:
|
127
126
|
specification_version: 4
|
128
127
|
summary: A framework for building data transformation pipelines
|
129
128
|
test_files: []
|