advance 0.3.18 → 0.4.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/Gemfile.lock +5 -5
- data/advance.gemspec +1 -1
- data/bin/split_csv.rb +6 -4
- data/lib/advance.rb +20 -11
- data/lib/advance/version.rb +1 -1
- metadata +5 -11
- data/bin/console +0 -14
- data/bin/remove_logs_and_sub_dirs +0 -4
- data/bin/setup +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2c3b98570bb73f6b6c2052d8c14c45c69897edf76b6ae610983eef6a6feaf4e4
|
4
|
+
data.tar.gz: 9e8e3f845512b8b139b2a11d572a226680986f1ff5bdf6816d7c329ebe402ae6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1e334afa520ae03ab4ec4a4a974242044d582f6892e18aadfd0e260ffef22bd5892276856249988e596af34cb0b5e964ac636980f7cc64949488186f244074c0
|
7
|
+
data.tar.gz: 102ee97e718c637ac18ec6240e313695023b5fd9ca0c77cf5ad15ab3ddf9c781006105f6d0670445f1908c04d3d97d4e96975de1695c9643f897ef46ae5a7b8b
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
advance (0.
|
4
|
+
advance (0.4.5)
|
5
5
|
team_effort
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
9
9
|
specs:
|
10
10
|
minitest (5.10.1)
|
11
|
-
rake (
|
12
|
-
team_effort (1.
|
11
|
+
rake (13.0.1)
|
12
|
+
team_effort (1.1.1)
|
13
13
|
|
14
14
|
PLATFORMS
|
15
15
|
ruby
|
@@ -18,7 +18,7 @@ DEPENDENCIES
|
|
18
18
|
advance!
|
19
19
|
bundler (~> 1.16)
|
20
20
|
minitest (~> 5.0)
|
21
|
-
rake (~>
|
21
|
+
rake (~> 13.0)
|
22
22
|
|
23
23
|
BUNDLED WITH
|
24
|
-
1.17.
|
24
|
+
1.17.3
|
data/advance.gemspec
CHANGED
@@ -30,6 +30,6 @@ step become the input to the next step.
|
|
30
30
|
|
31
31
|
spec.add_runtime_dependency "team_effort"
|
32
32
|
spec.add_development_dependency "bundler", "~> 1.16"
|
33
|
-
spec.add_development_dependency "rake", "~>
|
33
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
34
34
|
spec.add_development_dependency "minitest", "~> 5.0"
|
35
35
|
end
|
data/bin/split_csv.rb
CHANGED
@@ -9,13 +9,15 @@ end
|
|
9
9
|
csv_file = ARGV[0]
|
10
10
|
lines = ARGV[1]
|
11
11
|
csv_file_name = File.basename(csv_file)
|
12
|
-
|
12
|
+
header_file_name = "#{csv_file_name}_header"
|
13
|
+
system "head -n 1 #{csv_file} > #{header_file_name}"
|
13
14
|
system "tail -n +2 #{csv_file} | split -l #{lines} -a 3 - #{csv_file_name}_"
|
14
|
-
files = Dir.entries(".").reject { |f| f =~ %r{^(\.\.?|log)$} }
|
15
|
+
files = Dir.entries(".").reject { |f| f.end_with?(header_file_name) || f =~ %r{^(\.\.?|log)$} }
|
15
16
|
TeamEffort.work(files, 1) do |file|
|
16
17
|
tmp_file = "tmp_#{file}"
|
17
|
-
do_cmd "cat #{
|
18
|
+
do_cmd "cat #{header_file_name} #{file} >> #{tmp_file}"
|
18
19
|
do_cmd "mv #{tmp_file} #{file}"
|
19
20
|
end
|
20
|
-
do_cmd "rm #{
|
21
|
+
do_cmd "rm #{header_file_name}"
|
22
|
+
do_cmd "rename 's/^(.*)\.csv_(.*)$/$1_$2.csv/' *csv*"
|
21
23
|
puts ""
|
data/lib/advance.rb
CHANGED
@@ -80,8 +80,9 @@ module Advance
|
|
80
80
|
end
|
81
81
|
|
82
82
|
if File.exist?(".meta")
|
83
|
-
read_column_names_from_meta
|
84
|
-
|
83
|
+
if read_column_names_from_meta
|
84
|
+
return
|
85
|
+
end
|
85
86
|
end
|
86
87
|
|
87
88
|
previous_dir_path = get_previous_dir_path
|
@@ -98,10 +99,11 @@ module Advance
|
|
98
99
|
run.each do |step|
|
99
100
|
if step["columns"]
|
100
101
|
$column_names = step["columns"].map(&:to_sym)
|
101
|
-
return
|
102
|
+
return true
|
102
103
|
end
|
103
104
|
end
|
104
105
|
end
|
106
|
+
false
|
105
107
|
end
|
106
108
|
|
107
109
|
def advance(processing_mode, label, command)
|
@@ -137,6 +139,10 @@ module Advance
|
|
137
139
|
end
|
138
140
|
end
|
139
141
|
|
142
|
+
def pipeline(pipeline_path)
|
143
|
+
load pipeline_path
|
144
|
+
end
|
145
|
+
|
140
146
|
def count_files(dir)
|
141
147
|
file_count = 0
|
142
148
|
Find.find(dir) do |path|
|
@@ -194,14 +200,15 @@ module Advance
|
|
194
200
|
|
195
201
|
def single(command, previous_dir_path, dir_name)
|
196
202
|
work_in_sub_dir(dir_name) do
|
197
|
-
input_file_path = previous_file_path(previous_dir_path)
|
198
|
-
basename = File.basename(input_file_path)
|
199
|
-
root_file_name = basename.gsub(%r(\.[^.]+$), '')
|
200
203
|
command.gsub!("{input_dir}", previous_dir_path)
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
204
|
+
input_file_path = previous_file_path(previous_dir_path)
|
205
|
+
if input_file_path
|
206
|
+
basename = File.basename(input_file_path)
|
207
|
+
root_file_name = basename.gsub(%r(\.[^.]+$), '')
|
208
|
+
command.gsub!("{input_file}", input_file_path)
|
209
|
+
command.gsub!("{file_name}", basename)
|
210
|
+
command.gsub!("{file_name_without_extension}", root_file_name)
|
211
|
+
end
|
205
212
|
do_command command
|
206
213
|
end
|
207
214
|
end
|
@@ -212,7 +219,7 @@ module Advance
|
|
212
219
|
|
213
220
|
last_progress = ""
|
214
221
|
progress_proc = ->(index, max_index) do
|
215
|
-
latest_progress = sprintf("%3i
|
222
|
+
latest_progress = sprintf("%3i%%", index.to_f / max_index * 100)
|
216
223
|
puts latest_progress if last_progress != latest_progress
|
217
224
|
last_progress = latest_progress
|
218
225
|
end
|
@@ -338,6 +345,8 @@ module Advance
|
|
338
345
|
end
|
339
346
|
|
340
347
|
def ensure_bin_on_path
|
348
|
+
$LOAD_PATH << File.expand_path(File.join(caller_locations.first.path, "../../lib"))
|
349
|
+
|
341
350
|
advance_bin_path = File.expand_path(File.join(File.dirname(__FILE__), "../bin"))
|
342
351
|
add_dir_to_path(advance_bin_path)
|
343
352
|
|
data/lib/advance/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: advance
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- janemacfarlane
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-06-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: team_effort
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '13.0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '13.0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: minitest
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -77,12 +77,9 @@ email:
|
|
77
77
|
executables:
|
78
78
|
- concat_csv.rb
|
79
79
|
- concat_csv_nh.rb
|
80
|
-
- console
|
81
80
|
- csv_select.rb
|
82
81
|
- csv_select_nh.rb
|
83
82
|
- csv_split_on_change_nh.rb
|
84
|
-
- remove_logs_and_sub_dirs
|
85
|
-
- setup
|
86
83
|
- split_csv.rb
|
87
84
|
extensions: []
|
88
85
|
extra_rdoc_files: []
|
@@ -97,12 +94,9 @@ files:
|
|
97
94
|
- advance.gemspec
|
98
95
|
- bin/concat_csv.rb
|
99
96
|
- bin/concat_csv_nh.rb
|
100
|
-
- bin/console
|
101
97
|
- bin/csv_select.rb
|
102
98
|
- bin/csv_select_nh.rb
|
103
99
|
- bin/csv_split_on_change_nh.rb
|
104
|
-
- bin/remove_logs_and_sub_dirs
|
105
|
-
- bin/setup
|
106
100
|
- bin/split_csv.rb
|
107
101
|
- lib/advance.rb
|
108
102
|
- lib/advance/version.rb
|
@@ -128,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
128
122
|
version: '0'
|
129
123
|
requirements: []
|
130
124
|
rubyforge_project:
|
131
|
-
rubygems_version: 2.7.
|
125
|
+
rubygems_version: 2.7.8
|
132
126
|
signing_key:
|
133
127
|
specification_version: 4
|
134
128
|
summary: A framework for building data transformation pipelines
|
data/bin/console
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require "bundler/setup"
|
4
|
-
require "advance"
|
5
|
-
|
6
|
-
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
-
# with your gem easier. You can also use a different console, if you like.
|
8
|
-
|
9
|
-
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
-
# require "pry"
|
11
|
-
# Pry.start
|
12
|
-
|
13
|
-
require "irb"
|
14
|
-
IRB.start(__FILE__)
|