advance 0.4.2 → 0.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f5e8405ec63f8137e9bd384d8548e381a04140e69144ce86ce7a48221e0e03b
-  data.tar.gz: 3d31957d005b36f42d3174fc5bf402e3797c54587c06a3db51c7d7af03ecd864
+  metadata.gz: 375512e62a9c6d6004938ed1e6324e87a0370c6b9b41514a65fc97de879aa413
+  data.tar.gz: 30877f6ade5427ea453394cbc31c8526199754e7ba0af2bbb24f9b3288cf86fd
 SHA512:
-  metadata.gz: 355d0dd1fb1b76f218034cdbcf8de445cf6e84ec20e5dd57c22fc9ff05efc4bd70ab1ebf4d902e482a062b0b30a40fd8782303f4d38e18db04c5f011cd43c444
-  data.tar.gz: 51542a803ce7b3830a9d3f25775cb9805c7ea6ceee39c5ccd886715db6641f72ccf77e2f529589f3cbea419771b37b6893f25d430df600c93264363917d5da6d
+  metadata.gz: 6a392c9abcca6b356f2429bd6288e9f2a2335d1d85b933f4db848911a8a595541090363585a30486619c031400a4df763189aeea5c65867c2a027ae960a6e02d
+  data.tar.gz: fa285b94002bf9741fa3e610f676581908953968b4a5e2cb1c5f74476ca84f5e763a992c9a2f8e9d5e95e33d0f612a9b2ac70cd8abb5ffbd82b5fa62b6209358

data/.gitignore

@@ -9,3 +9,4 @@
 .idea
 .ruby-gemset
 .ruby-version
+.DS_Store

data/Gemfile.lock

@@ -1,15 +1,15 @@
 PATH
   remote: .
   specs:
-    advance (0.4.2)
+    advance (0.4.7)
       team_effort
 
 GEM
   remote: https://rubygems.org/
   specs:
     minitest (5.10.1)
-    rake (10.4.2)
-    team_effort (1.0.1)
+    rake (13.0.1)
+    team_effort (1.1.1)
 
 PLATFORMS
   ruby
@@ -18,7 +18,7 @@ DEPENDENCIES
   advance!
   bundler (~> 1.16)
   minitest (~> 5.0)
-  rake (~> 10.0)
+  rake (~> 13.0)
 
 BUNDLED WITH
    1.17.3

data/advance.gemspec

@@ -30,6 +30,6 @@ step become the input to the next step.
 
   spec.add_runtime_dependency "team_effort"
   spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
 end

data/bin/split_csv.rb

@@ -9,13 +9,15 @@ end
 csv_file = ARGV[0]
 lines = ARGV[1]
 csv_file_name = File.basename(csv_file)
-system "head -n 1 #{csv_file} > #{csv_file_name}_header"
+header_file_name = "#{csv_file_name}_header"
+system "head -n 1 #{csv_file} > #{header_file_name}"
 system "tail -n +2 #{csv_file} | split -l #{lines} -a 3 - #{csv_file_name}_"
-files = Dir.entries(".").reject { |f| f =~ %r{^(\.\.?|log)$} }
+files = Dir.entries(".").reject { |f| f.end_with?(header_file_name) || f =~ %r{^(\.\.?|log)$} }
 TeamEffort.work(files, 1) do |file|
   tmp_file = "tmp_#{file}"
-  do_cmd "cat #{csv_file_name}_header #{file} >> #{tmp_file}"
+  do_cmd "cat #{header_file_name} #{file} >> #{tmp_file}"
   do_cmd "mv #{tmp_file} #{file}"
 end
-do_cmd "rm #{csv_file_name}_header"
+do_cmd "rm #{header_file_name}"
+do_cmd "rename 's/^(.*)\.csv_(.*)$/$1_$2.csv/' *csv*"
 puts ""

data/lib/advance.rb

@@ -21,6 +21,18 @@ module Advance
   WHITE="\e[1;37m"
   YELLOW="\e[33m"
 
+  def env_is?(env_var_name, default)
+    case ENV[env_var_name]
+    when "true"; true
+    when "false"; false
+    when nil; default
+    else
+      puts "env variable #{env_var_name} should be 'true', 'false', or not present (defaults to '#{default}')"
+      puts "currently set to >#{ENV[env_var_name]}<"
+      false
+    end
+  end
+
   def self.included(pipeline_module)
     $pipeline = caller_locations.first.path
     meta =
@@ -33,17 +45,6 @@ module Advance
     $run_number = last_run_number + 1
     $cores=`nproc`.to_i
     puts "Multi steps will use #{$cores} cores"
-
-    $verbose_logging = case ENV["ADVANCE_VERBOSE_LOGGING"]
-                       when "true"; true
-                       when "false"; false
-                       when nil
-                         puts "For detailed logging of multi steps, rerun after 'export ADVANCE_VERBOSE_LOGGING=true'"
-                         false
-                       else
-                         puts "env variable ADVANCE_VERBOSE_LOGGING should be 'true', 'false', or not present (defaults to 'false')"
-                         puts "currently set to >#{ENV["ADVANCE_VERBOSE_LOGGING"]}<"
-                       end
   end
 
   def update_meta(step_number, processing_mode, label, command, start_time, duration, file_count)
@@ -132,7 +133,7 @@ module Advance
     end
     previous_dir_path = previous_dir_path.gsub(/\.tgz$/, "")
     if File.basename(previous_dir_path) =~ /^step_/
-      if !File.exist?("#{previous_dir_path}.tgz")
+      if env_is?("ADVANCE_SAVE_HISTORY", true) && !File.exist?("#{previous_dir_path}.tgz")
         do_command_wo_log "tar czf #{previous_dir_path}.tgz #{File.basename(previous_dir_path)}"
       end
       do_command_wo_log "rm -rf #{previous_dir_path}"
@@ -219,7 +220,7 @@ module Advance
 
       last_progress = ""
       progress_proc = ->(index, max_index) do
-        latest_progress = sprintf("%3i%", index.to_f / max_index * 100)
+        latest_progress = sprintf("%3i%%", index.to_f / max_index * 100)
         puts latest_progress if last_progress != latest_progress
         last_progress = latest_progress
       end
@@ -235,9 +236,9 @@ module Advance
           command.gsub!("{input_file}", file_path)
           command.gsub!("{file_name}", basename)
           command.gsub!("{file_name_without_extension}", root_file_name)
-          puts "#{YELLOW}#{command}#{RESET}  " if $verbose_logging
+          puts "#{YELLOW}#{command}#{RESET}  " if env_is?("ADVANCE_VERBOSE_LOGGING", false)
           work_in_sub_dir(new_dir_name) do
-            do_command command, $verbose_logging
+            do_command command, env_is?("ADVANCE_VERBOSE_LOGGING", false)
           end
         rescue
           puts "%%%% error while processing >>#{file_path}<<"
@@ -308,7 +309,7 @@ module Advance
   end
 
   def do_command(command, feedback = true)
-    puts "#{YELLOW}#{command}#{RESET}  " #if feedback
+    puts "#{YELLOW}#{command}#{RESET}  " if feedback
     start_time = Time.now
     stdout, stderr, status = Open3.capture3(command)
     elapsed_time = Time.now - start_time

data/lib/advance/version.rb

@@ -1,3 +1,3 @@
 module Advance
-  VERSION = "0.4.2"
+  VERSION = "0.4.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: advance
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.7
 platform: ruby
 authors:
 - janemacfarlane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-23 00:00:00.000000000 Z
+date: 2020-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: team_effort
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ homepage: https://github.com/doctorjane/advance
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -121,9 +121,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
-signing_key: 
+rubygems_version: 3.0.6
+signing_key:
 specification_version: 4
 summary: A framework for building data transformation pipelines
 test_files: []