advance 0.4.3 → 0.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 341c54201a649459fbd7f7c3e2900f9f27045759560919a0c583d19088e40577
-  data.tar.gz: 4f62f027c8cf2f5b35f48220a1243cf356274136600390cf8a5a40b9e2a34fa0
+  metadata.gz: 0dff13c708f480234a0e209cd4cedcfff790f180894447681a064e7973211412
+  data.tar.gz: c27f9f4a5f20c1748df540106286cdad9fc6581cef7d26426dec3275ed671b2f
 SHA512:
-  metadata.gz: b476fb178dbaaf0ec54a48d7bb3d64ac9a754513f29096e2a4c5528edc2e28491dedf2dc63f983411063ed71f8d48163c32b2aed9e1787d87472fce6f02a5384
-  data.tar.gz: 57e71827cc95371d9337558a8756497e0f6049e74648464e449ef459990a596b0345fac24841ae472262941ffe19a771f24e64749d917acfde8539f9a24d18db
+  metadata.gz: c2f4b67acf76dc75dbcc93724bdb05f60738ce1a6b16779d7f2ffed6f3bfe1759c4fea05ff726af83be0f6bec77f1888ecfd6214758d3efece44f30fdbe9d4f6
+  data.tar.gz: aae16995c3cc409f4f51ad6bf53dae7cdfa4f834355f8debc87af806a87fbbf0548b58fae9c486da08de250c8c2e2047254a52ab6ccd98b89ecd9e4338ab0146

data/.gitignore

@@ -9,3 +9,4 @@
 .idea
 .ruby-gemset
 .ruby-version
+.DS_Store

data/Gemfile.lock

@@ -1,15 +1,15 @@
 PATH
   remote: .
   specs:
-    advance (0.4.2)
+    advance (0.4.8)
       team_effort
 
 GEM
   remote: https://rubygems.org/
   specs:
     minitest (5.10.1)
-    rake (10.4.2)
-    team_effort (1.0.1)
+    rake (13.0.1)
+    team_effort (1.1.1)
 
 PLATFORMS
   ruby
@@ -18,7 +18,7 @@ DEPENDENCIES
   advance!
   bundler (~> 1.16)
   minitest (~> 5.0)
-  rake (~> 10.0)
+  rake (~> 13.0)
 
 BUNDLED WITH
    1.17.3

data/advance.gemspec

@@ -30,6 +30,6 @@ step become the input to the next step.
 
   spec.add_runtime_dependency "team_effort"
   spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
 end

data/bin/split_csv.rb

@@ -9,13 +9,15 @@ end
 csv_file = ARGV[0]
 lines = ARGV[1]
 csv_file_name = File.basename(csv_file)
-system "head -n 1 #{csv_file} > #{csv_file_name}_header"
+header_file_name = "#{csv_file_name}_header"
+system "head -n 1 #{csv_file} > #{header_file_name}"
 system "tail -n +2 #{csv_file} | split -l #{lines} -a 3 - #{csv_file_name}_"
-files = Dir.entries(".").reject { |f| f =~ %r{^(\.\.?|log)$} }
+files = Dir.entries(".").reject { |f| f.end_with?(header_file_name) || f =~ %r{^(\.\.?|log)$} }
 TeamEffort.work(files, 1) do |file|
   tmp_file = "tmp_#{file}"
-  do_cmd "cat #{csv_file_name}_header #{file} >> #{tmp_file}"
+  do_cmd "cat #{header_file_name} #{file} >> #{tmp_file}"
   do_cmd "mv #{tmp_file} #{file}"
 end
-do_cmd "rm #{csv_file_name}_header"
+do_cmd "rm #{header_file_name}"
+do_cmd "rename 's/^(.*)\.csv_(.*)$/$1_$2.csv/' *csv*"
 puts ""

data/lib/advance.rb

@@ -21,6 +21,18 @@ module Advance
   WHITE="\e[1;37m"
   YELLOW="\e[33m"
 
+  def env_is?(env_var_name, default)
+    case ENV[env_var_name]
+    when "true"; true
+    when "false"; false
+    when nil; default
+    else
+      puts "env variable #{env_var_name} should be 'true', 'false', or not present (defaults to '#{default}')"
+      puts "currently set to >#{ENV[env_var_name]}<"
+      false
+    end
+  end
+
   def self.included(pipeline_module)
     $pipeline = caller_locations.first.path
     meta =
@@ -33,17 +45,6 @@ module Advance
     $run_number = last_run_number + 1
     $cores=`nproc`.to_i
     puts "Multi steps will use #{$cores} cores"
-
-    $verbose_logging = case ENV["ADVANCE_VERBOSE_LOGGING"]
-                       when "true"; true
-                       when "false"; false
-                       when nil
-                         puts "For detailed logging of multi steps, rerun after 'export ADVANCE_VERBOSE_LOGGING=true'"
-                         false
-                       else
-                         puts "env variable ADVANCE_VERBOSE_LOGGING should be 'true', 'false', or not present (defaults to 'false')"
-                         puts "currently set to >#{ENV["ADVANCE_VERBOSE_LOGGING"]}<"
-                       end
   end
 
   def update_meta(step_number, processing_mode, label, command, start_time, duration, file_count)
@@ -132,7 +133,7 @@ module Advance
     end
     previous_dir_path = previous_dir_path.gsub(/\.tgz$/, "")
     if File.basename(previous_dir_path) =~ /^step_/
-      if !File.exist?("#{previous_dir_path}.tgz")
+      if env_is?("ADVANCE_SAVE_HISTORY", true) && !File.exist?("#{previous_dir_path}.tgz")
         do_command_wo_log "tar czf #{previous_dir_path}.tgz #{File.basename(previous_dir_path)}"
       end
       do_command_wo_log "rm -rf #{previous_dir_path}"
@@ -235,10 +236,11 @@ module Advance
           command.gsub!("{input_file}", file_path)
           command.gsub!("{file_name}", basename)
           command.gsub!("{file_name_without_extension}", root_file_name)
-          puts "#{YELLOW}#{command}#{RESET}  " if $verbose_logging
+          puts "#{YELLOW}#{command}#{RESET}  " if env_is?("ADVANCE_VERBOSE_LOGGING", false)
           work_in_sub_dir(new_dir_name) do
-            do_command command, $verbose_logging
+            do_command command, env_is?("ADVANCE_VERBOSE_LOGGING", false)
           end
+          do_command_wo_log "rm #{file_path}" if !env_is?("ADVANCE_SAVE_HISTORY", true)
         rescue
           puts "%%%% error while processing >>#{file_path}<<"
           raise
@@ -308,7 +310,7 @@ module Advance
   end
 
   def do_command(command, feedback = true)
-    puts "#{YELLOW}#{command}#{RESET}  " #if feedback
+    puts "#{YELLOW}#{command}#{RESET}  " if feedback
     start_time = Time.now
     stdout, stderr, status = Open3.capture3(command)
     elapsed_time = Time.now - start_time

data/lib/advance/version.rb

@@ -1,3 +1,3 @@
 module Advance
-  VERSION = "0.4.3"
+  VERSION = "0.4.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: advance
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.8
 platform: ruby
 authors:
 - janemacfarlane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-07-22 00:00:00.000000000 Z
+date: 2020-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: team_effort
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ homepage: https://github.com/doctorjane/advance
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -121,9 +121,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.9
-signing_key: 
+rubygems_version: 3.0.6
+signing_key:
 specification_version: 4
 summary: A framework for building data transformation pipelines
 test_files: []