advance 0.4.1 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 04bcff52a5d936e0a93127116ace3eb0f47354b645ed05de6e68cf281765d04c
-  data.tar.gz: 5729aff69b7bb72c11eb337e54007552240112311bce5d8e878882e1ddd7a3c7
+  metadata.gz: ca35e055d27b22f73b7187a6f3fb2d1f33aa34038fa6460ade966edf48b69847
+  data.tar.gz: 2f5025dfcd45580a7b8b8c52e671de06d3c45de2e8e25d0769e3126426548dcd
 SHA512:
-  metadata.gz: 2938d12bf9c47fd3c5e9320b4a7e116c70cd6b95a8981b8c03a3c6429aa619265bfe382e02688fdc89dadad38ecbe3ed9e56356e97dfcfbaca1af993f26ac879
-  data.tar.gz: '07790af69ba69662154f9fc51969517708893a9f0492e2cc593a869f5ea1cddaa457458b5e95afa0d0b1a7025e30bff98e5861d988510aa808ae95c9be8833bc'
+  metadata.gz: 4a7b9a3685f2e6fad1b73f730801b83f0873e844f5203733b2d45cc43950571fd43745c9a86efae35ae67b4c5f503fccae7ef11dd481187fcfff5805878df83f
+  data.tar.gz: ab02b802b9a533f48999122789d0d4e7526a4cd8da84e83d440acfbc6dc457d5be41089e1fca51b08d53e47edc38e57bffef61e7169e1427fe8f0382996cf949

data/.gitignore

@@ -9,3 +9,4 @@
 .idea
 .ruby-gemset
 .ruby-version
+.DS_Store

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    advance (0.4.1)
+    advance (0.4.6)
       team_effort
 
 GEM
   remote: https://rubygems.org/
   specs:
     minitest (5.10.1)
-    rake (10.4.2)
+    rake (13.0.1)
     team_effort (1.0.1)
 
 PLATFORMS
@@ -18,7 +18,7 @@ DEPENDENCIES
   advance!
   bundler (~> 1.16)
   minitest (~> 5.0)
-  rake (~> 10.0)
+  rake (~> 13.0)
 
 BUNDLED WITH
    1.17.3

data/advance.gemspec

@@ -30,6 +30,6 @@ step become the input to the next step.
 
   spec.add_runtime_dependency "team_effort"
   spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
 end

data/bin/split_csv.rb

@@ -9,13 +9,15 @@ end
 csv_file = ARGV[0]
 lines = ARGV[1]
 csv_file_name = File.basename(csv_file)
-system "head -n 1 #{csv_file} > #{csv_file_name}_header"
+header_file_name = "#{csv_file_name}_header"
+system "head -n 1 #{csv_file} > #{header_file_name}"
 system "tail -n +2 #{csv_file} | split -l #{lines} -a 3 - #{csv_file_name}_"
-files = Dir.entries(".").reject { |f| f =~ %r{^(\.\.?|log)$} }
+files = Dir.entries(".").reject { |f| f.end_with?(header_file_name) || f =~ %r{^(\.\.?|log)$} }
 TeamEffort.work(files, 1) do |file|
   tmp_file = "tmp_#{file}"
-  do_cmd "cat #{csv_file_name}_header #{file} >> #{tmp_file}"
+  do_cmd "cat #{header_file_name} #{file} >> #{tmp_file}"
   do_cmd "mv #{tmp_file} #{file}"
 end
-do_cmd "rm #{csv_file_name}_header"
+do_cmd "rm #{header_file_name}"
+do_cmd "rename 's/^(.*)\.csv_(.*)$/$1_$2.csv/' *csv*"
 puts ""

data/lib/advance.rb

@@ -37,13 +37,21 @@ module Advance
     $verbose_logging = case ENV["ADVANCE_VERBOSE_LOGGING"]
                        when "true"; true
                        when "false"; false
-                       when nil
-                         puts "For detailed logging of multi steps, rerun after 'export ADVANCE_VERBOSE_LOGGING=true'"
-                         false
+                       when nil; false
                        else
                          puts "env variable ADVANCE_VERBOSE_LOGGING should be 'true', 'false', or not present (defaults to 'false')"
                          puts "currently set to >#{ENV["ADVANCE_VERBOSE_LOGGING"]}<"
+                         false
                        end
+    $save_history = case ENV["ADVANCE_SAVE_HISTORY"]
+                    when "true"; true
+                    when "false"; false
+                    when nil; true
+                    else
+                      puts "env variable ADVANCE_SAVE_HISTORY should be 'true', 'false', or not present (defaults to 'true')"
+                      puts "currently set to >#{ENV["ADVANCE_SAVE_HISTORY"]}<"
+                      true
+                    end
   end
 
   def update_meta(step_number, processing_mode, label, command, start_time, duration, file_count)
@@ -132,7 +140,7 @@ module Advance
     end
     previous_dir_path = previous_dir_path.gsub(/\.tgz$/, "")
     if File.basename(previous_dir_path) =~ /^step_/
-      if !File.exist?("#{previous_dir_path}.tgz")
+      if $save_history && !File.exist?("#{previous_dir_path}.tgz")
         do_command_wo_log "tar czf #{previous_dir_path}.tgz #{File.basename(previous_dir_path)}"
       end
       do_command_wo_log "rm -rf #{previous_dir_path}"
@@ -140,8 +148,7 @@ module Advance
   end
 
   def pipeline(pipeline_path)
-    expanded_path = File.expand_path(pipeline_path)
-    load expanded_path
+    load pipeline_path
   end
 
   def count_files(dir)
@@ -220,7 +227,7 @@ module Advance
 
       last_progress = ""
       progress_proc = ->(index, max_index) do
-        latest_progress = sprintf("%3i%", index.to_f / max_index * 100)
+        latest_progress = sprintf("%3i%%", index.to_f / max_index * 100)
         puts latest_progress if last_progress != latest_progress
         last_progress = latest_progress
       end
@@ -309,7 +316,7 @@ module Advance
   end
 
   def do_command(command, feedback = true)
-    puts "#{YELLOW}#{command}#{RESET}  " #if feedback
+    puts "#{YELLOW}#{command}#{RESET}  " if feedback
     start_time = Time.now
     stdout, stderr, status = Open3.capture3(command)
     elapsed_time = Time.now - start_time
@@ -346,6 +353,8 @@ module Advance
   end
 
   def ensure_bin_on_path
+    $LOAD_PATH << File.expand_path(File.join(caller_locations.first.path, "../../lib"))
+
     advance_bin_path = File.expand_path(File.join(File.dirname(__FILE__), "../bin"))
     add_dir_to_path(advance_bin_path)
 

data/lib/advance/version.rb

@@ -1,3 +1,3 @@
 module Advance
-  VERSION = "0.4.1"
+  VERSION = "0.4.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: advance
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.6
 platform: ruby
 authors:
 - janemacfarlane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-23 00:00:00.000000000 Z
+date: 2020-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: team_effort
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ homepage: https://github.com/doctorjane/advance
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -121,9 +121,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
-signing_key: 
+rubygems_version: 3.0.6
+signing_key:
 specification_version: 4
 summary: A framework for building data transformation pipelines
 test_files: []