advance 0.4.5 → 0.4.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c3b98570bb73f6b6c2052d8c14c45c69897edf76b6ae610983eef6a6feaf4e4
-  data.tar.gz: 9e8e3f845512b8b139b2a11d572a226680986f1ff5bdf6816d7c329ebe402ae6
+  metadata.gz: 36440c8a7f47c017df2840a92518a542a8b92d6b5cece7ae5edb9c8a6b8e7399
+  data.tar.gz: 88940f75013c5f390a4855a859dbc13175e6cc8dddeb2ee4b72ca28c3d43d029
 SHA512:
-  metadata.gz: 1e334afa520ae03ab4ec4a4a974242044d582f6892e18aadfd0e260ffef22bd5892276856249988e596af34cb0b5e964ac636980f7cc64949488186f244074c0
-  data.tar.gz: 102ee97e718c637ac18ec6240e313695023b5fd9ca0c77cf5ad15ab3ddf9c781006105f6d0670445f1908c04d3d97d4e96975de1695c9643f897ef46ae5a7b8b
+  metadata.gz: d025732524521d9f3b6ec95e26e3e0113bc083a472a875b2d5d83ec7ca6dfd243245089e30a12a8541b1da50527a38c5e1f3850142a82c3fcb19eff98552ef89
+  data.tar.gz: 9673c1369a07047d779634e37ba746f8f305cc2547f271c85e1c251aba06e231548408021f6c0410873f430968abe93133a0fa4ea6df8af3fa783c6fa85b0291

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    advance (0.4.5)
+    advance (0.4.10)
       team_effort
 
 GEM
@@ -21,4 +21,4 @@ DEPENDENCIES
   rake (~> 13.0)
 
 BUNDLED WITH
-   1.17.3
+   2.1.4

data/lib/advance.rb

@@ -21,6 +21,18 @@ module Advance
   WHITE="\e[1;37m"
   YELLOW="\e[33m"
 
+  def env_is?(env_var_name, default)
+    case ENV[env_var_name]
+    when "true"; true
+    when "false"; false
+    when nil; default
+    else
+      puts "env variable #{env_var_name} should be 'true', 'false', or not present (defaults to '#{default}')"
+      puts "currently set to >#{ENV[env_var_name]}<"
+      false
+    end
+  end
+
   def self.included(pipeline_module)
     $pipeline = caller_locations.first.path
     meta =
@@ -33,17 +45,6 @@ module Advance
     $run_number = last_run_number + 1
     $cores=`nproc`.to_i
     puts "Multi steps will use #{$cores} cores"
-
-    $verbose_logging = case ENV["ADVANCE_VERBOSE_LOGGING"]
-                       when "true"; true
-                       when "false"; false
-                       when nil
-                         puts "For detailed logging of multi steps, rerun after 'export ADVANCE_VERBOSE_LOGGING=true'"
-                         false
-                       else
-                         puts "env variable ADVANCE_VERBOSE_LOGGING should be 'true', 'false', or not present (defaults to 'false')"
-                         puts "currently set to >#{ENV["ADVANCE_VERBOSE_LOGGING"]}<"
-                       end
   end
 
   def update_meta(step_number, processing_mode, label, command, start_time, duration, file_count)
@@ -125,6 +126,11 @@ module Advance
       end
       previous_dir_path = previous_dir_path.gsub(/\.tgz$/, "")
       start_time = Time.now
+
+      puts "#{GRAY}storage summary"
+      system('df -h .')
+      puts "---------#{RESET}"
+
       send(processing_mode, command, previous_dir_path, dir_name)
       file_count = count_files(dir_name)
       duration = Time.now - start_time
@@ -132,7 +138,7 @@ module Advance
     end
     previous_dir_path = previous_dir_path.gsub(/\.tgz$/, "")
     if File.basename(previous_dir_path) =~ /^step_/
-      if !File.exist?("#{previous_dir_path}.tgz")
+      if env_is?("ADVANCE_SAVE_HISTORY", true) && !File.exist?("#{previous_dir_path}.tgz")
         do_command_wo_log "tar czf #{previous_dir_path}.tgz #{File.basename(previous_dir_path)}"
       end
       do_command_wo_log "rm -rf #{previous_dir_path}"
@@ -235,10 +241,11 @@ module Advance
           command.gsub!("{input_file}", file_path)
           command.gsub!("{file_name}", basename)
           command.gsub!("{file_name_without_extension}", root_file_name)
-          puts "#{YELLOW}#{command}#{RESET}  " if $verbose_logging
+          puts "#{YELLOW}#{command}#{RESET}  " if env_is?("ADVANCE_VERBOSE_LOGGING", false)
           work_in_sub_dir(new_dir_name) do
-            do_command command, $verbose_logging
+            do_command command, env_is?("ADVANCE_VERBOSE_LOGGING", false)
           end
+          do_command_wo_log("rm #{file_path}", false) if !env_is?("ADVANCE_SAVE_HISTORY", true)
         rescue
           puts "%%%% error while processing >>#{file_path}<<"
           raise
@@ -311,16 +318,20 @@ module Advance
     puts "#{YELLOW}#{command}#{RESET}  " if feedback
     start_time = Time.now
     stdout, stderr, status = Open3.capture3(command)
-    elapsed_time = Time.now - start_time
-    File.open("log", "w") do |f|
-      f.puts "%%% command: >#{command}<"
-      f.puts "%%% returned status: >#{status}<"
-      f.puts "%%% elapsed time: #{elapsed_time} seconds"
-      f.puts "%%% stdout:"
-      f.puts stdout
-      f.puts "%%% stderr:"
-      f.puts stderr
+
+    if env_is?("ADVANCE_LOG_FILES", true)
+      elapsed_time = Time.now - start_time
+      File.open("log", "w") do |f|
+        f.puts "%%% command: >#{command}<"
+        f.puts "%%% returned status: >#{status}<"
+        f.puts "%%% elapsed time: #{elapsed_time} seconds"
+        f.puts "%%% stdout:"
+        f.puts stdout
+        f.puts "%%% stderr:"
+        f.puts stderr
+      end
     end
+
     if !status.success?
       raise "step #{$step} failed with #{status}\n#{stderr}"
     end

data/lib/advance/version.rb

@@ -1,3 +1,3 @@
 module Advance
-  VERSION = "0.4.5"
+  VERSION = "0.4.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: advance
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.11
 platform: ruby
 authors:
 - janemacfarlane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-28 00:00:00.000000000 Z
+date: 2020-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: team_effort
@@ -106,7 +106,7 @@ homepage: https://github.com/doctorjane/advance
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -121,9 +121,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: A framework for building data transformation pipelines
 test_files: []