advance 0.3.18 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e0ac05e79a457d37a7bf18aae36c8276ecb029493bbc19a03c6410d9774dd3e
-  data.tar.gz: b915a139b4d433be5c3751c3b9cb980bd6d4a0352d6f2fe7900d459dda962080
+  metadata.gz: 2c3b98570bb73f6b6c2052d8c14c45c69897edf76b6ae610983eef6a6feaf4e4
+  data.tar.gz: 9e8e3f845512b8b139b2a11d572a226680986f1ff5bdf6816d7c329ebe402ae6
 SHA512:
-  metadata.gz: bd189c4214a299871a06704bd4424dc6b0e6ef8b41acf8bfdac5e02400af80d916909b88e80d1c3e44688292282ad549090fc8f432ab0b79cbd743e13b28f4ff
-  data.tar.gz: 4f8e340937f6801a90a02f9d700fbab7502e8e021dc6138b6b54fb544cc662d79fba449d82bc8eaeb7ed47c6cb2ed3c88ee486f4d345f92d3178c1288895696c
+  metadata.gz: 1e334afa520ae03ab4ec4a4a974242044d582f6892e18aadfd0e260ffef22bd5892276856249988e596af34cb0b5e964ac636980f7cc64949488186f244074c0
+  data.tar.gz: 102ee97e718c637ac18ec6240e313695023b5fd9ca0c77cf5ad15ab3ddf9c781006105f6d0670445f1908c04d3d97d4e96975de1695c9643f897ef46ae5a7b8b

data/.gitignore

@@ -9,3 +9,4 @@
 .idea
 .ruby-gemset
 .ruby-version
+.DS_Store

data/Gemfile.lock

@@ -1,15 +1,15 @@
 PATH
   remote: .
   specs:
-    advance (0.3.18)
+    advance (0.4.5)
       team_effort
 
 GEM
   remote: https://rubygems.org/
   specs:
     minitest (5.10.1)
-    rake (10.4.2)
-    team_effort (1.0.1)
+    rake (13.0.1)
+    team_effort (1.1.1)
 
 PLATFORMS
   ruby
@@ -18,7 +18,7 @@ DEPENDENCIES
   advance!
   bundler (~> 1.16)
   minitest (~> 5.0)
-  rake (~> 10.0)
+  rake (~> 13.0)
 
 BUNDLED WITH
-   1.17.1
+   1.17.3

data/advance.gemspec

@@ -30,6 +30,6 @@ step become the input to the next step.
 
   spec.add_runtime_dependency "team_effort"
   spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
 end

data/bin/split_csv.rb

@@ -9,13 +9,15 @@ end
 csv_file = ARGV[0]
 lines = ARGV[1]
 csv_file_name = File.basename(csv_file)
-system "head -n 1 #{csv_file} > #{csv_file_name}_header"
+header_file_name = "#{csv_file_name}_header"
+system "head -n 1 #{csv_file} > #{header_file_name}"
 system "tail -n +2 #{csv_file} | split -l #{lines} -a 3 - #{csv_file_name}_"
-files = Dir.entries(".").reject { |f| f =~ %r{^(\.\.?|log)$} }
+files = Dir.entries(".").reject { |f| f.end_with?(header_file_name) || f =~ %r{^(\.\.?|log)$} }
 TeamEffort.work(files, 1) do |file|
   tmp_file = "tmp_#{file}"
-  do_cmd "cat #{csv_file_name}_header #{file} >> #{tmp_file}"
+  do_cmd "cat #{header_file_name} #{file} >> #{tmp_file}"
   do_cmd "mv #{tmp_file} #{file}"
 end
-do_cmd "rm #{csv_file_name}_header"
+do_cmd "rm #{header_file_name}"
+do_cmd "rename 's/^(.*)\.csv_(.*)$/$1_$2.csv/' *csv*"
 puts ""

data/lib/advance.rb

@@ -80,8 +80,9 @@ module Advance
     end
 
     if File.exist?(".meta")
-      read_column_names_from_meta
-      return
+      if read_column_names_from_meta
+        return
+      end
     end
 
     previous_dir_path = get_previous_dir_path
@@ -98,10 +99,11 @@ module Advance
       run.each do |step|
         if step["columns"]
           $column_names = step["columns"].map(&:to_sym)
-          return
+          return true
         end
       end
     end
+    false
   end
 
   def advance(processing_mode, label, command)
@@ -137,6 +139,10 @@ module Advance
     end
   end
 
+  def pipeline(pipeline_path)
+    load pipeline_path
+  end
+
   def count_files(dir)
     file_count = 0
     Find.find(dir) do |path|
@@ -194,14 +200,15 @@ module Advance
 
   def single(command, previous_dir_path, dir_name)
     work_in_sub_dir(dir_name) do
-      input_file_path = previous_file_path(previous_dir_path)
-      basename = File.basename(input_file_path)
-      root_file_name = basename.gsub(%r(\.[^.]+$), '')
       command.gsub!("{input_dir}", previous_dir_path)
-      command.gsub!("{input_file}", input_file_path)
-      command.gsub!("{file_name}", basename)
-      command.gsub!("{file_name_without_extension}", root_file_name)
-
+      input_file_path = previous_file_path(previous_dir_path)
+      if input_file_path
+        basename = File.basename(input_file_path)
+        root_file_name = basename.gsub(%r(\.[^.]+$), '')
+        command.gsub!("{input_file}", input_file_path)
+        command.gsub!("{file_name}", basename)
+        command.gsub!("{file_name_without_extension}", root_file_name)
+      end
       do_command command
     end
   end
@@ -212,7 +219,7 @@ module Advance
 
       last_progress = ""
       progress_proc = ->(index, max_index) do
-        latest_progress = sprintf("%3i%", index.to_f / max_index * 100)
+        latest_progress = sprintf("%3i%%", index.to_f / max_index * 100)
         puts latest_progress if last_progress != latest_progress
         last_progress = latest_progress
       end
@@ -338,6 +345,8 @@ module Advance
   end
 
   def ensure_bin_on_path
+    $LOAD_PATH << File.expand_path(File.join(caller_locations.first.path, "../../lib"))
+
     advance_bin_path = File.expand_path(File.join(File.dirname(__FILE__), "../bin"))
     add_dir_to_path(advance_bin_path)
 

data/lib/advance/version.rb

@@ -1,3 +1,3 @@
 module Advance
-  VERSION = "0.3.18"
+  VERSION = "0.4.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: advance
 version: !ruby/object:Gem::Version
-  version: 0.3.18
+  version: 0.4.5
 platform: ruby
 authors:
 - janemacfarlane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-04 00:00:00.000000000 Z
+date: 2020-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: team_effort
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -77,12 +77,9 @@ email:
 executables:
 - concat_csv.rb
 - concat_csv_nh.rb
-- console
 - csv_select.rb
 - csv_select_nh.rb
 - csv_split_on_change_nh.rb
-- remove_logs_and_sub_dirs
-- setup
 - split_csv.rb
 extensions: []
 extra_rdoc_files: []
@@ -97,12 +94,9 @@ files:
 - advance.gemspec
 - bin/concat_csv.rb
 - bin/concat_csv_nh.rb
-- bin/console
 - bin/csv_select.rb
 - bin/csv_select_nh.rb
 - bin/csv_split_on_change_nh.rb
-- bin/remove_logs_and_sub_dirs
-- bin/setup
 - bin/split_csv.rb
 - lib/advance.rb
 - lib/advance/version.rb
@@ -128,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.9
+rubygems_version: 2.7.8
 signing_key: 
 specification_version: 4
 summary: A framework for building data transformation pipelines

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "advance"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start(__FILE__)

data/bin/remove_logs_and_sub_dirs

@@ -1,4 +0,0 @@
-#!/bin/bash
-input_dir="$1"
-find $input_dir -type f \! -name 'log' -exec cp -vn {} . \;
-

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here