remi 0.3.0 → 0.3.1

data/lib/remi/loader.rb

@@ -4,11 +4,12 @@ module Remi
   # define specific ways to load data.
   class Loader
 
-    def initialize(*args, logger: Remi::Settings.logger, **kargs, &block)
+    def initialize(*args, context: nil, logger: Remi::Settings.logger, **kargs, &block)
+      @context = context
       @logger = logger
     end
 
-    attr_accessor :logger
+    attr_accessor :logger, :context
 
     # Any child classes need to define a load method that loads data from
     # the given dataframe into the target system.
@@ -18,5 +19,16 @@ module Remi
       raise NoMethodError, "#{__method__} not defined for #{self.class.name}"
     end
 
+    # If autoload is set to true, then any loaders are called at the moment
+    # a dataframe is assigned to a target (e.g., `my_target.df = some_df` will
+    # call `#load` on any loaders associated with `my_target`).
+    def autoload
+      false
+    end
+
+    # @return [Remi::Fields] The fields defined in the context
+    def fields
+      context && context.respond_to?(:fields) ? context.fields : Remi::Fields.new({})
+    end
   end
 end

data/lib/remi/testing/business_rules.rb

@@ -521,19 +521,22 @@ module Remi::Testing::BusinessRules
       @table = table
     end
 
+    def parse_formula(value)
+      parsed_value = ParseFormula.parse(value)
+      case parsed_value
+      when '\nil'
+        nil
+      else
+        parsed_value
+      end
+    end
+
     def to_df(seed_hash, field_symbolizer:)
       table_headers = @table.headers.map { |h| h.symbolize(field_symbolizer) }
       df = Daru::DataFrame.new([], order: seed_hash.keys | table_headers)
       @table.hashes.each do |example_row|
         example_row_sym = example_row.reduce({}) do |h, (k,v)|
-          formula_value = ParseFormula.parse(v)
-          value = case formula_value
-            when '\nil'
-              nil
-            else
-              formula_value
-            end
-          h[k.symbolize(field_symbolizer)] = value
+          h[k.symbolize(field_symbolizer)] = parse_formula(v)
           h
         end
         df.add_row(seed_hash.merge(example_row_sym))
@@ -546,7 +549,7 @@ module Remi::Testing::BusinessRules
     def column_hash
       @table.hashes.reduce({}) do |h, row|
         row.each do |k,v|
-          (h[k.symbolize] ||= []) << v
+          (h[k.symbolize] ||= []) << parse_formula(v)
         end
         h
       end

data/lib/remi/transform.rb

@@ -514,6 +514,11 @@ module Remi
         type == :string ? '' : nil
       end
 
+      def truthy(value)
+        @truthy ||= Truthy.new(allow_nils: false).to_proc
+        @truthy.call(value)
+      end
+
       def transform(value)
         if value.blank? && type != :json
           blank_handler(value)
@@ -537,6 +542,10 @@ module Remi
             else
               value.is_a?(Hash) || value.is_a?(Array) ? value : JSON.parse(value)
             end
+          when :boolean
+            # Ugh, there is a bug with Daru 0.1.4 that converts false values to nil when joining
+            # For now, we'll just standardize boolean values (#to_s)
+            truthy(value).to_s
           else
             raise ArgumentError, "Unknown type enforcement: #{type}"
           end

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.3.0'
+  VERSION = '0.3.1'
 end

data/spec/data_subject_spec.rb

@@ -339,7 +339,9 @@ describe DataTarget do
 
   before do
     allow(my_loader).to receive(:load)
+    allow(my_loader).to receive(:context=)
     allow(my_loader2).to receive(:load)
+    allow(my_loader2).to receive(:context=)
     allow(my_encoder).to receive(:encode) .and_return 'encoded data'
   end
 
@@ -378,7 +380,6 @@ describe DataTarget do
       end
     end
 
-
     context '#field_symbolizer' do
       context 'field_symbolizer called before encoder' do
         let(:before_encoder) do
@@ -448,15 +449,15 @@ describe DataTarget do
   end
 
   context '#loader' do
-    before { data_target.loader 'my_loader' }
+    before { data_target.loader my_loader }
 
     it 'adds a loader to the list of loaders' do
-      expect(data_target.loaders).to eq ['my_loader']
+      expect(data_target.loaders).to eq [my_loader]
     end
 
     it 'allows for multiple loaders to be defined' do
-      data_target.loader 'my_loader2'
-      expect(data_target.loaders).to eq ['my_loader', 'my_loader2']
+      data_target.loader my_loader2
+      expect(data_target.loaders).to eq [my_loader, my_loader2]
     end
   end
 
@@ -505,4 +506,21 @@ describe DataTarget do
       end
     end
   end
+
+  context '#df=' do
+    before do
+      data_target.encoder my_encoder
+      data_target.loader my_loader
+      data_target.loader my_loader2
+
+      allow(my_loader).to receive(:autoload) { false }
+      allow(my_loader2).to receive(:autoload) { true }
+    end
+
+    it 'loads any loaders set to autoload' do
+      expect(my_loader).not_to receive :load
+      expect(my_loader2).to receive :load
+      data_target.df = Remi::DataFrame::Daru.new([])
+    end
+  end
 end

data/spec/data_subjects/file_system_spec.rb

@@ -5,15 +5,15 @@ describe Extractor::FileSystem do
     now = Time.new
 
     example_files = [
-      { pathname: "pdir/ApplicantsA-9.csv", create_time: now - 10.minutes },
-      { pathname: "pdir/ApplicantsA-3.csv", create_time: now - 5.minutes },
-      { pathname: "pdir/ApplicantsA-5.csv", create_time: now - 1.minutes },
-      { pathname: "pdir/ApplicantsB-7.csv", create_time: now - 10.minutes },
-      { pathname: "pdir/ApplicantsB-6.csv", create_time: now - 5.minutes },
-      { pathname: "pdir/ApplicantsB-2.csv", create_time: now - 1.minutes },
-      { pathname: "pdir/ApplicantsB-2.txt", create_time: now - 0.minutes },
-      { pathname: "pdir/Apples.csv",        createtime: now - 1.minutes },
-      { pathname: "otherdir/ApplicantsA-11.csv", createtime: now - 1.minutes },
+      { pathname: "pdir/ApplicantsA-9.csv",      create_time: now - 10.minutes },
+      { pathname: "pdir/ApplicantsA-3.csv",      create_time: now - 5.minutes  },
+      { pathname: "pdir/ApplicantsA-5.csv",      create_time: now - 1.minutes  },
+      { pathname: "pdir/ApplicantsB-7.csv",      create_time: now - 10.minutes },
+      { pathname: "pdir/ApplicantsB-6.csv",      create_time: now - 5.minutes  },
+      { pathname: "pdir/ApplicantsB-2.csv",      create_time: now - 1.minutes  },
+      { pathname: "pdir/ApplicantsB-2.txt",      create_time: now - 0.minutes  },
+      { pathname: "pdir/Apples.csv",             create_time: now - 1.minutes  },
+      { pathname: "otherdir/ApplicantsA-11.csv", create_time: now - 1.minutes  },
     ]
 
     remote_path = 'pdir'
@@ -89,6 +89,40 @@ describe Extractor::FileSystem do
   end
 
 
+  context 'extracting the most recent file by create time' do
+    before do
+      @params.merge!({
+         most_recent_within_n: 1.hour,
+         most_recent_only: true
+      })
+    end
+
+    it 'extracts the files within n hours of creation' do
+      expect(file_system.entries.map(&:name)).to match_array([
+        "ApplicantsB-2.txt"
+      ])
+    end
+  end
+
+  context 'extracting all recent files by create time' do
+    before do
+      @params.merge!({
+         created_within: 0.02.hours,
+         most_recent_only: false
+      })
+    end
+
+    it 'extracts the files within n hours of creation' do
+      puts @params
+      expect(file_system.entries.map(&:name)).to match_array([
+        "Apples.csv",
+        "ApplicantsA-5.csv",
+        "ApplicantsB-2.csv",
+        "ApplicantsB-2.txt"
+      ])
+    end
+  end
+
   context 'extracting files matching a pattern with a by group' do
     before do
       @params.merge!({

data/spec/data_subjects/gsheet_spec.rb

@@ -0,0 +1,133 @@
+require 'remi_spec'
+require 'remi/data_subjects/gsheet'
+
+describe Extractor::Gsheet do
+
+  let(:remote_path) { '' }
+  let(:credentials) {
+    {
+      :client_id        => 'some_client_id',
+      :access_token     => 'some_access_token',
+      :refresh_token    => 'some_refresh_token',
+      :client_secret    => 'some_client_secret',
+      :application_name => 'some_application_name',
+      :project_id       => 'some_project_id',
+      :expiration_time  => '123456789'
+    }
+  }
+
+
+  let(:params) {
+    {
+      credentials: credentials,
+      folder_id:   'some_google_folder_id',
+      remote_path: remote_path
+    }
+  }
+
+  let(:gsheet_file) {
+    Extractor::Gsheet.new(params)
+  }
+
+  let(:response) { double('response') }
+  let(:remote_filenames) {["test_file_1","test_file_2"]}
+  let(:remote_files) do
+    [{name: "test_file_1", create_time:Date.current, id: "1234"},
+     {name: "test_file_2", create_time:Date.current, id: "5678"}]
+
+  end
+
+  context '.new' do
+    it 'creates an instance with valid parameters' do
+      gsheet_file
+    end
+
+    it 'requires a client_id' do
+      credentials.delete(:client_id)
+      expect { gsheet_file }.to raise_error KeyError
+    end
+
+    it 'requires an access_token' do
+      credentials.delete(:access_token)
+      expect { gsheet_file }.to raise_error KeyError
+    end
+
+    it 'requires a client_secret' do
+      credentials.delete(:client_secret)
+      expect { gsheet_file }.to raise_error KeyError
+    end
+
+    it 'requires a refresh_token' do
+      credentials.delete(:refresh_token)
+      expect { gsheet_file }.to raise_error KeyError
+    end
+
+    it 'requires a folder id' do
+      params.delete(:credentials)
+      expect { gsheet_file }.to raise_error ArgumentError
+    end
+
+    it 'requires an application name' do
+      credentials.delete(:application_name)
+      expect { gsheet_file }.to raise_error KeyError
+    end
+
+    it 'requires a project id' do
+      credentials.delete(:project_id)
+      expect { gsheet_file }.to raise_error KeyError
+    end
+
+  end
+
+  context '#all_entires' do
+    it 'returns all entries' do
+
+      allow(response).to receive(:files) { remote_files }
+      allow(gsheet_file).to receive(:service_list_files) { response }
+
+      expect(gsheet_file.all_entries.map(&:name)).to eq remote_filenames
+
+    end
+  end
+
+  context '#extract' do
+    it 'downloads files from google' do
+
+      allow(response).to receive(:files) { remote_files }
+      allow(gsheet_file).to receive(:service_list_files) { response }
+      expect(gsheet_file).to receive(:get_spreadsheet_vals).exactly(remote_filenames.size).times
+      gsheet_file.extract
+
+    end
+
+  end
+end
+
+describe Parser::Gsheet do
+
+  let(:parser) { Parser::Gsheet.new }
+  let(:gs_extract) { double('gs_extract') }
+  let(:example_data) do
+    [{"headers" => ["header_1", "header_2", "header_3"],
+      "row 1"   => ["value 1", "value 2", "value 3"]
+    }]
+  end
+
+  before do
+    allow(gs_extract).to receive(:data) { example_data }
+  end
+
+  it 'converts Google Sheets response data into a dataframe' do
+    expect(parser.parse gs_extract).to be_a Remi::DataFrame::Daru
+  end
+
+  it 'converted data into the correct dataframe' do
+    expected_df = Daru::DataFrame.new(
+      :header_1 => ['value 1'],
+      :header_2 => ['value 2'],
+      :header_3 => ['value 3'],
+    )
+    expect(parser.parse(gs_extract).to_a).to eq expected_df.to_a
+  end
+
+end

data/spec/data_subjects/sub_job_spec.rb

@@ -4,7 +4,10 @@ describe 'sub jobs' do
   before :each do
     Object.send(:remove_const, :MySubJob) if Object.constants.include?(:MySubJob)
     class MySubJob < Job
-      source(:sub_source) {}
+      source :sub_source do
+        extractor Extractor::None.new
+        fields({ a: { from_sub_job: true, to_overwrite: 'from_sub_job' } })
+      end
       target(:sub_target) {}
     end
   end
@@ -13,21 +16,50 @@ describe 'sub jobs' do
 
 
   describe Extractor::SubJob do
-    let(:extractor) { Extractor::SubJob.new(sub_job: sub_job, data_subject: :sub_target) }
+    let(:target_extractor) { Extractor::SubJob.new(sub_job: sub_job, data_subject: :sub_target) }
+    let(:source_extractor) { Extractor::SubJob.new(sub_job: sub_job, data_subject: :sub_source) }
 
     it 'returns the data from the sub-job' do
-      allow(sub_job.job.sub_target).to receive(:df) { 'sub target df' }
-      expect(extractor.extract).to eq 'sub target df'
+      allow(sub_job.sub_job.sub_target).to receive(:df) { 'sub target df' }
+      expect(target_extractor.extract).to eq 'sub target df'
+    end
+
+    it 'executes the sub job when target data is requested' do
+      expect(sub_job).to receive(:execute).once
+      target_extractor.extract
+    end
+
+    it 'does not execute the sub job when source data is requested' do
+      expect(sub_job).not_to receive(:execute)
+      source_extractor.extract
     end
   end
 
   describe Loader::SubJob do
-    let(:loader) { Loader::SubJob.new(sub_job: sub_job, data_subject: :sub_source) }
+    let(:data_target) { DataTarget.new }
+    let(:loader) { Loader::SubJob.new(context: data_target, sub_job: sub_job, data_subject: :sub_source) }
+    let(:my_data_frame) { Daru::DataFrame.new({ a: [1,2,3] }) }
 
     it 'populates the sub-job data frame' do
-      some_data_frame = Daru::DataFrame.new({ a: [1,2,3] })
-      loader.load(some_data_frame)
-      expect(sub_job.job.sub_source.df).to eq some_data_frame
+      loader.load(my_data_frame)
+      expect(sub_job.sub_job.sub_source.df).to eq my_data_frame
+    end
+
+    it 'merges fields from the parent source when requested' do
+      data_target.fields({ a: { from_parent: :true, to_overwrite: 'from_parent' } })
+      loader.load(my_data_frame)
+      expect(sub_job.sub_job.sub_source.fields).to eq MySubJob.new.sub_source.fields.merge data_target.fields
+    end
+
+    it 'does not merge fields from the parent source when requested' do
+      loader.merge_fields = false
+      data_target.fields({ a: { from_parent: :true, to_overwrite: 'from_parent' } })
+      loader.load(my_data_frame)
+      expect(sub_job.sub_job.sub_source.fields).to eq MySubJob.new.sub_source.fields
+    end
+
+    it 'autoloads the target' do
+      expect(loader.autoload).to be true
     end
   end
 end

data/spec/job_spec.rb

@@ -70,11 +70,11 @@ describe Job do
         class MyJob
           sub_job(:my_sub_job) { 'something' }
         end
-        expect { job.my_sub_job.job }.to raise_error ArgumentError
+        expect { job.my_sub_job.sub_job }.to raise_error ArgumentError
       end
 
       it 'returns a Remi job' do
-        expect(job.my_sub_job.job).to be_a Remi::Job
+        expect(job.my_sub_job.sub_job).to be_a Remi::Job
       end
     end
 
@@ -276,6 +276,11 @@ describe Job do
         expect(job.params[:my_param]).to eq 'instantiated'
       end
 
+      it 'works with booleans too' do
+        other_job = MyJob.new(my_param: false)
+        expect(other_job.params[:my_param]).to eq false
+      end
+
       it 'does not affect the values of other instances' do
         job
         other_job = MyJob.new
@@ -295,17 +300,12 @@ describe Job do
   context '#execute' do
     before do
       class MyJob
-        transform :transform_one do
-        end
-
-        transform :transform_two do
-        end
-
-        target :target_one do
-        end
-
-        target :target_two do
-        end
+        transform(:transform_one) {}
+        transform(:transform_two) {}
+        sub_job(:sub_job_one) { Remi::Job.new }
+        sub_job(:sub_job_two) { Remi::Job.new }
+        target(:target_one) {}
+        target(:target_two) {}
       end
     end
 
@@ -314,6 +314,11 @@ describe Job do
       job.execute
     end
 
+    it 'executes all subjobs' do
+      expect(job).to receive(:execute_sub_jobs)
+      job.execute
+    end
+
     it 'executes load all targets' do
       expect(job).to receive(:execute_load_targets)
       job.execute
@@ -330,12 +335,39 @@ describe Job do
         job.execute(:transforms)
       end
 
+      it 'does not execute all sub jobs' do
+        expect(job).not_to receive(:execute_sub_jobs)
+        job.execute(:transforms)
+      end
+
       it 'does not load all targets' do
         expect(job).not_to receive(:execute_load_targets)
         job.execute(:transforms)
       end
     end
 
+    context '#execute(:sub_jobs)' do
+      it 'executes all sub_jobs' do
+        [:sub_job_one, :sub_job_two].each do |sub_job_name|
+          sub_job = instance_double(Job::SubJob)
+          expect(sub_job).to receive(:execute)
+          expect(job).to receive(sub_job_name) .and_return(sub_job)
+        end
+
+        job.execute(:sub_jobs)
+      end
+
+      it 'does not execute all transforms' do
+        expect(job).not_to receive(:execute_transforms)
+        job.execute(:sub_jobs)
+      end
+
+      it 'does not load all targets' do
+        expect(job).not_to receive(:execute_load_targets)
+        job.execute(:sub_jobs)
+      end
+    end
+
     context '#execute(:load_targets)' do
       it 'loads all targets' do
         [:target_one, :target_two].each do |target_name|
@@ -351,6 +383,11 @@ describe Job do
         expect(job).not_to receive(:execute_transforms)
         job.execute(:load_targets)
       end
+
+      it 'does not execute all sub jobs' do
+        expect(job).not_to receive(:execute_sub_jobs)
+        job.execute(:load_targets)
+      end
     end
   end
 
@@ -469,9 +506,9 @@ describe Job do
       Job::SubJob.new { scoped_sub_job }
     end
 
-    context '#job' do
+    context '#sub_job' do
       it 'returns the job instance for the sub job' do
-        expect(job_sub_job.job).to eq sub_job
+        expect(job_sub_job.sub_job).to eq sub_job
       end
     end
 
@@ -494,6 +531,12 @@ describe Job do
         expect(sub_job).to receive(:execute)
         job_sub_job.execute
       end
+
+      it 'only executes the sub job once' do
+        expect(sub_job).to receive(:execute).once
+        job_sub_job.execute
+        job_sub_job.execute
+      end
     end
 
     context '#execute_transforms' do