remi 0.2.42 → 0.3.0

data/spec/data_subjects/csv_file_spec.rb

@@ -0,0 +1,69 @@
+require_relative '../remi_spec'
+
+describe Parser::CsvFile do
+
+  let(:basic_file) { 'spec/fixtures/basic.csv' }
+  let(:basic_dataframe) do
+    Remi::DataFrame::Daru.new(
+      {
+        column_a: ['value 1A', 'value 2A'],
+        column_b: ['value 1B', 'value 2B']
+      }
+    )
+  end
+
+  it 'converts a CSV into a dataframe' do
+    csv = Parser::CsvFile.new
+    expect(csv.parse(basic_file).to_a).to eq basic_dataframe.to_a
+  end
+
+  it 'adds filename when requested' do
+    csv = Parser::CsvFile.new(
+      filename_field: :from_file
+    )
+
+    expected_files = [Pathname.new(basic_file).to_s] * 2
+    expect(csv.parse(basic_file)[:from_file].to_a).to eq expected_files
+  end
+
+  it 'preprocesses records when required' do
+    csv = Parser::CsvFile.new(
+      preprocessor: ->(line) { line.gsub(/\\"/,'""') }
+    )
+
+    bad_escape_file = 'spec/fixtures/unsupported_escape.csv'
+
+    expected_df = Remi::DataFrame::Daru.new(
+      {
+        column_a: ['value 1A', 'value 2A'],
+        column_b: ['value "1B"', 'value "2B"']
+      }
+    )
+    expect(csv.parse(bad_escape_file).to_a).to eq expected_df.to_a
+  end
+
+  it 'accepts standard Ruby CSV options' do
+    csv = Parser::CsvFile.new(
+      preprocessor: ->(line) { line.gsub(/,/,'|') },
+      csv_options: { col_sep: '|' }
+    )
+
+    expect(csv.parse(basic_file).to_a).to eq basic_dataframe.to_a
+  end
+
+  it 'combines multiple csv files into a single dataframe' do
+    csv = Parser::CsvFile.new
+    two_files = ['spec/fixtures/basic.csv', 'spec/fixtures/basic2.csv']
+
+    expected_df = Remi::DataFrame::Daru.new(
+      {
+        column_a: ['value 1A', 'value 2A', 'value 1A', 'value 2A'],
+        column_b: ['value 1B', 'value 2B', nil, nil],
+        column_c: [nil, nil, 'value 1C', 'value 2C']
+      }
+    )
+
+    expect(csv.parse(two_files).to_a).to eq expected_df.to_a
+  end
+
+end

data/spec/data_subjects/data_frame_spec.rb

@@ -0,0 +1,52 @@
+require_relative '../remi_spec'
+
+describe Extractor::DataFrame do
+  let(:extractor) { Extractor::DataFrame.new(data: data) }
+  let(:data) { 'some_data' }
+
+  context '#data' do
+    it 'returns the raw data' do
+      expect(extractor.extract.data).to eq data
+    end
+  end
+end
+
+describe Parser::DataFrame do
+  let(:fields) do
+    {
+      brewer: { type: 'text' },
+      style: { type: 'text' }
+    }
+  end
+  let(:parser) { Parser::DataFrame.new(fields: fields) }
+  let(:df_extract) { double('df_extract') }
+  let(:data) {
+    [
+      [ 'Baerlic', 'IPA' ],
+      [ 'Ex Novo', 'Red' ]
+    ]
+  }
+
+  before do
+    allow(df_extract).to receive(:data) { data }
+  end
+
+  it 'converts the data array into a dataframe' do
+    expect(parser.parse df_extract).to be_a Remi::DataFrame::Daru
+  end
+
+  it 'converts the data array into the dataframe' do
+    expected_df = Daru::DataFrame.new(
+      :brewer => ['Baerlic', 'Ex Novo'],
+      :style  => ['IPA', 'Red']
+    )
+    expect(parser.parse(df_extract).to_a).to eq expected_df.to_a
+  end
+
+end
+
+describe Encoder::DataFrame, skip: 'todo' do
+end
+
+describe Loader::DataFrame, skip: 'todo' do
+end

data/spec/data_subjects/none_spec.rb

@@ -0,0 +1,41 @@
+require_relative '../remi_spec'
+
+describe Extractor::None do
+  let(:extractor) { Extractor::None.new }
+
+  context '#extract' do
+    it 'does nothing' do
+      expect(extractor.extract).to be nil
+    end
+  end
+end
+
+describe Parser::None do
+  let(:parser) { Parser::None.new }
+
+  context '#parse' do
+    it 'returns what it is given' do
+      expect(parser.parse('some data')).to eq 'some data'
+    end
+  end
+end
+
+describe Encoder::None do
+  let(:encoder) { Encoder::None.new }
+
+  context '#encode' do
+    it 'returns what it is given' do
+      expect(encoder.encode('some data')).to eq 'some data'
+    end
+  end
+end
+
+describe Loader::None do
+  let(:loader) { Loader::None.new }
+
+  context '#loader' do
+    it 'does nothing' do
+      expect(loader.load('some data')).to be true
+    end
+  end
+end

data/spec/data_subjects/postgres_spec.rb

@@ -0,0 +1,80 @@
+require_relative '../remi_spec'
+
+describe Extractor::Postgres do
+  let(:extractor) { Extractor::Postgres.new(credentials: {}, query: 'some_query') }
+  let(:pg_conn) { double('pg_conn') }
+  let(:data) { 'some postgres data' }
+
+  before do
+    allow(pg_conn).to receive(:exec) { data }
+    allow(extractor).to receive(:connection) { pg_conn }
+  end
+
+  context '#data' do
+    it 'returns extracted data' do
+      expect(extractor.extract.data).to eq data
+    end
+  end
+end
+
+
+describe Parser::Postgres do
+  let(:parser) { Parser::Postgres.new }
+  let(:pg_extract) { double('pg_extract') }
+  let(:data) do
+    [
+      { 'brewer' => 'Baerlic', 'style' => 'IPA', 'quantity' => 5 },
+      { 'brewer' => 'Ex Novo', 'style' => 'Red', 'quantity' => 3 }
+    ]
+  end
+
+  before do
+    allow(pg_extract).to receive(:data) { data }
+  end
+
+  it 'converts postgres response data into a dataframe' do
+    expect(parser.parse pg_extract).to be_a Remi::DataFrame::Daru
+  end
+
+  it 'converted data into the correct dataframe' do
+    expected_df = Daru::DataFrame.new(
+      :brewer => ['Baerlic', 'Ex Novo'],
+      :style  => ['IPA', 'Red'],
+      :quantity => [5, 3]
+    )
+    expect(parser.parse(pg_extract).to_a).to eq expected_df.to_a
+  end
+end
+
+
+describe Encoder::Postgres do
+  let(:fields) do
+    {
+      brewer: { type: 'text' },
+      style: { type: 'text' },
+      quantity: { type: 'integer' }
+    }
+  end
+  let(:encoder) { Encoder::Postgres.new(fields: fields) }
+  let(:dataframe) do
+    expected_df = Daru::DataFrame.new(
+      :brewer => ['Baerlic', 'Ex Novo'],
+      :style  => ['IPA', 'Red'],
+      :quantity => [5, 3]
+    )
+  end
+
+  it 'converts the dataframe into an array of strings to be used by the loader' do
+    expect(encoder.encode(dataframe).values).to eq [
+      "Baerlic\tIPA\t5",
+      "Ex Novo\tRed\t3"
+    ]
+  end
+
+  it 'builds the field ddl' do
+    expect(encoder.encode(dataframe).ddl_fields).to eq 'brewer text, style text, quantity integer'
+  end
+end
+
+describe Loader::Postgres, skip: 'todo' do
+end

data/spec/data_subjects/salesforce_spec.rb

@@ -0,0 +1,117 @@
+require_relative '../remi_spec'
+require 'remi/data_subjects/salesforce.rb'
+
+describe Extractor::Salesforce do
+  let(:extractor) { Extractor::Salesforce.new(object: :Contact, credentials: {}, query: '') }
+  let(:sf_bulk) { double('sf_bulk') }
+  let(:data) do
+    {
+      'batches' => [
+        {
+          'id' => ['751160000065e2BAAQ'],
+          'state' => [ 'Completed' ]
+        }
+      ]
+    }
+  end
+
+  before do
+    allow(extractor).to receive(:sf_bulk) { sf_bulk }
+    allow(sf_bulk).to receive(:query) { data }
+  end
+
+  context '#data' do
+    it 'returns extracted data' do
+      expect(extractor.extract.data).to eq data
+    end
+  end
+
+
+  it 'raises an error if the batch fails' do
+    data['batches'].first['state'] = ['Error']
+    expect { extractor.extract }.to raise_error Extractor::Salesforce::ExtractError
+  end
+end
+
+
+describe Parser::Salesforce do
+  let(:parser) { Parser::Salesforce.new }
+  let(:sf_extract) { double('sf_extract') }
+  let(:data) do
+    {
+      'batches' => [
+        {
+          'id' => ['751160000065e2BAAQ'],
+          'state' => [ 'Completed' ],
+          'response' => [
+            {
+              "xsi:type" => "sObject",
+              "type" => [
+                "Contact"
+              ],
+              "Id" => [
+                "003G000001cKYaUIA4",
+                "003G000001cKYaUIA4"
+              ],
+              "Student_ID__c" => [
+                "FJD385628"
+              ]
+            },
+            {
+              "xsi:type" => "sObject",
+              "type" => [
+                "Contact"
+              ],
+              "Id" => [
+                "003G000001cKYbXIA4",
+                "003G000001cKYbXIA4"
+              ],
+              "Student_ID__c" => [
+                { 'xsi:nil' => 'true' }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  end
+
+  before do
+    allow(sf_extract).to receive(:data) { data }
+  end
+
+  it 'converts SalesforceBulkApi response data into a dataframe' do
+    expect(parser.parse sf_extract).to be_a Remi::DataFrame::Daru
+  end
+
+  it 'converted data into the correct dataframe' do
+    expected_df = Daru::DataFrame.new(
+      :Id            => ['003G000001cKYaUIA4', '003G000001cKYbXIA4'],
+      :Student_ID__c => ['FJD385628', nil]
+    )
+    expect(parser.parse(sf_extract).to_a).to eq expected_df.to_a
+  end
+end
+
+
+describe Encoder::Salesforce do
+  let(:encoder) { Encoder::Salesforce.new }
+  let(:dataframe) do
+    Daru::DataFrame.new(
+      :Id            => ['003G000001cKYaUIA4', '003G000001cKYbXIA4'],
+      :Student_ID__c => ['FJD385628', nil]
+    )
+  end
+
+  it 'converts the dataframe into an array of hashes' do
+    expected_result = [
+      { :Id => '003G000001cKYaUIA4', :Student_ID__c => 'FJD385628' },
+      { :Id => '003G000001cKYbXIA4', :Student_ID__c => nil },
+    ]
+    expect(encoder.encode dataframe).to eq expected_result
+  end
+end
+
+
+describe Loader::Salesforce, skip: 'todo' do
+end

data/spec/{extractor → data_subjects}/sftp_file_spec.rb

@@ -82,3 +82,19 @@ describe Extractor::SftpFile do
     end
   end
 end
+
+
+describe Loader::SftpFile do
+  let(:loader) { Loader::SftpFile.new(credentials: {}, remote_path: 'some_path') }
+  let(:data) { double('some_data') }
+  let(:sftp_session) { instance_double('Net:SFTP::Session') }
+
+  before do
+    allow(Net::SFTP).to receive(:start).and_yield sftp_session
+  end
+
+  it 'loads a csv to a target sftp filesystem' do
+    expect(sftp_session).to receive(:upload!).with(data, 'some_path')
+    loader.load data
+  end
+end

data/spec/data_subjects/sub_job_spec.rb

@@ -0,0 +1,33 @@
+require_relative '../remi_spec'
+
+describe 'sub jobs' do
+  before :each do
+    Object.send(:remove_const, :MySubJob) if Object.constants.include?(:MySubJob)
+    class MySubJob < Job
+      source(:sub_source) {}
+      target(:sub_target) {}
+    end
+  end
+
+  let(:sub_job) { Job::SubJob.new { MySubJob.new } }
+
+
+  describe Extractor::SubJob do
+    let(:extractor) { Extractor::SubJob.new(sub_job: sub_job, data_subject: :sub_target) }
+
+    it 'returns the data from the sub-job' do
+      allow(sub_job.job.sub_target).to receive(:df) { 'sub target df' }
+      expect(extractor.extract).to eq 'sub target df'
+    end
+  end
+
+  describe Loader::SubJob do
+    let(:loader) { Loader::SubJob.new(sub_job: sub_job, data_subject: :sub_source) }
+
+    it 'populates the sub-job data frame' do
+      some_data_frame = Daru::DataFrame.new({ a: [1,2,3] })
+      loader.load(some_data_frame)
+      expect(sub_job.job.sub_source.df).to eq some_data_frame
+    end
+  end
+end

data/spec/encoder_spec.rb

@@ -0,0 +1,38 @@
+require_relative 'remi_spec'
+
+describe Remi::Encoder do
+  let(:field_symbolizer) { double('field_symbolizer') }
+  let(:context) { double('context') }
+  let(:fields) { double('fields') }
+  let(:encoder) { Encoder.new(context: context, fields: fields, field_symbolizer: field_symbolizer) }
+
+  context '#encode' do
+    it 'has an encode method' do
+      expect(encoder).respond_to? :encode
+    end
+  end
+
+  context '#field_symbolizer' do
+    it 'can be set in the constructor' do
+      expect(encoder.field_symbolizer).to eq field_symbolizer
+    end
+
+    it 'the field_symbolizer defined in the context takes priority' do
+      symbolizer_from_context = double('symbolizer_from_context')
+      allow(context).to receive(:field_symbolizer) { symbolizer_from_context }
+      expect(encoder.field_symbolizer).to eq symbolizer_from_context
+    end
+  end
+
+  context '#fields' do
+    it 'can be set in the constructor' do
+      expect(encoder.fields).to eq fields
+    end
+
+    it 'the field_symbolizer defined in the context takes priority' do
+      fields_from_context = double('fields_from_context')
+      allow(context).to receive(:fields) { fields_from_context }
+      expect(encoder.fields).to eq fields_from_context
+    end
+  end
+end