drudgery 0.0.1

data/spec/drudgery/extractors/active_record_extractor_spec.rb

@@ -0,0 +1,79 @@
+require 'spec_helper'
+require 'active_record'
+
+describe Drudgery::Extractors::ActiveRecordExtractor do
+  class Record < ActiveRecord::Base; end
+
+  describe '#initialize' do
+    it 'sets model to provided argument' do
+      model = mock
+
+      extractor = Drudgery::Extractors::ActiveRecordExtractor.new(model)
+      extractor.instance_variable_get('@model').must_equal model
+    end
+  end
+
+  describe '#extract' do
+    it 'finds records using model' do
+      model = mock
+      model.expects(:find_each)
+
+      extractor = Drudgery::Extractors::ActiveRecordExtractor.new(model)
+      extractor.extract
+    end
+
+    it 'yields each record as a hash' do
+      record1 = mock
+      record1.expects(:attributes).returns({ :a => 1 })
+
+      record2 = mock
+      record2.expects(:attributes).returns({ :b => 2 })
+
+      model = mock
+      model.stubs(:find_each).multiple_yields([record1], [record2])
+
+      extractor = Drudgery::Extractors::ActiveRecordExtractor.new(model)
+
+      records = []
+      extractor.extract do |record|
+        records << record
+      end
+
+      records[0].must_equal({ :a => 1 })
+      records[1].must_equal({ :b => 2 })
+    end
+
+    describe 'without stubs' do
+      before(:each) do
+        ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
+        ActiveRecord::Base.connection.create_table(:records) do |t|
+          t.integer :a
+          t.integer :b
+        end
+
+        Record.create!({ :a => 1, :b => 2 })
+        Record.create!({ :a => 3, :b => 4 })
+        Record.create!({ :a => 5, :b => 6 })
+      end
+
+      after(:each) do
+        ActiveRecord::Base.clear_active_connections!
+      end
+
+      it 'yields each record as a hash' do
+        extractor = Drudgery::Extractors::ActiveRecordExtractor.new(Record)
+
+        records = []
+        extractor.extract do |record|
+          records << record
+        end
+
+        records.must_equal([
+          { 'id' => 1, 'a' => 1, 'b' => 2 },
+          { 'id' => 2, 'a' => 3, 'b' => 4 },
+          { 'id' => 3, 'a' => 5, 'b' => 6 }
+        ])
+      end
+    end
+  end
+end

data/spec/drudgery/extractors/csv_extractor_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+
+describe Drudgery::Extractors::CSVExtractor do
+  describe '#initialize' do
+    it 'sets filepath to provided filepath' do
+      extractor = Drudgery::Extractors::CSVExtractor.new('file.csv')
+      extractor.instance_variable_get('@filepath').must_equal 'file.csv'
+    end
+
+    it 'initializes options hash' do
+      extractor = Drudgery::Extractors::CSVExtractor.new('file.csv')
+      extractor.instance_variable_get('@options').must_equal({ :headers => true })
+    end
+
+    it 'merges provided options with default options' do
+      options = { :col_sep => '|', :headers => %w[id name email] }
+
+      extractor = Drudgery::Extractors::CSVExtractor.new('file.csv', options)
+      extractor.instance_variable_get('@options').must_equal({ :col_sep => '|', :headers => %w[id name email] })
+    end
+  end
+
+  describe '#extract' do
+    it 'parses records from file' do
+      CSV.expects(:foreach).with('file.csv', :headers => true)
+
+      extractor = Drudgery::Extractors::CSVExtractor.new('file.csv')
+      extractor.extract
+    end
+
+    it 'yields each record as a hash' do
+      record1 = mock
+      record1.expects(:to_hash).returns({ :a => 1 })
+
+      record2 = mock
+      record2.expects(:to_hash).returns({ :b => 2 })
+
+      CSV.stubs(:foreach).multiple_yields([record1], [record2])
+
+      extractor = Drudgery::Extractors::CSVExtractor.new('file.csv')
+
+      records = []
+      extractor.extract do |record|
+        records << record
+      end
+
+      records[0].must_equal({ :a => 1 })
+      records[1].must_equal({ :b => 2 })
+    end
+
+    describe 'without stubs' do
+      before(:each) do
+        File.delete('file.csv') if File.exists?('file.csv')
+      end
+
+      after(:each) do
+        File.delete('file.csv') if File.exists?('file.csv')
+      end
+
+      it 'writes hash keys as header and records as rows' do
+        File.open('file.csv', 'w') do |f|
+          f.puts 'a,b'
+          f.puts '1,2'
+          f.puts '3,4'
+          f.puts '5,6'
+        end
+
+        extractor = Drudgery::Extractors::CSVExtractor.new('file.csv')
+
+        records = []
+        extractor.extract do |record|
+          records << record
+        end
+
+        records.must_equal([
+          { 'a' => '1', 'b' => '2' },
+          { 'a' => '3', 'b' => '4' },
+          { 'a' => '5', 'b' => '6' }
+        ])
+      end
+    end
+  end
+end

data/spec/drudgery/extractors/sqlite3_extractor_spec.rb

@@ -0,0 +1,161 @@
+require 'spec_helper'
+require 'sqlite3'
+
+describe Drudgery::Extractors::SQLite3Extractor do
+  describe '#initialize' do
+    it 'sets db and table to provided arguments' do
+      db = mock
+      db.expects(:results_as_hash=).with(true)
+      db.expects(:type_translation=).with(true)
+
+      extractor = Drudgery::Extractors::SQLite3Extractor.new(db, 'tablename')
+      extractor.instance_variable_get('@db').must_equal db
+      extractor.instance_variable_get('@table').must_equal 'tablename'
+    end
+
+    it 'initializes clauses hash' do
+      db = mock
+      db.stubs(:results_as_hash=)
+      db.stubs(:type_translation=)
+
+      extractor = Drudgery::Extractors::SQLite3Extractor.new(db, 'tablename')
+      extractor.instance_variable_get('@clauses').must_equal({})
+    end
+  end
+
+  describe 'query building' do
+    before(:each) do
+      db = mock
+      db.stubs(:results_as_hash=)
+      db.stubs(:type_translation=)
+
+      @extractor = Drudgery::Extractors::SQLite3Extractor.new(db, 'tablename')
+    end
+
+    describe '#select' do
+      it 'sets select clause with provided expressions' do
+        @extractor.select('id', "(first_name || ' ' || last_name) AS name", 'email')
+        @extractor.instance_variable_get('@clauses').must_equal({ :select => "id, (first_name || ' ' || last_name) AS name, email" })
+      end
+    end
+
+    describe '#from' do
+      it 'sets from clause with provided expression' do
+        @extractor.from('table AS t')
+        @extractor.instance_variable_get('@clauses').must_equal({ :from => 'table AS t' })
+      end
+    end
+
+    describe '#joins' do
+      it 'sets join clauses with provided clauses' do
+        joins = [
+          'JOIN table2 t2 ON t2.my_id = t1.id',
+          'LEFT OUTER JOIN table3 t3 ON t3.my_id = t2.id'
+        ]
+
+        @extractor.joins(joins[0], joins[1])
+        @extractor.instance_variable_get('@clauses').must_equal({ :joins => joins })
+      end
+    end
+
+    describe '#group' do
+      it 'sets group clause with provided expressions' do
+        @extractor.group('id', 'email')
+        @extractor.instance_variable_get('@clauses').must_equal({ :group => 'id, email' })
+      end
+    end
+
+    describe '#where' do
+      it 'sets where clause with provided condition' do
+        @extractor.where('age >= 18 AND age < 50')
+        @extractor.instance_variable_get('@clauses').must_equal({ :where => 'age >= 18 AND age < 50' })
+      end
+    end
+
+    describe '#having' do
+      it 'sets having clause with provided condition' do
+        @extractor.having('count(*) > 1')
+        @extractor.instance_variable_get('@clauses').must_equal({ :having => 'count(*) > 1' })
+      end
+    end
+
+    describe '#order' do
+      it 'sets order clause with provided expressions' do
+        @extractor.order('id', 'email DESC')
+        @extractor.instance_variable_get('@clauses').must_equal({ :order => 'id, email DESC' })
+      end
+    end
+  end
+
+  describe '#extract' do
+    it 'selects records from db using defined query' do
+      db = mock
+      db.stubs(:results_as_hash=)
+      db.stubs(:type_translation=)
+      db.expects(:execute).with('SELECT * FROM tablename')
+      db.expects(:execute).with('SELECT age, count(*) AS nr_ages FROM tablename t table2 t2 ON t2.my_id = t.id WHERE age > 10 GROUP BY age HAVING count(*) > 1 ORDER BY nr_ages')
+
+      extractor = Drudgery::Extractors::SQLite3Extractor.new(db, 'tablename')
+      extractor.extract
+
+      extractor.select('age', 'count(*) AS nr_ages')
+      extractor.from('tablename t')
+      extractor.joins('table2 t2 ON t2.my_id = t.id')
+      extractor.where('age > 10')
+      extractor.group('age')
+      extractor.having('count(*) > 1')
+      extractor.order('nr_ages')
+
+      extractor.extract
+    end
+
+    it 'yields each record as a hash' do
+      record1 = { :a => 1 }
+      record2 = { :b => 2 }
+
+      db = mock
+      db.stubs(:results_as_hash=)
+      db.stubs(:type_translation=)
+      db.stubs(:execute).multiple_yields([record1], [record2])
+
+      extractor = Drudgery::Extractors::SQLite3Extractor.new(db, 'tablename')
+
+      records = []
+      extractor.extract do |record|
+        records << record
+      end
+
+      records[0].must_equal({ :a => 1 })
+      records[1].must_equal({ :b => 2 })
+    end
+
+    describe 'without stubs' do
+      before(:each) do
+        @db = SQLite3::Database.new(':memory:')
+        @db.execute('CREATE TABLE records (a INTEGER, b INTEGER)')
+        @db.execute('INSERT INTO records (a, b) VALUES (1, 2)');
+        @db.execute('INSERT INTO records (a, b) VALUES (3, 4)');
+        @db.execute('INSERT INTO records (a, b) VALUES (5, 6)');
+      end
+
+      after(:each) do
+        @db.close
+      end
+
+      it 'yields each record as a hash' do
+        extractor = Drudgery::Extractors::SQLite3Extractor.new(@db, 'records')
+
+        records = []
+        extractor.extract do |record|
+          records << record
+        end
+
+        records.must_equal([
+          { 'a' => 1, 'b' => 2 },
+          { 'a' => 3, 'b' => 4 },
+          { 'a' => 5, 'b' => 6 }
+        ])
+      end
+    end
+  end
+end

data/spec/drudgery/job_spec.rb

@@ -0,0 +1,144 @@
+require 'spec_helper'
+
+describe Drudgery::Job do
+  describe '#initialize' do
+    before(:each) do
+      @extractor = mock
+      @transformer = mock
+      @loader = mock
+
+      @job = Drudgery::Job.new(:extractor => @extractor, :transformer => @transformer, :loader => @loader)
+    end
+
+    it 'sets extractor, transformer, and loader with provided arguments' do
+      @job.instance_variable_get('@extractor').must_equal @extractor
+      @job.instance_variable_get('@transformer').must_equal @transformer
+      @job.instance_variable_get('@loader').must_equal @loader
+    end
+
+    it 'initializes extractor, transformer, and loader if none provided' do
+      job = Drudgery::Job.new
+      job.instance_variable_get('@extractor').must_be_nil
+      job.instance_variable_get('@transformer').must_be_instance_of(Drudgery::Transformer)
+      job.instance_variable_get('@loader').must_be_nil
+    end
+
+    it 'initializes records as array' do
+      @job.instance_variable_get('@records').must_equal []
+    end
+
+    it 'initializes batch_size as 1000' do
+      @job.instance_variable_get('@batch_size').must_equal 1000
+    end
+  end
+
+  describe '#batch_size' do
+    it 'sets batch_size to provided value' do
+      job = Drudgery::Job.new
+      job.batch_size 2
+      job.instance_variable_get('@batch_size').must_equal 2
+    end
+  end
+
+  describe '#extract' do
+    it 'instantiates extractor with type and args' do
+      Drudgery::Extractors.expects(:instantiate).with(:csv, 'filename.csv', :col_sep => '|')
+
+      job = Drudgery::Job.new
+      job.extract(:csv, 'filename.csv', :col_sep => '|')
+    end
+
+    it 'sets extractor' do
+      extractor = mock
+
+      Drudgery::Extractors.stubs(:instantiate).returns(extractor)
+
+      job = Drudgery::Job.new
+      job.extract(:csv, 'filename.csv', :col_sep => '|')
+
+      job.instance_variable_get('@extractor').must_equal extractor
+    end
+  end
+
+  describe '#transform' do
+    it 'registers provided proc with transformer' do
+      block = Proc.new { |data, cache| data }
+
+      transformer = mock
+      transformer.expects(:register).with(block)
+
+      job = Drudgery::Job.new(:transformer => transformer)
+      job.transform(&block)
+    end
+
+    it 'registers provided block with transformer' do
+      transformer = mock
+      transformer.expects(:register).with { |data, cache| data }
+
+      job = Drudgery::Job.new(:transformer => transformer)
+      job.transform { |data, cache| data }
+    end
+  end
+
+  describe '#load' do
+    it 'instantiates loader with type with args' do
+      Drudgery::Loaders.expects(:instantiate).with(:sqlite3, 'db.sqlite3', 'tablename')
+
+      job = Drudgery::Job.new
+      job.load(:sqlite3, 'db.sqlite3', 'tablename')
+    end
+
+    it 'sets extractor' do
+      loader = mock
+
+      Drudgery::Loaders.expects(:instantiate).with(:sqlite3, 'db.sqlite3', 'tablename').returns(loader)
+
+      job = Drudgery::Job.new
+      job.load(:sqlite3, 'db.sqlite3', 'tablename')
+      job.instance_variable_get('@loader').must_equal loader
+    end
+  end
+
+  describe '#perform' do
+    it 'extracts records from extractor' do
+      extractor = mock
+      extractor.expects(:extract).yields({ 'a' => 1 })
+
+      loader = mock
+      loader.stubs(:load)
+
+      job = Drudgery::Job.new(:extractor => extractor, :loader => loader)
+
+      job.perform
+    end
+
+    it 'transforms records with transformer' do
+      extractor = mock
+      extractor.stubs(:extract).yields({ 'a' => 1 })
+
+      transformer = mock
+      transformer.expects(:transform).with({ 'a' => 1 }).returns({ :a => 1 })
+
+      loader = mock
+      loader.stubs(:load)
+
+      job = Drudgery::Job.new(:extractor => extractor, :transformer => transformer, :loader => loader)
+
+      job.perform
+    end
+
+    it 'loads records with loader in batches' do
+      extractor = mock
+      extractor.stubs(:extract).multiple_yields([{ 'a' => 1 }], [{ 'b' => 2 }], [{ 'c' => 3 }])
+
+      loader = mock
+      loader.expects(:load).with([{ :a => 1 }, { :b => 2 }])
+      loader.expects(:load).with([{ :c => 3 }])
+
+      job = Drudgery::Job.new(:extractor => extractor, :loader => loader)
+      job.batch_size 2
+
+      job.perform
+    end
+  end
+end

data/spec/drudgery/loaders/active_record_import_loader_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+require 'active_record'
+require 'activerecord-import'
+
+describe Drudgery::Loaders::ActiveRecordImportLoader do
+  class Record < ActiveRecord::Base; end
+
+  describe '#initialize' do
+    it 'sets model to provided argument' do
+      model = mock
+
+      loader = Drudgery::Loaders::ActiveRecordImportLoader.new(model)
+      loader.instance_variable_get('@model').must_equal model
+    end
+  end
+
+  describe '#load' do
+    it 'write records using model.import' do
+      record1 = { :a => 1, :b => 2 }
+      record2 = { :a => 3, :b => 4 }
+
+      model = mock
+      model.expects(:import).with([:a, :b], [[1, 2], [3, 4]], :validate => false)
+
+      loader = Drudgery::Loaders::ActiveRecordImportLoader.new(model)
+      loader.load([record1, record2])
+    end
+
+    describe 'without stubs' do
+      before(:each) do
+        ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
+        ActiveRecord::Base.connection.create_table(:records) do |t|
+          t.integer :a
+          t.integer :b
+        end
+      end
+
+      after(:each) do
+        ActiveRecord::Base.clear_active_connections!
+      end
+
+      it 'yields each record as a hash' do
+        record1 = { :a => 1, :b => 2 }
+        record2 = { :a => 3, :b => 4 }
+
+        loader = Drudgery::Loaders::ActiveRecordImportLoader.new(Record)
+        loader.load([record1, record2])
+
+        records = Record.all.map(&:attributes)
+        records.must_equal([
+          { 'id' => 1, 'a' => 1, 'b' => 2 },
+          { 'id' => 2, 'a' => 3, 'b' => 4 }
+        ])
+      end
+    end
+  end
+end

data/spec/drudgery/loaders/active_record_loader_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+require 'active_record'
+
+describe Drudgery::Loaders::ActiveRecordLoader do
+  class Record < ActiveRecord::Base; end
+
+  describe '#initialize' do
+    it 'sets model to provided argument' do
+      model = mock
+
+      loader = Drudgery::Loaders::ActiveRecordLoader.new(model)
+      loader.instance_variable_get('@model').must_equal model
+    end
+  end
+
+  describe '#load' do
+    it 'write records using model.create' do
+      record1 = { :a => 1 }
+      record2 = { :a => 2 }
+
+      obj1 = mock
+      obj2 = mock
+
+      model = mock
+      model.expects(:new).with(record1).returns(obj1)
+      model.expects(:new).with(record2).returns(obj2)
+
+      obj1.expects(:save).with(:validate => false)
+      obj2.expects(:save).with(:validate => false)
+
+      loader = Drudgery::Loaders::ActiveRecordLoader.new(model)
+      loader.load([record1, record2])
+    end
+
+    describe 'without stubs' do
+      before(:each) do
+        ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
+        ActiveRecord::Base.connection.create_table(:records) do |t|
+          t.integer :a
+          t.integer :b
+        end
+      end
+
+      after(:each) do
+        ActiveRecord::Base.clear_active_connections!
+      end
+
+      it 'yields each record as a hash' do
+        record1 = { :a => 1, :b => 2 }
+        record2 = { :a => 3, :b => 4 }
+
+        loader = Drudgery::Loaders::ActiveRecordLoader.new(Record)
+        loader.load([record1, record2])
+
+        records = Record.all.map(&:attributes)
+        records.must_equal([
+          { 'id' => 1, 'a' => 1, 'b' => 2 },
+          { 'id' => 2, 'a' => 3, 'b' => 4 }
+        ])
+      end
+    end
+  end
+end

data/spec/drudgery/loaders/csv_loader_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+describe Drudgery::Loaders::CSVLoader do
+  describe '#initialize' do
+    it 'sets filepath to provided filepath' do
+      loader = Drudgery::Loaders::CSVLoader.new('file.csv')
+      loader.instance_variable_get('@filepath').must_equal 'file.csv'
+    end
+
+    it 'initializes write_headers boolean' do
+      loader = Drudgery::Loaders::CSVLoader.new('file.csv')
+      loader.instance_variable_get('@write_headers').must_equal true
+    end
+
+    it 'sets options to provided options' do
+      options = { :col_sep => '|' }
+
+      loader = Drudgery::Loaders::CSVLoader.new('file.csv', options)
+      loader.instance_variable_get('@options').must_equal({ :col_sep => '|' })
+    end
+  end
+
+  describe '#load' do
+    it 'opens CSV file to append records' do
+      CSV.expects(:open).with('file.csv', 'a', :col_sep => '|')
+
+      loader = Drudgery::Loaders::CSVLoader.new('file.csv', :col_sep => '|')
+      loader.load([{}])
+    end
+
+    it 'writes hash keys as header and records as rows' do
+      record1 = { :a => 1, :b => 2 }
+      record2 = { :a => 3, :b => 4 }
+      record3 = { :a => 5, :b => 6 }
+
+      csv = mock
+      csv.expects(:<<).with([:a, :b])
+      csv.expects(:<<).with([1, 2])
+      csv.expects(:<<).with([3, 4])
+      csv.expects(:<<).with([5, 6])
+
+      CSV.expects(:open).with('file.csv', 'a', {}).yields(csv).times(2)
+
+      loader = Drudgery::Loaders::CSVLoader.new('file.csv')
+      loader.load([record1, record2])
+      loader.load([record3])
+    end
+
+    describe 'without stubs' do
+      before(:each) do
+        File.delete('file.csv') if File.exists?('file.csv')
+      end
+
+      after(:each) do
+        File.delete('file.csv') if File.exists?('file.csv')
+      end
+
+      it 'writes hash keys as header and records as rows' do
+        record1 = { :a => 1, :b => 2 }
+        record2 = { :a => 3, :b => 4 }
+        record3 = { :a => 5, :b => 6 }
+
+        loader = Drudgery::Loaders::CSVLoader.new('file.csv')
+        loader.load([record1, record2])
+        loader.load([record3])
+
+        records = File.readlines('file.csv').map { |line| line.strip.split(',') }
+        records.must_equal [%w[a b], %w[1 2], %w[3 4], %w[5 6]]
+      end
+    end
+  end
+end