drudgery 0.0.3 → 0.1.0

data/spec/drudgery/job_spec.rb

@@ -3,13 +3,20 @@ require 'spec_helper'
 describe Drudgery::Job do
   describe '#initialize' do
     before(:each) do
-      @extractor = mock
-      @transformer = mock
-      @loader = mock
+      @extractor = stub('extractor')
+      @transformer = stub('transformer')
+      @loader = stub('loader')
+
+      @now = Time.now
+      Time.stubs(:now).returns(@now)
 
       @job = Drudgery::Job.new(:extractor => @extractor, :transformer => @transformer, :loader => @loader)
     end
 
+    it 'sets job id to nsec time' do
+      @job.id.must_equal @now.nsec
+    end
+
     it 'sets extractor, transformer, and loader with provided arguments' do
       @job.instance_variable_get('@extractor').must_equal @extractor
       @job.instance_variable_get('@transformer').must_equal @transformer
@@ -38,6 +45,16 @@ describe Drudgery::Job do
     end
   end
 
+  describe '#name' do
+    it 'returns <extractor name> => <loader name>' do
+      extractor = stub('extractor', :name => 'csv:file.csv')
+      loader = stub('loader', :name => 'sqlite3:memory.tablename')
+
+      job = Drudgery::Job.new(:extractor => extractor, :loader => loader)
+      job.name.must_equal 'csv:file.csv => sqlite3:memory.tablename'
+    end
+  end
+
   describe '#batch_size' do
     it 'sets batch_size to provided value' do
       job = Drudgery::Job.new
@@ -56,7 +73,7 @@ describe Drudgery::Job do
       end
 
       it 'yields extractor if block_given' do
-        extractor = mock
+        extractor = mock('extractor')
         extractor.expects(:col_sep).with('|')
 
         Drudgery::Extractors.stubs(:instantiate).returns(extractor)
@@ -68,7 +85,7 @@ describe Drudgery::Job do
       end
 
       it 'sets extractor' do
-        extractor = mock
+        extractor = stub('extractor')
 
         Drudgery::Extractors.stubs(:instantiate).returns(extractor)
 
@@ -81,7 +98,7 @@ describe Drudgery::Job do
 
     describe 'when extractor provided' do
       it 'does not instantiat extractor with type and args' do
-        extractor = mock
+        extractor = stub('extractor')
 
         Drudgery::Extractors.expects(:instantiate).never
 
@@ -90,7 +107,7 @@ describe Drudgery::Job do
       end
 
       it 'yields extractor if block_given' do
-        extractor = mock
+        extractor = mock('extractor')
         extractor.expects(:col_sep).with('|')
 
         job = Drudgery::Job.new
@@ -100,7 +117,7 @@ describe Drudgery::Job do
       end
 
       it 'sets extractor' do
-        extractor = mock
+        extractor = stub('extractor')
 
         job = Drudgery::Job.new
         job.extract(extractor)
@@ -113,8 +130,7 @@ describe Drudgery::Job do
   describe '#transform' do
     describe 'when transformer provided' do
       it 'sets transformer to provided transformer' do
-        transformer = mock
-        transformer.stubs(:register)
+        transformer = stub('transformer', :register => nil)
 
         job = Drudgery::Job.new
         job.transform(transformer)
@@ -125,7 +141,7 @@ describe Drudgery::Job do
       it 'registers provided proc with provided transformer' do
         block = Proc.new { |data, cache| data }
 
-        transformer = mock
+        transformer = mock('transformer')
         transformer.expects(:register).with(block)
 
         job = Drudgery::Job.new
@@ -133,7 +149,7 @@ describe Drudgery::Job do
       end
 
       it 'registers provided block with provided transformer' do
-        transformer = mock
+        transformer = mock('transformer')
         transformer.expects(:register).with { |data, cache| data }
 
         job = Drudgery::Job.new
@@ -143,11 +159,10 @@ describe Drudgery::Job do
 
     describe 'when no transformer provided' do
       it 'sets transformer to default transformer' do
-        transformer = mock
-        transformer.stubs(:register)
-        
+        transformer = stub('transformer', :register => nil)
+
         Drudgery::Transformer.expects(:new).returns(transformer)
-        
+
         job = Drudgery::Job.new
         job.transform
 
@@ -157,9 +172,9 @@ describe Drudgery::Job do
       it 'registers provided proc with default transformer' do
         block = Proc.new { |data, cache| data }
 
-        transformer = mock
+        transformer = mock('transformer')
         transformer.expects(:register).with(block)
-        
+
         Drudgery::Transformer.stubs(:new).returns(transformer)
 
         job = Drudgery::Job.new
@@ -188,7 +203,7 @@ describe Drudgery::Job do
       end
 
       it 'yields loader if block_given' do
-        loader = mock
+        loader = mock('loader')
         loader.expects(:select).with('a', 'b', 'c')
 
         Drudgery::Loaders.stubs(:instantiate).with(:sqlite3, 'db.sqlite3', 'tablename').returns(loader)
@@ -200,7 +215,7 @@ describe Drudgery::Job do
       end
 
       it 'sets loader' do
-        loader = mock
+        loader = stub('loader')
 
         Drudgery::Loaders.expects(:instantiate).with(:sqlite3, 'db.sqlite3', 'tablename').returns(loader)
 
@@ -212,7 +227,7 @@ describe Drudgery::Job do
 
     describe 'when loader provided' do
       it 'does not instantiate loader with type with args' do
-        loader = mock
+        loader = stub('loader')
 
         Drudgery::Loaders.expects(:instantiate).never
 
@@ -221,7 +236,7 @@ describe Drudgery::Job do
       end
 
       it 'yields loader if block_given' do
-        loader = mock
+        loader = mock('loader')
         loader.expects(:select).with('a', 'b', 'c')
 
         job = Drudgery::Job.new
@@ -231,7 +246,7 @@ describe Drudgery::Job do
       end
 
       it 'sets loader' do
-        loader = mock
+        loader = stub('loader')
 
         job = Drudgery::Job.new
         job.load(loader)
@@ -240,13 +255,25 @@ describe Drudgery::Job do
     end
   end
 
+  def mock_logger
+    stub('job_logger', :log => nil, :log_with_progress => nil)
+  end
+
+  def stub_logging
+    
+  end
+
   describe '#perform' do
+    before(:each) do 
+      Drudgery.show_progress = false
+      Drudgery::JobLogger.stubs(:new).returns(mock_logger)
+    end
+
     it 'extracts records from extractor' do
-      extractor = mock
-      extractor.expects(:extract).yields({ 'a' => 1 })
+      extractor = stub('extractor', :record_count => 1, :name => 'extractor')
+      extractor.expects(:extract).yields([{ 'a' => 1 }, 0])
 
-      loader = mock
-      loader.stubs(:load)
+      loader = stub('loader', :name => 'loader', :load => nil)
 
       job = Drudgery::Job.new(:extractor => extractor, :loader => loader)
 
@@ -254,25 +281,51 @@ describe Drudgery::Job do
     end
 
     it 'transforms records with transformer' do
-      extractor = mock
-      extractor.stubs(:extract).yields({ 'a' => 1 })
+      extractor = stub('extractor', :record_count => 1, :name => 'extractor')
+      extractor.stubs(:extract).yields([{ 'a' => 1 }, 0])
 
-      transformer = mock
+      transformer = mock('transformer')
       transformer.expects(:transform).with({ 'a' => 1 }).returns({ :a => 1 })
 
-      loader = mock
-      loader.stubs(:load)
+      loader = stub('loader', :name => 'loader', :load => nil)
 
       job = Drudgery::Job.new(:extractor => extractor, :transformer => transformer, :loader => loader)
 
       job.perform
     end
 
+    it 'skips nil records' do
+      extractor = stub('extractor', :record_count => 1, :name => 'extractor')
+      extractor.stubs(:extract).yields([{ 'a' => 1 }, 0])
+
+      transformer = mock('transformer')
+      transformer.expects(:transform).with({ 'a' => 1 }).returns(nil)
+
+      loader = stub('loader', :name => 'loader')
+      loader.expects(:load).with([{ '1' => 1 }]).never
+
+      job = Drudgery::Job.new(:extractor => extractor, :transformer => transformer, :loader => loader)
+
+      job.perform
+    end
+
+    it 'does not load empty records' do
+      extractor = stub('extractor', :record_count => 1, :name => 'extractor')
+      extractor.stubs(:extract)
+
+      loader = stub('loader', :name => 'loader')
+      loader.expects(:load).with([]).never
+
+      job = Drudgery::Job.new(:extractor => extractor, :loader => loader)
+
+      job.perform
+    end
+
     it 'loads records with loader in batches' do
-      extractor = mock
-      extractor.stubs(:extract).multiple_yields([{ 'a' => 1 }], [{ 'b' => 2 }], [{ 'c' => 3 }])
+      extractor = stub('extractor', :record_count => 3, :name => 'extractor')
+      extractor.stubs(:extract).multiple_yields([{ 'a' => 1 }, 0], [{ 'b' => 2 }, 1], [{ 'c' => 3 }, 2])
 
-      loader = mock
+      loader = stub('loader', :name => 'loader')
       loader.expects(:load).with([{ 'a' => 1 }, { 'b' => 2 }])
       loader.expects(:load).with([{ 'c' => 3 }])
 
@@ -281,5 +334,80 @@ describe Drudgery::Job do
 
       job.perform
     end
+
+    describe 'with progress on' do
+      it 'tracks progress information' do
+        Drudgery.show_progress = true
+
+        extractor = stub('extractor', :record_count => 3, :name => 'extractor')
+        extractor.stubs(:extract).multiple_yields([{ 'a' => 1 }, 0], [{ 'b' => 2 }, 1], [{ 'c' => 3 }, 2])
+
+        loader = stub('loader', :name => 'loader', :load => nil)
+
+        job = Drudgery::Job.new(:extractor => extractor, :loader => loader)
+
+        progress = mock('job_progress') do
+          expects(:inc).times(3)
+          expects(:finish)
+        end
+        Drudgery::JobProgress.stubs(:new).with(job.id, 3).returns(progress)
+
+        job.perform
+      end
+    end
+
+    describe 'with progress off' do
+      it 'does not track progress information' do
+        extractor = stub('extractor', :record_count => 3, :name => 'extractor')
+        extractor.stubs(:extract).multiple_yields([{ 'a' => 1 }, 0], [{ 'b' => 2 }, 1], [{ 'c' => 3 }, 2])
+
+        loader = stub('loader', :name => 'loader', :load => nil)
+
+        job = Drudgery::Job.new(:extractor => extractor, :loader => loader)
+
+        Drudgery::JobProgress.expects(:new).never
+
+        job.perform
+      end
+    end
+
+    it 'logs job details' do
+      extractor = stub('extractor', :record_count => 3, :name => 'extractor')
+      extractor.stubs(:extract).multiple_yields([{ 'a' => 1 }, 0], [{ 'b' => 2 }, 1], [{ 'c' => 3 }, 2])
+
+      loader = stub('loader', :name => 'loader', :load => nil)
+
+      job = Drudgery::Job.new(:extractor => extractor, :loader => loader)
+      job.batch_size 2
+
+      Benchmark.stubs(:realtime).returns(1.25333).yields
+
+      logger = mock_logger
+      logger.expects(:log_with_progress).with(:info,  "extractor => loader")
+
+      logger.expects(:log).with(:debug, "Extracting Record -- Index: 0")
+      logger.expects(:log).with(:debug, "#{{ 'a' => 1 }.inspect}")
+      logger.expects(:log).with(:debug, "Transforming Record -- Index: 0")
+      logger.expects(:log).with(:debug, "#{{ 'a' => 1 }.inspect}")
+      logger.expects(:log).with(:debug, "Extracting Record -- Index: 1")
+      logger.expects(:log).with(:debug, "#{{ 'b' => 2 }.inspect}")
+      logger.expects(:log).with(:debug, "Transforming Record -- Index: 1")
+      logger.expects(:log).with(:debug, "#{{ 'b' => 2 }.inspect}")
+      logger.expects(:log).with(:debug, "Loading Records -- Count: 2")
+      logger.expects(:log).with(:debug, "#{[{ 'a' => 1 }, { 'b' => 2 }].inspect}")
+
+      logger.expects(:log).with(:debug, "Extracting Record -- Index: 2")
+      logger.expects(:log).with(:debug, "#{{ 'c' => 3 }.inspect}")
+      logger.expects(:log).with(:debug, "Transforming Record -- Index: 2")
+      logger.expects(:log).with(:debug, "#{{ 'c' => 3 }.inspect}")
+      logger.expects(:log).with(:debug, "Loading Records -- Count: 1")
+      logger.expects(:log).with(:debug, "#{[{ 'c' => 3 }].inspect}")
+
+      logger.expects(:log_with_progress).with(:info,  "Completed in 1.25s\n\n")
+
+      Drudgery::JobLogger.stubs(:new).returns(logger)
+
+      job.perform
+    end
   end
 end

data/spec/drudgery/loaders/active_record_import_loader_spec.rb

@@ -5,13 +5,22 @@ require 'activerecord-import'
 describe Drudgery::Loaders::ActiveRecordImportLoader do
   class Record < ActiveRecord::Base; end
 
+  def mock_model
+    stub('model', :name => 'Record')
+  end
+
   describe '#initialize' do
     it 'sets model to provided argument' do
-      model = mock
+      model = mock_model
 
       loader = Drudgery::Loaders::ActiveRecordImportLoader.new(model)
       loader.instance_variable_get('@model').must_equal model
     end
+
+    it 'sets name to active_record_import:<model name>' do
+      loader = Drudgery::Loaders::ActiveRecordImportLoader.new(mock_model)
+      loader.name.must_equal 'active_record_import:Record'
+    end
   end
 
   describe '#load' do
@@ -19,27 +28,29 @@ describe Drudgery::Loaders::ActiveRecordImportLoader do
       record1 = { :a => 1, :b => 2 }
       record2 = { :a => 3, :b => 4 }
 
-      model = mock
+      model = mock_model
       model.expects(:import).with([:a, :b], [[1, 2], [3, 4]], :validate => false)
 
       loader = Drudgery::Loaders::ActiveRecordImportLoader.new(model)
       loader.load([record1, record2])
     end
+  end
 
-    describe 'without stubs' do
-      before(:each) do
-        ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
-        ActiveRecord::Base.connection.create_table(:records) do |t|
-          t.integer :a
-          t.integer :b
-        end
+  describe 'without stubs' do
+    before(:each) do
+      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
+      ActiveRecord::Base.connection.create_table(:records) do |t|
+        t.integer :a
+        t.integer :b
       end
+    end
 
-      after(:each) do
-        ActiveRecord::Base.clear_active_connections!
-      end
+    after(:each) do
+      ActiveRecord::Base.clear_active_connections!
+    end
 
-      it 'yields each record as a hash' do
+    describe '#load' do
+      it 'writes records using model.import' do
         record1 = { :a => 1, :b => 2 }
         record2 = { :a => 3, :b => 4 }
 

data/spec/drudgery/loaders/active_record_loader_spec.rb

@@ -4,13 +4,22 @@ require 'active_record'
 describe Drudgery::Loaders::ActiveRecordLoader do
   class Record < ActiveRecord::Base; end
 
+  def mock_model
+    stub('model', :name => 'Record')
+  end
+
   describe '#initialize' do
     it 'sets model to provided argument' do
-      model = mock
+      model = mock_model
 
       loader = Drudgery::Loaders::ActiveRecordLoader.new(model)
       loader.instance_variable_get('@model').must_equal model
     end
+
+    it 'sets name to active_record:<model name>' do
+      loader = Drudgery::Loaders::ActiveRecordLoader.new(mock_model)
+      loader.name.must_equal 'active_record:Record'
+    end
   end
 
   describe '#load' do
@@ -18,10 +27,10 @@ describe Drudgery::Loaders::ActiveRecordLoader do
       record1 = { :a => 1 }
       record2 = { :a => 2 }
 
-      obj1 = mock
-      obj2 = mock
+      obj1 = mock('obj1')
+      obj2 = mock('obj2')
 
-      model = mock
+      model = mock_model
       model.expects(:new).with(record1).returns(obj1)
       model.expects(:new).with(record2).returns(obj2)
 
@@ -31,21 +40,23 @@ describe Drudgery::Loaders::ActiveRecordLoader do
       loader = Drudgery::Loaders::ActiveRecordLoader.new(model)
       loader.load([record1, record2])
     end
+  end
 
-    describe 'without stubs' do
-      before(:each) do
-        ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
-        ActiveRecord::Base.connection.create_table(:records) do |t|
-          t.integer :a
-          t.integer :b
-        end
+  describe 'without stubs' do
+    before(:each) do
+      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
+      ActiveRecord::Base.connection.create_table(:records) do |t|
+        t.integer :a
+        t.integer :b
       end
+    end
 
-      after(:each) do
-        ActiveRecord::Base.clear_active_connections!
-      end
+    after(:each) do
+      ActiveRecord::Base.clear_active_connections!
+    end
 
-      it 'yields each record as a hash' do
+    describe '#load' do
+      it 'writes records using model.create' do
         record1 = { :a => 1, :b => 2 }
         record2 = { :a => 3, :b => 4 }
 

data/spec/drudgery/loaders/csv_loader_spec.rb

@@ -18,6 +18,11 @@ describe Drudgery::Loaders::CSVLoader do
       loader = Drudgery::Loaders::CSVLoader.new('file.csv', options)
       loader.instance_variable_get('@options').must_equal({ :col_sep => '|' })
     end
+
+    it 'sets name to csv:<file base name>' do
+      loader = Drudgery::Loaders::CSVLoader.new('tmp/file.csv')
+      loader.name.must_equal 'csv:file.csv'
+    end
   end
 
   describe '#load' do
@@ -33,7 +38,7 @@ describe Drudgery::Loaders::CSVLoader do
       record2 = { :a => 3, :b => 4 }
       record3 = { :a => 5, :b => 6 }
 
-      csv = mock
+      csv = mock('csv')
       csv.expects(:<<).with([:a, :b])
       csv.expects(:<<).with([1, 2])
       csv.expects(:<<).with([3, 4])
@@ -45,16 +50,18 @@ describe Drudgery::Loaders::CSVLoader do
       loader.load([record1, record2])
       loader.load([record3])
     end
+  end
 
-    describe 'without stubs' do
-      before(:each) do
-        File.delete('file.csv') if File.exists?('file.csv')
-      end
+  describe 'without stubs' do
+    before(:each) do
+      File.delete('file.csv') if File.exists?('file.csv')
+    end
 
-      after(:each) do
-        File.delete('file.csv') if File.exists?('file.csv')
-      end
+    after(:each) do
+      File.delete('file.csv') if File.exists?('file.csv')
+    end
 
+    describe '#load' do
       it 'writes hash keys as header and records as rows' do
         record1 = { :a => 1, :b => 2 }
         record2 = { :a => 3, :b => 4 }

data/spec/drudgery/loaders/sqlite3_loader_spec.rb

@@ -1,14 +1,37 @@
 require 'spec_helper'
+require 'sqlite3'
 
 describe Drudgery::Loaders::SQLite3Loader do
+
+  def mock_db
+    stub('db', :database_list => [{ 'name' => 'main', 'file' => '' }], :results_as_hash= => nil, :type_translation= => nil)
+  end
+
   describe '#initialize' do
     it 'sets db to SQLite3 database using provided db name and table to provided table name' do
-      db = mock
+      db = mock_db
 
       loader = Drudgery::Loaders::SQLite3Loader.new(db, 'tablename')
       loader.instance_variable_get('@db').must_equal db
       loader.instance_variable_get('@table').must_equal 'tablename'
     end
+
+    describe 'with in memory db' do
+      it 'sets name to sqlite3:memory:<table name>' do
+        loader = Drudgery::Loaders::SQLite3Loader.new(mock_db, 'tablename')
+        loader.name.must_equal 'sqlite3:memory.tablename'
+      end
+    end
+
+    describe 'with file based db' do
+      it 'sets name to sqlite3:<main db name>:<table name>' do
+        db = mock_db
+        db.expects(:database_list).returns([{ 'name' => 'main', 'file' => 'db/test.sqlite3.db' }])
+
+        loader = Drudgery::Loaders::SQLite3Loader.new(db, 'tablename')
+        loader.name.must_equal 'sqlite3:test.tablename'
+      end
+    end
   end
 
   describe '#load' do
@@ -16,7 +39,7 @@ describe Drudgery::Loaders::SQLite3Loader do
       record1 = { :a => 1, :b => 2 }
       record2 = { :a => 3, :b => 4 }
 
-      db = mock
+      db = mock_db
       db.expects(:transaction).yields(db)
       db.expects(:execute).with('INSERT INTO tablename (a, b) VALUES (?, ?)', [1, 2])
       db.expects(:execute).with('INSERT INTO tablename (a, b) VALUES (?, ?)', [3, 4])
@@ -24,17 +47,26 @@ describe Drudgery::Loaders::SQLite3Loader do
       loader = Drudgery::Loaders::SQLite3Loader.new(db, 'tablename')
       loader.load([record1, record2])
     end
+  end
 
-    describe 'without stubs' do
-      before(:each) do
-        @db = SQLite3::Database.new(':memory:')
-        @db.execute('CREATE TABLE records (a INTEGER, b INTEGER)')
-      end
+  describe 'without stubs' do
+    before(:each) do
+      @db = SQLite3::Database.new(':memory:')
+      @db.execute('CREATE TABLE records (a INTEGER, b INTEGER)')
+    end
 
-      after(:each) do
-        @db.close
+    after(:each) do
+      @db.close
+    end
+
+    describe '#initialize' do
+      it 'sets name to sqlite3:memory:records' do
+        loader = Drudgery::Loaders::SQLite3Loader.new(@db, 'records')
+        loader.name.must_equal 'sqlite3:memory.records'
       end
+    end
 
+    describe '#load' do
       it 'writes each record in single transaction' do
         record1 = { :a => 1, :b => 2 }
         record2 = { :a => 3, :b => 4 }
@@ -48,8 +80,8 @@ describe Drudgery::Loaders::SQLite3Loader do
         end
 
         results.must_equal([
-          [1, 2],
-          [3, 4]
+          { 'a' => 1, 'b' => 2, 0 => 1, 1 => 2 },
+          { 'a' => 3, 'b' => 4, 0 => 3, 1 => 4}
         ])
       end
     end

data/spec/drudgery/manager_spec.rb

@@ -13,7 +13,7 @@ describe Drudgery::Manager do
 
   describe '#prepare' do
     it 'adds obj to jobs array' do
-      job = mock
+      job = stub('job')
 
       @manager.prepare(job)
       @manager.instance_variable_get('@jobs').must_include job
@@ -22,13 +22,9 @@ describe Drudgery::Manager do
 
   describe '#run' do
     it 'performs each job' do
-      job1 = mock
-      job1.expects(:perform)
-
-      job2 = mock
-      job2.expects(:perform)
-
-      job3 = mock
+      job1 = mock('job1', :perform => nil)
+      job2 = mock('job2', :perform => nil)
+      job3 = mock('job3')
       job3.expects(:perform).never
 
       @manager.prepare(job1)