drudgery 0.1.0 → 0.2.0

data/README.md

@@ -79,7 +79,7 @@ db = SQLite3::Database.new('db.sqlite3')
 m = Drudgery::Manager.new
 
 m.prepare do |job|
-  job.batch_size 5000
+  job.batch_size = 5000
 
   job.extract :sqlite3, db, 'addresses' do |extractor|
     extractor.select(
@@ -108,35 +108,6 @@ end
 m.run
 ```
 
-Logging
--------
-
-Provide Drudgery with a logger and info will be logged about each job.
-
-When log level is `INFO` expect to see basic output for each job (e.g.
-when it starts and completes).
-
-```ruby
-logger = Logger.new('log/etl.log')
-logger.level = Logger::INFO # Logger defaults to log level DEBUG
-
-Drudgery.logger = logger
-```
-
-When log level is `DEBUG` expect to see output for each record
-extracted, transformed and loaded (VERY NOISY).
-
-Progress
---------
-
-Drudgery also provides progress output to STDERR courtesty of the
-`progressbar` gem.  Progress output is on by default, but can be
-disabled with the following:
-
-```ruby
-Drudgery.show_progress = false
-```
-
 Extractors
 ----------
 
@@ -255,7 +226,7 @@ m = Drudgery::Manager.new
 
 m.prepare do |job|
   m.extract :csv, 'source.csv'
-  m.transform( CustomTransformer.new)
+  m.transform CustomTransformer.new
   m.load :csv, 'destination.csv'
 end
 ```
@@ -332,6 +303,84 @@ m.prepare do |job|
 end
 ```
 
+Event Hooks
+-----------
+
+Drudgery provides hooks so that you can listen for events and execute
+your own code (e.g. logging and progress).
+
+The following events are provided:
+
+ * `:before_job` - Fired before the jobs starts.
+ * `:after_job` - Fired after the jobs completes.
+ * `:after_extract` - Fired after each record is extracted.
+ * `:after_transform` - Fired after each record is transformed.
+ * `:after_load` - Fired after each batch of records are loaded.
+
+Logging
+-------
+
+Support for logging is not provided explicitly.  Here is an example
+using the hooks provided:
+
+```ruby
+require 'logger'
+logger = Logger.new('drudgery.log')
+
+# before_job yields the job
+Drudgery.subscribe :before_job do |job|
+  logger.info "## JOB #{job.id}: #{job.name}"
+end
+
+# after_extract yields the job, record, and record index
+Drudgery.subscribe :after_extract do |job, record, index|
+  logger.debug "## JOB #{job.id}: Extracting Record -- Index: #{index}"
+  logger.debug "## JOB #{job.id}: #{record.inspect}"
+end
+
+# after_transform yields the job, record, and record index
+Drudgery.subscribe :after_transform do |job, record, index|
+  logger.debug "## JOB #{job.id}: Transforming Record -- Index: #{index}"
+  logger.debug "## JOB #{job.id}: #{record.inspect}"
+end
+
+# after_load yields the job and records that were loaded
+Drudgery.subscribe :after_load do |job, records|
+  logger.debug "## JOB #{job.id}: Loading Records -- Count: #{records.size}"
+  logger.debug "## JOB #{job.id}: #{records.inspect}"
+end
+
+# after_job yields the job
+Drudgery.subscribe :after_job do |job|
+  logger.info "## JOB #{job.id}: Completed at #{job.completed_at}"
+end
+```
+
+Progress
+--------
+
+Support for progress indication is not provided explicitly.  Here is an example
+using the hooks provided:
+
+```ruby
+require 'rubygems'
+require 'progressbar'
+
+progress = {}
+
+Drudgery.subscribe :before_job do |job|
+  progress[job.id] ||= ProgressBar.new("## JOB #{job.id}", job.record_count)
+end
+
+Drudgery.subscribe :after_extract do |job, record, index|
+  progress[job.id].inc
+end
+
+Drudgery.subscribe :after_job do |job|
+  progress[job.id].finish
+end
+```
+
 Contributing
 ------------
 

data/lib/drudgery/extractors/csv_extractor.rb

@@ -10,6 +10,14 @@ module Drudgery
         @name = "csv:#{File.basename(@filepath)}"
       end
 
+      def col_sep
+        @options[:col_sep]
+      end
+
+      def col_sep=(char)
+        @options[:col_sep] = char
+      end
+
       def extract
         index = 0
 

data/lib/drudgery/job.rb

@@ -1,6 +1,7 @@
 module Drudgery
   class Job
-    attr_reader :id
+    attr_reader :id, :started_at, :completed_at
+    attr_accessor :extractor, :loader, :transformer, :batch_size
 
     def initialize(options={})
       @id           = Time.now.nsec
@@ -16,8 +17,10 @@ module Drudgery
       "#{@extractor.name} => #{@loader.name}"
     end
 
-    def batch_size(size)
-      @batch_size = size
+    def record_count
+      if @extractor
+        @record_count ||= @extractor.record_count
+      end
     end
 
     def extract(*args)
@@ -33,7 +36,7 @@ module Drudgery
     end
 
     def transform(transformer=Drudgery::Transformer.new, &processor)
-      transformer.register(processor)
+      transformer.register(processor) if processor
 
       @transformer = transformer
     end
@@ -51,36 +54,30 @@ module Drudgery
     end
 
     def perform
-      logger.log_with_progress :info, name
-
-      elapsed = Benchmark.realtime do
-        extract_records do |record|
-          @records << record
+      @started_at = Time.now
+      Drudgery.notify :before_job, self
 
-          if @records.size == @batch_size
-            load_records
-          end
+      extract_records do |record|
+        @records << record
 
-          progress.inc if Drudgery.show_progress
+        if @records.size == @batch_size
+          load_records
         end
-
-        load_records
-
-        progress.finish if Drudgery.show_progress
       end
 
-      logger.log_with_progress :info, "Completed in #{"%.2f" % elapsed}s\n\n"
+      load_records
+
+      @completed_at = Time.now
+      Drudgery.notify :after_job, self
     end
 
     private
     def extract_records
       @extractor.extract do |data, index|
-        logger.log :debug, "Extracting Record -- Index: #{index}"
-        logger.log :debug, data.inspect
+        Drudgery.notify :after_extract, self, data, index
 
         record = transform_data(data)
-        logger.log :debug, "Transforming Record -- Index: #{index}"
-        logger.log :debug, data.inspect
+        Drudgery.notify :after_transform, self, record, index
 
         if record.nil?
           next
@@ -91,10 +88,9 @@ module Drudgery
     end
 
     def load_records
-      logger.log :debug, "Loading Records -- Count: #{@records.size}"
-      logger.log :debug, @records.inspect
-
       @loader.load(@records) unless @records.empty?
+      Drudgery.notify :after_load, self, @records
+
       @records.clear
     end
 
@@ -105,13 +101,5 @@ module Drudgery
         data
       end
     end
-
-    def progress
-      @progress ||= Drudgery::JobProgress.new(id, @extractor.record_count)
-    end
-
-    def logger
-      @logger ||= Drudgery::JobLogger.new(id)
-    end
   end
 end

data/lib/drudgery/loaders/csv_loader.rb

@@ -12,6 +12,14 @@ module Drudgery
         @name = "csv:#{File.basename(@filepath)}"
       end
 
+      def col_sep
+        @options[:col_sep]
+      end
+
+      def col_sep=(char)
+        @options[:col_sep] = char
+      end
+
       def load(records)
         columns = records.first.keys.sort { |a,b| a.to_s <=> b.to_s }
 

data/lib/drudgery/version.rb

@@ -1,3 +1,3 @@
 module Drudgery
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

data/lib/drudgery.rb

@@ -1,10 +1,6 @@
-require 'benchmark'
 require 'csv'
-require 'progressbar'
 
 require 'drudgery/version'
-require 'drudgery/job_progress'
-require 'drudgery/job_logger'
 require 'drudgery/manager'
 require 'drudgery/job'
 require 'drudgery/transformer'
@@ -20,10 +16,22 @@ require 'drudgery/loaders/sqlite3_loader'
 
 module Drudgery
   class << self
-    attr_accessor :logger, :show_progress
+    def listeners
+      @listeners ||= Hash.new { |hash, key| hash[key] = [] }
+    end
+
+    def subscribe(event, &block)
+      listeners[event] << block
+    end
 
-    def log(mode, message)
-      logger.send(mode, message) if logger
+    def unsubscribe(event)
+      listeners[event].clear
+    end
+
+    def notify(event, *args)
+      listeners[event].each do |listener|
+        listener.call(*args)
+      end
     end
   end
 
@@ -57,5 +65,3 @@ module Drudgery
     end
   end
 end
-
-Drudgery.show_progress = true

data/spec/drudgery/extractors/active_record_extractor_spec.rb

@@ -1,102 +1,59 @@
 require 'spec_helper'
-require 'active_record'
 
-describe Drudgery::Extractors::ActiveRecordExtractor do
-  class Record < ActiveRecord::Base; end
+class Record < ActiveRecord::Base; end
 
-  def mock_model
-    stub('model', :name => 'Record')
-  end
-
-  describe '#initialize' do
-    it 'sets model to provided argument' do
-      model = mock_model
-
-      extractor = Drudgery::Extractors::ActiveRecordExtractor.new(model)
-      extractor.instance_variable_get('@model').must_equal model
-    end
-
-    it 'sets name to active_record:<model name>' do
-      extractor = Drudgery::Extractors::ActiveRecordExtractor.new(mock_model)
-      extractor.name.must_equal 'active_record:Record'
-    end
-  end
-
-  describe '#extract' do
-    it 'finds records using model' do
-      model = mock_model
-      model.expects(:find_each)
-
-      extractor = Drudgery::Extractors::ActiveRecordExtractor.new(model)
-      extractor.extract
-    end
-
-    it 'yields each record hash and index' do
-      record1 = mock('record1', :attributes => { :a => 1 })
-      record2 = mock('record2', :attributes => { :b => 2 })
-
-      model = mock_model
-      model.stubs(:find_each).multiple_yields([record1], [record2])
+module Drudgery
+  module Extractors
+    describe ActiveRecordExtractor do
+      before do
+        ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
+        ActiveRecord::Base.connection.create_table(:records) do |t|
+          t.integer :a
+          t.integer :b
+        end
 
-      extractor = Drudgery::Extractors::ActiveRecordExtractor.new(model)
+        Record.create!({ :a => 1, :b => 2 })
+        Record.create!({ :a => 3, :b => 4 })
+        Record.create!({ :a => 5, :b => 6 })
 
-      records = []
-      indexes = []
-      extractor.extract do |record, index|
-        records << record
-        indexes << index
+        @extractor = ActiveRecordExtractor.new(Record)
       end
 
-      records[0].must_equal({ :a => 1 })
-      records[1].must_equal({ :b => 2 })
-
-      indexes.must_equal [0, 1]
-    end
+      after do
+        ActiveRecord::Base.clear_active_connections!
+      end
 
-  end
 
-  describe 'without stubs' do
-    before(:each) do
-      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
-      ActiveRecord::Base.connection.create_table(:records) do |t|
-        t.integer :a
-        t.integer :b
+      describe '#name' do
+        it 'returns active_record:<model name>' do
+          @extractor.name.must_equal 'active_record:Record'
+        end
       end
 
-      Record.create!({ :a => 1, :b => 2 })
-      Record.create!({ :a => 3, :b => 4 })
-      Record.create!({ :a => 5, :b => 6 })
-    end
+      describe '#extract' do
+        it 'yields each record hash and index' do
+          records, indexes = [], []
 
-    after(:each) do
-      ActiveRecord::Base.clear_active_connections!
-    end
+          @extractor.extract do |record, index|
+            records << record
+            indexes << index
+          end
 
-    describe '#extract' do
-      it 'yields each record hash and index' do
-        extractor = Drudgery::Extractors::ActiveRecordExtractor.new(Record)
+          records.must_equal([
+            { 'id' => 1, 'a' => 1, 'b' => 2 },
+            { 'id' => 2, 'a' => 3, 'b' => 4 },
+            { 'id' => 3, 'a' => 5, 'b' => 6 }
+          ])
 
-        records = []
-        indexes = []
-        extractor.extract do |record, index|
-          records << record
-          indexes << index
+          indexes.must_equal [0, 1, 2]
         end
-
-        records.must_equal([
-          { 'id' => 1, 'a' => 1, 'b' => 2 },
-          { 'id' => 2, 'a' => 3, 'b' => 4 },
-          { 'id' => 3, 'a' => 5, 'b' => 6 }
-        ])
-
-        indexes.must_equal [0, 1, 2]
       end
-    end
 
-    describe '#record_count' do
-      it 'returns model count' do
-        extractor = Drudgery::Extractors::ActiveRecordExtractor.new(Record)
-        extractor.record_count.must_equal 3
+      describe '#record_count' do
+        it 'returns model count' do
+          @extractor = ActiveRecordExtractor.new(Record)
+          @extractor.record_count.must_equal 3
+        end
       end
     end
   end

data/spec/drudgery/extractors/csv_extractor_spec.rb

@@ -1,104 +1,72 @@
 require 'spec_helper'
 
-describe Drudgery::Extractors::CSVExtractor do
-  describe '#initialize' do
-    it 'sets filepath to provided filepath' do
-      extractor = Drudgery::Extractors::CSVExtractor.new('file.csv')
-      extractor.instance_variable_get('@filepath').must_equal 'file.csv'
-    end
-
-    it 'initializes options hash' do
-      extractor = Drudgery::Extractors::CSVExtractor.new('file.csv')
-      extractor.instance_variable_get('@options').must_equal({ :headers => true })
-    end
-
-    it 'merges provided options with default options' do
-      options = { :col_sep => '|', :headers => %w[id name email] }
-
-      extractor = Drudgery::Extractors::CSVExtractor.new('file.csv', options)
-      extractor.instance_variable_get('@options').must_equal({ :col_sep => '|', :headers => %w[id name email] })
-    end
-
-    it 'sets name to csv:<file base name>' do
-      extractor = Drudgery::Extractors::CSVExtractor.new('tmp/file.csv')
-      extractor.name.must_equal 'csv:file.csv'
-    end
-  end
-
-  describe '#extract' do
-    it 'parses records from file' do
-      CSV.expects(:foreach).with('file.csv', :headers => true)
-
-      extractor = Drudgery::Extractors::CSVExtractor.new('file.csv')
-      extractor.extract
-    end
-
-    it 'yields each record hash and index' do
-      record1 = mock('record1')
-      record1.expects(:to_hash).returns({ :a => 1 })
-
-      record2 = mock('record2')
-      record2.expects(:to_hash).returns({ :b => 2 })
-
-      CSV.stubs(:foreach).multiple_yields([record1], [record2])
-
-      extractor = Drudgery::Extractors::CSVExtractor.new('file.csv')
-
-      records = []
-      indexes = []
-      extractor.extract do |record, index|
-        records << record
-        indexes << index
+module Drudgery
+  module Extractors
+    describe CSVExtractor do
+      before do
+        @file = 'tmp/test.csv'
+        File.delete(@file) if File.exists?(@file)
+
+        File.open(@file, 'w') do |f|
+          f.puts 'a,b'
+          f.puts '1,2'
+          f.puts '3,4'
+          f.puts '5,6'
+        end
       end
 
-      records[0].must_equal({ :a => 1 })
-      records[1].must_equal({ :b => 2 })
-
-      indexes.must_equal [0, 1]
-    end
-  end
+      after do
+        File.delete(@file) if File.exists?(@file)
+      end
 
-  describe 'without stubs' do
-    before(:each) do
-      File.delete('file.csv') if File.exists?('file.csv')
+      describe '#name' do
+        it 'returns csv:<file base name>' do
+          extractor = CSVExtractor.new('tmp/people.csv')
+          extractor.name.must_equal 'csv:people.csv'
+        end
+      end
 
-      File.open('file.csv', 'w') do |f|
-        f.puts 'a,b'
-        f.puts '1,2'
-        f.puts '3,4'
-        f.puts '5,6'
+      describe '#col_sep' do
+        it 'returns col_sep option' do
+          extractor = CSVExtractor.new('tmp/people.csv', :col_sep => '|')
+          extractor.col_sep.must_equal '|'
+        end
       end
-    end
 
-    after(:each) do
-      File.delete('file.csv') if File.exists?('file.csv')
-    end
+      describe '#col_sep=' do
+        it 'sets col_sep to provided character' do
+          extractor = CSVExtractor.new('tmp/people.csv')
+          extractor.col_sep = '|'
+          extractor.col_sep.must_equal '|'
+        end
+      end
 
-    describe '#extract' do
-      it 'yields each record hash and index' do
-        extractor = Drudgery::Extractors::CSVExtractor.new('file.csv')
+      describe '#extract' do
+        it 'yields each record hash and index' do
+          extractor = CSVExtractor.new(@file)
 
-        records = []
-        indexes = []
-        extractor.extract do |record, index|
-          records << record
-          indexes << index
-        end
+          records = []
+          indexes = []
+          extractor.extract do |record, index|
+            records << record
+            indexes << index
+          end
 
-        records.must_equal([
-          { 'a' => '1', 'b' => '2' },
-          { 'a' => '3', 'b' => '4' },
-          { 'a' => '5', 'b' => '6' }
-        ])
+          records.must_equal([
+            { 'a' => '1', 'b' => '2' },
+            { 'a' => '3', 'b' => '4' },
+            { 'a' => '5', 'b' => '6' }
+          ])
 
-        indexes.must_equal [0, 1, 2]
+          indexes.must_equal [0, 1, 2]
+        end
       end
-    end
 
-    describe '#record_count' do
-      it 'returns count of CSV rows' do
-        extractor = Drudgery::Extractors::CSVExtractor.new('file.csv')
-        extractor.record_count.must_equal 3
+      describe '#record_count' do
+        it 'returns count of CSV rows' do
+          extractor = CSVExtractor.new(@file)
+          extractor.record_count.must_equal 3
+        end
       end
     end
   end