drudgery 0.0.3 → 0.1.0

data/README.md

@@ -108,24 +108,66 @@ end
 m.run
 ```
 
+Logging
+-------
+
+Provide Drudgery with a logger and info will be logged about each job.
+
+When log level is `INFO` expect to see basic output for each job (e.g.
+when it starts and completes).
+
+```ruby
+logger = Logger.new('log/etl.log')
+logger.level = Logger::INFO # Logger defaults to log level DEBUG
+
+Drudgery.logger = logger
+```
+
+When log level is `DEBUG` expect to see output for each record
+extracted, transformed and loaded (VERY NOISY).
+
+Progress
+--------
+
+Drudgery also provides progress output to STDERR courtesty of the
+`progressbar` gem.  Progress output is on by default, but can be
+disabled with the following:
+
+```ruby
+Drudgery.show_progress = false
+```
+
 Extractors
 ----------
 
 The following extractors are provided: `:csv`, `:sqlite3`, `:active_record`
 
-You can use your own extractors if you would like.  They need only
-implement an `#extract` method that yields each record:
+You can use your own extractors if you would like.  They need to
+implement the following methods:
+
+ * `#name` - returns extractor's name
+ * `#record_count` - returns count of records in source
+ * `#extract` - must yield each record and record index
 
 ```ruby
 class ArrayExtractor
+  attr_reader :name
+
   def initialize(source)
     @source = source
+    @name = 'array'
   end
 
   def extract
+    index = 0
     @source.each do |record|
-      yield record
-    end    
+      yield [record, index]
+      index += 1
+    end
+  end
+
+  def record_count
+    @source.size
   end
 end
 
@@ -146,15 +188,24 @@ namespace:
 module Drudgery
   module Extractors
     class ArrayExtractor
+      attr_reader :name
+
       def initialize(source)
         @source = source
+        @name = 'array'
       end
 
       def extract
+        index = 0
         @source.each do |record|
-          yield record
+          yield [record, index]
+          index += 1
         end
       end
+
+      def record_count
+        @source.size
+      end
     end
   end
 end
@@ -219,14 +270,20 @@ The following loaders are provided:
  * `:active_record`
  * `:active_record_import`
 
-You can use your own loaders if you would like.  They need only
-implement a `#load` method that accepts an array of records as an
-argument and then writes/inserts them to the destination.
+You can use your own loaders if you would like.  They need to implement
+the following methods:
+
+* `#name` - returns the loader's name
+* `#load` - accepts an array of records and then write them to the
+  destination
 
 ```ruby
 class ArrayLoader
+  attr_reader :name
+
   def initialize(destination)
     @destination = destination
+    @name = 'array'
   end
 
   def load(records)
@@ -251,8 +308,11 @@ namespace:
 module Drudgery
   module Loaders
     class ArrayLoader
+      attr_reader :name
+
       def initialize(destination)
         @destination = destination
+        @name = 'array'
       end
 
       def load(records)

data/lib/drudgery/extractors/active_record_extractor.rb

@@ -1,15 +1,26 @@
 module Drudgery
   module Extractors
     class ActiveRecordExtractor
+      attr_reader :name
+
       def initialize(model)
         @model = model
+        @name = "active_record:#{@model.name}"
       end
 
       def extract
+        index = 0
+
         @model.find_each do |record|
-          yield record.attributes
+          yield [record.attributes, index]
+
+          index += 1
         end
       end
+
+      def record_count
+        @record_count ||= @model.count
+      end
     end
   end
 end

data/lib/drudgery/extractors/csv_extractor.rb

@@ -1,18 +1,38 @@
-require 'csv'
-
 module Drudgery
   module Extractors
     class CSVExtractor
+      attr_reader :name
+
       def initialize(filepath, options={})
         @filepath = filepath
-        @options = { :headers => true }
-        @options.merge!(options)
+        @options = { :headers => true }.merge(options)
+
+        @name = "csv:#{File.basename(@filepath)}"
       end
 
       def extract
+        index = 0
+
         CSV.foreach(@filepath, @options) do |row|
-          yield row.to_hash
+          yield [row.to_hash, index]
+
+          index += 1
+        end
+      end
+
+      def record_count
+        @record_count ||= calculate_record_count
+      end
+
+      private
+      def calculate_record_count
+        record_count = 0
+
+        extract do |data, index|
+          record_count += 1
         end
+
+        record_count
       end
     end
   end

data/lib/drudgery/extractors/sqlite3_extractor.rb

@@ -1,6 +1,8 @@
 module Drudgery
   module Extractors
     class SQLite3Extractor
+      attr_reader :name
+
       def initialize(db, table)
         @db = db
         @db.results_as_hash = true
@@ -8,6 +10,8 @@ module Drudgery
 
         @table = table
         @clauses = {}
+
+        @name = "sqlite3:#{main_db_name}.#{@table}"
       end
 
       def select(*expressions)
@@ -39,12 +43,20 @@ module Drudgery
       end
 
       def extract
+        index = 0
+
         @db.execute(sql) do |row|
           row.reject! { |key, value| key.kind_of?(Integer) }
-          yield row
+          yield [row, index]
+
+          index += 1
         end
       end
 
+      def record_count
+        @record_count ||= @db.get_first_value(count_sql)
+      end
+
       private
       def sql
         clauses = [
@@ -63,6 +75,24 @@ module Drudgery
 
         clauses.join(' ')
       end
+
+      def count_sql
+        if @clauses.empty?
+          "SELECT COUNT(*) FROM #{@table}"
+        else
+          "SELECT COUNT(*) FROM (#{sql})"
+        end
+      end
+
+      def main_db_name
+        main = @db.database_list.detect { |list| list['name'] == 'main' }
+
+        if main['file'].empty?
+          'memory'
+        else
+          File.basename(main['file']).split('.').first
+        end
+      end
     end
   end
 end

data/lib/drudgery/job.rb

@@ -1,6 +1,9 @@
 module Drudgery
   class Job
+    attr_reader :id
+
     def initialize(options={})
+      @id           = Time.now.nsec
       @extractor    = options[:extractor]
       @loader       = options[:loader]
       @transformer  = options[:transformer]
@@ -9,6 +12,10 @@ module Drudgery
       @records = []
     end
 
+    def name
+      "#{@extractor.name} => #{@loader.name}"
+    end
+
     def batch_size(size)
       @batch_size = size
     end
@@ -44,29 +51,50 @@ module Drudgery
     end
 
     def perform
-      extract_records do |record|
-        @records << record
+      logger.log_with_progress :info, name
+
+      elapsed = Benchmark.realtime do
+        extract_records do |record|
+          @records << record
 
-        if @records.size == @batch_size
-          load_records
+          if @records.size == @batch_size
+            load_records
+          end
+
+          progress.inc if Drudgery.show_progress
         end
+
+        load_records
+
+        progress.finish if Drudgery.show_progress
       end
 
-      load_records
+      logger.log_with_progress :info, "Completed in #{"%.2f" % elapsed}s\n\n"
     end
 
     private
     def extract_records
-      @extractor.extract do |data|
+      @extractor.extract do |data, index|
+        logger.log :debug, "Extracting Record -- Index: #{index}"
+        logger.log :debug, data.inspect
+
         record = transform_data(data)
-        next if record.nil?
+        logger.log :debug, "Transforming Record -- Index: #{index}"
+        logger.log :debug, data.inspect
 
-        yield record
+        if record.nil?
+          next
+        else
+          yield record
+        end
       end
     end
 
     def load_records
-      @loader.load(@records)
+      logger.log :debug, "Loading Records -- Count: #{@records.size}"
+      logger.log :debug, @records.inspect
+
+      @loader.load(@records) unless @records.empty?
       @records.clear
     end
 
@@ -77,5 +105,13 @@ module Drudgery
         data
       end
     end
+
+    def progress
+      @progress ||= Drudgery::JobProgress.new(id, @extractor.record_count)
+    end
+
+    def logger
+      @logger ||= Drudgery::JobLogger.new(id)
+    end
   end
 end

data/lib/drudgery/job_logger.rb

@@ -0,0 +1,21 @@
+module Drudgery
+  class JobLogger
+    def initialize(job_id)
+      @prefix = "## JOB #{job_id}"
+    end
+
+    def log_with_progress(mode, message)
+      STDERR.puts format_message(message) if Drudgery.show_progress
+      log(mode, message)
+    end
+
+    def log(mode, message)
+      Drudgery.log mode, format_message(message)
+    end
+
+    private
+    def format_message(message)
+      "#{@prefix}: #{message}"
+    end
+  end
+end

data/lib/drudgery/job_progress.rb

@@ -0,0 +1,11 @@
+module Drudgery
+  class JobProgress < ProgressBar
+    def initialize(job_id, total)
+      title = "## JOB #{job_id}"
+
+      super(title, total)
+
+      @title_width = title.length + 1
+    end
+  end
+end

data/lib/drudgery/loaders/active_record_import_loader.rb

@@ -1,8 +1,11 @@
 module Drudgery
   module Loaders
     class ActiveRecordImportLoader
+      attr_reader :name
+
       def initialize(model)
         @model = model
+        @name = "active_record_import:#{@model.name}"
       end
 
       def load(records)

data/lib/drudgery/loaders/active_record_loader.rb

@@ -1,8 +1,11 @@
 module Drudgery
   module Loaders
     class ActiveRecordLoader
+      attr_reader :name
+
       def initialize(model)
         @model = model
+        @name = "active_record:#{@model.name}"
       end
 
       def load(records)

data/lib/drudgery/loaders/csv_loader.rb

@@ -1,13 +1,15 @@
-require 'csv'
-
 module Drudgery
   module Loaders
     class CSVLoader
+      attr_reader :name
+
       def initialize(filepath, options={})
         @filepath = filepath
         @options = options
 
         @write_headers = true
+
+        @name = "csv:#{File.basename(@filepath)}"
       end
 
       def load(records)

data/lib/drudgery/loaders/sqlite3_loader.rb

@@ -1,9 +1,16 @@
 module Drudgery
   module Loaders
     class SQLite3Loader
+      attr_reader :name
+
       def initialize(db, table)
         @db = db
+        @db.results_as_hash = true
+        @db.type_translation = true
+
         @table = table
+
+        @name = "sqlite3:#{main_db_name}.#{@table}"
       end
 
       def load(records)
@@ -20,6 +27,16 @@ module Drudgery
       def sql(columns)
         "INSERT INTO #{@table} (#{columns.map { |column| column }.join(', ')}) VALUES (#{columns.map { |column| '?' }.join(', ')})"
       end
+
+      def main_db_name
+        main = @db.database_list.detect { |list| list['name'] == 'main' }
+
+        if main['file'].empty?
+          'memory'
+        else
+          File.basename(main['file']).split('.').first
+        end
+      end
     end
   end
 end

data/lib/drudgery/version.rb

@@ -1,3 +1,3 @@
 module Drudgery
-  VERSION = '0.0.3'
+  VERSION = '0.1.0'
 end

data/lib/drudgery.rb

@@ -1,4 +1,10 @@
+require 'benchmark'
+require 'csv'
+require 'progressbar'
+
 require 'drudgery/version'
+require 'drudgery/job_progress'
+require 'drudgery/job_logger'
 require 'drudgery/manager'
 require 'drudgery/job'
 require 'drudgery/transformer'
@@ -13,6 +19,14 @@ require 'drudgery/loaders/csv_loader'
 require 'drudgery/loaders/sqlite3_loader'
 
 module Drudgery
+  class << self
+    attr_accessor :logger, :show_progress
+
+    def log(mode, message)
+      logger.send(mode, message) if logger
+    end
+  end
+
   module Extractors
     def self.instantiate(type, *args)
       case type
@@ -43,3 +57,5 @@ module Drudgery
     end
   end
 end
+
+Drudgery.show_progress = true

data/spec/drudgery/extractors/active_record_extractor_spec.rb

@@ -4,68 +4,83 @@ require 'active_record'
 describe Drudgery::Extractors::ActiveRecordExtractor do
   class Record < ActiveRecord::Base; end
 
+  def mock_model
+    stub('model', :name => 'Record')
+  end
+
   describe '#initialize' do
     it 'sets model to provided argument' do
-      model = mock
+      model = mock_model
 
       extractor = Drudgery::Extractors::ActiveRecordExtractor.new(model)
       extractor.instance_variable_get('@model').must_equal model
     end
+
+    it 'sets name to active_record:<model name>' do
+      extractor = Drudgery::Extractors::ActiveRecordExtractor.new(mock_model)
+      extractor.name.must_equal 'active_record:Record'
+    end
   end
 
   describe '#extract' do
     it 'finds records using model' do
-      model = mock
+      model = mock_model
       model.expects(:find_each)
 
       extractor = Drudgery::Extractors::ActiveRecordExtractor.new(model)
       extractor.extract
     end
 
-    it 'yields each record as a hash' do
-      record1 = mock
-      record1.expects(:attributes).returns({ :a => 1 })
-
-      record2 = mock
-      record2.expects(:attributes).returns({ :b => 2 })
+    it 'yields each record hash and index' do
+      record1 = mock('record1', :attributes => { :a => 1 })
+      record2 = mock('record2', :attributes => { :b => 2 })
 
-      model = mock
+      model = mock_model
       model.stubs(:find_each).multiple_yields([record1], [record2])
 
       extractor = Drudgery::Extractors::ActiveRecordExtractor.new(model)
 
       records = []
-      extractor.extract do |record|
+      indexes = []
+      extractor.extract do |record, index|
         records << record
+        indexes << index
       end
 
       records[0].must_equal({ :a => 1 })
       records[1].must_equal({ :b => 2 })
+
+      indexes.must_equal [0, 1]
     end
 
-    describe 'without stubs' do
-      before(:each) do
-        ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
-        ActiveRecord::Base.connection.create_table(:records) do |t|
-          t.integer :a
-          t.integer :b
-        end
+  end
 
-        Record.create!({ :a => 1, :b => 2 })
-        Record.create!({ :a => 3, :b => 4 })
-        Record.create!({ :a => 5, :b => 6 })
+  describe 'without stubs' do
+    before(:each) do
+      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
+      ActiveRecord::Base.connection.create_table(:records) do |t|
+        t.integer :a
+        t.integer :b
       end
 
-      after(:each) do
-        ActiveRecord::Base.clear_active_connections!
-      end
+      Record.create!({ :a => 1, :b => 2 })
+      Record.create!({ :a => 3, :b => 4 })
+      Record.create!({ :a => 5, :b => 6 })
+    end
+
+    after(:each) do
+      ActiveRecord::Base.clear_active_connections!
+    end
 
-      it 'yields each record as a hash' do
+    describe '#extract' do
+      it 'yields each record hash and index' do
         extractor = Drudgery::Extractors::ActiveRecordExtractor.new(Record)
 
         records = []
-        extractor.extract do |record|
+        indexes = []
+        extractor.extract do |record, index|
           records << record
+          indexes << index
         end
 
         records.must_equal([
@@ -73,6 +88,15 @@ describe Drudgery::Extractors::ActiveRecordExtractor do
           { 'id' => 2, 'a' => 3, 'b' => 4 },
           { 'id' => 3, 'a' => 5, 'b' => 6 }
         ])
+
+        indexes.must_equal [0, 1, 2]
+      end
+    end
+
+    describe '#record_count' do
+      it 'returns model count' do
+        extractor = Drudgery::Extractors::ActiveRecordExtractor.new(Record)
+        extractor.record_count.must_equal 3
       end
     end
   end