drudgery 0.0.2 → 0.0.3

data/README.md

@@ -55,7 +55,7 @@ m = Drudgery::Manager.new
 m.prepare do |job|
   job.extract :csv, 'src/addresses.csv'
 
-  job.transform do |data|
+  job.transform do |data, cache|
     first_name, last_name = data.delete(:name).split(' ')
 
     data[:first_name] = first_name
@@ -93,7 +93,7 @@ m.prepare do |job|
     extractor.order('name')
   end
 
-  job.transform do |data|
+  job.transform do |data, cache|
     first_name, last_name = data.delete(:name).split(' ')
 
     data[:first_name] = first_name
@@ -132,9 +132,9 @@ end
 source = []
 
 m = Drudgery::Manager.new
-job = Drudgery::Job.new(:extractor => ArrayExtractor.new(source))
 
-m.prepare(job) do |job|
+m.prepare do |job|
+  m.extract ArrayExtractor.new(source)
   m.load :csv, 'destination.csv'
 end
 ```
@@ -173,12 +173,12 @@ Transformers
 ------------
 
 Drudgery comes with a basic Transformer class.  It symbolizes the keys of
-each record and allows you to register processors to process data.  Registered
-processors should implement a `#call` method and return a `Hash` or `nil`.
+each record and allows you to register a processor to process data.  The
+processor should implement a `#call` method and return a `Hash` or `nil`.
 
 ```ruby
 custom_processor = Proc.new do |data, cache|
-  data[:initials] = data[:name].split(' ').map(&:capitalize)
+  data[:initials] = data[:name].split(' ').map(&:capitalize).join()
   data
 end
 
@@ -190,7 +190,7 @@ transformer.transform({ :name => 'John Doe' }) # == { :name => 'John Doe', :init
 
 You could also implement your own transformer if you need more custom
 processing power.  If you inherit from `Drudgery::Transfomer`, you need
-only implement the `#transform` method that accepts a hash as an
+only implement the `#transform` method that accepts a hash argument as an
 argument and returns a `Hash` or `nil`.
 
 ```ruby
@@ -201,10 +201,10 @@ class CustomTransformer < Drudgery::Transformer
 end
 
 m = Drudgery::Manager.new
-job = Drudgery::Job.new(:transformer => CustomTransformer.new)
 
-m.prepare(job) do |job|
+m.prepare do |job|
   m.extract :csv, 'source.csv'
+  m.transform( CustomTransformer.new)
   m.load :csv, 'destination.csv'
 end
 ```
@@ -237,10 +237,10 @@ end
 destination = []
 
 m = Drudgery::Manager.new
-job = Drudgery::Job.new(:loader => ArrayLoader.new(destination))
 
-m.prepare(job) do |job|
+m.prepare do |job|
   m.extract :csv, 'source.csv'
+  m.load ArrayLoader.new(destination)
 end
 ```
 

data/lib/drudgery/job.rb

@@ -3,29 +3,40 @@ module Drudgery
     def initialize(options={})
       @extractor    = options[:extractor]
       @loader       = options[:loader]
-      @transformer  = options[:transformer] || Drudgery::Transformer.new
+      @transformer  = options[:transformer]
+      @batch_size   = options[:batch_size] || 1000
 
-      @batch_size, @records = 1000, []
+      @records = []
     end
 
     def batch_size(size)
       @batch_size = size
     end
 
-    def extract(type, *args)
-      extractor = Drudgery::Extractors.instantiate(type, *args)
+    def extract(*args)
+      if args.first.kind_of?(Symbol)
+        extractor = Drudgery::Extractors.instantiate(*args)
+      else
+        extractor = args.first
+      end
 
       yield extractor if block_given?
 
       @extractor = extractor
     end
 
-    def transform(&processor)
-      @transformer.register(processor)
+    def transform(transformer=Drudgery::Transformer.new, &processor)
+      transformer.register(processor)
+
+      @transformer = transformer
     end
 
-    def load(type, *args)
-      loader = Drudgery::Loaders.instantiate(type, *args)
+    def load(*args)
+      if args.first.kind_of?(Symbol)
+        loader = Drudgery::Loaders.instantiate(*args)
+      else
+        loader = args.first
+      end
 
       yield loader if block_given?
 
@@ -47,7 +58,7 @@ module Drudgery
     private
     def extract_records
       @extractor.extract do |data|
-        record = @transformer.transform(data)
+        record = transform_data(data)
         next if record.nil?
 
         yield record
@@ -58,5 +69,13 @@ module Drudgery
       @loader.load(@records)
       @records.clear
     end
+
+    def transform_data(data)
+      if @transformer
+        @transformer.transform(data)
+      else
+        data
+      end
+    end
   end
 end

data/lib/drudgery/transformer.rb

@@ -1,23 +1,17 @@
 module Drudgery
   class Transformer
     def initialize
-      @processors = []
       @cache = {}
     end
 
     def register(processor)
-      @processors << processor
+      @processor = processor
     end
 
     def transform(data)
       symbolize_keys!(data)
 
-      @processors.each do |processor|
-        data = processor.call(data, @cache)
-        break if data.nil?
-      end
-
-      data
+      @processor ? @processor.call(data, @cache) : data
     end
 
     private

data/lib/drudgery/version.rb

@@ -1,3 +1,3 @@
 module Drudgery
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

data/spec/drudgery/job_spec.rb

@@ -16,10 +16,15 @@ describe Drudgery::Job do
       @job.instance_variable_get('@loader').must_equal @loader
     end
 
+    it 'sets batch_size with provided argument' do
+      job = Drudgery::Job.new(:batch_size => 100)
+      job.instance_variable_get('@batch_size').must_equal(100)
+    end
+
     it 'initializes extractor, transformer, and loader if none provided' do
       job = Drudgery::Job.new
       job.instance_variable_get('@extractor').must_be_nil
-      job.instance_variable_get('@transformer').must_be_instance_of(Drudgery::Transformer)
+      job.instance_variable_get('@transformer').must_be_nil
       job.instance_variable_get('@loader').must_be_nil
     end
 
@@ -27,7 +32,8 @@ describe Drudgery::Job do
       @job.instance_variable_get('@records').must_equal []
     end
 
-    it 'initializes batch_size as 1000' do
+
+    it 'initializes batch_size as 1000 if none provided' do
       @job.instance_variable_get('@batch_size').must_equal 1000
     end
   end
@@ -41,85 +47,196 @@ describe Drudgery::Job do
   end
 
   describe '#extract' do
-    it 'instantiates extractor with type and args' do
-      Drudgery::Extractors.expects(:instantiate).with(:csv, 'filename.csv', :col_sep => '|')
+    describe 'when type and args provided' do
+      it 'instantiates extractor with type and args' do
+        Drudgery::Extractors.expects(:instantiate).with(:csv, 'filename.csv', :col_sep => '|')
 
-      job = Drudgery::Job.new
-      job.extract(:csv, 'filename.csv', :col_sep => '|')
-    end
+        job = Drudgery::Job.new
+        job.extract(:csv, 'filename.csv', :col_sep => '|')
+      end
 
-    it 'yields extractor if block_given' do
-      extractor = mock
-      extractor.expects(:col_sep).with('|')
+      it 'yields extractor if block_given' do
+        extractor = mock
+        extractor.expects(:col_sep).with('|')
 
-      Drudgery::Extractors.stubs(:instantiate).returns(extractor)
+        Drudgery::Extractors.stubs(:instantiate).returns(extractor)
 
-      job = Drudgery::Job.new
-      job.extract(:csv, 'filename.csv') do |extractor|
-        extractor.col_sep '|'
+        job = Drudgery::Job.new
+        job.extract(:csv, 'filename.csv') do |extractor|
+          extractor.col_sep '|'
+        end
+      end
+
+      it 'sets extractor' do
+        extractor = mock
+
+        Drudgery::Extractors.stubs(:instantiate).returns(extractor)
+
+        job = Drudgery::Job.new
+        job.extract(:csv, 'filename.csv', :col_sep => '|')
+
+        job.instance_variable_get('@extractor').must_equal extractor
       end
     end
 
-    it 'sets extractor' do
-      extractor = mock
+    describe 'when extractor provided' do
+      it 'does not instantiat extractor with type and args' do
+        extractor = mock
 
-      Drudgery::Extractors.stubs(:instantiate).returns(extractor)
+        Drudgery::Extractors.expects(:instantiate).never
 
-      job = Drudgery::Job.new
-      job.extract(:csv, 'filename.csv', :col_sep => '|')
+        job = Drudgery::Job.new
+        job.extract(extractor)
+      end
+
+      it 'yields extractor if block_given' do
+        extractor = mock
+        extractor.expects(:col_sep).with('|')
+
+        job = Drudgery::Job.new
+        job.extract(extractor) do |ext|
+          ext.col_sep '|'
+        end
+      end
 
-      job.instance_variable_get('@extractor').must_equal extractor
+      it 'sets extractor' do
+        extractor = mock
+
+        job = Drudgery::Job.new
+        job.extract(extractor)
+
+        job.instance_variable_get('@extractor').must_equal extractor
+      end
     end
   end
 
   describe '#transform' do
-    it 'registers provided proc with transformer' do
-      block = Proc.new { |data, cache| data }
+    describe 'when transformer provided' do
+      it 'sets transformer to provided transformer' do
+        transformer = mock
+        transformer.stubs(:register)
 
-      transformer = mock
-      transformer.expects(:register).with(block)
+        job = Drudgery::Job.new
+        job.transform(transformer)
+
+        job.instance_variable_get('@transformer').must_equal transformer
+      end
+
+      it 'registers provided proc with provided transformer' do
+        block = Proc.new { |data, cache| data }
+
+        transformer = mock
+        transformer.expects(:register).with(block)
+
+        job = Drudgery::Job.new
+        job.transform(transformer, &block)
+      end
 
-      job = Drudgery::Job.new(:transformer => transformer)
-      job.transform(&block)
+      it 'registers provided block with provided transformer' do
+        transformer = mock
+        transformer.expects(:register).with { |data, cache| data }
+
+        job = Drudgery::Job.new
+        job.transform(transformer) { |data, cache| data }
+      end
     end
 
-    it 'registers provided block with transformer' do
-      transformer = mock
-      transformer.expects(:register).with { |data, cache| data }
+    describe 'when no transformer provided' do
+      it 'sets transformer to default transformer' do
+        transformer = mock
+        transformer.stubs(:register)
+        
+        Drudgery::Transformer.expects(:new).returns(transformer)
+        
+        job = Drudgery::Job.new
+        job.transform
+
+        job.instance_variable_get('@transformer').must_equal transformer
+      end
+
+      it 'registers provided proc with default transformer' do
+        block = Proc.new { |data, cache| data }
+
+        transformer = mock
+        transformer.expects(:register).with(block)
+        
+        Drudgery::Transformer.stubs(:new).returns(transformer)
+
+        job = Drudgery::Job.new
+        job.transform(&block)
+      end
 
-      job = Drudgery::Job.new(:transformer => transformer)
-      job.transform { |data, cache| data }
+      it 'registers provided block with default transformer' do
+        transformer = Drudgery::Transformer.new
+        transformer.expects(:register).with { |data, cache| data }
+
+        Drudgery::Transformer.stubs(:new).returns(transformer)
+
+        job = Drudgery::Job.new
+        job.transform { |data, cache| data }
+      end
     end
   end
 
   describe '#load' do
-    it 'instantiates loader with type with args' do
-      Drudgery::Loaders.expects(:instantiate).with(:sqlite3, 'db.sqlite3', 'tablename')
+    describe 'when type and args provided' do
+      it 'instantiates loader with type with args' do
+        Drudgery::Loaders.expects(:instantiate).with(:sqlite3, 'db.sqlite3', 'tablename')
 
-      job = Drudgery::Job.new
-      job.load(:sqlite3, 'db.sqlite3', 'tablename')
-    end
+        job = Drudgery::Job.new
+        job.load(:sqlite3, 'db.sqlite3', 'tablename')
+      end
 
-    it 'yields loader if block_given' do
-      loader = mock
-      loader.expects(:select).with('a', 'b', 'c')
+      it 'yields loader if block_given' do
+        loader = mock
+        loader.expects(:select).with('a', 'b', 'c')
 
-      Drudgery::Loaders.stubs(:instantiate).with(:sqlite3, 'db.sqlite3', 'tablename').returns(loader)
+        Drudgery::Loaders.stubs(:instantiate).with(:sqlite3, 'db.sqlite3', 'tablename').returns(loader)
 
-      job = Drudgery::Job.new
-      job.load(:sqlite3, 'db.sqlite3', 'tablename') do |loader|
-        loader.select('a', 'b', 'c')
+        job = Drudgery::Job.new
+        job.load(:sqlite3, 'db.sqlite3', 'tablename') do |loader|
+          loader.select('a', 'b', 'c')
+        end
+      end
+
+      it 'sets loader' do
+        loader = mock
+
+        Drudgery::Loaders.expects(:instantiate).with(:sqlite3, 'db.sqlite3', 'tablename').returns(loader)
+
+        job = Drudgery::Job.new
+        job.load(:sqlite3, 'db.sqlite3', 'tablename')
+        job.instance_variable_get('@loader').must_equal loader
       end
     end
 
-    it 'sets extractor' do
-      loader = mock
+    describe 'when loader provided' do
+      it 'does not instantiate loader with type with args' do
+        loader = mock
 
-      Drudgery::Loaders.expects(:instantiate).with(:sqlite3, 'db.sqlite3', 'tablename').returns(loader)
+        Drudgery::Loaders.expects(:instantiate).never
 
-      job = Drudgery::Job.new
-      job.load(:sqlite3, 'db.sqlite3', 'tablename')
-      job.instance_variable_get('@loader').must_equal loader
+        job = Drudgery::Job.new
+        job.load(loader)
+      end
+
+      it 'yields loader if block_given' do
+        loader = mock
+        loader.expects(:select).with('a', 'b', 'c')
+
+        job = Drudgery::Job.new
+        job.load(loader) do |loader|
+          loader.select('a', 'b', 'c')
+        end
+      end
+
+      it 'sets loader' do
+        loader = mock
+
+        job = Drudgery::Job.new
+        job.load(loader)
+        job.instance_variable_get('@loader').must_equal loader
+      end
     end
   end
 
@@ -156,8 +273,8 @@ describe Drudgery::Job do
       extractor.stubs(:extract).multiple_yields([{ 'a' => 1 }], [{ 'b' => 2 }], [{ 'c' => 3 }])
 
       loader = mock
-      loader.expects(:load).with([{ :a => 1 }, { :b => 2 }])
-      loader.expects(:load).with([{ :c => 3 }])
+      loader.expects(:load).with([{ 'a' => 1 }, { 'b' => 2 }])
+      loader.expects(:load).with([{ 'c' => 3 }])
 
       job = Drudgery::Job.new(:extractor => extractor, :loader => loader)
       job.batch_size 2

data/spec/drudgery/transformer_spec.rb

@@ -6,21 +6,19 @@ describe Drudgery::Transformer do
   end
 
   describe '#initialize' do
-    it 'initializes processors array' do
-      @transformer.instance_variable_get('@processors').must_equal []
-    end
-
     it 'initializes cache hash' do
       @transformer.instance_variable_get('@cache').must_equal({})
     end
   end
 
   describe '#register' do
-    it 'adds processor to processors array' do
+    it 'sets processor with provided proc' do
       processor = Proc.new { |data, cache| data }
 
       @transformer.register(processor)
-      @transformer.instance_variable_get('@processors').must_include processor
+
+      # must_equal bug with comparing procs, so use assert_equal instead
+      assert_equal @transformer.instance_variable_get('@processor'), processor
     end
   end
 
@@ -29,14 +27,14 @@ describe Drudgery::Transformer do
       @transformer.transform({ 'a' => 1 }).must_equal({ :a => 1 })
     end
 
-    it 'processes data in each processor' do
+    it 'processes data with processor' do
       processor = Proc.new { |data, cache| data[:b] = 2; data }
 
       @transformer.register(processor)
       @transformer.transform({ 'a' => 1 }).must_equal({ :a => 1, :b => 2 })
     end
 
-    it 'allows processors to use cache' do
+    it 'allows processor to use cache' do
       processor = Proc.new do |data, cache|
         cache[:a] ||= 0
         cache[:a] += data[:a]
@@ -49,12 +47,10 @@ describe Drudgery::Transformer do
       @transformer.instance_variable_get('@cache').must_equal({ :a => 6 })
     end
 
-    it 'skips remaining processors and returns nil if any processor returns nil' do
-      processor1 = Proc.new { |data, cache| nil }
-      processor2 = Proc.new { |data, cache| raise 'should not get here' }
+    it 'returns nil if any processor returns nil' do
+      processor = Proc.new { |data, cache| nil }
 
-      @transformer.register(processor1)
-      @transformer.register(processor2)
+      @transformer.register(processor)
       @transformer.transform({ 'a' => 1 }).must_be_nil
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: drudgery
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-03 00:00:00.000000000 Z
+date: 2012-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70334397325460 !ruby/object:Gem::Requirement
+  requirement: &70230466813560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70334397325460
+  version_requirements: *70230466813560
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70334397324780 !ruby/object:Gem::Requirement
+  requirement: &70230466812880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '1.1'
   type: :development
   prerelease: false
-  version_requirements: *70334397324780
+  version_requirements: *70230466812880
 - !ruby/object:Gem::Dependency
   name: mocha
-  requirement: &70334397324280 !ruby/object:Gem::Requirement
+  requirement: &70230466812240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '0.10'
   type: :development
   prerelease: false
-  version_requirements: *70334397324280
+  version_requirements: *70230466812240
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70334397323700 !ruby/object:Gem::Requirement
+  requirement: &70230466811700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '0.6'
   type: :development
   prerelease: false
-  version_requirements: *70334397323700
+  version_requirements: *70230466811700
 - !ruby/object:Gem::Dependency
   name: guard-minitest
-  requirement: &70334397323080 !ruby/object:Gem::Requirement
+  requirement: &70230466811240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: '0.5'
   type: :development
   prerelease: false
-  version_requirements: *70334397323080
+  version_requirements: *70230466811240
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &70334397322440 !ruby/object:Gem::Requirement
+  requirement: &70230466810720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: '3.0'
   type: :development
   prerelease: false
-  version_requirements: *70334397322440
+  version_requirements: *70230466810720
 - !ruby/object:Gem::Dependency
   name: activerecord-import
-  requirement: &70334397321700 !ruby/object:Gem::Requirement
+  requirement: &70230466810220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +87,10 @@ dependencies:
         version: 0.2.9
   type: :development
   prerelease: false
-  version_requirements: *70334397321700
+  version_requirements: *70230466810220
 - !ruby/object:Gem::Dependency
   name: sqlite3
-  requirement: &70334397321080 !ruby/object:Gem::Requirement
+  requirement: &70230466809620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,7 +98,7 @@ dependencies:
         version: '1.3'
   type: :development
   prerelease: false
-  version_requirements: *70334397321080
+  version_requirements: *70230466809620
 description: A simple ETL library that supports CSV, SQLite3, and ActiveRecord sources
   and destinations.
 email: