drudgery 0.0.1

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Jeremy Israelsen
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,306 @@
+Drudgery [![Build Status](https://secure.travis-ci.org/jisraelsen/drudgery.png?branch=master)](http://travis-ci.org/jisraelsen/drudgery)
+========
+
+A simple ETL library that supports the following sources/destinations:
+
+ * CSV and other delimited file formats (e.g. pipe, tab, etc)
+ * SQLite3
+ * ActiveRecord (bulk insert support using activerecord-import)
+
+Supported Rubies:
+
+ * Ruby 1.9.2, 1.9.3
+
+Install
+-------
+
+Install the gem directly:
+
+```bash
+gem install drudgery
+```
+
+Or, add it to your Gemfile:
+
+```ruby
+gem 'drudgery'
+```
+
+And, if using the `:sqlite3` extractor or loader:
+
+```ruby
+gem 'sqlite3', '~> 1.3'
+```
+
+And, if using the `:active_record` extractor or loader:
+
+```ruby
+gem 'activerecord', '~> 3.0'
+```
+
+And, if using the `:active_record_import` loader:
+
+```ruby
+gem 'activerecord-import', '>= 0.2.9'
+```
+
+Usage
+-----
+
+Extracting from CSV and loading into ActiveRecord:
+
+```ruby
+m = Drudgery::Manager.new
+
+m.prepare do |job|
+  job.extract :csv, 'src/addresses.csv'
+
+  job.transform do |data|
+    first_name, last_name = data.delete(:name).split(' ')
+
+    data[:first_name] = first_name
+    data[:last_name]  = last_name
+    data[:state]      = data.delete(:state_abbr)
+
+    data
+  end
+
+  job.load :active_record, Address
+end
+
+m.run
+```
+
+Extracting from SQLite3 and bulk loading into ActiveRecord:
+
+```ruby
+db = SQLite3::Database.new('db.sqlite3')
+
+m = Drudgery::Manager.new
+
+m.prepare do |job|
+  job.batch_size 5000
+
+  job.extract :sqlite3, db, 'addresses' do |extractor|
+    extractor.select(
+      'name',
+      'street_address',
+      'city',
+      'state_abbr AS state',
+      'zip'
+    )
+    extractor.where("state LIKE 'A%'")
+    extractor.order('name')
+  end
+
+  job.transform do |data|
+    first_name, last_name = data.delete(:name).split(' ')
+
+    data[:first_name] = first_name
+    data[:last_name]  = last_name
+
+    data
+  end
+
+  job.load :active_record_import, Address
+end
+
+m.run
+```
+
+Extractors
+----------
+
+The following extractors are provided: `:csv`, `:sqlite3`, `:active_record`
+
+You can use your own extractors if you would like.  They need only
+implement an `#extract` method that yields each record:
+
+```ruby
+class ArrayExtractor
+  def initialize(source)
+    @source = source
+  end
+
+  def extract
+    @source.each do |record|
+      yield record
+    end    
+  end
+end
+
+source = []
+
+m = Drudgery::Manager.new
+job = Drudgery::Job.new(:extractor => ArrayExtractor.new(source))
+
+m.prepare(job) do |job|
+  m.load :csv, 'destination.csv'
+end
+```
+
+Or, if you define your custom extractor under the Drudgery::Extractors
+namespace:
+
+```ruby
+module Drudgery
+  module Extractors
+    class ArrayExtractor
+      def initialize(source)
+        @source = source
+      end
+
+      def extract
+        @source.each do |record|
+          yield record
+        end
+      end
+    end
+  end
+end
+
+source = []
+
+m = Drudgery::Manager.new
+
+m.prepare do |job|
+  m.extract :array, source
+  m.load :csv, 'destination.csv'
+end
+```
+
+Transformers
+------------
+
+Drudgery comes with a basic Transformer class.  It symbolizes the keys of
+each record and allows you to register processors to process data.  Registered
+processors should implement a `#call` method and return a `Hash` or `nil`.
+
+```ruby
+custom_processor = Proc.new do |data, cache|
+  data[:initials] = data[:name].split(' ').map(&:capitalize)
+  data
+end
+
+transformer = Drudgery::Transformer.new
+transformer.register(custom_processor)
+
+transformer.transform({ :name => 'John Doe' }) # == { :name => 'John Doe', :initials => 'JD' }
+```
+
+You could also implement your own transformer if you need more custom
+processing power.  If you inherit from `Drudgery::Transfomer`, you need
+only implement the `#transform` method that accepts a hash as an
+argument and returns a `Hash` or `nil`.
+
+```ruby
+class CustomTransformer < Drudgery::Transformer
+  def transform(data)
+    # do custom processing here
+  end
+end
+
+m = Drudgery::Manager.new
+job = Drudgery::Job.new(:transformer => CustomTransformer.new)
+
+m.prepare(job) do |job|
+  m.extract :csv, 'source.csv'
+  m.load :csv, 'destination.csv'
+end
+```
+
+Loaders
+-------
+
+The following loaders are provided:
+
+ * `:csv`
+ * `:sqlite3`
+ * `:active_record`
+ * `:active_record_import`
+
+You can use your own loaders if you would like.  They need only
+implement a `#load` method that accepts an array of records as an
+argument and then writes/inserts them to the destination.
+
+```ruby
+class ArrayLoader
+  def initialize(destination)
+    @destination = destination
+  end
+
+  def load(records)
+    @destination.push(*records)
+  end
+end
+
+destination = []
+
+m = Drudgery::Manager.new
+job = Drudgery::Job.new(:loader => ArrayLoader.new(destination))
+
+m.prepare(job) do |job|
+  m.extract :csv, 'source.csv'
+end
+```
+
+Or, if you define your custom loader under the Drudgery::Loaders
+namespace:
+
+```ruby
+module Drudgery
+  module Loaders
+    class ArrayLoader
+      def initialize(destination)
+        @destination = destination
+      end
+
+      def load(records)
+        @destination.push(*records)
+      end
+    end
+  end
+end
+
+destination = []
+
+m = Drudgery::Manager.new
+
+m.prepare do |job|
+  m.extract :csv, 'source.csv'
+  m.load :array, destination
+end
+```
+
+Contributing
+------------
+
+Pull requests are welcome.  Just make sure to include tests!
+
+To run tests, install some dependencies:
+
+```bash
+bundle install
+```
+
+Then, run tests with:
+
+```bash
+rake test
+```
+
+Or, If you want to check coverage:
+
+```bash
+COVERAGE=true rake test
+```
+
+Issues
+------
+
+Please use GitHub's [issue tracker](http://github.com/jisraelsen/drudgery/issues).
+
+Author
+------
+
+[Jeremy Israelsen](http://github.com/jisraelsen)

data/lib/drudgery/extractors/active_record_extractor.rb

@@ -0,0 +1,15 @@
+module Drudgery
+  module Extractors
+    class ActiveRecordExtractor
+      def initialize(model)
+        @model = model
+      end
+
+      def extract
+        @model.find_each do |record|
+          yield record.attributes
+        end
+      end
+    end
+  end
+end

data/lib/drudgery/extractors/csv_extractor.rb

@@ -0,0 +1,19 @@
+require 'csv'
+
+module Drudgery
+  module Extractors
+    class CSVExtractor
+      def initialize(filepath, options={})
+        @filepath = filepath
+        @options = { :headers => true }
+        @options.merge!(options)
+      end
+
+      def extract
+        CSV.foreach(@filepath, @options) do |row|
+          yield row.to_hash
+        end
+      end
+    end
+  end
+end

data/lib/drudgery/extractors/sqlite3_extractor.rb

@@ -0,0 +1,68 @@
+module Drudgery
+  module Extractors
+    class SQLite3Extractor
+      def initialize(db, table)
+        @db = db
+        @db.results_as_hash = true
+        @db.type_translation = true
+
+        @table = table
+        @clauses = {}
+      end
+
+      def select(*expressions)
+        @clauses[:select] = expressions.join(', ')
+      end
+
+      def from(expression)
+        @clauses[:from] = expression
+      end
+
+      def joins(*clauses)
+        @clauses[:joins] = clauses
+      end
+
+      def where(condition)
+        @clauses[:where] = condition
+      end
+
+      def group(*expressions)
+        @clauses[:group] = expressions.join(', ')
+      end
+
+      def having(condition)
+        @clauses[:having] = condition
+      end
+
+      def order(*expressions)
+        @clauses[:order] = expressions.join(', ')
+      end
+
+      def extract
+        @db.execute(sql) do |row|
+          row.reject! { |key, value| key.kind_of?(Integer) }
+          yield row
+        end
+      end
+
+      private
+      def sql
+        clauses = [
+          "SELECT #{@clauses[:select] || '*'}",
+          "FROM #{@clauses[:from] || @table}"
+        ]
+
+        (@clauses[:joins] || []).each do |join|
+          clauses << join
+        end
+
+        clauses << "WHERE #{@clauses[:where]}"    if @clauses[:where]
+        clauses << "GROUP BY #{@clauses[:group]}" if @clauses[:group]
+        clauses << "HAVING #{@clauses[:having]}"  if @clauses[:having]
+        clauses << "ORDER BY #{@clauses[:order]}" if @clauses[:order]
+
+        clauses.join(' ')
+      end
+    end
+  end
+end

data/lib/drudgery/job.rb

@@ -0,0 +1,54 @@
+module Drudgery
+  class Job
+    def initialize(options={})
+      @extractor    = options[:extractor]
+      @loader       = options[:loader]
+      @transformer  = options[:transformer] || Drudgery::Transformer.new
+
+      @batch_size, @records = 1000, []
+    end
+
+    def batch_size(size)
+      @batch_size = size
+    end
+
+    def extract(type, *args)
+      @extractor = Drudgery::Extractors.instantiate(type, *args)
+    end
+
+    def transform(&processor)
+      @transformer.register(processor)
+    end
+
+    def load(type, *args)
+      @loader = Drudgery::Loaders.instantiate(type, *args)
+    end
+
+    def perform
+      extract_records do |record|
+        @records << record
+
+        if @records.size == @batch_size
+          load_records
+        end
+      end
+
+      load_records
+    end
+
+    private
+    def extract_records
+      @extractor.extract do |data|
+        record = @transformer.transform(data)
+        next if record.nil?
+
+        yield record
+      end
+    end
+
+    def load_records
+      @loader.load(@records)
+      @records.clear
+    end
+  end
+end

data/lib/drudgery/loaders/active_record_import_loader.rb

@@ -0,0 +1,16 @@
+module Drudgery
+  module Loaders
+    class ActiveRecordImportLoader
+      def initialize(model)
+        @model = model
+      end
+
+      def load(records)
+        columns = records.first.keys
+        values = records.map { |record| columns.map { |column| record[column] } }
+
+        @model.import(columns, values, :validate => false)
+      end
+    end
+  end
+end

data/lib/drudgery/loaders/active_record_loader.rb

@@ -0,0 +1,15 @@
+module Drudgery
+  module Loaders
+    class ActiveRecordLoader
+      def initialize(model)
+        @model = model
+      end
+
+      def load(records)
+        records.each do |record|
+          @model.new(record).save(:validate => false)
+        end
+      end
+    end
+  end
+end

data/lib/drudgery/loaders/csv_loader.rb

@@ -0,0 +1,29 @@
+require 'csv'
+
+module Drudgery
+  module Loaders
+    class CSVLoader
+      def initialize(filepath, options={})
+        @filepath = filepath
+        @options = options
+
+        @write_headers = true
+      end
+
+      def load(records)
+        columns = records.first.keys.sort { |a,b| a.to_s <=> b.to_s }
+
+        CSV.open(@filepath, 'a', @options) do |csv|
+          if @write_headers
+            csv << columns
+            @write_headers = false
+          end
+
+          records.each do |record|
+            csv << columns.map { |column| record[column] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/drudgery/loaders/sqlite3_loader.rb

@@ -0,0 +1,25 @@
+module Drudgery
+  module Loaders
+    class SQLite3Loader
+      def initialize(db, table)
+        @db = db
+        @table = table
+      end
+
+      def load(records)
+        columns = records.first.keys
+
+        @db.transaction do |db|
+          records.each do |record|
+            db.execute(sql(columns), columns.map { |column| record[column] })
+          end
+        end
+      end
+
+      private
+      def sql(columns)
+        "INSERT INTO #{@table} (#{columns.map { |column| column }.join(', ')}) VALUES (#{columns.map { |column| '?' }.join(', ')})"
+      end
+    end
+  end
+end

data/lib/drudgery/manager.rb

@@ -0,0 +1,17 @@
+module Drudgery
+  class Manager
+    def initialize
+      @jobs = []
+    end
+
+    def prepare(job=Drudgery::Job.new)
+      yield job if block_given?
+
+      @jobs << job
+    end
+
+    def run
+      @jobs.each { |job| job.perform }
+    end
+  end
+end

data/lib/drudgery/transformer.rb

@@ -0,0 +1,30 @@
+module Drudgery
+  class Transformer
+    def initialize
+      @processors = []
+      @cache = {}
+    end
+
+    def register(processor)
+      @processors << processor
+    end
+
+    def transform(data)
+      symbolize_keys!(data)
+
+      @processors.each do |processor|
+        data = processor.call(data, @cache)
+        break if data.nil?
+      end
+
+      data
+    end
+
+    private
+    def symbolize_keys!(data)
+      data.keys.each do |key|
+        data[(key.to_sym rescue key) || key] = data.delete(key)
+      end
+    end
+  end
+end

data/lib/drudgery/version.rb

@@ -0,0 +1,3 @@
+module Drudgery
+  VERSION = '0.0.1'
+end

data/lib/drudgery.rb

@@ -0,0 +1,45 @@
+require 'drudgery/version'
+require 'drudgery/manager'
+require 'drudgery/job'
+require 'drudgery/transformer'
+
+require 'drudgery/extractors/active_record_extractor'
+require 'drudgery/extractors/csv_extractor'
+require 'drudgery/extractors/sqlite3_extractor'
+
+require 'drudgery/loaders/active_record_import_loader'
+require 'drudgery/loaders/active_record_loader'
+require 'drudgery/loaders/csv_loader'
+require 'drudgery/loaders/sqlite3_loader'
+
+module Drudgery
+  module Extractors
+    def self.instantiate(type, *args)
+      case type
+      when :csv
+        extractor = Drudgery::Extractors::CSVExtractor
+      when :sqlite3
+        extractor = Drudgery::Extractors::SQLite3Extractor
+      else
+        extractor = Drudgery::Extractors.const_get("#{type.to_s.split('_').map(&:capitalize).join}Extractor")
+      end
+
+      extractor.new(*args)
+    end
+  end
+
+  module Loaders
+    def self.instantiate(type, *args)
+      case type
+      when :csv
+        loader = Drudgery::Loaders::CSVLoader
+      when :sqlite3
+        loader = Drudgery::Loaders::SQLite3Loader
+      else
+        loader = Drudgery::Loaders.const_get("#{type.to_s.split('_').map(&:capitalize).join}Loader")
+      end
+
+      loader.new(*args)
+    end
+  end
+end