petl 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c4aef860e78445984a2fa4c4c32216a699abd257
+  data.tar.gz: 8fb4ea2f027fa88819d368c1cfe5e4c9d3a63458
+SHA512:
+  metadata.gz: 19567edbfd6e82b3cd1bb7626317820fdc91611b23021837f28d2b5a4bcd64486287cca1c225a98768126d6b2ca500d5064660258e163c0fc5b7f17b1e720c0b
+  data.tar.gz: 70019a5f685728ce2651df6929ecf01eb3058096fb40d20102a772e6e959e0f6c9627b2de126b9ced1c00303a9b84f958f4f1385de43259901d99706317faf8b

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in petl.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 ZestFinance
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# petl [![Build Status](https://travis-ci.org/ZestFinance/petl.png?branch=master)](https://travis-ci.org/ZestFinance/petl) [![Code Climate](https://codeclimate.com/github/ZestFinance/petl.png)](https://codeclimate.com/github/ZestFinance/petl)
+
+Pretty good ETL framework
+
+## Features
+1. Batching support
+2. Automatic validity check
+3. Logging of running times
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'petl'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install petl
+
+## Usage
+
+```ruby
+require 'petl'
+
+module ETL::Example
+  extend Petl::ETL
+  extend self
+
+  def extract
+    # Grab all data from source(s) here.
+    # Perferrably return an array of hashes.
+  end
+
+  def transform rows
+    # Manipulate the data extracted by the previous extract method.
+  end
+
+  def load rows
+    # Load the transformed data here into the destination(s).
+  end
+
+  def source_count
+    # Count the number of records from your source(s).
+  end
+
+  def destination_count
+    # Same as #source_count but with your destination(s).
+  end
+end
+
+# Run it!
+ETL::Example.perform
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = '--color'
+  t.pattern = 'spec/**/*_spec.rb'
+end
+
+task default: :spec

data/lib/petl.rb

@@ -0,0 +1,2 @@
+require "petl/version"
+require "petl/etl"

data/lib/petl/etl.rb

@@ -0,0 +1,71 @@
+module Petl
+  module ETL
+    def perform logger = Rails.logger
+      if batch
+        batch_perform logger
+      else
+        not_batch_perform logger
+      end
+    end
+
+    def extract
+      raise NotImplementedError.new "#{self}#extract not implemented."
+    end
+
+    def transform
+      raise NotImplementedError.new "#{self}#transform not implemented."
+    end
+
+    def load
+      raise NotImplementedError.new "#{self}#load not implemented."
+    end
+
+    def source_count
+      raise NotImplementedError.new "#{self}#source_count not implemented."
+    end
+
+    def destination_count
+      raise NotImplementedError.new "#{self}#destination_count not implemented."
+    end
+
+    def verify logger = Rails.logger
+      if source_count != destination_count
+        logger.error "#{self}: counts don't match"
+      end
+
+      logger.info "#{self}: source count        #{source_count}"
+      logger.info "#{self}: destination count   #{destination_count}"
+    end
+
+    def batch
+      false
+    end
+
+    private
+
+    def not_batch_perform logger = Rails.logger
+      with_profiling logger do
+        load(transform(extract))
+        verify logger
+      end
+    end
+
+    def batch_perform logger = Rails.logger
+      with_profiling logger do
+        extract do |batch|
+          load(transform batch)
+        end
+        verify logger
+      end
+    end
+
+    def with_profiling logger, &block
+      start_time = Time.now
+      logger.info "#{self} starting at #{start_time}"
+
+      yield block
+
+      logger.info "#{self} finished at #{Time.now}. Took #{(Time.now - start_time).round} seconds"
+    end
+  end
+end

data/lib/petl/version.rb

@@ -0,0 +1,3 @@
+module Petl
+  VERSION = "0.0.1"
+end

data/petl.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'petl/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "petl"
+  spec.version       = Petl::VERSION
+  spec.authors       = ["Alexander Tamoykin", "Chris Rosario"]
+  spec.email         = ["at@zestfinance.com", "car@zestfinance.com"]
+  spec.description   = %q{Pretty good ETL framework}
+  spec.summary       = %q{Pretty good ETL framework}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

data/spec/lib/petl/etl_spec.rb

@@ -0,0 +1,116 @@
+require 'spec_helper'
+
+describe "Petl::ETL" do
+
+  module TestETL
+    extend Petl::ETL
+    extend self
+
+    def source_count
+      1
+    end
+
+    def destination_count
+      1
+    end
+
+    def extract
+      []
+    end
+
+    def transform data
+      data
+    end
+
+    def load data
+    end
+  end
+
+  describe TestETL do
+
+    describe '#verify' do
+      before do
+        @logger = double
+      end
+
+      context "counts don't match" do
+        before do
+          TestETL.stub(:source_count).and_return(1)
+          TestETL.stub(:destination_count).and_return(2)
+        end
+
+        it "logs the error" do
+          @logger.should_receive(:error).with(/counts don't match/i).once
+          @logger.should_receive(:info).twice
+          described_class.verify @logger
+        end
+      end
+
+      it "logs the source count" do
+        @logger.should_receive(:info).with(/source count\s*1/).once
+        @logger.should_receive(:info).once
+        described_class.verify @logger
+      end
+
+      it "logs the destination count" do
+        @logger.should_receive(:info).once
+        @logger.should_receive(:info).with(/destination count\s*1/).once
+        described_class.verify @logger
+      end
+    end
+
+    context '#perform' do
+      before do
+        @logger = double
+      end
+
+      context "with batching" do
+        module BatchETL
+          extend Petl::ETL
+          extend self
+
+          def batch
+            true
+          end
+
+          def source_count
+            1
+          end
+
+          def destination_count
+            1
+          end
+
+          def extract &block
+            yield []
+          end
+
+          def transform data
+            data
+          end
+
+          def load data
+          end
+        end
+
+        before do
+          @logger.stub(:info)
+        end
+
+        it 'yields to extract' do
+          BatchETL.should_receive(:extract).and_yield []
+          BatchETL.perform @logger
+        end
+      end
+
+      it 'logs the start time, end time and time elapsed' do
+        @logger.should_receive(:info).with(/starting/i).once
+        @logger.should_receive(:info).once
+        @logger.should_receive(:info).once
+        @logger.should_receive(:info).with(/finished.*took/i).once
+
+        described_class.perform @logger
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+$:.unshift File.expand_path '../../lib', __FILE__
+Bundler.require
+require 'petl'
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: petl
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Alexander Tamoykin
+- Chris Rosario
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Pretty good ETL framework
+email:
+- at@zestfinance.com
+- car@zestfinance.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/petl.rb
+- lib/petl/etl.rb
+- lib/petl/version.rb
+- petl.gemspec
+- spec/lib/petl/etl_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.2
+signing_key: 
+specification_version: 4
+summary: Pretty good ETL framework
+test_files:
+- spec/lib/petl/etl_spec.rb
+- spec/spec_helper.rb