setl 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5a69608ef7938b0e490055db9bc3cf5d1924f9e0
+  data.tar.gz: 66b1a67765d78496ab1791d65a4c5f699d4b178f
+SHA512:
+  metadata.gz: 5ff6e5d5cce17d475f08d5ad2965dc1c22d75343b4d4f04ceee9be83a15e0957a5b75a3de58f6c9c9ed374dfcb3e3a703eacd4be872549bb20a5b5393fd286f7
+  data.tar.gz: d3123a6e871b3b6ff720ff0fbd271cdbde7b4b14032317c58240a39d4bad09f6cb710a8b0942c800095852a94dd03817627eca39c9991de446386b6ea8231410

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.2.2

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in setl.gemspec
+gemspec
+
+gem 'byebug', require: false

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Leonard Garvey
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+# Setl
+
+A simple framework for writing testable ETL systems. Takes concepts from Kiba but reorganises them to use simple objects, composition and dependency injection. This means your ETL stays simple, reliable and testable.
+
+DSLs are cool, but composable, testable objects are cooler.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'setl'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install setl
+
+## Usage
+
+1. Define an object that is responsible for sourcing the data. Must respond to `#each` and `yield` to the provided block.
+2. Define your transformations. These are simple objects that will receive one of the things provided by the source object.
+3. Provide your transforms to a controller object. Setl provides `Setl::Controller` and you can simply add your transforms into this object. Can actually be any object that responds to `#call`
+4. Define a destination object. Must respond to `#call` and sends output of each transformed row of data to a destination.
+
+```ruby
+source = [{id: 1, name: 'foo'},{id: 2, name: 'bar'}]
+destination = lambda {|d| puts d.inspect }
+
+transform = lambda do |d|
+  d[:name].upcase!
+  d
+end
+
+Setl::ETL.new(source, destination).process(transform)
+  #=> {id: 1, name: 'FOO'}
+  #=> {id: 2, name: 'BAR'}
+```
+
+See the examples folder for some more extensive, and realistic, implementations.
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/setl/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/rake

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('rake', 'rake')

data/bin/rspec

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('rspec-core', 'rspec')

data/lib/setl/controller.rb

@@ -0,0 +1,16 @@
+module Setl
+  class Controller
+    def initialize(*pipeline)
+      @pipeline = pipeline
+    end
+
+    def call(row)
+      pipeline.each { |t| row = t.(row) }
+      row
+    end
+
+    private
+
+    attr_reader :pipeline
+  end
+end

data/lib/setl/etl.rb

@@ -0,0 +1,45 @@
+module Setl
+  class ETL
+    def initialize(source, destination, stop_on_errors: false, error_handler: nil)
+      @source = source
+      @destination = destination
+      @stop_on_errors = stop_on_errors
+      @error_handler = error_handler || DefaultHandler.new(stop_on_errors)
+    end
+
+    def process(transform)
+      source.each do |row|
+        row_copy = row.dup
+
+        begin
+          destination.(transform.(row_copy))
+        rescue StandardError => e
+          error_handler.(row, e)
+        end
+      end
+    end
+
+    private
+
+    attr_reader :source, :destination, :stop_on_errors, :error_handler
+  end
+
+  class ProcessingError < StandardError
+  end
+
+  class DefaultHandler
+    def initialize(reraise)
+      @reraise = reraise
+    end
+
+    def call(row, exception)
+      raise ProcessingError, "Failed to process #{row}" if reraise?
+    end
+
+    private
+
+    def reraise?
+      @reraise
+    end
+  end
+end

data/lib/setl/version.rb

@@ -0,0 +1,3 @@
+module Setl
+  VERSION = "0.0.2"
+end

data/lib/setl.rb

@@ -0,0 +1,3 @@
+require "setl/version"
+require "setl/etl"
+require "setl/controller"

data/setl.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'setl/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "setl"
+  spec.version       = Setl::VERSION
+  spec.authors       = ["Leonard Garvey"]
+  spec.email         = ["lengarvey@gmail.com"]
+  spec.summary       = %q{Simple Extract Transform & Load - setl}
+  spec.description   = %q{Can you setl for a tool that barely provides anything?}
+  spec.homepage      = "https://github.com/lengarvey/setl"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject {|f| f.match /examples/}
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/spec/controller_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+require 'setl/controller'
+
+module Setl
+  RSpec.describe 'controller' do
+    let(:controller) { Controller.new(transform1, transform2) }
+    let(:transform1) { double('Transform1', call: 'output') }
+    let(:transform2) { double('Transform2', call: 'final result') }
+    let(:row) { double('Row') }
+
+    describe 'processing a row of data' do
+      it 'creates a pipeline of transforms' do
+        expect(transform1).to receive(:call).with(row)
+        expect(transform2).to receive(:call).with('output')
+
+        controller.call(row)
+      end
+
+      it 'returns the result of all the transformations' do
+        expect(controller.call(row)).to eq 'final result'
+      end
+    end
+  end
+end

data/spec/error_handling_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+require 'setl/etl'
+
+module Setl
+  RSpec.describe 'error handling' do
+    let(:etl) { ETL.new(source, destination) }
+    let(:source) { [1, 2] }
+    let(:destination) { double('Destination', call: true) }
+    let(:transform) do
+      Class.new do
+        def self.call(row)
+          raise 'nope' if row == 1
+        end
+      end
+    end
+
+    let(:process) { etl.process(transform) }
+
+    context 'by default' do
+      it 'rescues the error' do
+        expect { process }.to_not raise_error
+      end
+
+      it 'processes the next row' do
+        allow(transform).to receive(:call).with(1)
+        expect(transform).to receive(:call).with(2)
+        process
+      end
+    end
+
+    context 'when configured to stop on errors' do
+      let(:etl) { ETL.new(source, destination, stop_on_errors: true) }
+
+      it 'stops processing when a processing error occurs' do
+        expect { process }.to raise_error(ProcessingError, 'Failed to process 1')
+      end
+
+      it 'wraps the original error' do
+        begin
+          process
+        rescue ProcessingError => e
+          expect(e.cause).to be_a RuntimeError
+        end
+      end
+    end
+
+    context 'when provided an error handler' do
+      let(:handler) { double('Error Handler') }
+      let(:etl) { ETL.new(source, destination, error_handler: handler) }
+
+      it 'sends the row and exception to the handler' do
+        expect(handler).to receive(:call).with(1, an_instance_of(RuntimeError))
+
+        process
+      end
+    end
+  end
+end

data/spec/etl_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+require 'setl/etl'
+
+module Setl
+  RSpec.describe 'etl' do
+    let(:etl) { ETL.new(source, destination) }
+
+    describe 'processing a row' do
+      let(:row) { 'hello' }
+      let(:source) { [row] }
+      let(:destination) { double('Destination', call: true) }
+      let(:processed_data) { double('Processed row') }
+      let(:transform) { double('Transform', call: processed_data) }
+
+      before do
+        etl.process(transform)
+      end
+
+      it 'delegates the processing to the transform' do
+        expect(transform).to have_received(:call).with(row)
+      end
+
+      it 'sends the processed row to the destination' do
+        expect(destination).to have_received(:call).with(processed_data)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,96 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: setl
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Leonard Garvey
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Can you setl for a tool that barely provides anything?
+email:
+- lengarvey@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/rake
+- bin/rspec
+- lib/setl.rb
+- lib/setl/controller.rb
+- lib/setl/etl.rb
+- lib/setl/version.rb
+- setl.gemspec
+- spec/controller_spec.rb
+- spec/error_handling_spec.rb
+- spec/etl_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/lengarvey/setl
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Simple Extract Transform & Load - setl
+test_files:
+- spec/controller_spec.rb
+- spec/error_handling_spec.rb
+- spec/etl_spec.rb
+- spec/spec_helper.rb