reindeer-etl 0.1.0

data/lib/reindeer-etl/transforms/simple_transforms.rb

@@ -0,0 +1,33 @@
+module ReindeerETL::Transforms
+    module SimpleTransforms
+        def st_initialize opts={}
+            @only_cols = (opts.delete(:only) || []).to_set
+            @require_cols = (opts.delete(:require) || []).to_set
+        end
+       
+        ## 
+        # Configurable transforms in source
+        def simple_transforms row
+            st_only_cols(row) unless @only_cols.empty?
+            st_require_cols(row) unless @require_cols.empty?
+            row
+        end
+        
+        ##
+        # Filter out everything except these columns
+        def st_only_cols dict
+            (dict.keys.to_set - @only_cols).each{|col|dict.delete(col)}
+            dict
+        end
+        
+        ## 
+        # require these columns
+        def st_require_cols dict
+            dcols = dict.keys.to_set
+            unless @require_cols.subset? dict.keys.to_set
+                missing_cols = (@require_cols - dcols).to_a
+                raise ReindeerETL::Errors::RecordInvalid.new("Missing required columns: #{missing_cols}")
+            end
+        end
+    end
+end

data/lib/reindeer-etl/version.rb

@@ -0,0 +1,3 @@
+module ReindeerETL
+  VERSION = "0.1.0"
+end

data/reindeer-etl.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'reindeer-etl/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "reindeer-etl"
+  spec.version       = ReindeerETL::VERSION
+  spec.authors       = ["William Hatt", "Patrick Chung"]
+  spec.email         = ["hattb@ohsu.edu", "chungp@ohsu.edu"]
+
+  spec.summary       = %q{A simple ETL pipeline for use with project reindeer and LimeSurvey}
+  spec.description   = %q{An ETL pipeline tool for automatic data modifications to LimeSurvey}
+  spec.homepage      = "https://github.com/OHSU-FM/reindeer-etl"
+  spec.license       = "GPL"
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  #if spec.respond_to?(:metadata)
+  #  spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  #else
+  #  raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  #end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.test_files    = Dir.glob("test/**/*.rb")
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rest-client", "~> 1.8"
+  spec.add_development_dependency "minitest", "~> 5.7"
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "pry", "~> 0.10.1"
+
+end

data/test/lib/sources_base_source_test.rb

@@ -0,0 +1,17 @@
+require 'minitest_helper'
+
+describe ReindeerETL::Sources::BaseSource do
+    it 'must remove columns when asked' do
+        transform = ReindeerETL::Sources::BaseSource.new '', :only=>[:aa, :bb]
+        row = {aa: 1, bb: 2, cc: 3}
+        transform.simple_transforms row
+        row.keys.must_equal [:aa, :bb]
+    end
+
+    it 'must raise error if required fields missing' do
+        transform = ReindeerETL::Sources::BaseSource.new '', :require=>[:aa]
+        row = {cc: 3}
+        assert_raises(ReindeerETL::Errors::RecordInvalid){transform.simple_transforms row}
+    end
+
+end

data/test/lib/sources_csv_source_test.rb

@@ -0,0 +1,24 @@
+require 'minitest_helper'
+
+def loads_as_hash path, opts={}
+    source = ReindeerETL::Sources::CSVSource.new(path, opts)
+    counter = 0
+    source.each do |row|
+       row.keys.must_equal %w{a b c d e f g h i}
+       counter += 1
+       counter.must_equal row['a'].to_i
+    end
+    counter.must_equal 13 
+end
+
+describe ReindeerETL::Sources::CSVSource do
+    it 'must yield each line of a csv file as a hash' do
+        path = "#{$dir}/fixtures/comma_delimited.csv" 
+        loads_as_hash(path, :col_sep=>',')
+    end
+
+    it 'must yield each line of a tabbed csv file as a hash' do
+        path = "#{$dir}/fixtures/tab_delimited.csv" 
+        loads_as_hash(path, :col_sep=>"\t")
+    end
+end

data/test/lib/sources_multi_source_test.rb

@@ -0,0 +1,20 @@
+require 'minitest_helper'
+
+describe ReindeerETL::Sources::MultiSource do
+    it 'must merge data from multiple sources' do
+        klass = ReindeerETL::Sources::CSVSource
+        path1 = "#{$dir}/fixtures/comma_delimited.csv" 
+        path2 = "#{$dir}/fixtures/comma_delimited_join_on_a.csv" 
+        source = ReindeerETL::Sources::MultiSource.new('a', [path1, path2], :klass=>klass)
+        rows = []
+        keys = %w[a b c d e f g h i j k l]
+        source.each do |row|
+            rows.push row
+            row.keys.must_equal keys
+        end
+        rows.count.must_equal 13
+    end
+
+end
+
+

data/test/lib/transforms_recode_test.rb

@@ -0,0 +1,34 @@
+require 'minitest_helper'
+
+describe ReindeerETL::Transforms::Recode do
+
+    it 'must recode values' do
+        row = {g: :a}
+        transform = ReindeerETL::Transforms::Recode.new :codes=>{:a=>:b}, :cols=>[:g]
+        transform.process row
+        row[:g].must_equal :b
+    end
+    
+    it 'must raise an error if the column does not exist' do
+        row = {hhh: :a}
+        transform = ReindeerETL::Transforms::Recode.new codes: {a: :b}, cols: [:g] 
+        
+        assert_raises(ReindeerETL::Errors::RecordInvalid){ 
+            transform.process row
+        }
+    end
+    
+    it 'must raise an error if parameters are missing' do
+        assert_raises(ArgumentError){ 
+            ReindeerETL::Transforms::Recode.new codes: {}
+        }
+        assert_raises(ArgumentError){ 
+            ReindeerETL::Transforms::Recode.new cols: {}
+        }
+        ReindeerETL::Transforms::Recode.new codes: {a: :b}, cols: [:hhh]
+    end
+
+
+end
+
+

data/test/lib/transforms_response_status_test.rb

@@ -0,0 +1,22 @@
+require 'minitest_helper'
+
+describe ReindeerETL::Transforms::RenameFields do
+    before do
+        cols = {:aaa=>:bbb, :ccc=>:ddd}
+        @transform = ReindeerETL::Transforms::RenameFields.new cols
+    end
+
+    it 'must raise an error when the column is not in the row' do
+        row = {}
+        assert_raises(ReindeerETL::Errors::RecordInvalid){ @transform.process row }
+    end
+    
+    it 'must rename fields' do
+        row = {:aaa=>555, :ccc=>888}
+        row = @transform.process row
+        row.keys.include?(:aaa).must_equal false
+        row.keys.include?(:bbb).must_equal true
+    end
+
+end
+

data/test/minitest_helper.rb

@@ -0,0 +1,7 @@
+require 'pry'
+require 'minitest/autorun'
+require 'minitest/unit'
+require 'minitest/pride'
+
+require File.expand_path('../../lib/reindeer-etl.rb', __FILE__)
+$dir = File.dirname(File.expand_path(__FILE__))

data/test/reindeer_waterworks_test.rb

@@ -0,0 +1,8 @@
+require 'minitest_helper'
+
+class TestReindeerETL < Minitest::Test
+  def test_that_it_has_a_version_number
+    refute_nil ::ReindeerETL::VERSION
+  end
+
+end

metadata

@@ -0,0 +1,156 @@
+--- !ruby/object:Gem::Specification
+name: reindeer-etl
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- William Hatt
+- Patrick Chung
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-07-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.7'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+description: An ETL pipeline tool for automatic data modifications to LimeSurvey
+email:
+- hattb@ohsu.edu
+- chungp@ohsu.edu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- License.md
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- etl/.gitignore
+- lib/reindeer-etl.rb
+- lib/reindeer-etl/destinations/csv_dest.rb
+- lib/reindeer-etl/errors.rb
+- lib/reindeer-etl/sources/base_source.rb
+- lib/reindeer-etl/sources/csv_source.rb
+- lib/reindeer-etl/sources/multi_source.rb
+- lib/reindeer-etl/transforms/recode.rb
+- lib/reindeer-etl/transforms/rename_fields.rb
+- lib/reindeer-etl/transforms/response_status.rb
+- lib/reindeer-etl/transforms/simple_transforms.rb
+- lib/reindeer-etl/version.rb
+- reindeer-etl.gemspec
+- test/lib/destinations_csv_dest_test.rb
+- test/lib/sources_base_source_test.rb
+- test/lib/sources_csv_source_test.rb
+- test/lib/sources_multi_source_test.rb
+- test/lib/transforms_recode_test.rb
+- test/lib/transforms_rename_fields_test.rb
+- test/lib/transforms_response_status_test.rb
+- test/minitest_helper.rb
+- test/reindeer_waterworks_test.rb
+homepage: https://github.com/OHSU-FM/reindeer-etl
+licenses:
+- GPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: A simple ETL pipeline for use with project reindeer and LimeSurvey
+test_files:
+- test/lib/transforms_recode_test.rb
+- test/lib/transforms_response_status_test.rb
+- test/lib/sources_csv_source_test.rb
+- test/lib/transforms_rename_fields_test.rb
+- test/lib/sources_multi_source_test.rb
+- test/lib/sources_base_source_test.rb
+- test/lib/destinations_csv_dest_test.rb
+- test/reindeer_waterworks_test.rb
+- test/minitest_helper.rb