reindeer-etl 0.1.0

data/README.md

@@ -0,0 +1,112 @@
+[![Gem Version](https://badge.fury.io/rb/reindeer-etl.svg)](http://badge.fury.io/rb/reindeer-etl)
+[![License](https://img.shields.io/badge/license-GPL-blue.svg)](License.md)
+![Downloads](https://img.shields.io/gem/dt/reindeer-etl.svg)
+
+# ReindeerETL
+
+Sources, Transforms and Destinations for the [Kiba](https://github.com/thbar/kiba) ETL gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'reindeer-etl'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install reindeer-etl
+
+## Usage
+
+### Simple Example
+
+If you have a csv file that looks like this:
+
+input.csv
+
+```
+a,b,c
+1,2,3
+4,5,6
+```
+
+In your kiba ETL script you can now do this:
+
+
+```ruby
+require 'reindeer-etl'
+
+only_fields = ['a', 'b']
+
+# Open a csv file
+source(ReindeerETL::Sources::CSVSource, './input.csv', 
+    require: only_fields, only: only_fields)
+
+# rename a column
+transform(ReindeerETL::Transforms::RenameFields, {'b'=>'c'}
+
+# Recode all 777 values as 888
+transform ReindeerETL::Transforms::Recode, cols: ['a'],
+    codes: {'777'=>'888'}, ignore_all: true
+
+# Write the file to disk
+destination ReindeerETL::Destinations::CSVDest, './output.csv'
+```
+
+### Joining data from multiple sources
+
+A slightly more complex example is where you have data in multiple CSV files and 
+you would like to join that information into a single ETL job.
+
+input1.csv
+```
+a,b,c
+1,2,3
+4,5,6
+```
+
+input2.csv
+```
+a,e,f
+1,7,8
+4,10,11
+```
+
+reindeer.etl
+```ruby
+# Open a csv file
+source(ReindeerETL::Sources::MultiSource, ['./input1.csv', './input2.csv'], key: 'a')
+
+# Write the file to disk
+destination ReindeerETL::Destinations::CSVDest, './output.csv'
+
+```
+
+output.csv
+```
+a,b,c,e,f
+1,2,3,7,8
+4,5,6,10,11
+```
+
+### More examples coming soon
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/reindeer-etl/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+    t.libs << 'lib/reindeer-etl'
+    t.libs << 'test'
+    t.test_files = FileList[
+        "test/*_test.rb",
+        "test/lib/*_test.rb"
+    ]
+    t.verbose = true
+end
+
+task default: :test

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "reindeer-etl"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require "pry"
+Pry.config.color = true
+Pry.start
+
+#require "irb"
+#IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/etl/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

data/lib/reindeer-etl.rb

@@ -0,0 +1,31 @@
+module ReindeerETL
+  # Your code goes here...
+
+  module Sources
+  end
+
+  module Transforms
+  end
+
+  module Destinations
+  end
+
+end
+
+require 'rest-client'
+require 'pp'
+
+require 'reindeer-etl/version'
+require 'reindeer-etl/errors'
+
+require 'reindeer-etl/transforms/simple_transforms'
+require 'reindeer-etl/transforms/recode'
+require 'reindeer-etl/transforms/rename_fields'
+require 'reindeer-etl/transforms/response_status'
+
+require 'reindeer-etl/sources/base_source'
+require 'reindeer-etl/sources/csv_source'
+require 'reindeer-etl/sources/multi_source'
+
+require 'reindeer-etl/destinations/csv_dest'
+require 'reindeer-etl/destinations/lime_survey_curl'

data/lib/reindeer-etl/destinations/csv_dest.rb

@@ -0,0 +1,21 @@
+require 'csv'
+
+module ReindeerETL::Destinations
+    class CSVDest
+        def initialize(output_file, delimiter=',')
+          @csv = CSV.open(output_file, 'w', {col_sep: delimiter})
+        end
+        
+        def write(row)
+          unless @headers_written
+            @headers_written = true
+            @csv << row.keys
+          end
+          @csv << row.values
+        end
+        
+        def close
+          @csv.close
+        end
+    end
+end

data/lib/reindeer-etl/errors.rb

@@ -0,0 +1,3 @@
+module ReindeerETL::Errors
+    class RecordInvalid < StandardError; end
+end

data/lib/reindeer-etl/sources/base_source.rb

@@ -0,0 +1,13 @@
+require 'set'
+
+module ReindeerETL::Sources
+    class BaseSource
+        include ReindeerETL::Transforms::SimpleTransforms
+        
+        def initialize path, opts={}
+            @path = path
+            st_initialize(opts)
+        end
+       
+    end
+end

data/lib/reindeer-etl/sources/csv_source.rb

@@ -0,0 +1,25 @@
+require 'csv'
+
+module ReindeerETL::Sources
+    class CSVSource < BaseSource
+        def initialize path, opts = {}
+            super
+            @csv_opts = {headers: true, col_sep: ','}.merge(opts)
+        end
+            
+        def each
+            first_run = true
+            CSV.foreach(@path, @csv_opts) do |row|
+                if first_run
+                    first_run = false
+                    if row.headers.count != row.headers.uniq.count
+                        raise ReindeerETL::Errors::RecordInvalid.new('Duplicate columns')
+                    end
+                end
+                row = row.to_hash
+                simple_transforms(row)
+                yield(row)
+            end
+        end
+    end
+end

data/lib/reindeer-etl/sources/multi_source.rb

@@ -0,0 +1,51 @@
+require 'set'
+
+module ReindeerETL::Sources
+
+    class MultiSource
+        def initialize key, paths, opts={}
+            @klass = opts[:klass] || ReindeerETL::Sources::CSVSource
+            @key = key
+            @sources = paths.map{|path|
+                @klass.new path
+            }
+        end
+        
+        def each
+            rows = []
+            all_keys = Set.new
+            @sources.each_with_index do |source, source_idx|
+                first_row = false
+                idx = 0
+                source.each do |row|
+                    unless first_row
+                        first_row = true
+                        all_keys += row.keys
+                        unless row.keys.include? @key
+                            raise ReindeerETL::Errors::RecordInvalid.new("Path#1 missing key: #{@key}")
+                        end
+                    end
+                        
+                    if source_idx == 0
+                        # first source?
+                        rows.push row
+                    else
+                        rindex = rows.index{|arow|arow[@key] == row[@key]}
+                        begin
+                            rows[rindex] = rows[rindex].merge(row)
+                        rescue TypeError
+                            raise ReindeerETL::Errors::RecordInvalid.new("Unable to Join source##{source_idx} - row##{idx}")
+                        end
+                    end
+                    idx += 1
+                end
+            end
+            
+            rows.each do |row|
+                (all_keys - row.keys).each{|k|row[k] = nil}
+                yield row
+            end
+        end
+    end
+
+end

data/lib/reindeer-etl/transforms/recode.rb

@@ -0,0 +1,61 @@
+module ReindeerETL::Transforms
+    class Recode
+        attr_accessor :cols
+
+        def initialize opts={}
+            @cols = opts[:cols]
+            @except = (opts[:except] || []).to_set
+            @codes = opts[:codes] || {}
+            @ignore_vals = (opts[:ignore] || [])
+            @ignore_all = (opts[:ignore_all] || false)
+            @error_on_unknown = !@ignore_all
+            
+            if @cols.nil? && opts.keys.include?(:cols)
+                raise ArgumentError.new(':cols array is empty')
+            end
+            @cols = @cols.to_set unless @cols.nil?
+            raise ArgumentError.new(':codes hash is empty') if @codes.empty?
+            @acceptable_keys = (@codes.keys + @ignore_vals).to_set
+            @counter = 0
+        end
+        
+        def process row
+            @cols ||= row.keys.to_set - @except
+             
+            # Raise error unless all columns are present
+            rset = row.keys.to_set
+            unless @cols.subset?(rset)
+                m_cols = @cols - rset
+                raise ReindeerETL::Errors::RecordInvalid.new("Missing columns: #{m_cols.to_a}")
+            end
+            
+            # Run recode
+            @cols.each do |col| 
+                val = row[col]
+                _validate_val(val)
+                _update_row(row, col, val)
+            end
+            
+            @counter += 1
+            row
+        end
+        
+        private
+
+        def _validate_val(val)
+            if @error_on_unkown && !@any_val.include?(val)
+                # Raise error if we don't recognize this value
+                raise ReindeerETL::Errors::RecordInvalid.new("Bad value: #{val}")
+            end
+        end
+        
+        def _update_row(row, col, val)
+            if @acceptable_keys.include? val
+                row[col] = @codes[val] if @codes.has_key?(val)
+            elsif @error_on_unknown
+                raise ReindeerETL::Errors::RecordInvalid.new("Invalid value in recode: row# #{@counter} {#{col}:#{val}}")
+            end
+        end
+
+    end
+end

data/lib/reindeer-etl/transforms/rename_fields.rb

@@ -0,0 +1,26 @@
+module ReindeerETL::Transforms
+    ##
+    # A simple transform that renames columns
+    class RenameFields
+        def initialize cols
+            @cols = cols
+        end
+        
+        def process(row)
+            counter=0
+            row_keys = row.keys.to_set
+            req_keys = @cols.keys.to_set
+
+            # raise an error unless all of req is in row
+            unless req_keys.subset?(row_keys)
+                raise ReindeerETL::Errors::RecordInvalid.new('Missing columns in rename')
+            end
+            @cols.each do |k, v|
+                next if k == v
+                row[v] = row.delete(k)
+            end
+            row
+        end
+    end
+
+end

data/lib/reindeer-etl/transforms/response_status.rb

@@ -0,0 +1,46 @@
+module ReindeerETL::Transforms
+    ##
+    # Swap out old error codes with REP_CODE, add new columns with error codes
+    class ResponseStatus
+        ERROR_CODES = %w{222 444 555 777 888 998 999}
+        NO_CODE = '111'
+        
+        # What to replace a code with if one is found
+        REP_CODE = '{question_not_shown}'
+        REP_COL_PREFIX = 'responseStatus_'
+        
+        def initialize opts={}
+            @except_cols = (opts[:except] || []).to_set
+        end
+        
+        def process(row)
+            row_keys = row.keys.to_set
+            unless @except_cols.subset? row_keys
+                x_cols = (@except_cols - row_keys).to_a
+                raise ReindeerETL::Errors::RecordInvalid.new("Missing except keys: #{x_cols}")
+            end
+            oldrow = row.dup
+            (row_keys - @except_cols).each do |k|
+                new_col = "#{REP_COL_PREFIX}#{k.gsub('_','')}" 
+                if row_keys.include? new_col 
+                    raise ReindeerETL::Errors::RecordInvalid.new("Column #{new_col} already exists")
+                end
+                val = row[k]
+                if _has_code?(val)
+                    row[k] = REP_CODE
+                    ecode = val.to_s
+                else
+                    ecode = NO_CODE
+                end
+                row[new_col] = "E#{ecode}E"
+            end
+
+            row
+        end
+        
+        private
+        def _has_code? val
+            ERROR_CODES.include?(val.to_s)
+        end
+    end
+end