ingestion_engine 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bb45d11b839f5c50bf93f06d0cdb531896170f85
+  data.tar.gz: 739c9202ed5b8016ae2795545c2c84ac3f53a747
+SHA512:
+  metadata.gz: c521895788ea57ed62c767f231775a8119682c18ffdb9254df809b1d7c6478082754798886b4d9eef001080b5cdccfc3283265918b7b960352d29cf28a125657
+  data.tar.gz: b06c97b99a0452335dd69e424dc37a2b96777a1775193ab9996affcd1ef0e2a63be761799d51a1aaa484cb0ed6138e8aa12c70c691563c7e50829ebd88430952

data/.gitignore

@@ -0,0 +1,2 @@
+.bundle
+vendor/ruby

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+gem 'rspec'
+gem 'sqlite3'
+gem 'activerecord', require: 'active_record'
+gem 'pry-byebug'
+gem 'awesome_print'
+gem 'database_cleaner'

data/Gemfile.lock

@@ -0,0 +1,65 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (4.2.1)
+      activesupport (= 4.2.1)
+      builder (~> 3.1)
+    activerecord (4.2.1)
+      activemodel (= 4.2.1)
+      activesupport (= 4.2.1)
+      arel (~> 6.0)
+    activesupport (4.2.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    arel (6.0.0)
+    awesome_print (1.6.1)
+    builder (3.2.2)
+    byebug (4.0.4)
+      columnize (= 0.9.0)
+    coderay (1.1.0)
+    columnize (0.9.0)
+    database_cleaner (1.4.1)
+    diff-lcs (1.2.5)
+    i18n (0.7.0)
+    json (1.8.2)
+    method_source (0.8.2)
+    minitest (5.5.1)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-byebug (3.1.0)
+      byebug (~> 4.0)
+      pry (~> 0.10)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.2)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
+    slop (3.6.0)
+    sqlite3 (1.3.10)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  awesome_print
+  database_cleaner
+  pry-byebug
+  rspec
+  sqlite3

data/bin/byebug

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'byebug' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('byebug', 'byebug')

data/bin/coderay

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'coderay' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('coderay', 'coderay')

data/bin/htmldiff

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'htmldiff' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('diff-lcs', 'htmldiff')

data/bin/ldiff

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'ldiff' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('diff-lcs', 'ldiff')

data/bin/pry

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'pry' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('pry', 'pry')

data/bin/rspec

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('rspec-core', 'rspec')

data/ingestion_engine.gemspec

@@ -0,0 +1,9 @@
+Gem::Specification.new do |spec|
+  spec.name = "ingestion_engine"
+  spec.version = '0.0.1'
+  spec.authors = ['Brandon Mathis']
+  spec.email = ['BeMathis@gmail.com']
+  spec.summary = 'Ingest and massage csv data'
+  spec.license = 'MIT'
+  spec.files = `git ls-files`.split("\n")
+end

data/invalid.csv

@@ -0,0 +1,3 @@
+username,email,first_name,last_name,errors
+BeMathis, bemathis<at>gmail.com, brandon, mathis, Email is invalid
+Carrion, carrion@gmail, carrion, fost, Email is invalid

data/lib/ingestion_engine/base.rb

@@ -0,0 +1,43 @@
+module IngestionEngine
+  class Base
+    attr_reader :rows, :klass, :headers
+
+    def initialize(klass, file)
+      @klass = klass
+      @rows = CSV.read(file)
+      @headers = @rows.shift.map!(&:strip)
+    end
+
+    def ingest
+      entities = []
+      init entities
+      dump_invalid entities
+      save entities
+    end
+
+    private
+
+    def init(entities)
+      rows.each do |row|
+        obj = klass.new
+        headers.each_with_index do |header, index|
+          obj.send("#{header}=", row[index].strip)
+        end
+        entities << obj
+      end
+    end
+
+    def dump_invalid(entities)
+      invalids = entities.select(&:invalid?).each do |invalid_entity|
+        entities.delete(invalid_entity)
+      end
+      IngestionEngine::Reporter.new(invalids, headers).log do |invalid|
+        invalid.errors.full_messages.to_sentence
+      end
+    end
+
+    def save(entities)
+      entities.each { |entity| entity.save }
+    end
+  end
+end

data/lib/ingestion_engine/reporter.rb

@@ -0,0 +1,21 @@
+module IngestionEngine
+  class Reporter
+    attr_reader :entities, :headers
+
+    def initialize(entities, headers)
+      @entities = entities
+      @headers = headers
+    end
+
+    def log(&block)
+      File.open('invalid.csv', 'w') do |f|
+        f.puts (headers + ['errors']).join(',')
+        entities.each do |entity|
+          msg = yield(entity)
+          values = headers.map{ |header| entity.send(header) }
+          f.puts (values + [msg.gsub(/,/, '')]).join(', ')
+        end
+      end
+    end
+  end
+end

data/lib/ingestion_engine.rb

@@ -0,0 +1,6 @@
+require_relative 'ingestion_engine/base'
+require_relative 'ingestion_engine/reporter'
+require 'csv'
+
+module IngestionEngine
+end

data/spec/lib/ingestion_engine/base_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+class User < ActiveRecord::Base
+  validates_presence_of :first_name
+  validates :email, format: { with: /\A([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})\z/i, on: :create }
+end
+
+describe IngestionEngine::Base do
+  describe '#ingest' do
+    let(:csv) { File.open('spec/sample_csvs/users.csv') }
+    let(:emails) { User.all.map(&:email) }
+    let(:usernames) { User.all.map(&:username) }
+
+    it 'saves the given items' do
+      IngestionEngine::Base.new(User, csv).ingest
+      expect(User.count).to eq 3
+      expect(usernames).to include 'BeMathis'
+      expect(usernames).to include 'Carrion'
+      expect(usernames).to include 'durrhurrdurr'
+      expect(emails).to include 'bemathis@gmail.com'
+      expect(emails).to include 'carrion@gmail.com'
+      expect(emails).to include 'durrhurrdurr@gmail.com'
+    end
+
+    context 'with missing attr that is required' do
+      let(:csv) { File.open('spec/sample_csvs/users_with_missing_first_name.csv') }
+
+      it 'does not save invalid items' do
+        IngestionEngine::Base.new(User, csv).ingest
+        expect(User.count).to eq 2
+        expect(usernames).to include 'durrhurrdurr'
+        expect(usernames).to include 'Carrion'
+        expect(usernames).to_not include 'BeMathis'
+      end
+
+      it 'dumps the bad entry into invalid records csv' do
+        IngestionEngine::Base.new(User, csv).ingest
+        CSV.foreach('invalid.csv', {headers: true}) do |row|
+          expect(row['username']).to eq 'BeMathis'
+        end
+      end
+    end
+
+    context 'with bad formatted email' do
+      let(:csv) { File.open('spec/sample_csvs/users_with_bad_email.csv') }
+
+      it 'dumps the bad entry into invalid records csv' do
+        IngestionEngine::Base.new(User, csv).ingest
+        invalids = CSV.parse(File.open('invalid.csv'), headers: true, header_converters: :symbol)
+        expect(invalids[0][:username]).to eq 'BeMathis'
+        expect(invalids[1][:username]).to eq 'Carrion'
+        expect(invalids[0][:errors].strip).to eq 'Email is invalid'
+        expect(invalids[1][:errors].strip).to eq 'Email is invalid'
+      end
+    end
+  end
+end

data/spec/sample_csvs/users.csv

@@ -0,0 +1,4 @@
+username, email, first_name, last_name
+BeMathis, bemathis@gmail.com, brandon, mathis
+Carrion, carrion@gmail.com, carrion, fost
+durrhurrdurr, durrhurrdurr@gmail.com, durr, hurrdurr

data/spec/sample_csvs/users_with_bad_email.csv

@@ -0,0 +1,4 @@
+username, email, first_name, last_name
+BeMathis, bemathis<at>gmail.com, brandon, mathis
+Carrion, carrion@gmail, carrion, fost
+durrhurrdurr, durrhurrdurr@gmail.com, durr, hurrdurr

data/spec/sample_csvs/users_with_missing_first_name.csv

@@ -0,0 +1,4 @@
+username, email, first_name, last_name
+BeMathis, bemathis@gmail.com, , mathis
+Carrion, carrion@gmail.com, carrion, fost
+durrhurrdurr, durrhurrdurr@gmail.com, durr, hurrdurr

data/spec/spec_helper.rb

@@ -0,0 +1,39 @@
+Bundler.require
+require 'ingestion_engine'
+require 'rspec/core'
+
+
+ActiveRecord::Base.establish_connection(
+  :adapter => "sqlite3",
+  :database  => "/tmp/ingestion_engine.sqlite"
+)
+
+connection = ActiveRecord::Base.connection
+
+RSpec.configure do |config|
+  config.mock_with :rspec
+
+  config.before(:suite) do
+    DatabaseCleaner.strategy = :transaction
+    DatabaseCleaner.clean_with(:truncation)
+  end
+
+  config.around(:each) do |example|
+    DatabaseCleaner.cleaning do
+      example.run
+    end
+  end
+
+  config.before(:all) do
+    connection.create_table :users do |t|
+      t.string :first_name
+      t.string :last_name
+      t.string :email
+      t.string :username
+    end
+  end
+
+  config.after(:all) do
+    connection.drop_table :users
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: ingestion_engine
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Brandon Mathis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-28 00:00:00.000000000 Z
+dependencies: []
+description: 
+email:
+- BeMathis@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- bin/byebug
+- bin/coderay
+- bin/htmldiff
+- bin/ldiff
+- bin/pry
+- bin/rspec
+- ingestion_engine.gemspec
+- invalid.csv
+- lib/ingestion_engine.rb
+- lib/ingestion_engine/base.rb
+- lib/ingestion_engine/reporter.rb
+- spec/lib/ingestion_engine/base_spec.rb
+- spec/sample_csvs/users.csv
+- spec/sample_csvs/users_with_bad_email.csv
+- spec/sample_csvs/users_with_missing_first_name.csv
+- spec/spec_helper.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Ingest and massage csv data
+test_files: []