forklift_etl 1.0.11 → 1.0.12

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MDY0YjJmN2E1M2I0MTBkZTNjZjc4MmIwOTEwMWVjYWQ4ZjNkMGM5Yw==
+    N2E0NGRiYzA0ZGVmYzIyMDY5MDRjMDI3ODI5NGNmMTZkMjJiNzc2OQ==
   data.tar.gz: !binary |-
-    OGQwMzk1NzcwMDA1NDA5ZWE5YzVhMmNjODlhYTNmZDA1MzQzMDAwNQ==
+    ZDI5Y2VjYWRmNWRmZGY2ZWZjZDBjNDk5N2RlNTQ5MzBhMjBlOWU5ZA==
 SHA512:
   metadata.gz: !binary |-
-    NzNlZjM4MDg5ZWQ5NzA2ZGIwMGI2MTUwNTY2NDA5NWZlZGNmZDkyZGJkMTBl
-    YzNhMmIzZWU5MDE4MDFlODRiNDg1MDU1ZDI5MTk5ODRiYzQ3YjkyMDUyMWNm
-    ZDI4OTI5YmU0ZDRkNjZiZjNlOTFjN2I1N2ZlMGY4MTY4ODg1MDE=
+    YjZkZWZhMzk3MjE1NjUzMDg3ODg0ZmZkMDFkMGViMjQ0NGQ3NzRjZTZlNzBl
+    MGJlNmVkZTI2ZjdkZWNhZGUzZjQ5OTcwMGM3ZmVjMWEyOGFkNzE5NGRlMDA3
+    NzI5ZjZmNzI2MDNiZDIzMzY1MzM3MTI1YWZiMDFiYmFmMTJhZWU=
   data.tar.gz: !binary |-
-    YWE0ZjMxMjQzZjY5ZmM2MjBmZTEyMGJmOGY1MzYzZDliOTQ0YWIzZjI5ZDBj
-    YzgxODJiMGQyYThjMDQxZDQ0OTIyNjM4ZjUxZTRjNzZmNjMwMGMxYjUyZDE3
-    MzU3ZmE5MjAzOGNlOGZmZmUwM2I5ZjAyNTEzOWMxZThlOTk2Mjc=
+    NDk3NDU5NDQwZjBjYWMwZmQ1MjYyNjU4ZTI5NmMwMjFiMWY2YjQwYWU3Nzdj
+    MmI1YjM1MzJlYmQ4ZjAwZjUzYTMzZDBiYWI5MWI3MTJhOGRlNDRjOTgwZDY1
+    NmZlYjJkZTcyYzgzMTUzODY5ZDdhNGQ2NDI0YWU1ZGIwNjA2OTQ=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    forklift_etl (1.0.11)
+    forklift_etl (1.0.12)
       activesupport (~> 4.0, >= 4.0.0)
       elasticsearch (~> 1.0, >= 1.0.0)
       lumberjack (~> 1.0, >= 1.0.0)

data/bin/forklift

@@ -15,6 +15,8 @@ def generate
   Dir.mkdir "#{p}/config"
   Dir.mkdir "#{p}/config/connections"
   Dir.mkdir "#{p}/config/connections/mysql"
+  Dir.mkdir "#{p}/config/connections/elasticsearch"
+  Dir.mkdir "#{p}/config/connections/csv"
   Dir.mkdir "#{p}/log"
   Dir.mkdir "#{p}/pid"
   Dir.mkdir "#{p}/template"

data/example/config/connections/csv/csv.yml

@@ -0,0 +1 @@
+:file: /path/to/file.csv

data/lib/forklift/transports/csv.rb

@@ -0,0 +1,70 @@
+require 'csv'
+require 'fileutils'
+
+module Forklift
+  module Connection
+    class Csv < Forklift::Base::Connection
+
+      def initialize(config, forklift)
+        @config = config
+        @forklift = forklift
+      end
+
+      def config
+        @config
+      end
+
+      def forklift
+        @forklift
+      end
+
+      def read(size=1000)
+        data = []
+        CSV.foreach(config[:file], :headers => true, :converters => :all) do |row|
+          data << row.to_hash.symbolize_keys
+          if(data.length == size)
+            if block_given?
+              yield data
+              data = []
+            else
+              return data
+            end
+          end
+        end
+
+        if block_given?
+          yield data
+        else
+          return data
+        end
+      end
+
+      def write(data, append=true)
+        if (append == false)
+          FileUtils.rm(config[:file], {:force => true})
+        end
+
+        if( !File.exists?(config[:file]) )
+          keys = data.first.keys
+          row = {}
+          keys.each do |k|
+            row[k] = k
+          end
+          data = [row] + data
+        end
+
+        CSV.open(config[:file],'a') do |file|
+          data.each do |row|
+            file << row.values
+          end
+        end
+
+      end
+
+      private
+
+      #/private
+
+    end
+  end
+end

data/lib/forklift/version.rb

@@ -1,3 +1,3 @@
 module Forklift
-  VERSION = "1.0.11"
+  VERSION = "1.0.12"
 end

data/readme.md

@@ -6,13 +6,7 @@ Moving heavy databases around. [![Gem Version](https://badge.fury.io/rb/forklift
 
 ## What?
 
-[Forklift](https://github.com/taskrabbit/forklift) is a ruby gem that makes it easy for you to move your data around.  Forklift can be an integral part of your datawarehouse pipeline or a backup too.  Forklift can collect and collapse data from multiple sources or accross a source.  In forklift's first version, it was only a MySQL tool.  Now, you can create [transports]() to deal with the data of your choice.
-
-## What does TaskRabbit use this for?
-
-At TaskRabbit, the website you see at [www.taskrabbit.com](https://www.taskrabbit.com) is actually made up of many [smaller rails applications](http://en.wikipedia.org/wiki/Service-oriented_architecture).  When analyzing our site, we need to collect all of this data into one place so we can easily join across it.
-
-We replicate all of our databases into one server in our office, and then use Forklift to extract the data we want into a common place.  This gives us the option to both look at live data and to have a more accessible transformed set which we create on a rolling basis. Our "Forklift Loop" also git-pulls to check for any new transformations before each run.
+[Forklift](https://github.com/taskrabbit/forklift) is a ruby gem that makes it easy for you to move your data around.  Forklift can be an integral part of your datawarehouse pipeline or a backup tool.  Forklift can collect and collapse data from multiple sources or across a single source.  In forklift's first version, it was only a MySQL tool but now, you can create transports to deal with the data of your choice.
 
 ## Set up
 
@@ -42,6 +36,8 @@ Forklift expects your project to be arranged like:
 |       ├── (DB).yml
 |   ├── elasticsearch/
 |       ├── (DB).yml
+|   ├── csv/
+|       ├── (file).yml
 ├── log/
 ├── pid/
 ├── template/
@@ -53,11 +49,13 @@ Forklift expects your project to be arranged like:
 ```
 
 To enable a foklift connection, all you need to do is place the yml config file for it within `/config/connections/(type)/(name).yml`
-File you place withing `/patterns/` or `connections/(type)/` will be loaded automatically.
+Files you place within `/patterns/` or `connections/(type)/` will be loaded automatically.
+
+## Examples 
 
-## Example plans
+### Example Project
 
-Visit the [`/example`]() directory to see a whole forklift project.
+Visit the [`/example`](https://github.com/taskrabbit/forklift/tree/master/example) directory to see a whole forklift project.
 
 ### Simple extract and load (no transformations)
 
@@ -145,7 +143,7 @@ plan.do! do
 end
 ```
 
-### Forklift Emails
+## Forklift Emails
 
 #### Setup
 Put this at the end of your plan inside the `do!` block.
@@ -239,7 +237,7 @@ end
 
 ### Steps
 
-You can optionally devide up your forklift plan into steps:
+You can optionally divide up your forklift plan into steps:
 ```ruby
 plan = Forklift::Plan.new
 plan.do! do
@@ -265,15 +263,15 @@ plan.do! do
 end  
 ```
 
-When you use steps, you can run your whole plan, or just part if it with command line arguments.  For example, `forklift plan.rb "Elasticsearch Import"` would just run that one portion of the plan.  Note that any parts of your plan not within a step will be run each time. 
+When you use steps, you can run your whole plan, or just part if it with command line arguments.  For example, `forklift plan.rb "Elasticsearch Import"` would just run that single portion of the plan.  Note that any parts of your plan not within a step will be run each time. 
 
 ## Transports
 
-Transports are how you interact with your data.  Every transport defines `read` and `write` methods which handle arrays of data objects (and helper methods required).  
+Transports are how you interact with your data.  Every transport defines `read` and `write` methods which handle arrays of data objects (and the helper methods required).  
 
 Each transport should have a config file in `./config/connections/#{transport}/`. It will be loaded at boot.
 
-Transports optinally define a methods which are a shortcut to copy data within a transport, like the mysql `pipe` methods (eg `insert into #{to_db}.#{to_table}; select * from #{from_db}.#{from_table}` for MySQL). A trasport may also define other helpers like how to create a MySQL dump).  These should be defined in `/patterns/#{type}.rb` within the `Forklift::Patterns::#{type}` namespace.
+Transports optionally define helper methods which are a shortcut to copy data *within* a transport, like the mysql `pipe` methods (i.e.: `insert into #{to_db}.#{to_table}; select * from #{from_db}.#{from_table})`. A transport may also define other helpers (like how to create a MySQL dump).  These should be defined in `/patterns/#{type}.rb` within the `Forklift::Patterns::#{type}` namespace.
 
 ### Creating your own transport
 
@@ -375,6 +373,13 @@ end
 
 - delete_index(index)
 
+### Csv
+
+#### Forklift methods
+
+- read(size)
+- write(data, append=true)
+
 ## Transformations
 
 Forklift allows you to create both Ruby transformations and script transformations.
@@ -407,4 +412,4 @@ end
 - If testing locally, mailcatcher (https://github.com/sj26/mailcatcher) is a helpful gem to test your email sending
 
 ## Contributing and Testing
-To run this test suite, you will need access to both a mysql and elasticsearch database. Test configurations are saved in `/spec/config/connections`.
+To run this test suite, you will need access to both a mysql and elasticsearch database. Test configurations are saved in `/spec/config/connections`.

data/spec/config/connections/csv/forklift_test_destination.yml

@@ -0,0 +1 @@
+:file: /tmp/destination.csv

data/spec/config/connections/csv/forklift_test_source.yml

@@ -0,0 +1 @@
+:file: /tmp/source.csv

data/spec/integration/csv_spec.rb

@@ -0,0 +1,84 @@
+require 'spec_helper'
+require 'csv'
+
+describe 'csv' do  
+
+  after(:each) do
+    SpecSeeds.setup_csv
+  end
+
+  it "can read data (simple)" do
+    plan = SpecPlan.new
+    @rows = []
+
+    plan.do! {
+      source = plan.connections[:csv][:forklift_test_source]
+      source.read {|data| 
+        @rows = (@rows + data)
+      }
+    }
+
+    expect(@rows.length).to eql 5
+    expect(@rows.first[:vendor_id]).to eql 1
+    expect(@rows.last[:vendor_id]).to eql 5
+  end
+
+  it "can read partial data" do
+    plan = SpecPlan.new
+    @rows = []
+
+    plan.do! {
+      source = plan.connections[:csv][:forklift_test_source]
+      @rows = source.read(3)
+    }
+
+    expect(@rows.length).to eql 3
+    expect(@rows.first[:vendor_id]).to eql 1
+    expect(@rows.last[:vendor_id]).to eql 3
+  end
+
+  it "can write data (simple)" do
+    plan = SpecPlan.new
+    data = [
+      {:thing => 1, :when => Time.now},
+      {:thing => 2, :when => Time.now},
+    ]
+
+    plan.do! {
+      destination = plan.connections[:csv][:forklift_test_destination]
+      destination.write(data)
+    }
+
+    @rows = SpecClient.csv('/tmp/destination.csv')
+    expect(@rows.length).to eql 2
+    expect(@rows.first[:thing]).to eql 1
+    expect(@rows.last[:thing]).to eql 2
+  end
+
+  it "can append data" do
+    plan = SpecPlan.new
+
+    plan.do! {
+      destination = plan.connections[:csv][:forklift_test_destination]
+
+      data = [
+        {:thing => 1, :when => Time.now},
+        {:thing => 2, :when => Time.now},
+      ]
+
+      destination.write(data)
+
+      data = [
+        {:thing => 3, :when => Time.now},
+      ]
+
+      destination.write(data)
+    }
+
+    @rows = SpecClient.csv('/tmp/destination.csv')
+    expect(@rows.length).to eql 3
+    expect(@rows.first[:thing]).to eql 1
+    expect(@rows.last[:thing]).to eql 3
+  end
+
+end

data/spec/spec_helper.rb

@@ -25,6 +25,7 @@ RSpec.configure do |config|
     FileUtils.rmdir(piddir) if File.exists?(piddir)
     SpecSeeds.setup_mysql
     SpecSeeds.setup_elasticsearch
+    SpecSeeds.setup_csv
   end
 
 end

data/spec/support/dumps/csv/source.csv

@@ -0,0 +1,6 @@
+vendor_id,vendor_name,created_at,updated_at
+1,Evan's Hats,2000-01-01 00:00:01,2000-01-01 00:00:01
+2,Aaron's Scarves,2000-01-01 00:00:02,2000-01-01 00:00:02
+3,Pablo's Shirts,2000-01-01 00:00:03,2000-01-01 00:00:03
+4,Kevin's Headies,2000-01-01 00:00:04,2000-01-01 00:00:04
+5,Brian's Boots,2000-01-01 00:00:05,2000-01-01 00:00:05

data/spec/support/spec_client.rb

@@ -27,4 +27,8 @@ class SpecClient
     ::Elasticsearch::Client.new(config)
   end
 
+  def self.csv(file)
+    CSV.read(file, :headers => true, :converters => :all).map {|r| r = r.to_hash.symbolize_keys }
+  end
+
 end

data/spec/support/spec_seeds.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'fileutils'
 
 class SpecSeeds
 
@@ -66,4 +67,13 @@ class SpecSeeds
     end
   end
 
+  def self.setup_csv
+    seed        = File.join(File.dirname(__FILE__), '..', 'support', 'dumps', 'csv', "source.csv")
+    source      = '/tmp/source.csv'
+    destination = '/tmp/destination.csv'
+    FileUtils.rm(source, {:force => true})
+    FileUtils.rm(destination, {:force => true})
+    FileUtils.copy(seed, source)
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forklift_etl
 version: !ruby/object:Gem::Version
-  version: 1.0.11
+  version: 1.0.12
 platform: ruby
 authors:
 - Evan Tahler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-07 00:00:00.000000000 Z
+date: 2014-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -127,6 +127,7 @@ files:
 - bin/forklift
 - example/Gemfile
 - example/Gemfile.lock
+- example/config/connections/csv/csv.yml
 - example/config/connections/elasticsearch/source.yml
 - example/config/connections/mysql/destination.yml
 - example/config/connections/mysql/source.yml
@@ -147,10 +148,13 @@ files:
 - lib/forklift/patterns/elasticsearch_patterns.rb
 - lib/forklift/patterns/mysql_patterns.rb
 - lib/forklift/plan.rb
+- lib/forklift/transports/csv.rb
 - lib/forklift/transports/elasticsearch.rb
 - lib/forklift/transports/mysql.rb
 - lib/forklift/version.rb
 - readme.md
+- spec/config/connections/csv/forklift_test_destination.yml
+- spec/config/connections/csv/forklift_test_source.yml
 - spec/config/connections/elasticsearch/forklift_test.yml
 - spec/config/connections/mysql/forklift_test_destination.yml
 - spec/config/connections/mysql/forklift_test_source_a.yml
@@ -158,12 +162,14 @@ files:
 - spec/config/connections/mysql/forklift_test_working.yml
 - spec/config/email.yml
 - spec/integration/basic_spec.rb
+- spec/integration/csv_spec.rb
 - spec/integration/elasticsearch_patterns_spec.rb
 - spec/integration/elasticsearch_spec.rb
 - spec/integration/multi_transport_spec.rb
 - spec/integration/mysql_patterns_spec.rb
 - spec/integration/mysql_spec.rb
 - spec/spec_helper.rb
+- spec/support/dumps/csv/source.csv
 - spec/support/dumps/elasticsearch/forklift_test.json
 - spec/support/dumps/mysql/forklift_test_source_a.sql
 - spec/support/dumps/mysql/forklift_test_source_b.sql
@@ -205,6 +211,8 @@ signing_key:
 specification_version: 4
 summary: ! 'Forklift: Moving big databases around. A ruby ETL tool.'
 test_files:
+- spec/config/connections/csv/forklift_test_destination.yml
+- spec/config/connections/csv/forklift_test_source.yml
 - spec/config/connections/elasticsearch/forklift_test.yml
 - spec/config/connections/mysql/forklift_test_destination.yml
 - spec/config/connections/mysql/forklift_test_source_a.yml
@@ -212,12 +220,14 @@ test_files:
 - spec/config/connections/mysql/forklift_test_working.yml
 - spec/config/email.yml
 - spec/integration/basic_spec.rb
+- spec/integration/csv_spec.rb
 - spec/integration/elasticsearch_patterns_spec.rb
 - spec/integration/elasticsearch_spec.rb
 - spec/integration/multi_transport_spec.rb
 - spec/integration/mysql_patterns_spec.rb
 - spec/integration/mysql_spec.rb
 - spec/spec_helper.rb
+- spec/support/dumps/csv/source.csv
 - spec/support/dumps/elasticsearch/forklift_test.json
 - spec/support/dumps/mysql/forklift_test_source_a.sql
 - spec/support/dumps/mysql/forklift_test_source_b.sql