ingestor 0.1.2 → 0.1.3

data/README.md

@@ -71,7 +71,8 @@ Add the following to your Rakefile
     end
 
   It can handle XML, JSON, and more... 
-  
+
+    require 'ingestor/parser/xml'
     ingest("http://example.com/books.xml") do
       parser :xml
       parser_options xpath: '//book'
@@ -99,8 +100,56 @@ Add the following to your Rakefile
       }
     end  
 
+  CSV Example
+
+    require 'ingestor/parser/csv'
+    ingest "./samples/contracts.csv" do
+      parser :csv
+      
+      # all options come directly from Ruby core CSV class
+      parser_options :headers => true,
+        :col_sep            => ",",
+        :row_sep            => :auto,
+        :quote_char         => '"',
+        :field_size_limit   => nil,
+        :converters         => nil,
+        :unconverted_fields => nil,
+        :return_headers     => false,
+        :header_converters  => nil,
+        :skip_blanks        => false,
+        :force_quotes       => false    
+
+      # How to map out the columns from text to AR
+      map_attributes do |row|
+        {
+          id:                 row[0],
+          seller_name:        row[1],
+          customer_name:      row[2],
+          commencement_date:  row[7],
+          termination_date:   row[8]
+        }
+      end
+      
+      # before{|attrs| attrs}
+      
+      # Your strategy for finding or instantiating a new object to be handled by the processor block
+      finder{|attrs|
+        Contract.new
+      }
+
+      processor{|attrs,record|
+        # ... custom processor here ...
+        record.update_attributes attrs
+      }
+      
+      after{|record| 
+        puts "Created: #{record.summary}"
+      }
+    end  
+
   JSON Example
 
+    require 'ingestor/parser/json'
     ingest("http://example.com/people.json") do
       parser :json
       parser_options collection: lambda{|document|
@@ -236,6 +285,7 @@ Coming soon...
 
 
 ## Todos
+* Deprecate plain_text (this was the first thing I created)
 * rdoc http://rdoc.rubyforge.org/RDoc/Markup.html
 * Move includes_header to CSV, PlainText
 * Mongoid Support

data/examples/contracts_csv.rb

@@ -0,0 +1,71 @@
+#! /usr/bin/env ruby
+require 'rubygems'
+require 'bundler/setup'
+require 'ingestor'
+require 'ingestor/parser/csv'
+
+# Set up a bogus active model
+require 'active_model'
+class Contract
+  include ActiveModel::Naming
+  def persisted?
+    true
+  end
+  # Make a fake active model
+  attr_accessor :id, :seller_name, :customer_name, :commencement_date, :termination_date
+  def summary
+    [:seller_name, :customer_name, :commencement_date, :termination_date].map{|key|
+      send(key)
+    }.join(' ')
+  end
+  def update_attributes(attributes = {})
+    attributes.each do |name, value|
+        send("#{name}=", value)
+    end
+    true    
+  end
+end
+# end bogusness
+
+ingest "./samples/contracts.csv" do
+  parser :csv
+  #sample true
+  parser_options :headers => true
+  #   :col_sep            => ",",
+  #   :row_sep            => :auto,
+  #   :quote_char         => '"',
+  #   :field_size_limit   => nil,
+  #   :converters         => nil,
+  #   :unconverted_fields => nil,
+  #   :return_headers     => false,
+  #   :header_converters  => nil,
+  #   :skip_blanks        => false,
+  #   :force_quotes       => false    
+
+  # How to map out the columns from text to AR
+  map_attributes do |row|
+    {
+      id:                 row[0],
+      seller_name:        row[1],
+      customer_name:      row[2],
+      commencement_date:  row[7],
+      termination_date:   row[8]
+    }
+  end
+  
+  # before{|attrs| attrs}
+  
+  # Your strategy for finding or instantiating a new object to be handled by the processor block
+  finder{|attrs|
+    Contract.new
+  }
+
+  processor{|attrs,record|
+    # ... custom processor here ...
+    record.update_attributes attrs
+  }
+  
+  after{|record| 
+    puts "Created: #{record.summary}"
+  }
+end

data/examples/hotel_chains_plain_text.rb

@@ -21,7 +21,7 @@ class HotelChain
 end
 # end bogusness
 
-ingest "https://www.ian.com/affiliatecenter/include/V2/ChainList.zip" do
+ingest "./samples/ChainList.zip" do
   parser :plain_text
   compressed true
   includes_header true

data/lib/ingestor/parser/csv.rb

@@ -1,6 +1,34 @@
+require 'csv'
 module Ingestor
   module Parser
     class Csv
+      include Ingestor::Parser::Base      
+      def options(opts={})
+        @options = {
+          :col_sep            => ",",
+          :row_sep            => :auto,
+          :quote_char         => '"',
+          :field_size_limit   => nil,
+          :converters         => nil,
+          :unconverted_fields => nil,
+          :headers            => false,
+          :return_headers     => false,
+          :header_converters  => nil,
+          :skip_blanks        => false,
+          :force_quotes       => false          
+        }.merge(opts)
+      end
+
+      def sample!
+        puts CSV.parse( @document.read, @options ).first
+      end      
+
+      def process!
+        CSV.parse( @document.read, @options ).each do |row|
+          @proxy.process_entry @proxy.options[:map_attributes].call( row )
+        end
+      end      
+
     end
   end
 end

data/lib/ingestor/version.rb

@@ -1,3 +1,3 @@
 module Ingestor
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/samples/contracts.csv

@@ -0,0 +1,23 @@
+contract_id,seller_company_name,customer_company_name,customer_duns_number,contract_affiliate,FERC_tariff_reference,contract_service_agreement_id,contract_execution_date,contract_commencement_date,contract_termination_date,actual_termination_date,extension_provision_description,class_name,term_name,increment_name,increment_peaking_name,product_type_name,product_name,quantity,units_for_contract,rate,rate_minimum,rate_maximum,rate_description,units_for_rate,point_of_receipt_control_area,point_of_receipt_specific_location,point_of_delivery_control_area,point_of_delivery_specific_location,begin_date,end_date,time_zone
+C71,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Original Volume No. 10,2,2/15/2001,2/15/2001,,,Evergreen,N/A,N/A,N/A,N/A,MB,ENERGY,0,, , , ,Market Based,,,,,,,,ES
+C72,The Electric Company,Utility A,38495837,n,FERC Electric Tariff Original Volume No. 10,15,7/25/2001,8/1/2001,,,Evergreen,N/A,N/A,N/A,N/A,MB,ENERGY,0,, , , ,Market Based,,,,,,,,ES
+C73,The Electric Company,Utility B,493758794,N,FERC Electric Tariff Original Volume No. 10,7,6/8/2001,7/6/2001,,,Evergreen,N/A,N/A,N/A,N/A,MB,ENERGY,0,, , , ,Market Based,,,, , ,,,ep
+C74,The Electric Company,Utility C,594739573,n,FERC Electric Tariff Original Volume No. 10,25,6/8/2001,7/6/2001,,,Evergreen,N/A,N/A,N/A,N/A,MB,ENERGY,0,, , , ,Market Based,,,, , ,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,ENERGY,2000,KWh,.1475, , ,Max amount of capacity and energy to be transmitted.  Bill based on monthly max delivery to City.,$/KWh,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,point-to-point agreement,2000,KW,0.01, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,network,2000,KW,0.2, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,BLACK START SERVICE,2000,KW,0.22, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,CAPACITY,2000,KW,0.04, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,regulation & frequency response,2000,KW,0.1, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,real power transmission loss,2000,KW,7, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C76,The Electric Company,The Power Company,456534333,N,FERC Electric Tariff Original Volume No. 10,132,12/15/2001,1/1/2002,12/31/2004,12/31/2004,None,F,LT,M,FP,MB,CAPACITY,70,MW,3750, , ,70MW for each and every hour over the term of the agreement (7x24 schedule).,$/MW,,,,,,,ep
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,35, , ,,$/MWH,,,PJM,Bus 4321,20020101,20030101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,37, , ,,$/MWH,,,PJM,Bus 4321,20030101,20040101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,39, , ,,$/MWH,,,PJM,Bus 4321,20040101,20050101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,41, , ,,$/MWH,,,PJM,Bus 4321,20050101,20060101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,43, , ,,$/MWH,,,PJM,Bus 4321,20060101,20070101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,45, , ,,$/MWH,,,PJM,Bus 4321,20070101,20080101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,47, , ,,$/MWH,,,PJM,Bus 4321,20080101,20090101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,49, , ,,$/MWH,,,PJM,Bus 4321,20090101,20100101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,51, , ,,$/MWH,,,PJM,Bus 4321,20100101,20110101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,53, , ,,$/MWH,,,PJM,Bus 4321,20110101,20120101,EP

data/spec/lib/ingestor/parser/csv_spec.rb

@@ -1,5 +1,24 @@
+require 'spec_helper'
+require 'ingestor/parser/csv'
 
-# require 'spec_helper'
-# describe Ingestor::Parser::Csv do
-#   pending 'write it'
-# end
+describe Ingestor::Parser::Csv do
+  describe '#process!' do
+    before do
+      @proxy = ingest("./samples/contracts.csv") do
+        parser :csv
+        parser_options headers: true
+          
+        finder{|attrs| Dummy.new}
+        map_attributes{|row| 
+          {
+            :name => row[1]
+          } 
+        }
+      end
+    end
+    
+    it 'should be able to process a JSON file' do
+      Dummy.first.name.should eq "The Electric Company"
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ingestor
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docile
-  requirement: &70218534200000 !ruby/object:Gem::Requirement
+  requirement: &70215971566020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70218534200000
+  version_requirements: *70215971566020
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &70218534199320 !ruby/object:Gem::Requirement
+  requirement: &70215971565380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70218534199320
+  version_requirements: *70215971565380
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &70218534198520 !ruby/object:Gem::Requirement
+  requirement: &70215971564500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70218534198520
+  version_requirements: *70215971564500
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70218534197600 !ruby/object:Gem::Requirement
+  requirement: &70215971563640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.5.6
   type: :runtime
   prerelease: false
-  version_requirements: *70218534197600
+  version_requirements: *70215971563640
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70218534196500 !ruby/object:Gem::Requirement
+  requirement: &70215971563080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: 3.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *70218534196500
+  version_requirements: *70215971563080
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70218534195400 !ruby/object:Gem::Requirement
+  requirement: &70215971562260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,7 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70218534195400
+  version_requirements: *70215971562260
 description: Ingesting local and remote data files into ActiveRecord
 email:
 - github@coryodaniel.com
@@ -94,6 +94,7 @@ files:
 - bin/ingest
 - examples/books_xml.rb
 - examples/colors_json.rb
+- examples/contracts_csv.rb
 - examples/hotel_chains_plain_text.rb
 - examples/people_json.rb
 - ingestor.gemspec
@@ -101,15 +102,18 @@ files:
 - lib/ingestor/dsl.rb
 - lib/ingestor/parser/base.rb
 - lib/ingestor/parser/csv.rb
+- lib/ingestor/parser/http.rb
 - lib/ingestor/parser/json.rb
 - lib/ingestor/parser/plain_text.rb
 - lib/ingestor/parser/xml.rb
 - lib/ingestor/proxy.rb
 - lib/ingestor/tasks.rb
 - lib/ingestor/version.rb
+- samples/ChainList.zip
 - samples/animals.csv
 - samples/books.xml
 - samples/colors.json
+- samples/contracts.csv
 - samples/flags.txt
 - samples/people.json
 - spec/cassettes/remote-zipped-files.yml