ingestor 0.1.1 → 0.1.2

data/Gemfile

@@ -10,9 +10,7 @@ gem "guard-rspec"
 gem 'debugger', '1.2.0'
 gem 'vcr'
 gem 'fakeweb'
-gem 'nokogiri'
 gem 'activerecord', "~>3.2.0"
-gem 'activesupport', "~>3.2.0"
 gem 'mysql2'
 gem 'rb-fsevent'
 gem 'ruby_gntp'

data/README.md

@@ -99,6 +99,33 @@ Add the following to your Rakefile
       }
     end  
 
+  JSON Example
+
+    ingest("http://example.com/people.json") do
+      parser :json
+      parser_options collection: lambda{|document|
+        document['people']
+      }
+      map_attributes do |values|
+        {
+          name:         values["first_name"] + " " + values["last_name"]
+          age:          values['age'],
+          address:      values['address']
+        }
+      end
+
+      # current lines values
+      finder{|attrs| 
+        Person.where(name: attrs[:name]).first || Person.new
+      }
+
+      processor{|attrs,record|
+        record.update_attributes(attrs)
+        record.send_junk_mail!
+      }
+    end    
+
+
 ## Advanced Usage
 DSL Options
   * parser - the parser to use on the file
@@ -178,7 +205,13 @@ Writing parsers is simple ([see examples](https://github.com/coryodaniel/ingesto
     * Default libxml2 best guess
 
 ### JSON Parser
-Coming soon...
+  Parses a JSON document
+
+  Options
+  * collection - receives the document and narrows it down to the collection you are interested in
+    * Proc(Hash)
+    * Returns Hash | Array
+    * Required
 
 ### CSV Parser
 Coming soon...
@@ -203,9 +236,10 @@ Coming soon...
 
 
 ## Todos
-* rdoc lib/ folder
+* rdoc http://rdoc.rubyforge.org/RDoc/Markup.html
 * Move includes_header to CSV, PlainText
 * Mongoid Support
 * sort/limit options
+* configure travis
 * A way to sample a file without building an ingestor first
   * bin/ingestor --sample --path=./my.xml --parser xml --parser_options_xpath '//book'

data/examples/colors_json.rb

@@ -0,0 +1,57 @@
+#! /usr/bin/env ruby
+require 'rubygems'
+require 'bundler/setup'
+require 'ingestor'
+require 'ingestor/parser/json'
+
+# Set up a bogus active model
+require 'active_model'
+class Color
+  include ActiveModel::Naming
+  def persisted?
+    true
+  end
+  # Make a fake active model
+  attr_accessor :name, :hex
+  def update_attributes(attributes = {})
+    attributes.each do |name, value|
+        send("#{name}=", value)
+    end
+    true    
+  end
+end
+# end bogusness
+
+ingest "./samples/colors.json" do
+  parser :json
+
+  # Receives the full document and narrows it down to the collection to process.
+  parser_options collection: lambda{|document|
+    document
+  }
+  
+  # How to map out the columns from text to AR
+  map_attributes do |values|
+    {
+      name: values['color'],
+      hex:  values['value']
+    }
+  end
+  
+  # before{|attrs| values}
+
+  # Your strategy for finding or instantiating a new object to be handled by the processor block
+  finder{|attrs|
+    # Book.find( attrs['id'] ) || Book.new
+    Color.new
+  }
+
+  processor{|attrs,record|
+    # ... custom processor here ...
+    record.update_attributes attrs
+  }
+  
+  after{|record| 
+    puts "Created: #{record.name}"
+  }
+end

data/examples/people_json.rb

@@ -0,0 +1,55 @@
+#! /usr/bin/env ruby
+require 'rubygems'
+require 'bundler/setup'
+require 'ingestor'
+require 'ingestor/parser/json'
+
+# Set up a bogus active model
+require 'active_model'
+class Person
+  include ActiveModel::Naming
+  def persisted?
+    true
+  end
+  # Make a fake active model
+  attr_accessor :id, :first_name, :last_name, :age, :address
+  def update_attributes(attributes = {})
+    attributes.each do |name, value|
+        send("#{name}=", value)
+    end
+    true    
+  end
+end
+# end bogusness
+
+ingest "./samples/people.json" do
+  parser :json
+
+  # Receives the full document and narrows it down to the collection to process.
+  parser_options collection: lambda{|document|
+    document['people']
+  }
+  #sample true
+  
+  # How to map out the columns from text to AR
+  map_attributes do |values|
+    values
+  end
+  
+  # before{|attrs| values}
+
+  # Your strategy for finding or instantiating a new object to be handled by the processor block
+  finder{|attrs|
+    # Book.find( attrs['id'] ) || Book.new
+    Person.new
+  }
+
+  processor{|attrs,record|
+    # ... custom processor here ...
+    record.update_attributes attrs
+  }
+  
+  after{|record| 
+    puts "Created: #{record.first_name} @ #{record.address}"
+  }
+end

data/ingestor.gemspec

@@ -19,5 +19,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "docile"
   gem.add_dependency "rubyzip"
   gem.add_dependency "thor"
+  gem.add_dependency "nokogiri", '~> 1.5.6'
   gem.add_dependency "activesupport", '>= 3.2.0'
+  gem.add_dependency 'multi_json', '~> 1.0'
 end

data/lib/ingestor/parser/json.rb

@@ -1,6 +1,31 @@
+require 'multi_json'
+require 'debugger'
 module Ingestor
   module Parser
     class Json
+      include Ingestor::Parser::Base      
+      def options(opts={})
+        @options = {
+          collection: nil
+        }.merge(opts)
+      end
+
+      def sample!
+        puts @options[:collection].call(document).first
+        #puts @options[:collection] ? @options[:collection].call(document).first : document.first
+      end      
+
+      def process!
+        @options[:collection].call(document).each do |attrs|
+          @proxy.process_entry @proxy.options[:map_attributes].call( attrs )
+        end
+      end      
+
+      protected
+
+      def document
+        MultiJson.load(@document.read)
+      end
     end
   end
 end

data/lib/ingestor/parser/xml.rb

@@ -17,19 +17,21 @@ module Ingestor
       end
 
       def sample!
-        doc = Nokogiri::XML(@document, nil, @options[:encoding])
-        puts Hash.from_xml( doc.xpath(@options[:xpath]).first.to_s )
+        puts Hash.from_xml( document.xpath(@options[:xpath]).first.to_s )
       end      
 
       def process!
-        doc = Nokogiri::XML(@document, nil, @options[:encoding])
-        
-        doc.xpath(@options[:xpath]).each do |node|
+        document.xpath(@options[:xpath]).each do |node|
           node_attrs = Hash.from_xml(node.to_s)
           attrs   = @proxy.options[:map_attributes].call( node_attrs )
           @proxy.process_entry attrs
         end
       end
+
+      protected
+      def document
+        Nokogiri::XML(@document, nil, @options[:encoding])
+      end
     end
   end
 end

data/lib/ingestor/version.rb

@@ -1,3 +1,3 @@
 module Ingestor
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/samples/people.json

@@ -1,8 +1,8 @@
 {"people":
   [
     {
-      "firstName": "John",
-      "lastName": "Smith",
+      "first_name": "John",
+      "last_name": "Smith",
       "age": 25,
       "address": {
         "streetAddress": "21 2nd Street",
@@ -12,11 +12,11 @@
       }
     },
     {
-      "firstName": "John",
-      "lastName": "Smith",
-      "age": 25,
+      "first_name": "Joanne",
+      "last_name": "Smith",
+      "age": 93,
       "address": {
-        "streetAddress": "21 2nd Street",
+        "streetAddress": "Westchester Street",
         "city": "New York",
         "state": "NY",
         "postalCode": 10021

data/spec/lib/ingestor/parser/json_spec.rb

@@ -1,5 +1,25 @@
+require 'spec_helper'
+require 'ingestor/parser/json'
 
-# require 'spec_helper'
-# describe Ingestor::Parser::Json do
-#   pending 'write it'
-# end
+describe Ingestor::Parser::Json do
+  describe '#process!' do
+    before do
+      @proxy = ingest("./samples/people.json") do
+        parser :json
+        parser_options collection: lambda{|document|
+          document['people']
+        }
+        finder{|attrs| Dummy.new}
+        map_attributes{|values| 
+          {
+            :name => [ values['first_name'], values["last_name"] ].join(' ')
+          } 
+        }
+      end
+    end
+    
+    it 'should be able to process a JSON file' do
+      Dummy.first.name.should eq "John Smith"
+    end
+  end
+end

data/spec/lib/ingestor/parser/xml_spec.rb

@@ -11,14 +11,12 @@ describe Ingestor::Parser::Xml do
         })
         finder{|attrs| Dummy.new}
         map_attributes{|values| 
-          puts values
           {:name => values['book']['title']} 
         }
       end
     end
     
     it 'should be able to process an XML file' do
-
       Dummy.first.name.should eq "XML Developer's Guide"
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ingestor
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docile
-  requirement: &70343549624980 !ruby/object:Gem::Requirement
+  requirement: &70218534200000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70343549624980
+  version_requirements: *70218534200000
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &70343549624400 !ruby/object:Gem::Requirement
+  requirement: &70218534199320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70343549624400
+  version_requirements: *70218534199320
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &70343549623860 !ruby/object:Gem::Requirement
+  requirement: &70218534198520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70343549623860
+  version_requirements: *70218534198520
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70218534197600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.6
+  type: :runtime
+  prerelease: false
+  version_requirements: *70218534197600
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70343549623160 !ruby/object:Gem::Requirement
+  requirement: &70218534196500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +65,18 @@ dependencies:
         version: 3.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *70343549623160
+  version_requirements: *70218534196500
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: &70218534195400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70218534195400
 description: Ingesting local and remote data files into ActiveRecord
 email:
 - github@coryodaniel.com
@@ -70,8 +92,10 @@ files:
 - README.md
 - Rakefile
 - bin/ingest
-- examples/text_parsing.rb
-- examples/xml_parsing.rb
+- examples/books_xml.rb
+- examples/colors_json.rb
+- examples/hotel_chains_plain_text.rb
+- examples/people_json.rb
 - ingestor.gemspec
 - lib/ingestor.rb
 - lib/ingestor/dsl.rb