ingestor 0.1.1

data/.gitignore

@@ -0,0 +1,21 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+log
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+spec/orm/database.yml
+.DS_Store
+**/.DS_Store

data/Gemfile

@@ -0,0 +1,18 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ingestor.gemspec
+gemspec
+gem 'rspec', '2.12.0'
+gem 'rb-fsevent'
+gem "guard", '1.6.1'
+gem "guard-bundler"
+gem "guard-rspec"
+gem 'debugger', '1.2.0'
+gem 'vcr'
+gem 'fakeweb'
+gem 'nokogiri'
+gem 'activerecord', "~>3.2.0"
+gem 'activesupport', "~>3.2.0"
+gem 'mysql2'
+gem 'rb-fsevent'
+gem 'ruby_gntp'

data/Guardfile

@@ -0,0 +1,11 @@
+guard 'bundler' do
+  watch('Gemfile')
+  # Uncomment next line if Gemfile contain `gemspec' command
+  # watch(/^.+\.gemspec/)
+end
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Cory O'Daniel
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,211 @@
+# Ingestor
+
+A simple DSL for importing data from text and csv files to ActiveRecord. This was originally designed to 
+continually import changing data from EAN and Geonames.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'ingestor'
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install ingestor
+
+Add the following to your Rakefile
+    require 'ingestor/tasks'
+    
+## Usage
+
+  Given a text file:
+
+    id|name|population
+    1|China|1,354,040,000
+    2|India|1,210,193,422
+    3|United States|315,550,000
+
+  And an AR Class:
+
+    class Country
+      attr_accessible :name, :population
+    end
+
+  Sync the file with AR:
+
+    ingest("path/to/countries.txt") do
+      map_attributes do |values|
+        {
+          id:           values[0],
+          name:         values[1],
+          population:  values[2]
+        }
+      end
+
+      # current lines values
+      finder{|attrs| 
+        Country.where(id: attrs[:id]).first || Country.new
+      }
+    end
+
+  It can handle remote files and zip files as well.
+
+    ingest("http://example.com/a_lot_of_countries.zip") do
+      compressed true
+      map_attributes do |values|
+        {
+          id:           values[0],
+          name:         values[1],
+          population:  values[2]
+        }
+      end
+
+      # current lines values
+      finder{|attrs| 
+        Country.where(id: attrs[:id]).first || Country.new
+      }
+    end
+
+  It can handle XML, JSON, and more... 
+  
+    ingest("http://example.com/books.xml") do
+      parser :xml
+      parser_options xpath: '//book'
+      map_attributes do |values|
+        {
+          id:           values['id'],
+          title:        values['title'],
+          author:       {
+            name: values['author']
+          }
+        }
+      end
+
+      # current lines values
+      finder{|attrs| 
+        Book.where(id: attrs[:id]).first || Book.new
+      }
+
+      processor{|attrs,record|
+        record.update_attributes(attrs)
+        record.reviews.create({
+          stars: 5,
+          comment: "Every book they sell is so great!"
+        })
+      }
+    end  
+
+## Advanced Usage
+DSL Options
+  * parser - the parser to use on the file
+    * Symbol
+    * Optional
+    * Default: :plain_text
+    * Available Values: :plain_text, :xml, :json, :csv, :html
+    * See 'Included Parsers' below
+  * parser_options - options for a specific parser
+    * Hash
+    * Optional
+    * Default: set per parser
+    * See 'Included Parsers' below
+  * sample - dump a single raw entry from the file to STDOUT and exit
+    * Boolean 
+    * Optional
+    * Default: false
+    (defaults: false) will 
+  * includes_header - Tells the parser that the first line is a header and should be ignored
+    * Boolean
+    * Optional
+    * Default: false
+  * compressed - Should the file be decompressed
+    * Boolean
+    * Optional
+    * Default: false
+  * working_directory - where to store remote or decompressed files for local processing
+    * String
+    * Optional
+    * Default: /tmp/ingestor
+  * before - callback that receives attributes for each record BEFORE call to [finder]
+    * Proc(attributes)
+    * Optional
+    * Default: nil
+  * finder - Arel finder for each object
+    * Proc(attributes)
+    * Returns: ~ActiveModel
+    * Required
+  * processor - What to do with the attributes and object
+    * Proc(attributes,record)
+    * Returns: ~ActiveModel
+    * Optional
+    * Default: Proc, calls #update_attributes on record without protection
+  * after - callback that receives each record after [processor]
+    * Proc(record)
+    * Optional  
+
+
+## Included Parsers
+
+Writing parsers is simple ([see examples](https://github.com/coryodaniel/ingestor/tree/master/lib/ingestor/parser])).
+
+### Plain Text Parser
+  Parses a plain text document.
+
+  Options
+  * delimiter - how to split up each line
+    * String
+    * Default: '|'
+    * Optional
+  * line\_processor - override default\_line\_processor. The default\_line\_processor simply splits the string using the delimiter
+    * Proc(string)
+    * Returns Array
+    * Default: nil
+    * Optional
+
+### XML Parser
+  Parses an XML document
+
+  Options
+  * selector - xpath selector to get the node collection
+    * String
+    * Required
+  * encoding - XML Encoding. See nokogiri encoding
+    * String
+    * Optional
+    * Default libxml2 best guess
+
+### JSON Parser
+Coming soon...
+
+### CSV Parser
+Coming soon...
+
+### HTML Parser
+Coming soon...
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Running Tests
+  
+  1. Copy spec/orm/database.example.yml => spec/orm/database.yml
+  2. Configure spec/orm/database.yml
+  3. bundle exec guard
+
+
+## Todos
+* rdoc lib/ folder
+* Move includes_header to CSV, PlainText
+* Mongoid Support
+* sort/limit options
+* A way to sample a file without building an ingestor first
+  * bin/ingestor --sample --path=./my.xml --parser xml --parser_options_xpath '//book'

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')
+
+# If you want to make this the default task
+task :default => :spec

data/bin/ingest

@@ -0,0 +1,73 @@
+#!/usr/bin/env ruby
+
+require 'ingestor'
+require 'thor'
+require 'active_support/inflector'
+
+module Ingestor
+  class CommandLine < Thor
+
+    desc "generate PATH", "Create a new ingestor for PATH. PATH should be an absolute path or URL"
+    def generate(path)
+      output_directory = 'script/ingestors'
+      FileUtils.mkdir_p(output_directory, mode: 0755) unless Dir.exists?(output_directory)
+      
+      file_name   = File.basename(path).underscore.parameterize.underscore
+      file_name   = %Q{#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_#{file_name}.rb}
+      
+      generated_file = File.join( output_directory, file_name )
+
+      File.open(generated_file, 'w+') do |f|
+        f.puts <<-HEREDOC
+#! /usr/bin/env ruby
+require 'ingestor'
+
+######################################
+#
+# Order of block execution is:
+#   * map_attributes
+#   * before
+#   * finder
+#   * processor
+#   * after
+#
+######################################
+
+ingest "#{path}" do
+  parser :plain_text
+  # compressed true                     # is the file compressed
+  # includes_header false               
+  # parser_options delimiter: '|'       # parser specific
+  # working_directory '/tmp/ingestor'   # where to store files that are compressed or remote
+  
+  # How to map out the columns from document to ActiveRecord
+  map_attributes do |values|
+    {
+      # ... create your attributes hash here for ActiveRecord/ActiveModel/etc.
+      # values may be a Hash (xml, json) or an Array (csv, plain_text)
+    }
+  end
+
+  # before{ |attrs| attrs}
+
+  # Your strategy for finding or instantiating a new object to be handled by the processor block
+  finder do |attrs|
+    MyClass.find( attrs[:id] ) || MyClass.new
+  end
+    
+  # The default processor simple calls update_attributes
+  # processor do |attrs,record|  
+  #  ... custom processor here ...
+  # end
+  
+  # after { |record| record}
+end
+HEREDOC
+      end
+
+      say "Generated #{generated_file}"
+    end
+  end
+end
+
+Ingestor::CommandLine.start

data/examples/text_parsing.rb

@@ -0,0 +1,56 @@
+#! /usr/bin/env ruby
+require 'rubygems'
+require 'bundler/setup'
+require 'ingestor'
+
+# Set up a bogus active model
+require 'active_model'
+class HotelChain
+  include ActiveModel::Naming
+  def persisted?
+    true
+  end
+  # Make a fake active model
+  attr_accessor :id, :name
+  def update_attributes(attributes = {})
+    attributes.each do |name, value|
+        send("#{name}=", value)
+    end
+    true    
+  end
+end
+# end bogusness
+
+ingest "https://www.ian.com/affiliatecenter/include/V2/ChainList.zip" do
+  parser :plain_text
+  compressed true
+  includes_header true
+  # sample true
+
+  parser_options delimiter: '|'
+
+  # How to map out the columns from text to AR
+  map_attributes do |values|
+    {
+      id:   values[0],
+      name: values[1]
+    }
+  end
+  
+  # before{|attrs| attrs}
+  
+  # Your strategy for finding or instantiating a new object to be handled by the processor block
+  finder{|attrs|
+    # Book.find( attrs['id'] ) || Book.new
+    HotelChain.new
+  }
+
+  processor{|attrs,record|
+    # ... custom processor here ...
+    record.update_attributes attrs
+  }
+  
+  after{|record| 
+    puts "Created: #{record.name}"
+  }
+end

data/examples/xml_parsing.rb

@@ -0,0 +1,52 @@
+#! /usr/bin/env ruby
+require 'rubygems'
+require 'bundler/setup'
+require 'ingestor'
+require 'ingestor/parser/xml'
+
+# Set up a bogus active model
+require 'active_model'
+class Book
+  include ActiveModel::Naming
+  def persisted?
+    true
+  end
+  # Make a fake active model
+  attr_accessor :id, :title, :author, :price, :genre, :publish_date, :description
+  def update_attributes(attributes = {})
+    attributes.each do |name, value|
+        send("#{name}=", value)
+    end
+    true    
+  end
+end
+# end bogusness
+
+ingest "./samples/books.xml" do
+  parser :xml
+  parser_options xpath: '//book'
+  #sample true
+  # compressed false
+
+  # How to map out the columns from text to AR
+  map_attributes do |values|
+    values['book']
+  end
+  
+  # before{|attrs| values}
+
+  # Your strategy for finding or instantiating a new object to be handled by the processor block
+  finder{|attrs|
+    # Book.find( attrs['id'] ) || Book.new
+    Book.new
+  }
+
+  processor{|attrs,record|
+    # ... custom processor here ...
+    record.update_attributes attrs
+  }
+  
+  after{|record| 
+    puts "Created: #{record.title}"
+  }
+end