ingestor 0.1.1

data/spec/lib/ingestor/parser/xml_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+require 'ingestor/parser/xml'
+
+describe Ingestor::Parser::Xml do
+  describe '#process!' do
+    before do
+      @proxy = ingest("./samples/books.xml") do
+        parser :xml
+        parser_options({
+          xpath: '//book'
+        })
+        finder{|attrs| Dummy.new}
+        map_attributes{|values| 
+          puts values
+          {:name => values['book']['title']} 
+        }
+      end
+    end
+    
+    it 'should be able to process an XML file' do
+
+      Dummy.first.name.should eq "XML Developer's Guide"
+    end
+  end
+end

data/spec/lib/ingestor/proxy_spec.rb

@@ -0,0 +1,129 @@
+require 'spec_helper'
+
+def default_test_ingestor
+  ingest("./samples/flags.txt") do
+    includes_header true
+    finder{|values| Country.new}
+    map_attributes{|values| {:name => values[0], :colors => values[1], :count => values[2]} }
+  end
+end
+
+describe Ingestor::Proxy do
+  describe 'loading local files' do
+    before :each do
+      @proxy = ingest("./samples/flags.txt") do
+        finder{|values| Country.new}
+        map_attributes{|values| {:name => values[0], :colors => values[1], :count => values[2]} }
+      end
+    end
+    it 'should know if a file is local' do
+      @proxy.should be_local
+      @proxy.should_not be_remote
+    end
+
+    it 'should know if a file is compressed' do
+      @proxy.should_not be_compressed
+    end    
+  end
+
+  describe 'loading remote files' do
+    use_vcr_cassette 'remote-zipped-files'
+
+    before :each do
+      @proxy  = ingest("https://www.ian.com/affiliatecenter/include/V2/ChainList.zip") do
+        finder{|values| Dummy.new}
+        map_attributes do |values|
+          {:id => values[0], :name => values[1]}
+        end
+        compressed true
+      end
+    end
+    it 'should know if a file is remote' do
+      @proxy.should_not be_local
+      @proxy.should be_remote
+    end
+
+    it 'should know if a file is compressed' do
+      @proxy.should be_compressed
+    end    
+
+    it 'should create a tempfile for remote files' do
+      File.exists?( @proxy.document.path ).should be true
+    end    
+  end
+
+  pending '#sample'
+
+  describe '#includes_header' do
+    it 'should include a header' do
+      ingest("./samples/flags.txt") do
+        includes_header true
+        finder{|values| Country.new}
+        map_attributes{|values| {:name => values[0], :colors => values[1], :count => values[2]} }        
+      end.header.should == "Country|Colors|Count|Secrets"
+    end
+
+    it 'should not include a header' do
+      ingest("./samples/flags.txt") do
+        includes_header false
+        finder{|values| Country.new}
+        map_attributes{|values| {:name => values[0], :colors => values[1], :count => values[2]} }
+      end.header.should be_nil
+    end    
+  end
+
+  describe '#before' do
+    before :each do
+      ingest("./samples/flags.txt") do
+        includes_header true
+        finder{|values| Country.new}
+        map_attributes{|values| {:name => values[0], :colors => values[1], :count => values[2], :secrets => values[3]} }
+        before{|attrs|
+          attrs[:name].reverse!
+          attrs
+        }
+      end
+    end
+    it 'should modify values in place when using a #before callback' do
+      Country.first.name.should == 'rodavlaS lE'
+    end
+  end
+
+  describe '#after' do
+    before :each do
+      @records = []
+      ingest("./samples/flags.txt") do
+        includes_header true
+        finder{|values| Country.new}
+        map_attributes{|values| {:name => values[0], :colors => values[1], :count => values[2], :secrets => values[3]} }
+        after{|record|
+          @records << record
+        }
+      end
+    end
+
+    it 'should pass the current record to an #after callback' do
+      @records.length.should be(11)
+    end
+  end  
+
+  describe '#processor' do
+    before do
+      ingest("./samples/flags.txt") do
+        includes_header true
+        finder{|values| Country.new}
+        map_attributes{|values| {:name => values[0], :colors => values[1], :count => values[2], :secrets => values[3]} }
+        processor{|attrs,record|
+          record.update_attributes attrs
+          record.secrets = "Squirrel Party"
+          record.save
+          record
+        }
+      end    
+    end
+
+    it 'should use the optional #processor when provided' do
+      Country.where(secrets: 'Squirrel Party').count.should be(11)
+    end
+  end
+end

data/spec/lib/ingestor_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+require 'ingestor/parser/xml'
+
+describe Ingestor do
+  it "should have a version" do
+    Ingestor::VERSION.should_not be_nil
+  end
+
+  it 'should have plain text as the default parser' do
+    Ingestor.parser_for(:plain_text).should be(Ingestor::Parser::PlainText)
+    Ingestor.parser_for(:xml).should be(Ingestor::Parser::Xml)
+  end
+
+  it 'should raise an exception for a bogus parser type' do
+    lambda{
+      Ingestor.parser_for(:bogus)
+    }.should raise_exception
+  end
+end

data/spec/orm/active_record.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+require 'active_record'
+
+ActiveRecord::Base.logger = ActiveSupport::BufferedLogger.new('log/test.log')
+ActiveRecord::Base.establish_connection YAML.load(File.open(File.join(File.dirname(__FILE__), 'database.yml')).read)[ENV['db'] || 'mysql']
+
+ActiveRecord::Migration.verbose = false
+
+class TestMigration < ActiveRecord::Migration
+  def self.up
+    create_table :countries, :force => true do |t|
+      t.column :name, :string
+      t.column :colors, :string
+      t.column :count, :integer
+      t.column :secrets, :string
+    end
+
+    create_table :dummies, :force => true do |t|
+      t.column :name, :string
+      t.timestamps
+    end    
+  end
+
+  def self.down
+    drop_table :countries
+    drop_table :dummies
+  end
+end
+
+class Dummy < ActiveRecord::Base;end;
+class Country < ActiveRecord::Base
+  attr_protected :secrets
+end

data/spec/orm/database.example.yml

@@ -0,0 +1,15 @@
+sqlite3: 
+  adapter: sqlite3 
+  database: ":memory:"
+postgresql: 
+  adapter: postgresql
+  database: ingestor_test
+  username: ingestor
+  password: ingestor
+  min_messages: WARNING
+mysql: 
+  adapter: mysql 
+  host: localhost 
+  database: ingestor_test
+  username: ingestor
+  password: ingestor

data/spec/spec_helper.rb

@@ -0,0 +1,21 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'ingestor'
+require 'vcr'
+require 'orm/active_record'
+
+VCR.configure do |c|
+  c.cassette_library_dir     = 'spec/cassettes'
+  c.stub_with                :fakeweb
+end
+
+RSpec.configure do |config|
+  config.extend VCR::RSpec::Macros
+  config.before(:all) { TestMigration.up }
+  config.after(:all) { TestMigration.down }
+  config.after(:each){ 
+    Country.delete_all
+    Dummy.delete_all
+  }
+end

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: ingestor
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+  prerelease: 
+platform: ruby
+authors:
+- Cory O'Daniel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: docile
+  requirement: &70343549624980 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70343549624980
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: &70343549624400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70343549624400
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: &70343549623860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70343549623860
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &70343549623160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70343549623160
+description: Ingesting local and remote data files into ActiveRecord
+email:
+- github@coryodaniel.com
+executables:
+- ingest
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/ingest
+- examples/text_parsing.rb
+- examples/xml_parsing.rb
+- ingestor.gemspec
+- lib/ingestor.rb
+- lib/ingestor/dsl.rb
+- lib/ingestor/parser/base.rb
+- lib/ingestor/parser/csv.rb
+- lib/ingestor/parser/json.rb
+- lib/ingestor/parser/plain_text.rb
+- lib/ingestor/parser/xml.rb
+- lib/ingestor/proxy.rb
+- lib/ingestor/tasks.rb
+- lib/ingestor/version.rb
+- samples/animals.csv
+- samples/books.xml
+- samples/colors.json
+- samples/flags.txt
+- samples/people.json
+- spec/cassettes/remote-zipped-files.yml
+- spec/lib/ingestor/dsl_spec.rb
+- spec/lib/ingestor/parser/.DS_Store
+- spec/lib/ingestor/parser/csv_spec.rb
+- spec/lib/ingestor/parser/json_spec.rb
+- spec/lib/ingestor/parser/plain_text_spec.rb
+- spec/lib/ingestor/parser/xml_spec.rb
+- spec/lib/ingestor/proxy_spec.rb
+- spec/lib/ingestor_spec.rb
+- spec/orm/active_record.rb
+- spec/orm/database.example.yml
+- spec/spec_helper.rb
+homepage: http://github.com/coryodaniel/ingestor
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Ingesting local and remote data files into ActiveRecord
+test_files:
+- spec/cassettes/remote-zipped-files.yml
+- spec/lib/ingestor/dsl_spec.rb
+- spec/lib/ingestor/parser/.DS_Store
+- spec/lib/ingestor/parser/csv_spec.rb
+- spec/lib/ingestor/parser/json_spec.rb
+- spec/lib/ingestor/parser/plain_text_spec.rb
+- spec/lib/ingestor/parser/xml_spec.rb
+- spec/lib/ingestor/proxy_spec.rb
+- spec/lib/ingestor_spec.rb
+- spec/orm/active_record.rb
+- spec/orm/database.example.yml
+- spec/spec_helper.rb