importer 0.4.1 → 0.4.2

data/README.rdoc

@@ -12,6 +12,7 @@ Docs: http://rdoc.info/projects/szajbus/importer
 
 * can import from XML and CSV formats, but it's possible to add custom parsers
 * reports how many new objects got imported, how many objects was modified and how many objects were invalid
+* includes ActiveRecord, DataMapper and MongoMapper adapters
 
 == Installation
 
@@ -36,6 +37,12 @@ Add to your model
     include Importer
   end
 
+  # DataMapper
+  class Product
+    include DataMapper::Resource
+    include Importer
+  end
+
 And start importing
 
   Product.import(path_to_xml_or_csv_file)

data/Rakefile

@@ -14,6 +14,10 @@ begin
     gem.add_dependency "fastercsv", ">= 1.5.0"
     gem.add_development_dependency "activerecord", ">= 0"
     gem.add_development_dependency "mongo_mapper", ">= 0.7.0"
+    gem.add_development_dependency "dm-core", ">= 0.10.2"
+    gem.add_development_dependency "dm-validations", ">= 0.10.2"
+    gem.add_development_dependency "dm-aggregates", ">= 0.10.2"
+    gem.add_development_dependency "do_sqlite3", ">= 0.10.1.1"
     gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
     gem.add_development_dependency 'sqlite3-ruby'
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings

data/VERSION

@@ -1 +1 @@
-0.4.1
+0.4.2

data/importer.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{importer}
-  s.version = "0.4.1"
+  s.version = "0.4.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Micha\305\202 Szajbe"]
-  s.date = %q{2010-04-16}
+  s.date = %q{2010-05-17}
   s.description = %q{Define new objects or modifications of existing ones in external file (xml, csv, etc) and import them to your application. Importer will not only import all the objects but also will give you detailed summary of the import process.}
   s.email = %q{michal.szajbe@gmail.com}
   s.extra_rdoc_files = [
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
      "importer.gemspec",
      "lib/importer.rb",
      "lib/importer/adapters/active_record_adapter.rb",
+     "lib/importer/adapters/data_mapper_adapter.rb",
      "lib/importer/adapters/mongo_mapper_adapter.rb",
      "lib/importer/import.rb",
      "lib/importer/imported_object.rb",
@@ -44,6 +45,7 @@ Gem::Specification.new do |s|
      "test/fixtures/products.xml",
      "test/helper.rb",
      "test/importer/adapters/active_record_adapter_test.rb",
+     "test/importer/adapters/data_mapper_adapter_test.rb",
      "test/importer/adapters/mongo_mapper_adapter_test.rb",
      "test/importer/import_test.rb",
      "test/importer/imported_object_test.rb",
@@ -60,6 +62,7 @@ Gem::Specification.new do |s|
     "test/factories.rb",
      "test/helper.rb",
      "test/importer/adapters/active_record_adapter_test.rb",
+     "test/importer/adapters/data_mapper_adapter_test.rb",
      "test/importer/adapters/mongo_mapper_adapter_test.rb",
      "test/importer/import_test.rb",
      "test/importer/imported_object_test.rb",
@@ -77,6 +80,10 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<fastercsv>, [">= 1.5.0"])
       s.add_development_dependency(%q<activerecord>, [">= 0"])
       s.add_development_dependency(%q<mongo_mapper>, [">= 0.7.0"])
+      s.add_development_dependency(%q<dm-core>, [">= 0.10.2"])
+      s.add_development_dependency(%q<dm-validations>, [">= 0.10.2"])
+      s.add_development_dependency(%q<dm-aggregates>, [">= 0.10.2"])
+      s.add_development_dependency(%q<do_sqlite3>, [">= 0.10.1.1"])
       s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
       s.add_development_dependency(%q<sqlite3-ruby>, [">= 0"])
     else
@@ -84,6 +91,10 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<fastercsv>, [">= 1.5.0"])
       s.add_dependency(%q<activerecord>, [">= 0"])
       s.add_dependency(%q<mongo_mapper>, [">= 0.7.0"])
+      s.add_dependency(%q<dm-core>, [">= 0.10.2"])
+      s.add_dependency(%q<dm-validations>, [">= 0.10.2"])
+      s.add_dependency(%q<dm-aggregates>, [">= 0.10.2"])
+      s.add_dependency(%q<do_sqlite3>, [">= 0.10.1.1"])
       s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
       s.add_dependency(%q<sqlite3-ruby>, [">= 0"])
     end
@@ -92,6 +103,10 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<fastercsv>, [">= 1.5.0"])
     s.add_dependency(%q<activerecord>, [">= 0"])
     s.add_dependency(%q<mongo_mapper>, [">= 0.7.0"])
+    s.add_dependency(%q<dm-core>, [">= 0.10.2"])
+    s.add_dependency(%q<dm-validations>, [">= 0.10.2"])
+    s.add_dependency(%q<dm-aggregates>, [">= 0.10.2"])
+    s.add_dependency(%q<do_sqlite3>, [">= 0.10.1.1"])
     s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
     s.add_dependency(%q<sqlite3-ruby>, [">= 0"])
   end

data/lib/importer.rb

@@ -1,4 +1,5 @@
 require 'importer/adapters/active_record_adapter'
+require 'importer/adapters/data_mapper_adapter'
 require 'importer/adapters/mongo_mapper_adapter'
 require 'importer/import'
 require 'importer/imported_object'
@@ -32,6 +33,8 @@ module Importer
         base.send(:include, Importer::Adapters::ActiveRecordAdapter)
       elsif defined?(MongoMapper) && (base.include?(MongoMapper::Document) || base.include?(MongoMapper::EmbeddedDocument))
         base.send(:include, Importer::Adapters::MongoMapperAdapter)
+      elsif defined?(DataMapper) && (base.include?(DataMapper::Resource))
+        base.send(:include, Importer::Adapters::DataMapperAdapter)
       else
         raise AdapterError.new("Can't determine adapter for #{base.class} class.")
       end

data/lib/importer/adapters/active_record_adapter.rb

@@ -64,7 +64,7 @@ module Importer
 
               unless object.save
                 imported_object.state             = "invalid_object"
-                imported_object.validation_errors = object.errors.full_messages
+                imported_object.validation_errors = object.errors.full_messages.uniq
               end
 
               imported_object.object = object

data/lib/importer/adapters/data_mapper_adapter.rb

@@ -0,0 +1,104 @@
+module Importer
+  module Adapters
+    # Adapter for DataMapper models
+    #
+    # Usage:
+    #
+    #   class Product
+    #     include DataMapper::Resource
+    #     include Importer
+    #   end
+    #
+    #   Product.import(path_to_xml_or_csv_file)
+    #
+    # It sends the given file to a parser and then imports detected objects.
+    # Instead of simply inserting all detected objects to database, the importer
+    # tries to determine wheter a detected object already exists. If so, the object
+    # is only updated, otherwise a new object is created.
+    #
+    # To change the way how importer checks for existing objects (or to turn off this
+    # behavior completely) override +find_on_import+ method. The default behavior now
+    # is to try to find existing object by detected object's id.
+    #
+    # By default the detected object's attributes hash is literally assigned to a
+    # soon-to-be-saved object. If there is a need for more sophisticated behavior,
+    # simply override +merge_attributes_on_import+ method.
+    module DataMapperAdapter
+
+      class << self
+        def included(base)
+          base.send(:include, InstanceMethods)
+          base.send(:extend,  ClassMethods)
+        end
+      end
+
+      module ClassMethods
+        # The import process is wrapped in a transaction, so if anything goes wrong there is no
+        # harm done.
+        # * +file+ - path to an XML or CVS file, you can also import from other data formats,
+        #   but you also need to provide a custom parser to read it
+        # Possible options:
+        # * +parser+ - by default the parser is determined from file extension, but you can force
+        #   the imported to use another one by passing it's class here
+        # * +import+ - by default importer returns instance of +Import+ class that contains
+        #   detailed report of import process, you can implement your own Import class and force
+        #   the importer to use it by passing it's class here
+        # * +import_options+ - options passed to Import instance on it's initialization
+        def import(file, options = {})
+          import = (options[:import] || Importer::Import).new(options[:import_options])
+          parser =  options[:parser] || Importer::Parser.get_klass(file)
+          data   = parser.run(file)
+
+          transaction do
+            data.each do |attributes|
+              imported_object = import.build_imported_object
+
+              if object = find_on_import(import, attributes)
+                imported_object.state = "existing_object"
+              else
+                object                = new
+                imported_object.state = "new_object"
+              end
+
+              imported_object.data = attributes
+              object.merge_attributes_on_import(import, attributes)
+
+              unless object.save
+                imported_object.state             = "invalid_object"
+                imported_object.validation_errors = object.errors.full_messages.uniq
+              end
+
+              imported_object.object = object
+
+              import.add_object(imported_object)
+            end
+          end
+
+          import
+        end
+
+        # Determines whether a detected object already exists in database.
+        # By default it tries to find an existing objects by id of the detected one.
+        # Returns the object or nil if it's not found.
+        # Override this method in your model to change that default behavior.
+        # * +import+ - current import
+        # * +attributes+ - detected object's attributes hash
+        def find_on_import(import, attributes)
+          get(attributes["id"])
+        end
+      end
+
+      module InstanceMethods
+        # Merges attributes of a detected object with current object's ones.
+        # By default it simply assigns detected attributes to the object.
+        # Override this method in your model to provide some more sophisticated
+        # behavior.
+        # * +import+ - current import
+        # * +attributes+ - detected object's attributes hash
+        def merge_attributes_on_import(import, attributes)
+          self.attributes = attributes
+        end
+      end
+    end
+  end
+end

data/lib/importer/adapters/mongo_mapper_adapter.rb

@@ -67,7 +67,7 @@ module Importer
 
             unless object.save
               imported_object.state             = "invalid_object"
-              imported_object.validation_errors = object.errors.full_messages
+              imported_object.validation_errors = object.errors.full_messages.uniq
             end
 
             imported_object.object = object

data/test/helper.rb

@@ -4,6 +4,9 @@ require 'shoulda'
 require 'rr'
 require 'active_record'
 require 'mongo_mapper'
+require 'dm-core'
+require 'dm-validations'
+require 'dm-aggregates'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
@@ -12,6 +15,7 @@ require 'importer'
 
 config = YAML::load(IO.read(File.join(File.dirname(__FILE__), 'database.yml')))
 ActiveRecord::Base.establish_connection(config['test'])
+DataMapper.setup(:default, config['test'])
 
 MongoMapper.connection = Mongo::Connection.new('127.0.0.1', 27017)
 MongoMapper.database = 'importer-test'

data/test/importer/adapters/data_mapper_adapter_test.rb

@@ -0,0 +1,129 @@
+require 'helper'
+
+class Importer::Adapters::DataMapperAdapterTest < Test::Unit::TestCase
+  def setup
+    def_class("Product") do
+      include DataMapper::Resource
+      include Importer
+
+      property :id,           DataMapper::Types::Serial
+      property :customid,     String
+      property :name,         String
+      property :description,  String
+      property :price,        Float
+
+      validates_is_number :price
+
+      def self.find_on_import(import, attributes)
+        first(:customid => attributes["customid"])
+      end
+    end
+
+    DataMapper.auto_migrate!
+  end
+
+  def teardown
+    undef_class("Product")
+  end
+
+  context "" do
+    setup do
+      @product = Product.create(:customid => "1", :name => "A pink ball", :description => "Round glass ball.", :price => 86)
+    end
+
+    context "importing objects" do
+      setup do
+        @new_object      = { "name" => "A red hat", "customid"=>"2", "price" => "114.00", "description" => "Party hat." }
+        @existing_object = { "name" => "A black ball", "customid"=>"1", "price" => "86.00", "description" => "Round glass ball." }
+        @invalid_object  = { "name" => "A white ribbon", "customid"=>"3", "price" => "oops", "description" => "A really long one." }
+
+        data = [ @new_object, @existing_object, @invalid_object ]
+
+        stub(Importer::Parser::Xml).run(fixture_file("products.xml")) { data }
+
+        @import = Product.import(fixture_file("products.xml"))
+      end
+
+      should_change("product's name", :from => "A pink ball", :to => "A black ball") { @product.reload.name }
+
+      should_change("products count", :by => 1) { Product.count }
+      should "correctly create new product" do
+        product = Product.last
+
+        assert_equal "A red hat", product.name
+        assert_equal "Party hat.", product.description
+        assert_equal 114, product.price
+        assert_equal "2", product.customid
+      end
+
+      should "correctly summarize the import process" do
+        assert_equal 1, @import.new_imported_objects.size
+        assert_equal 1, @import.existing_imported_objects.size
+        assert_equal 1, @import.invalid_imported_objects.size
+      end
+
+      should "correctly build imported objects" do
+        new_object      = @import.new_imported_objects.first
+        existing_object = @import.existing_imported_objects.first
+        invalid_object  = @import.invalid_imported_objects.first
+
+        assert_equal @new_object,       new_object.data
+        assert_equal 'new_object',      new_object.state
+        assert_equal Product.last,      new_object.object
+
+        assert_equal @existing_object,  existing_object.data
+        assert_equal 'existing_object', existing_object.state
+        assert_equal @product,          existing_object.object
+
+        assert_equal @invalid_object,   invalid_object.data
+        assert_equal 'invalid_object',  invalid_object.state
+        assert_equal ["Price must be a number"], invalid_object.validation_errors
+      end
+    end
+
+    context "when there is exception during import process" do
+      setup do
+        def_class("InvalidProduct", Product) do
+          storage_names[:default] = "products"
+
+          def self.find_on_import(import, attributes)
+            if attributes["customid"] == "3"
+              raise ::Exception.new("An error occured.")
+            else
+              super
+            end
+          end
+        end
+
+        begin
+          InvalidProduct.import(fixture_file("products.xml"))
+        rescue ::Exception => e
+          @exception = e
+        end
+      end
+
+      def teardown
+        undef_class("InvalidProduct")
+      end
+
+      should_not_change("product's name") { @product.reload.name }
+      should_not_change("products count") { InvalidProduct.count }
+      should "propagate exception" do
+        assert_equal "An error occured.", @exception.message
+      end
+    end
+  end
+
+  context "passing import_options to #import method" do
+    setup do
+      stub(Importer::Parser::Xml).run(fixture_file("empty.xml")) { [] }
+
+      @options = { :key => 'value' }
+      @import = Product.import(fixture_file("empty.xml"), :import_options => @options)
+    end
+
+    should "pass the options to Import instance" do
+      assert_equal @options, @import.options
+    end
+  end
+end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 1
-  version: 0.4.1
+  - 2
+  version: 0.4.2
 platform: ruby
 authors: 
 - "Micha\xC5\x82 Szajbe"
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-16 00:00:00 +02:00
+date: 2010-05-17 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -72,7 +72,7 @@ dependencies:
   type: :development
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: thoughtbot-shoulda
+  name: dm-core
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     requirements: 
@@ -80,11 +80,13 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
-        version: "0"
+        - 10
+        - 2
+        version: 0.10.2
   type: :development
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
-  name: sqlite3-ruby
+  name: dm-validations
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
     requirements: 
@@ -92,9 +94,64 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
-        version: "0"
+        - 10
+        - 2
+        version: 0.10.2
   type: :development
   version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: dm-aggregates
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 10
+        - 2
+        version: 0.10.2
+  type: :development
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: do_sqlite3
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 10
+        - 1
+        - 1
+        version: 0.10.1.1
+  type: :development
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: sqlite3-ruby
+  prerelease: false
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id010
 description: Define new objects or modifications of existing ones in external file (xml, csv, etc) and import them to your application. Importer will not only import all the objects but also will give you detailed summary of the import process.
 email: michal.szajbe@gmail.com
 executables: []
@@ -114,6 +171,7 @@ files:
 - importer.gemspec
 - lib/importer.rb
 - lib/importer/adapters/active_record_adapter.rb
+- lib/importer/adapters/data_mapper_adapter.rb
 - lib/importer/adapters/mongo_mapper_adapter.rb
 - lib/importer/import.rb
 - lib/importer/imported_object.rb
@@ -132,6 +190,7 @@ files:
 - test/fixtures/products.xml
 - test/helper.rb
 - test/importer/adapters/active_record_adapter_test.rb
+- test/importer/adapters/data_mapper_adapter_test.rb
 - test/importer/adapters/mongo_mapper_adapter_test.rb
 - test/importer/import_test.rb
 - test/importer/imported_object_test.rb
@@ -172,6 +231,7 @@ test_files:
 - test/factories.rb
 - test/helper.rb
 - test/importer/adapters/active_record_adapter_test.rb
+- test/importer/adapters/data_mapper_adapter_test.rb
 - test/importer/adapters/mongo_mapper_adapter_test.rb
 - test/importer/import_test.rb
 - test/importer/imported_object_test.rb