importer 0.3.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Michał Szajbe
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,70 @@
+= importer
+
+Importer is a tool to help you with importing objects to your database from various sources.
+
+=== Links
+
+Repository: http://github.com/szajbus/importer
+
+Docs: http://rdoc.info/projects/szajbus/importer
+
+== Main features:
+
+* can import from XML and CSV formats, but it's possible to add custom parsers
+* reports how many new objects got imported, how many objects was modified and how many objects were invalid
+* can save reports to the database, along with errors for later inspection
+
+== Installation
+
+Install the gem
+
+  gem install importer
+
+Or the plugin
+
+  script/plugin install git://github.com/szajbus/importer.git
+
+Generate the migrations and run them (this step may not be necessary, read below)
+
+  script/generate importer import
+  rake db:migrate
+
+Add to your model
+
+  class Product < ActiveRecord::Base
+    include Importer
+  end
+
+And start importing
+
+  Product.import(path_to_xml_or_csv_file)
+
+This will parse the file and import all products there are defined in it. Import summary will be saved in imports table, it will tell you how many products were created, modified or invalid. Exact information about each product (detected attributes and errors) will be saved in imported_objects table.
+
+If you don't want to save summaries to database you can force the importer to do a simple import with:
+
+  Product.import(path_to_xml_or_csv_file, :import => Importer::Import::Simple.create)
+
+== Customization
+
+You can create your own parser to import from sources other than XML or CSV files. Check the implementation of one of existing parsers to find out how to write your own. Then just pass parser class to import method:
+
+  Product.import(path_to_xml_or_csv_file, :parser => YourCustomParser)
+
+You can also create your custom Import and ImportedObject classes if you need the import summaries in other way than those currently offered (ActiveRecord which stores summaries in database and Simple which does not store them at all). Again check the current implementations to get the idea. To force the importer to use it:
+
+  Product.import(path_to_xml_or_csv_file, :import => YourCustomImport.new)
+
+== Note on Patches/Pull Requests
+
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Michal Szajbe. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,60 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "importer"
+    gem.summary = %Q{Import objects from XML files}
+    gem.description = %Q{Define new objects or modifications of existing ones in XML file and import them to your application.}
+    gem.email = "michal.szajbe@gmail.com"
+    gem.homepage = "http://github.com/szajbus/importer"
+    gem.authors = ["Michał Szajbe"]
+    gem.add_dependency "crack", ">= 0"
+    gem.add_dependency "fastercsv", ">= 0"
+    gem.add_dependency "activerecord", ">= 0"
+    gem.add_dependency "activesupport", ">= 0"
+    gem.add_dependency "workflow", ">= 0"
+    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+    gem.add_development_dependency "factory_girl", ">= 0"
+    gem.add_development_dependency 'sqlite3-ruby'
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "importer #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.3.0

data/importer.gemspec

@@ -0,0 +1,114 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{importer}
+  s.version = "0.3.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Micha\305\202 Szajbe"]
+  s.date = %q{2010-03-20}
+  s.description = %q{Define new objects or modifications of existing ones in XML file and import them to your application.}
+  s.email = %q{michal.szajbe@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "importer.gemspec",
+     "lib/importer.rb",
+     "lib/importer/import.rb",
+     "lib/importer/import/active_record.rb",
+     "lib/importer/import/simple.rb",
+     "lib/importer/imported_object.rb",
+     "lib/importer/imported_object/active_record.rb",
+     "lib/importer/imported_object/simple.rb",
+     "lib/importer/parser.rb",
+     "lib/importer/parser/base.rb",
+     "lib/importer/parser/csv.rb",
+     "lib/importer/parser/xml.rb",
+     "rails/init.rb",
+     "rails_generators/importer/importer_generator.rb",
+     "rails_generators/importer/templates/imported_objects_migration.rb",
+     "rails_generators/importer/templates/imports_migration.rb",
+     "test/database.yml",
+     "test/factories.rb",
+     "test/fixtures/empty.csv",
+     "test/fixtures/empty.xml",
+     "test/fixtures/product.csv",
+     "test/fixtures/product.xml",
+     "test/fixtures/products.csv",
+     "test/fixtures/products.xml",
+     "test/helper.rb",
+     "test/importer/import/active_record_test.rb",
+     "test/importer/import/simple_test.rb",
+     "test/importer/imported_object/active_record_test.rb",
+     "test/importer/imported_object/simple_test.rb",
+     "test/importer/imported_object_test.rb",
+     "test/importer/parser/csv_test.rb",
+     "test/importer/parser/xml_test.rb",
+     "test/importer/parser_test.rb",
+     "test/importer_test.rb"
+  ]
+  s.homepage = %q{http://github.com/szajbus/importer}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Import objects from XML files}
+  s.test_files = [
+    "test/factories.rb",
+     "test/helper.rb",
+     "test/importer/import/active_record_test.rb",
+     "test/importer/import/simple_test.rb",
+     "test/importer/imported_object/active_record_test.rb",
+     "test/importer/imported_object/simple_test.rb",
+     "test/importer/imported_object_test.rb",
+     "test/importer/parser/csv_test.rb",
+     "test/importer/parser/xml_test.rb",
+     "test/importer/parser_test.rb",
+     "test/importer_test.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<crack>, [">= 0"])
+      s.add_runtime_dependency(%q<fastercsv>, [">= 0"])
+      s.add_runtime_dependency(%q<activerecord>, [">= 0"])
+      s.add_runtime_dependency(%q<activesupport>, [">= 0"])
+      s.add_runtime_dependency(%q<workflow>, [">= 0"])
+      s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+      s.add_development_dependency(%q<factory_girl>, [">= 0"])
+      s.add_development_dependency(%q<sqlite3-ruby>, [">= 0"])
+    else
+      s.add_dependency(%q<crack>, [">= 0"])
+      s.add_dependency(%q<fastercsv>, [">= 0"])
+      s.add_dependency(%q<activerecord>, [">= 0"])
+      s.add_dependency(%q<activesupport>, [">= 0"])
+      s.add_dependency(%q<workflow>, [">= 0"])
+      s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+      s.add_dependency(%q<factory_girl>, [">= 0"])
+      s.add_dependency(%q<sqlite3-ruby>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<crack>, [">= 0"])
+    s.add_dependency(%q<fastercsv>, [">= 0"])
+    s.add_dependency(%q<activerecord>, [">= 0"])
+    s.add_dependency(%q<activesupport>, [">= 0"])
+    s.add_dependency(%q<workflow>, [">= 0"])
+    s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+    s.add_dependency(%q<factory_girl>, [">= 0"])
+    s.add_dependency(%q<sqlite3-ruby>, [">= 0"])
+  end
+end
+

data/lib/importer.rb

@@ -0,0 +1,102 @@
+require 'active_support'
+
+require 'importer/import'
+require 'importer/imported_object'
+require 'importer/import/active_record'
+require 'importer/import/simple'
+require 'importer/imported_object/active_record'
+require 'importer/imported_object/simple'
+require 'importer/parser'
+require 'importer/parser/base'
+require 'importer/parser/csv'
+require 'importer/parser/xml'
+
+# Importer is a tool to help you with importing objects to your database from various sources.
+#
+# Usage:
+#
+#   class Product < ActiveRecord::Base
+#     include Importer
+#   end
+#
+#   Product.import(path_to_xml_or_csv_file)
+#
+# Check README.rdoc file for more useful information
+#
+# Author:: Michal Szajbe
+# Copyright:: Copyright (c) 2010 Michal Szajbe
+# License:: check the LICENCE file
+module Importer
+
+  class << self
+    def included(base)
+      base.send(:include, Importer::InstanceMethods)
+      base.send(:extend,  Importer::ClassMethods)
+    end
+  end
+
+  module ClassMethods
+    # The import process is wrapped in a transaction, so if anything goes wrong there is no
+    # harm done.
+    # * +file+ - path to an XML or CVS file, you can also import from other data formats,
+    #   but you also need to provide a custom parser to read it
+    # Possible options:
+    # * +parser+ - by default the parser is determined from file extension, but you can force
+    #   the imported to use another one by passing it's class here
+    # * +import+ - by default importer tries to store import summary in database, so it uses
+    #   ActiveRecord import, to use other import type pass it's instance here
+    def import(file, options = {})
+      import = options[:import] || Importer::Import::ActiveRecord.create
+      parser = options[:parser] || Importer::Parser.get_klass(file)
+      data   = parser.run(file)
+
+      transaction do
+        import.start!
+
+        imported_object_klass = Importer::ImportedObject.get_klass(import)
+
+        data.each do |attributes|
+          imported_object = imported_object_klass.new(:import => import)
+
+          if object = find_on_import(import, attributes)
+            imported_object.state = "existing_object"
+          else
+            object                = new
+            imported_object.state = "new_object"
+          end
+
+          imported_object.data = attributes
+          object.merge_attributes_on_import(import, attributes)
+
+          unless object.save
+            imported_object.state             = "invalid_object"
+            imported_object.validation_errors = object.errors.full_messages
+          end
+
+          imported_object.object = object
+          imported_object.save
+        end
+
+        import.finish!
+      end
+
+      import
+    end
+
+    # Overload +find_on_import+ method to find existing objects while importing an object.
+    # This is used to determine wheter an imported should add a new object or just modify
+    # an existing one. By default it searches records by id.
+    def find_on_import(import, attributes)
+      find_by_id(attributes["id"])
+    end
+  end
+
+  module InstanceMethods
+    # Overload +merge_attributes_on_import+ method if you need the detected attributes
+    # merged to an object in some specific way. By default detected attributes are literally
+    # assigned to an object.
+    def merge_attributes_on_import(import, attributes)
+      self.attributes = attributes
+    end
+  end
+end

data/lib/importer/import.rb

@@ -0,0 +1,5 @@
+module Importer
+  module Import
+
+  end
+end

data/lib/importer/import/active_record.rb

@@ -0,0 +1,37 @@
+require 'active_record'
+require 'workflow'
+
+module Importer
+  module Import
+    # ActiveRecord model that stores import summaries in imports database table.
+    # has_many :+imported_objects+
+    #
+    # Attributes:
+    # * +new_objects_count+ - number of new objects created during the import
+    # * +existing_objects_count+ - number of objects modified during the import
+    # * +invalid_objects_count+ - number of objects that couldn't have been imported
+    # * +workflow_state+ - import may be in one of three states: ready, started or
+    #   finished. The state changes during the import process.
+    class ActiveRecord < ::ActiveRecord::Base
+      set_table_name "imports"
+
+      include Workflow
+
+      workflow do
+        state :ready do
+          event :start,   :transitions_to => :started
+        end
+        state :started do
+          event :finish,  :transitions_to => :finished
+        end
+        state :finished
+      end
+
+      named_scope :ready,    :conditions => { :workflow_state => "ready" }
+      named_scope :started,  :conditions => { :workflow_state => "started" }
+      named_scope :finished, :conditions => { :workflow_state => "finished" }
+
+      has_many :imported_objects, :class_name => "Importer::ImportedObject::ActiveRecord"
+    end
+  end
+end