importer 0.3.0

data/lib/importer/import/simple.rb

@@ -0,0 +1,57 @@
+module Importer
+  module Import
+    # Simple import summary. It's not stored in database (as with +ActiveRecord+ import).
+    #
+    # Attributes:
+    # * +new_objects_count+ - number of new objects created during the import
+    # * +existing_objects_count+ - number of objects modified during the import
+    # * +invalid_objects_count+ - number of objects that couldn't have been imported
+    # * +workflow_state+ - import may be in one of three states: ready, started or
+    #   finished. The state changes during the import process.
+    class Simple
+      attr_reader :state, :new_objects_count, :existing_objects_count, :invalid_objects_count, :imported_objects
+
+      class << self
+        def create
+          import = new
+          import.save
+          import
+        end
+      end
+
+      def initialize
+        @state                  = "ready"
+        @new_objects_count      = 0
+        @existing_objects_count = 0
+        @invalid_objects_count  = 0
+        @imported_objects       = []
+      end
+
+      def start!
+        @state = "started"
+      end
+
+      def finish!
+        @state = "finished"
+      end
+
+      def add_object(imported_object)
+        case imported_object.state
+        when "new_object"
+          @new_objects_count += 1
+        when "existing_object"
+          @existing_objects_count += 1
+        when "invalid_object"
+          @invalid_objects_count += 1
+        end
+        @imported_objects << imported_object
+      end
+
+      def save
+        true
+      end
+
+      alias_method :save!, :save
+    end
+  end
+end

data/lib/importer/imported_object.rb

@@ -0,0 +1,7 @@
+module Importer
+  module ImportedObject
+    def self.get_klass(import)
+      import.class.to_s.sub("Importer::Import", "Importer::ImportedObject").constantize
+    end
+  end
+end

data/lib/importer/imported_object/active_record.rb

@@ -0,0 +1,41 @@
+require 'active_record'
+
+module Importer
+  module ImportedObject
+    # ActiveRecord model that stores detailed information of imported objects in
+    # imported_objects database table.
+    #
+    # belongs_to :import - reference to import instance
+    # belongs_to :object - reference to imported object
+    #
+    # Attributes:
+    # * +data+ - object's detected attributes hash
+    # * +validation_errors+ - object's validation errors hash.
+    class ActiveRecord < ::ActiveRecord::Base
+      set_table_name "imported_objects"
+
+      named_scope :new_objects,       :conditions => { :state => "new_object" }
+      named_scope :existing_objects,  :conditions => { :state => "existing_object" }
+      named_scope :invalid_objects,   :conditions => { :state => "invalid_object" }
+
+      belongs_to  :import, :class_name => "Importer::Import::ActiveRecord"
+      belongs_to  :object, :polymorphic => true
+
+      after_save    :increment_counter
+      after_destroy :decrement_counter
+
+      serialize :data
+      serialize :validation_errors
+
+      protected
+
+      def increment_counter
+        import.increment!("#{state}s_count") if import
+      end
+
+      def decrement_counter
+        import.decrement!("#{state}s_count") if import
+      end
+    end
+  end
+end

data/lib/importer/imported_object/simple.rb

@@ -0,0 +1,25 @@
+module Importer
+  module ImportedObject
+    # Simple imported object details. It's not stored in database (as with
+    # +ActiveRecord+ imported object).
+    #
+    # Attributes:
+    # * +data+ - object's detected attributes hash
+    # * +validation_errors+ - object's validation errors hash
+    class Simple
+      attr_reader   :import
+      attr_accessor :state, :object, :data, :validation_errors
+
+      def initialize(attributes = {})
+        raise ArgumentError.new(":import attribute is required.") unless attributes[:import]
+        @import = attributes[:import]
+      end
+
+      def save
+        import.add_object(self)
+      end
+
+      alias_method :save!, :save
+    end
+  end
+end

data/lib/importer/parser.rb

@@ -0,0 +1,22 @@
+module Importer
+  class ParserNotFoundError < ::Exception; end
+
+  # Determines the parser needed to parse given +file+ basing on +file+ extension.
+  # Return Xml parser for .xml files, Csv parser for .csv file and so on.
+  module Parser
+    def self.get_klass(file)
+      extension = File.extname(file)[1..-1]
+
+      if extension
+        klass = extension.camelize
+
+        if Importer::Parser.const_defined?(klass.to_sym)
+          klass = "Importer::Parser::#{klass}".constantize
+          return klass
+        end
+      end
+
+      raise Importer::ParserNotFoundError.new("Can't find #{klass} parser.")
+    end
+  end
+end

data/lib/importer/parser/base.rb

@@ -0,0 +1,31 @@
+module Importer
+  module Parser
+    # Extend this class if you want to provide a custom parser.
+    # You only need to implement +run+ instance method in subclasses.
+    class Base
+
+      class << self
+        # Creates parser instance and processes the +file+
+        def run(file)
+          parser = new(file)
+          parser.run
+          parser.data
+        end
+      end
+
+      attr_reader :data
+
+      def initialize(file)
+        @file = file
+        @data = []
+      end
+
+      # This method must be implemented in subclasses.
+      # It is meant to process input @file, and store the results in @data instance
+      # variable.
+      def run
+        raise Exception.new("This method must be implemented in subclasses.")
+      end
+    end
+  end
+end

data/lib/importer/parser/csv.rb

@@ -0,0 +1,24 @@
+require 'fastercsv'
+
+# CSV parser
+# It uses fastercsv lib to parse the files.
+module Importer
+  module Parser
+    class Csv < Base
+      def run
+        @data = []
+
+        data = FasterCSV.read(@file, :skip_blanks => true)
+
+        unless data.empty?
+          attributes = data.shift
+          @data = data.map do |values|
+            Hash[*attributes.zip(values).flatten]
+          end
+        end
+
+        @data
+      end
+    end
+  end
+end

data/lib/importer/parser/xml.rb

@@ -0,0 +1,25 @@
+require 'crack/xml'
+
+# XML parser
+# It uses crack/xml lib to parse the files.
+module Importer
+  module Parser
+    class Xml < Base
+      def run
+        @data = []
+
+        file = File.new(@file)
+        data = Crack::XML.parse(file)
+
+        root = data.shift[1]
+
+        if root
+          objects = root.shift[1]
+          @data = objects.is_a?(Hash) ? [objects] : objects
+        end
+
+        @data
+      end
+    end
+  end
+end

data/rails/init.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), '..', 'lib', 'importer')

data/rails_generators/importer/importer_generator.rb

@@ -0,0 +1,18 @@
+module Rails
+  module Generator
+    class ImporterGenerator < NamedBase
+      def banner
+        "Usage: #{$0} importer import"
+      end
+
+      def manifest
+        record do |m|
+          time = Time.now
+          m.template "imports_migration.rb", "db/migrate/#{time.strftime('%Y%m%d%H%M%S')}_create_imports.rb"
+          time += 1
+          m.template "imported_objects_migration.rb", "db/migrate/#{time.strftime('%Y%m%d%H%M%S')}_create_imported_objects.rb"
+        end
+      end
+    end
+  end
+end

data/rails_generators/importer/templates/imported_objects_migration.rb

@@ -0,0 +1,19 @@
+class CreateImportedObjects < ActiveRecord::Migration
+  def self.up
+    create_table :imported_objects do |t|
+      t.integer :import_id,         :null => false
+      t.string  :object_type
+      t.integer :object_id
+      t.text    :data
+      t.text    :validation_errors
+      t.string  :state,             :null => false, :limit => 20
+      t.timestamps
+    end
+    add_index :imported_objects, :import_id
+  end
+
+  def self.down
+    remove_index :imported_objects, :import_id
+    drop_table :imported_objects
+  end
+end

data/rails_generators/importer/templates/imports_migration.rb

@@ -0,0 +1,17 @@
+class CreateImports < ActiveRecord::Migration
+  def self.up
+    create_table :imports do |t|
+      t.integer :new_objects_count,      :null => false, :default => 0
+      t.integer :existing_objects_count, :null => false, :default => 0
+      t.integer :invalid_objects_count,  :null => false, :default => 0
+      t.string  :workflow_state,         :null => false, :default => "ready", :limit => 10
+      t.timestamps
+    end
+    add_index :imports, :workflow_state
+  end
+
+  def self.down
+    remove_index :imports, :workflow_state
+    drop_table :imports
+  end
+end

data/test/database.yml

@@ -0,0 +1,3 @@
+test:
+  adapter: sqlite3
+  database: ":memory:"

data/test/factories.rb

@@ -0,0 +1,11 @@
+Factory.define :active_record_import, :class => Importer::Import::ActiveRecord do |f|
+end
+
+Factory.define :active_record_imported_object, :class => Importer::ImportedObject::ActiveRecord do |f|
+end
+
+Factory.define :simple_import, :class => Importer::Import::Simple do |f|
+end
+
+Factory.define :product do |f|
+end

data/test/fixtures/empty.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="utf-8"?>
+<products>
+</products>

data/test/fixtures/product.csv

@@ -0,0 +1,2 @@
+customid,name,description,price
+1,A black ball,Round glass ball.,86.00

data/test/fixtures/product.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<products>
+  <product>
+    <customid>1</customid>
+    <name>A black ball</name>
+    <description>Round glass ball.</description>
+    <price>86.00</price>
+  </product>
+</products>

data/test/fixtures/products.csv

@@ -0,0 +1,4 @@
+customid,name,description,price
+1,A black ball,Round glass ball.,86.00
+2,A red hat,Party hat.,114.00
+3,A white ribbon,A really long one.,oops

data/test/fixtures/products.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<products>
+  <product>
+    <customid>1</customid>
+    <name>A black ball</name>
+    <description>Round glass ball.</description>
+    <price>86.00</price>
+  </product>
+  <product>
+    <customid>2</customid>
+    <name>A red hat</name>
+    <description>Party hat.</description>
+    <price>114.00</price>
+  </product>
+  <product>
+    <customid>3</customid>
+    <name>A white ribbon</name>
+    <description>A really long one.</description>
+    <price>oops</price>
+  </product>
+</products>

data/test/helper.rb

@@ -0,0 +1,71 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'importer'
+
+config = YAML::load(IO.read(File.join(File.dirname(__FILE__), 'database.yml')))
+ActiveRecord::Base.establish_connection(config['test'])
+
+require 'factory_girl'
+require 'factories'
+
+class Test::Unit::TestCase
+  def setup
+    reset_tables
+  end
+
+  def reset_tables
+    ActiveRecord::Base.connection.create_table :products, { :force => true } do |t|
+      t.string  :customid
+      t.string  :name
+      t.string  :description
+      t.decimal :price
+    end
+
+    ActiveRecord::Base.connection.create_table :imports, { :force => true } do |t|
+      t.integer :new_objects_count,       :default => 0
+      t.integer :existing_objects_count,  :default => 0
+      t.integer :invalid_objects_count,   :default => 0
+      t.string  :workflow_state,          :default => "ready"
+    end
+
+    ActiveRecord::Base.connection.create_table :imported_objects, { :force => true } do |t|
+      t.integer :import_id
+      t.string  :object_type
+      t.integer :object_id
+      t.string  :data
+      t.string  :validation_errors
+      t.string  :state
+    end
+  end
+
+  # return path to a fixture file from test/fixtures dir
+  def fixture_file(file)
+    File.expand_path(File.dirname(__FILE__) + "/fixtures/#{file}")
+  end
+end
+
+class Product < ActiveRecord::Base
+  include Importer
+
+  validates_numericality_of :price
+
+  def self.find_on_import(import, attributes)
+    find_by_customid(attributes["customid"])
+  end
+end
+
+class InvalidProduct < Product
+  set_table_name "products"
+
+  def self.find_on_import(import, attributes)
+    if attributes["customid"] == "3"
+      raise ::Exception.new("An error occured.")
+    else
+      super
+    end
+  end
+end