importu 0.1.0

data/lib/importu/exceptions.rb

@@ -0,0 +1,34 @@
+module Importu
+  class ImportuException < StandardError
+    def name
+      self.class.name[/[^:]+$/]
+    end
+  end
+
+  class InvalidInput < ImportuException; end
+
+  class InvalidRecord < ImportuException
+    attr_reader :validation_errors
+
+    def initialize(message = nil, validation_errors = nil)
+      @validation_errors = validation_errors
+      super(message)
+    end
+  end
+
+  class FieldParseError < InvalidRecord; end
+  class DuplicateRecord < InvalidRecord; end
+
+  class MissingField < InvalidRecord
+    attr_reader :definition
+
+    def initialize(definition)
+      @definition = definition
+    end
+
+    def message
+      field = definition[:label] || definition[:name]
+      "missing field \"#{field}\" from source data"
+    end
+  end
+end

data/lib/importu/importer.rb

@@ -0,0 +1,119 @@
+require 'active_record/errors'
+
+class Importu::Importer
+  attr_reader :options, :infile, :outfile, :validation_errors
+  attr_reader :total, :invalid, :created, :updated, :unchanged
+
+  include Importu::Dsl
+  include Importu::Converters
+
+  def initialize(infile, options = {})
+    @options = options
+    @total = @invalid = @created = @updated = @unchanged = 0
+    @validation_errors = Hash.new(0) # counter for each validation error
+
+    @infile = infile.respond_to?(:readline) ? infile : File.open(infile, 'rb')
+  end
+
+  def records
+    [].to_enum # implement in a subclass
+  end
+
+  def outfile
+    @outfile ||= Tempfile.new('import', Rails.root.join('tmp'), 'wb+')
+  end
+
+  def import!(finder_scope = nil, &block)
+    # if a scope is passed in, that scope becomes the starting scope used by
+    # the finder, otherwise the model's default scope is used).
+
+    finder_scope ||= model_class.scoped
+    records.each {|r| import_record(r, finder_scope, &block) }
+  end
+
+  def result_msg
+    msg = <<-END.strip_heredoc
+      Total:     #{@total}
+      Created:   #{@created}
+      Updated:   #{@updated}
+      Invalid:   #{@invalid}
+      Unchanged: #{@unchanged}
+    END
+
+    if @validation_errors.any?
+      msg << "\nValidation Errors:\n"
+      msg << @validation_errors.map {|e,c| "  - #{e}: #{c}" }.join("\n")
+    end
+
+    msg
+  end
+
+
+  protected
+
+  def model_class
+    @model_class ||= model.constantize
+  end
+
+  def import_record(record, finder_scope, &block)
+    begin
+      object = find(finder_scope, record) || model_class.new
+      action = object.new_record? ? :create : :update
+      check_duplicate(object) if action == :update
+
+      case ([action] - allowed_actions).first
+        when :create then raise Importu::InvalidRecord, "#{model} not found"
+        when :update then raise Importu::InvalidRecord, "existing #{model} found"
+      end
+
+      record.assign_to(object, action, &block)
+
+      case record.save!
+        when :created   then @created   += 1
+        when :updated   then @updated   += 1
+        when :unchanged then @unchanged += 1
+      end
+
+    rescue Importu::InvalidRecord => e
+      if errors = e.validation_errors
+        # convention: assume data-specific error messages put data inside parens, e.g. 'Dupe record found (sysnum 5489x)'
+        errors.each {|error| @validation_errors[error.gsub(/ *\([^)]+\)/,'')] += 1 }     
+      else
+        @validation_errors["#{e.name}: #{e.message}"] += 1
+      end
+
+      @invalid += 1
+      raise
+
+    ensure
+      @total += 1
+    end
+  end
+
+  def find(scope, record)
+    # FIXME: find does not report if it finds more than one record matching
+    # the :find_by conditions passed in.  it just uses the first match for
+    # now.  what should be the correct behaviour?
+
+    field_groups = self.class.finder_fields or return
+    field_groups.each do |field_group|
+      if field_group.respond_to?(:call) # proc
+        object = scope.instance_exec(record, &field_group).first
+      else
+        conditions = Hash[field_group.map {|f| [f, record[f]]}]
+        object = scope.where(conditions).first
+      end
+
+      return object if object
+    end
+    nil
+  end
+
+  def check_duplicate(record)
+    return unless id = record.respond_to?(:id) && record.id
+    if ((@encountered||=Hash.new(0))[id] += 1) > 1
+      raise Importu::DuplicateRecord, 'matches a previously imported record'
+    end
+  end
+
+end

data/lib/importu/importer/csv.rb

@@ -0,0 +1,52 @@
+require 'csv'
+
+class Importu::Importer::Csv < Importu::Importer
+  def initialize(infile, options = {})
+    super
+
+    @csv_options = {
+      :headers        => true,
+      :return_headers => true,
+      :write_headers  => true,
+      :skip_blanks    => true,
+    }.merge(options[:csv_options]||{})
+
+    @reader = ::CSV.new(@infile, @csv_options)
+    @header = @reader.readline
+    @data_pos = @infile.pos
+  end
+
+  def records
+    @infile.pos = @data_pos
+    Enumerator.new do |yielder|
+      @reader.each do |row|
+        yielder.yield record_class.new(self, row.to_hash, row)
+      end
+    end
+  end
+
+  def import_record(record, finder_scope, &block)
+    begin
+      super
+    rescue Importu::MissingField => e
+      # if one record missing field, all are, major error
+      raise Importu::InvalidInput, "missing required field: #{e.message}"
+    rescue Importu::InvalidRecord => e
+      write_error(record.raw_data, e.message)
+    end
+  end
+
+
+  private
+
+  def write_error(data, msg)
+    unless @writer
+      @writer = ::CSV.new(outfile, @csv_options)
+      @header['_errors'] = '_errors'
+      @writer << @header
+    end
+
+    data['_errors'] = msg
+    @writer << data
+  end
+end

data/lib/importu/importer/json.rb

@@ -0,0 +1,45 @@
+require 'multi_json'
+
+class Importu::Importer::Json < Importu::Importer
+  def initialize(infile, options = {})
+    super
+
+    begin
+      infile.rewind
+      @reader = MultiJson.load(infile.read)
+    rescue MultiJson::DecodeError => e
+      raise Importu::InvalidInput, e.message
+    end
+  end
+
+  def import!(finder_scope = nil, &block)
+    result = super
+    outfile.write(JSON.pretty_generate(@error_records)) if @invalid > 0
+    result
+  end
+
+  def records(&block)
+    enum = Enumerator.new do |yielder|
+      @reader.each_with_index do |data,idx|
+        yielder.yield record_class.new(self, data, data)
+      end
+    end
+  end
+
+  def import_record(record, finder_scope, &block)
+    begin
+      super
+    rescue Importu::InvalidRecord => e
+      write_error(record.raw_data, e.message)
+    end
+  end
+
+
+  private
+
+  def write_error(data, msg)
+    @error_records ||= []
+    @error_records << data.merge('_errors' => msg)
+  end
+
+end

data/lib/importu/importer/xml.rb

@@ -0,0 +1,55 @@
+require 'nokogiri'
+
+class Importu::Importer::Xml < Importu::Importer
+  config_dsl :records_xpath
+
+  def initialize(infile, options = {})
+    super
+
+    xml_options = {}.merge(options[:xml_options]||{})
+    if reader.errors.any?
+      raise Importu::InvalidInput, reader.errors.join("\n")
+    end
+  end
+
+  def reader
+    @reader ||= Nokogiri::XML(infile)
+  end
+
+  def import!(finder_scope = nil, &block)
+    reader.xpath('//_errors').remove
+    result = super
+    outfile.write(reader) if @invalid > 0
+    result
+  end
+
+  def records
+    Enumerator.new do |yielder|
+      reader.xpath(records_xpath).each do |xml|
+        data = Hash[xml.elements.map {|e| [e.name, e.content]}]
+        yielder.yield record_class.new(self, data, xml)
+      end
+    end
+  end
+
+  def import_record(record, finder_scope, &block)
+    begin
+      super
+      record.raw_data.remove
+    rescue Importu::InvalidRecord => e
+      add_xml_record_error(record.raw_data, e.message)
+    end
+  end
+
+
+  private
+
+  def add_xml_record_error(xml, text)
+    unless node = xml.xpath('./_errors').first
+      node = Nokogiri::XML::Node.new '_errors', reader
+      xml.add_child(node)
+    end
+    node.content = text + ','
+  end
+
+end

data/lib/importu/record.rb

@@ -0,0 +1,124 @@
+require 'active_support/core_ext/module/delegation'
+
+class Importu::Record
+  attr_reader :importer, :data, :raw_data
+
+  include Enumerable
+
+  delegate :keys, :values, :each, :[], :key?, :to => :record_hash
+  delegate :preprocessor, :postprocessor, :to => :importer
+  delegate :definitions, :converters, :to => :importer
+
+  def initialize(importer, data, raw_data)
+    @importer, @data, @raw_data = importer, data, raw_data
+  end
+
+  def record_hash
+    @record_hash ||= generate_record_hash
+  end
+
+  def to_hash
+    record_hash
+  end
+
+  def convert(name, type, options = {})
+    type, options = type[:to], type if type.kind_of?(Hash)
+    converter = type ? converters[type] : options[:converter] \
+      or raise "converter not found: #{type}"
+
+    # TODO: defining options in field definition is deprecated
+    definition = definitions[name] || {}
+    options = definition.merge(options)
+
+    begin
+      value = instance_exec(name, options, &converter)
+      value.nil? ? options[:default] : value
+
+    rescue Importu::MissingField => e
+      raise e if options[:required]
+      options[:default]
+
+    rescue ArgumentError => e
+      # conversion of field value most likely failed
+      raise Importu::FieldParseError, "#{name}: #{e.message}"
+    end
+  end
+
+  def field_value(name, options = {})
+    definition = definitions[name] \
+      or raise "importer field not defined: #{name}"
+
+    convert(name, nil, definition.merge(options))
+  end
+
+  def assign_to(object, action, &block)
+    @object, @action = object, action
+
+    instance_eval(&preprocessor) if preprocessor
+    instance_exec(object, record_hash, &block) if block
+
+    # filter out any fields we're not allowed to copy for this action
+    allowed_fields = definitions.select {|n,d| d[action] }.keys
+    concrete_fields = definitions.reject {|n,d| d[:abstract] }.keys
+    field_names = record_hash.keys & allowed_fields & concrete_fields
+
+    unsupported = field_names.reject {|n| object.respond_to?("#{n}=") }
+    if unsupported.any?
+      raise "model does not support assigning fields: #{unsupported.to_sentence}"
+    end
+
+    (record_hash.keys & allowed_fields & concrete_fields).each do |name|
+      if object.respond_to?("#{name}=")
+        object.send("#{name}=", record_hash[name])
+      else
+      end
+    end
+
+    instance_eval(&postprocessor) if postprocessor
+
+    object
+  end
+
+  def save!
+    return :unchanged unless @object.changed?
+
+    begin
+      @object.save!
+      case @action
+        when :create then :created
+        when :update then :updated
+      end
+
+    rescue ActiveRecord::RecordInvalid => e
+      error_msgs = @object.errors.map do |name,message|
+        name = definitions[name][:label] if definitions[name]
+        name == 'base' ? message : "#{name} #{message}"
+      end.join(', ')
+
+      raise Importu::InvalidRecord, error_msgs, @object.errors.full_messages
+    end
+  end
+
+
+  private
+
+  attr_reader :object, :action # needed for exposing to instance_eval'd blocks
+
+  alias_method :record, :record_hash
+
+  def generate_record_hash
+    definitions.inject({}) do |hash,(name,definition)|
+      hash[name.to_sym] = field_value(name)
+      hash
+    end
+  end
+
+  def method_missing(meth, *args, &block)
+    if converters[meth]
+      convert(args[0], meth, args[1]||{}) # convert(name, type, options)
+    else
+      super
+    end
+  end
+
+end

data/lib/importu/version.rb

@@ -0,0 +1,3 @@
+module Importu
+  VERSION = '0.1.0'
+end

data/spec/factories/importer.rb

@@ -0,0 +1,12 @@
+FactoryGirl.define do
+  factory :importer, :class => Importu::Importer do
+    initialize_with do
+      Importu::Importer.new(infile, options)
+    end
+
+    ignore do
+      infile { StringIO.new }
+      options { Hash.new }
+    end
+  end
+end