importu 0.1.0 → 0.2.0

data/lib/importu/dsl.rb

@@ -1,127 +0,0 @@
-require 'active_support/core_ext/module/delegation'
-require 'active_support/core_ext/hash/deep_dup'
-require 'active_support/core_ext/hash/keys'
-require 'active_support/concern'
-
-require 'importu/core_ext/deep_freeze'
-
-# importer definition examples:
-#   allow_actions :create
-#   allow_actions :create, :update
-#
-#   find_by :id # match against a single field, :id (default)
-#   find_by [:name, :date] # match against multiple fields
-#   find_by :id, [:name, :date] # try name/date combo if no id match
-#   find_by nil # never try to look up records, assume :create
-#   find_by do |record|
-#     scoped.where(:foo => record[:name].downcase)
-#   end
-#
-#   field  :field1, :label => 'Field 1'
-#   fields :field1, :field2, :field3
-#   fields :field1, :field2, convert_to(:integer)
-#   fields :field1, :field2 do |data,definition|
-#     Time.strptime(data[definition[:label]], '%d/%m/%Y')
-#   end
-#
-# allow actions:
-#   :create - if an existing record can't be found, we can create it
-#   :update - if an existing record found, update its attributes
-#
-# field(s) definition options:
-#   :label - header/label/key/element name used in input file (default: field name)
-#   :required - must be present in input file (values can be blank, default: true)
-
-require 'active_support/concern'
-
-module Importu::Dsl
-  extend ActiveSupport::Concern
-
-  included do
-    config_dsl :record_class, :default => Importu::Record
-    config_dsl :model, :description
-    config_dsl :allowed_actions, :default => [:create]
-    config_dsl :finder_fields, :default => [[:id]]
-    config_dsl :definitions, :default => {}
-    config_dsl :preprocessor, :postprocessor
-    config_dsl :converters, :default => {}
-  end
-
-  module ClassMethods
-    def allow_actions(*actions)
-      @allowed_actions = actions
-    end
-
-    def find_by(*field_groups, &block)
-      @finder_fields = block ? [block] : field_groups.map {|g|g&&[*g]}.compact
-    end
-
-    def fields(*fields, &block)
-      block = fields.pop if fields.last.kind_of?(Proc)
-      options = fields.extract_options!.symbolize_keys!
-
-      @definitions ||= definitions.deep_dup
-      fields.compact.each do |field_name|
-        definition = (@definitions[field_name]||{}).merge(options)
-
-        definition[:name] = field_name
-        definition[:label] ||= (options['label'] || field_name).to_s
-        definition[:required] = true unless definition.key?(:required)
-        definition[:create] = true unless definition.key?(:create)
-        definition[:update] = true unless definition.key?(:update)
-
-        definition[:converter] = block if block
-        definition[:converter] ||= converters[:clean]
-
-        @definitions[field_name] = definition
-      end
-
-      return
-    end
-
-    alias_method :field, :fields
-
-    def preprocess(&block)
-      # gets executed just before record converted to object
-      @preprocessor = block
-    end
-
-    def postprocess(&block)
-      # gets executed just after record converted to object
-      @postprocessor = block
-    end
-
-    def converter(name, &block)
-      @converters = converters.merge(name => block)
-    end
-
-    def convert_to(type, options = {})
-      converters[type] # FIXME: raise error if not found?
-    end
-
-    def config_dsl(*methods)
-      options = methods.extract_options!
-      options.assert_valid_keys(:default)
-      default = (options[:default] || nil).deep_freeze
-
-      methods.each do |m|
-        instance_variable_set("@#{m}", default)
-
-        singleton_class.send(:define_method, m) do |*args,&block|
-          if block || !args.empty?
-            val = (block ? instance_eval(&block) : args[0])
-            instance_variable_set("@#{m}", val.deep_freeze)
-          else
-            instance_variable_defined?("@#{m}") \
-              ? instance_variable_get("@#{m}")
-              : superclass.send(m)
-          end
-        end
-      end
-
-      # make dsl methods available to importer instances
-      delegate *methods, :to => :singleton_class
-    end
-  end
-
-end

data/lib/importu/importer/csv.rb

@@ -1,52 +0,0 @@
-require 'csv'
-
-class Importu::Importer::Csv < Importu::Importer
-  def initialize(infile, options = {})
-    super
-
-    @csv_options = {
-      :headers        => true,
-      :return_headers => true,
-      :write_headers  => true,
-      :skip_blanks    => true,
-    }.merge(options[:csv_options]||{})
-
-    @reader = ::CSV.new(@infile, @csv_options)
-    @header = @reader.readline
-    @data_pos = @infile.pos
-  end
-
-  def records
-    @infile.pos = @data_pos
-    Enumerator.new do |yielder|
-      @reader.each do |row|
-        yielder.yield record_class.new(self, row.to_hash, row)
-      end
-    end
-  end
-
-  def import_record(record, finder_scope, &block)
-    begin
-      super
-    rescue Importu::MissingField => e
-      # if one record missing field, all are, major error
-      raise Importu::InvalidInput, "missing required field: #{e.message}"
-    rescue Importu::InvalidRecord => e
-      write_error(record.raw_data, e.message)
-    end
-  end
-
-
-  private
-
-  def write_error(data, msg)
-    unless @writer
-      @writer = ::CSV.new(outfile, @csv_options)
-      @header['_errors'] = '_errors'
-      @writer << @header
-    end
-
-    data['_errors'] = msg
-    @writer << data
-  end
-end

data/lib/importu/importer/json.rb

@@ -1,45 +0,0 @@
-require 'multi_json'
-
-class Importu::Importer::Json < Importu::Importer
-  def initialize(infile, options = {})
-    super
-
-    begin
-      infile.rewind
-      @reader = MultiJson.load(infile.read)
-    rescue MultiJson::DecodeError => e
-      raise Importu::InvalidInput, e.message
-    end
-  end
-
-  def import!(finder_scope = nil, &block)
-    result = super
-    outfile.write(JSON.pretty_generate(@error_records)) if @invalid > 0
-    result
-  end
-
-  def records(&block)
-    enum = Enumerator.new do |yielder|
-      @reader.each_with_index do |data,idx|
-        yielder.yield record_class.new(self, data, data)
-      end
-    end
-  end
-
-  def import_record(record, finder_scope, &block)
-    begin
-      super
-    rescue Importu::InvalidRecord => e
-      write_error(record.raw_data, e.message)
-    end
-  end
-
-
-  private
-
-  def write_error(data, msg)
-    @error_records ||= []
-    @error_records << data.merge('_errors' => msg)
-  end
-
-end

data/lib/importu/importer/xml.rb

@@ -1,55 +0,0 @@
-require 'nokogiri'
-
-class Importu::Importer::Xml < Importu::Importer
-  config_dsl :records_xpath
-
-  def initialize(infile, options = {})
-    super
-
-    xml_options = {}.merge(options[:xml_options]||{})
-    if reader.errors.any?
-      raise Importu::InvalidInput, reader.errors.join("\n")
-    end
-  end
-
-  def reader
-    @reader ||= Nokogiri::XML(infile)
-  end
-
-  def import!(finder_scope = nil, &block)
-    reader.xpath('//_errors').remove
-    result = super
-    outfile.write(reader) if @invalid > 0
-    result
-  end
-
-  def records
-    Enumerator.new do |yielder|
-      reader.xpath(records_xpath).each do |xml|
-        data = Hash[xml.elements.map {|e| [e.name, e.content]}]
-        yielder.yield record_class.new(self, data, xml)
-      end
-    end
-  end
-
-  def import_record(record, finder_scope, &block)
-    begin
-      super
-      record.raw_data.remove
-    rescue Importu::InvalidRecord => e
-      add_xml_record_error(record.raw_data, e.message)
-    end
-  end
-
-
-  private
-
-  def add_xml_record_error(xml, text)
-    unless node = xml.xpath('./_errors').first
-      node = Nokogiri::XML::Node.new '_errors', reader
-      xml.add_child(node)
-    end
-    node.content = text + ','
-  end
-
-end

data/spec/factories/importer.rb

@@ -1,12 +0,0 @@
-FactoryGirl.define do
-  factory :importer, :class => Importu::Importer do
-    initialize_with do
-      Importu::Importer.new(infile, options)
-    end
-
-    ignore do
-      infile { StringIO.new }
-      options { Hash.new }
-    end
-  end
-end

data/spec/factories/importer_record.rb

@@ -1,13 +0,0 @@
-FactoryGirl.define do
-  factory :importer_record, :class => Importu::Record do
-    initialize_with do
-      Importu::Record.new(importer, data, raw_data)
-    end
-
-    ignore do
-      importer { build(:importer) }
-      data { Hash.new }
-      raw_data { Hash.new }
-    end
-  end
-end

data/spec/factories/json_importer.rb

@@ -1,14 +0,0 @@
-FactoryGirl.define do
-  factory :json_importer, :class => Importu::Importer::Json do
-    initialize_with do
-      infile = StringIO.new(data) if data
-      Importu::Importer::Json.new(infile, options)
-    end
-
-    ignore do
-      data nil # string version of input file
-      infile { StringIO.new("[]") }
-      options { Hash.new }
-    end
-  end
-end

data/spec/factories/xml_importer.rb

@@ -1,12 +0,0 @@
-FactoryGirl.define do
-  factory :xml_importer, :class => Importu::Importer::Xml do
-    initialize_with do
-      Importu::Importer::Xml.new(infile, options)
-    end
-
-    ignore do
-      infile { StringIO.new("<r/>") }
-      options { Hash.new }
-    end
-  end
-end

data/spec/lib/importu/dsl_spec.rb

@@ -1,26 +0,0 @@
-require 'spec_helper'
-
-describe Importu::Importer do
-  describe "::record_class" do
-    it "returns Importu::Record by default" do
-      Importu::Importer.record_class.should eq Importu::Record
-    end
-
-    it "can be overridden globally" do
-      custom_record_class = Class.new(Importu::Record)
-      orig = Importu::Importer.record_class
-      Importu::Importer.record_class custom_record_class
-      Importu::Importer.record_class.should eq custom_record_class
-      Importu::Importer.record_class orig
-    end
-
-    it "can be overridden in a subclass" do
-      custom_record_class = Class.new(Importu::Record)
-      klass = Class.new(Importu::Importer) do
-        record_class custom_record_class
-      end
-
-      klass.record_class.should eq custom_record_class
-    end
-  end
-end

data/spec/lib/importu/importer/json_spec.rb

@@ -1,37 +0,0 @@
-require 'spec_helper'
-
-describe Importu::Importer::Json do
-  subject(:importer) { build(:json_importer, :data => data) }
-
-  context "input file is blank" do
-    let(:data) { "" }
-
-    it "raises an InvalidInput exception" do
-      expect { importer }.to raise_error Importu::InvalidInput
-    end
-  end
-
-  context "non-array root elements" do
-    %w({}, "foo", 3, 3.7, false, nil).each do |data|
-      it "raises InvalidInput exception if root is #{data}" do
-        expect { build(:json_importer, :data => "") }.to raise_error Importu::InvalidInput
-      end
-    end
-  end
-
-  context "input file is []" do
-    let(:data) { "[]" }
-
-    it "treats file as having 0 records" do
-      importer.records.should have(0).items
-    end
-  end
-
-  context "input file is [{},{}]" do
-    let(:data) { "[{},{}]" }
-
-    it "treats file as having 2 records" do
-      importer.records.should have(2).items
-    end
-  end
-end

data/spec/lib/importu/importer/xml_spec.rb

@@ -1,14 +0,0 @@
-require 'spec_helper'
-
-describe Importu::Importer::Xml do
-  subject(:importer) { build(:xml_importer, :infile => infile) }
-
-  context "input file is blank" do
-    let(:infile) { StringIO.new }
-
-    it "raises an InvalidInput exception" do
-      expect { importer }.to raise_error Importu::InvalidInput
-    end
-  end
-
-end