importu 0.1.0

data/.gitignore

@@ -0,0 +1,3 @@
+Gemfile.lock
+.bundle/
+vendor/bundle

data/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+
+# Specify your gem's dependencies in importu.gemspec
+gemspec

data/README.md

@@ -0,0 +1,163 @@
+## Overview
+Importu is a framework and DSL for simplifying the process of importing
+structured data into your application.  It is also a tool for separating
+import-related business logic from the rest of your code.
+
+Current supported source formats include CSV/TSV, XML and JSON.  It is fairly
+trivial to extend Importu to handle additional formats.  See the
+`lib/importu/importer` directory for implementations of supported importers.
+
+The current version of Importu depends on both ActiveRecord and ActiveSupport,
+which will become optional in a future release.
+
+## Example
+**Please read the tutorial in the
+[import-examples](https://github.com/dhedlund/importu-examples) repository for
+a more complete overview of available features.**
+
+Assuming you have the following data in the file `data.csv`:
+```
+"isbn10","title","author","release_date","pages"
+"0596516177","The Ruby Programming Language","David Flanagan and Yukihiro Matsumoto","Feb 1, 2008","448"
+"1449355978","Computer Science Programming Basics in Ruby","Ophir Frieder, Gideon Frieder and David Grossman","May 1, 2013","188"
+"0596523696","Ruby Cookbook"," Lucas Carlson and Leonard Richardson","Jul 26, 2006","910"
+```
+
+You can create a minimal importer to read the CSV data:
+```ruby
+class BookImporter < Importu::Importer::Csv
+  # fields we expect to find in the CSV file, field order is not important
+  fields :title, :author, :isbn10, :pages, :release_date
+end
+```
+
+And then load that data in your application:
+```ruby
+require 'importu'
+
+filename = File.expand_path('../data.csv', __FILE__)
+importer = BookImporter.new(filename)
+
+# importer.records returns an Enumerable
+importer.records.count # => 3
+importer.records.select {|r| r[:author] =~ /Matsumoto/ }.count # => 1
+importer.records.each do |record|
+  # ...
+end
+
+importer.records.map(&:to_hash)
+```
+
+A more complete example of the book importer above might look like the following:
+```ruby
+require 'importu'
+
+class BookImporter < Importu::Importer::Csv
+  # if you want to define multiple fields with similar rules, use 'fields'
+  # NOTE: ':required => true' is redundant in this example; any defined
+  # fields must have a corresponding column in the source data by default
+  fields :title, :isbn10, :authors, :required => true
+
+  # to mark a field as optional in the source data
+  field :pages, :required => false
+
+  # you can reference the same field multiple times and apply rules
+  # incrementally; this provides a lot of flexibility in describing your
+  # importer rules, such as grouping all the required fields together and
+  # explicitly stating that "these are required"; the importer becomes the
+  # reference document:
+  #
+  # fields :title, :isbn10, :authors, :release_date, :required => true
+  # fields :pages, :required => false
+  #
+  # ...or keep all the rules for that field with that field, whatever makes
+  # sense for your particular use case.
+
+  # if your field is not named the same as the source data, you can use
+  # :label => '...' to reference the correct field, where the label is what
+  # the field is labelled in the source data
+  field :authors, :label => 'author'
+
+  # you can convert fields using one of the built-in converters
+  field :pages, &convert_to(:integer)
+  field :release_date, &convert_to(:date) # date format is guessed
+
+  # some converters allow you to pass additional arguments; in the case of
+  # the date converter, you can pass an explicit format and it will raise an
+  # error if a date is encountered that doesn't match
+  field :release_date, &convert_to(:date, :format => '%b %d, %Y')
+
+  # passing a block to a field definition allows you to add your own logic
+  # for converting data or checking for unexpected values
+  field :authors do
+    value = clean(:authors) # apply :clean converter which strips whitespace
+    authors = value ? value.split(/(?:, )|(?: and )|(?: & )/i) : []
+
+    if authors.none?
+      # ArgumentError will be converted to an Importu::FieldParseError, which
+      # will include the name of the field affected
+      raise ArgumentError, "at least one author is required"
+    end
+
+    authors
+  end
+
+  # abstract fields that are not part of the original data set can be created
+  field :by_matz, :abstract => true do
+    # field conversion rules can reference other fields; the field value is
+    # what would be returned after referenced field's rules have been applied
+    field_value(:authors).include?('Yukihiro Matsumoto')
+  end
+end
+```
+
+A more condensed version of the above, with all the rules grouped into individual field definitions:
+```ruby
+class BookImporter < Importu::Importer::Csv
+  fields :title, :isbn10
+
+  field :authors, :label => 'author' do
+    authors = clean(:authors).to_s.split(/(?:, )|(?: and )|(?: & )/i)
+    raise ArgumentError, "at least one author is required" if authors.none?
+    
+    authors
+  end
+
+  field :pages, :required => false, &convert_to(:integer)
+  field :release_date, &convert_to(:date, :format => '%b %d, %Y') 
+
+  field :by_matz, :abstract => true do
+    field_value(:authors).include?('Yukihiro Matsumoto')
+  end
+end
+```
+
+### Rails / ActiveRecord
+If you define a model in the importer definition and the importer fields are
+named the same as the attributes in your model, Importu can iterate through and
+create or update records for you:
+
+```ruby
+class BookImporter < Importu::Importer::Csv
+  model 'Book'
+
+  # ...
+end
+
+filename = File.expand_path('../data.csv', __FILE__)
+importer = BookImporter.new(filename)
+
+importer.import!
+
+importer.total # => 3
+importer.invalid # => 0
+importer.created # => 3
+importer.updated # => 0
+importer.unchanged # => 0
+
+importer.import!
+
+importer.total # => 3
+importer.created # => 0
+importer.unchanged # => 3
+```

data/importu.gemspec

@@ -0,0 +1,29 @@
+$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
+require 'importu/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'importu'
+  s.version     = Importu::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Daniel Hedlund']
+  s.email       = ['daniel@lincracy.com']
+  s.homepage    = 'https://github.com/lincracy/importu'
+  s.summary     = 'A framework for importing data'
+  s.description = 'Importu is a framework for importing data'
+
+  s.files            = `git ls-files`.split("\n")
+  s.test_files       = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths    = ['lib']
+
+  s.licenses = ['MIT']
+  
+  s.add_dependency 'activesupport', ['>= 3.0.0']
+  s.add_dependency 'activerecord',  ['>= 3.0.0']
+  s.add_dependency 'multi_json',    ['~> 1.0']
+  s.add_dependency 'nokogiri'
+
+  s.add_development_dependency 'bundler',      ['>= 1.0.0']
+  s.add_development_dependency 'rspec',        ['>= 0']
+  s.add_development_dependency 'rdoc',         ['>= 0']
+  s.add_development_dependency 'factory_girl', ['>= 3.5.0']
+end

data/lib/importu.rb

@@ -0,0 +1,12 @@
+module Importu; end
+
+require 'importu/core_ext'
+
+require 'importu/dsl'
+require 'importu/exceptions'
+require 'importu/converters'
+require 'importu/record'
+require 'importu/importer'
+require 'importu/importer/csv'
+require 'importu/importer/json'
+require 'importu/importer/xml'

data/lib/importu/converters.rb

@@ -0,0 +1,82 @@
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/date_time/conversions'
+require 'active_support/concern'
+
+require 'bigdecimal'
+
+module Importu::Converters
+  extend ActiveSupport::Concern
+
+  included do
+    converter :raw do |name,options|
+      definition = definitions[name] \
+        or raise "importer field not defined: #{name}"
+
+      label = definition[:label]
+      raise Importu::MissingField, definition unless data.key?(label)
+      data[label]
+    end
+
+    converter :clean do |name,options|
+      value = convert(name, :raw, options)
+      value.is_a?(String) \
+        ? (value.blank? ? nil : value.strip)
+        : value
+    end
+
+    converter :string do |name,options|
+      convert(name, :clean, options).try(:to_s)
+    end
+
+    converter :integer do |name,options|
+      value = convert(name, :clean, options)
+      value.nil? ? nil : Integer(value)
+    end
+
+    converter :float do |name,options|
+      value = convert(name, :clean, options)
+      value.nil? ? nil : Float(value)
+    end
+
+    converter :decimal do |name,options|
+      value = convert(name, :clean, options)
+      case value
+        when nil then nil
+        when BigDecimal then value
+        when /\A-?\d+(?:\.\d+)?\Z/ then BigDecimal(value)
+        else raise ArgumentError, "invalid decimal value '#{value}'"
+      end
+    end
+
+    converter :boolean do |name,options|
+      value = convert(name, :clean, options)
+      case value
+        when nil then nil
+        when true, 'true', 'yes', '1', 1 then true
+        when false, 'false', 'no', '0', 0 then false
+        else raise ArgumentError, "invalid boolean value '#{value}'"
+      end
+    end
+
+    converter :date do |name,options|
+      if value = convert(name, :clean, options)
+        # TODO: options[:date_format] is deprecated
+        date_format = options[:date_format] || options[:format]
+        date_format \
+          ? Date.strptime(value, date_format)
+          : Date.parse(value)
+      end
+    end
+
+    converter :datetime do |name,options|
+      if value = convert(name, :clean, options)
+        # TODO: options[:date_format] is deprecated
+        date_format = options[:date_format] || options[:format]
+        date_format \
+          ? DateTime.strptime(value, date_format).utc
+          : DateTime.parse(value).utc
+      end
+    end
+
+  end
+end

data/lib/importu/core_ext.rb

@@ -0,0 +1,3 @@
+Dir["#{File.dirname(__FILE__)}/core_ext/*.rb"].sort.each do |path|
+  require "importu/core_ext/#{File.basename(path, '.rb')}"
+end

data/lib/importu/core_ext/array/deep_freeze.rb

@@ -0,0 +1,7 @@
+class Array
+  def deep_freeze
+    return self if frozen?
+    each(&:deep_freeze)
+    freeze
+  end
+end

data/lib/importu/core_ext/deep_freeze.rb

@@ -0,0 +1,3 @@
+require 'importu/core_ext/object/deep_freeze'
+require 'importu/core_ext/array/deep_freeze'
+require 'importu/core_ext/hash/deep_freeze'

data/lib/importu/core_ext/hash/deep_freeze.rb

@@ -0,0 +1,7 @@
+class Hash
+  def deep_freeze
+    return self if frozen?
+    values.each(&:deep_freeze)
+    freeze
+  end
+end

data/lib/importu/core_ext/object/deep_freeze.rb

@@ -0,0 +1,6 @@
+class Object
+  def deep_freeze
+    instance_variables.each{|v| instance_variable_get(v).freeze }
+    freeze
+  end
+end

data/lib/importu/dsl.rb

@@ -0,0 +1,127 @@
+require 'active_support/core_ext/module/delegation'
+require 'active_support/core_ext/hash/deep_dup'
+require 'active_support/core_ext/hash/keys'
+require 'active_support/concern'
+
+require 'importu/core_ext/deep_freeze'
+
+# importer definition examples:
+#   allow_actions :create
+#   allow_actions :create, :update
+#
+#   find_by :id # match against a single field, :id (default)
+#   find_by [:name, :date] # match against multiple fields
+#   find_by :id, [:name, :date] # try name/date combo if no id match
+#   find_by nil # never try to look up records, assume :create
+#   find_by do |record|
+#     scoped.where(:foo => record[:name].downcase)
+#   end
+#
+#   field  :field1, :label => 'Field 1'
+#   fields :field1, :field2, :field3
+#   fields :field1, :field2, convert_to(:integer)
+#   fields :field1, :field2 do |data,definition|
+#     Time.strptime(data[definition[:label]], '%d/%m/%Y')
+#   end
+#
+# allow actions:
+#   :create - if an existing record can't be found, we can create it
+#   :update - if an existing record found, update its attributes
+#
+# field(s) definition options:
+#   :label - header/label/key/element name used in input file (default: field name)
+#   :required - must be present in input file (values can be blank, default: true)
+
+require 'active_support/concern'
+
+module Importu::Dsl
+  extend ActiveSupport::Concern
+
+  included do
+    config_dsl :record_class, :default => Importu::Record
+    config_dsl :model, :description
+    config_dsl :allowed_actions, :default => [:create]
+    config_dsl :finder_fields, :default => [[:id]]
+    config_dsl :definitions, :default => {}
+    config_dsl :preprocessor, :postprocessor
+    config_dsl :converters, :default => {}
+  end
+
+  module ClassMethods
+    def allow_actions(*actions)
+      @allowed_actions = actions
+    end
+
+    def find_by(*field_groups, &block)
+      @finder_fields = block ? [block] : field_groups.map {|g|g&&[*g]}.compact
+    end
+
+    def fields(*fields, &block)
+      block = fields.pop if fields.last.kind_of?(Proc)
+      options = fields.extract_options!.symbolize_keys!
+
+      @definitions ||= definitions.deep_dup
+      fields.compact.each do |field_name|
+        definition = (@definitions[field_name]||{}).merge(options)
+
+        definition[:name] = field_name
+        definition[:label] ||= (options['label'] || field_name).to_s
+        definition[:required] = true unless definition.key?(:required)
+        definition[:create] = true unless definition.key?(:create)
+        definition[:update] = true unless definition.key?(:update)
+
+        definition[:converter] = block if block
+        definition[:converter] ||= converters[:clean]
+
+        @definitions[field_name] = definition
+      end
+
+      return
+    end
+
+    alias_method :field, :fields
+
+    def preprocess(&block)
+      # gets executed just before record converted to object
+      @preprocessor = block
+    end
+
+    def postprocess(&block)
+      # gets executed just after record converted to object
+      @postprocessor = block
+    end
+
+    def converter(name, &block)
+      @converters = converters.merge(name => block)
+    end
+
+    def convert_to(type, options = {})
+      converters[type] # FIXME: raise error if not found?
+    end
+
+    def config_dsl(*methods)
+      options = methods.extract_options!
+      options.assert_valid_keys(:default)
+      default = (options[:default] || nil).deep_freeze
+
+      methods.each do |m|
+        instance_variable_set("@#{m}", default)
+
+        singleton_class.send(:define_method, m) do |*args,&block|
+          if block || !args.empty?
+            val = (block ? instance_eval(&block) : args[0])
+            instance_variable_set("@#{m}", val.deep_freeze)
+          else
+            instance_variable_defined?("@#{m}") \
+              ? instance_variable_get("@#{m}")
+              : superclass.send(m)
+          end
+        end
+      end
+
+      # make dsl methods available to importer instances
+      delegate *methods, :to => :singleton_class
+    end
+  end
+
+end