active_import 0.0.2

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2011 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,56 @@
+= ActiveImport
+
+THIS DOCUMENTATION IS RATHER INCOMPLETE, I AM WORKING ON IT.
+
+== Installation
+
+Simple add the following to your Gemfile
+
+    gem 'active_import'
+
+Then run:
+
+    bundle install
+
+== Quick Example
+
+    rails g active_import:model_converter User
+
+This will create an model converter in the directory app/model_converters. You can read through this import file to see how the import works.
+
+This also creates a default data file in db/active_import. This will be the CSV used for this converter.
+
+== Usage
+
+There are three Rake tasks that allow you to use a converter on a file:
+
+    rake active_import:csv                   Load csv file into a model using a model converter
+    rake active_import:excel                 Load excel file into a model using a model converter
+    rake active_import:seed                  Seed a list of import files
+
+=== Examples
+
+rake active_import:csv user.csv CONVERTER=User CONVERTER_OPTIONS="give_admin_access=true"
+
+In this case the file in db/active_import/user.csv would be run through the converter UserConverter. The options specified are available within the converter. In this case @options["give_admin_access"] will evaluate to true.
+
+== Seeding
+
+Seeding allows you to import several files through different model converters in a single command. It involves the creation of a .seed file. Each file goes on a single line and the options are separated by pipe symbols.
+
+=== Example
+
+    user_info/user.xls | User | give_admin_access=false,send_email=true
+    user_info/roles.xls | Role
+    user_info/permissions.xls | UserPermission
+
+Note this will get the data files from a subdirectory: db/active_import/user_info. Also the top conversion uses options but as they are optional, the following two do not.
+
+== The Converter Class
+
+=== Column Setup
+
+==== Mandatory Columns
+==== Column Types
+
+=== Custom Type Conversions

data/Rakefile

@@ -0,0 +1,37 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+begin
+  require 'rdoc/task'
+rescue LoadError
+  require 'rdoc/rdoc'
+  require 'rake/rdoctask'
+  RDoc::Task = Rake::RDocTask
+end
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ActiveImport'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task :default => :test

data/lib/active_import.rb

@@ -0,0 +1,13 @@
+require "active_import/import_csv"
+require "active_import/import_excel"
+require 'active_import/model_converter'
+
+module ActiveImport
+  class Railtie < ::Rails::Railtie
+    railtie_name :active_import
+
+    rake_tasks do
+      load "tasks/active_import_tasks.rake"
+    end
+  end
+end

data/lib/active_import/import_csv.rb

@@ -0,0 +1,99 @@
+module ActiveImport
+  if RUBY_VERSION =~ /^1.9/
+    require 'csv'
+  else
+    require 'fastercsv'
+  end
+
+  class ImportCsv
+    attr_reader :data_file, :converter, :estimated_rows
+
+    def initialize(model_converter, data_file)
+      @converter = model_converter
+      @data_file = data_file
+    end
+
+    def all_headers_found(headers)
+      mappings = @converter.columns
+
+      @missing_headers_mandatory = []
+      @missing_headers_optional = []
+      found_at_least_one = false
+
+      mappings.each_pair do |column_name, mapping|
+        if headers[column_name].nil?
+          if mapping[:mandatory]
+            @missing_headers_mandatory << column_name
+          else
+            @missing_headers_optional << column_name
+          end
+        else
+          found_at_least_one = true
+        end
+      end
+      if found_at_least_one
+        @missing_headers_optional.each { |field| puts "Missing optional column #{field}".yellow }
+        @missing_headers_mandatory.each { |field| puts "Missing mandatory column #{field}".red }
+      end
+      return false unless @missing_headers_mandatory.empty?
+      true
+    end
+
+    def parse(&block)
+      # Get an estimate of the number of rows in the file
+      f = File.open(data_file)
+      @estimated_rows = f.readlines.size - 1
+      f.close
+
+      column_mappings = @converter.columns
+
+      headers = {}
+      header = true
+      data_count = 0
+      row_number = 0
+      csv_class = nil
+      if RUBY_VERSION =~ /^1.9/
+        csv_class = CSV
+        puts "Using built in Ruby 1.9 CSV parser".cyan
+      else
+        csv_class = FasterCSV
+        puts "Using FasterCSV parser".cyan
+      end
+
+      csv_class.foreach(@data_file) do |row|
+        row_number += 1
+        if (header)
+          column = 0
+          row.each do |column_value|
+            column += 1
+            column_mappings.each do |column_name, mapping|
+              match = mapping[:match] || mapping[:header]
+              if /#{match}/.match(column_value)
+                puts "Found header for #{column_name} at column #{column}".green
+                if (headers[column_name].nil?)
+                  headers[column_name] = column
+                else
+                  puts "Found duplicate header '#{column_name}' on columns #{column} and #{headers[column_name]}.".red
+                end
+              end
+            end
+          end
+          unless all_headers_found(headers)
+            puts "Missing headers".red
+            break
+          end
+          header = false
+        else
+          import_row = {}
+          headers.each_pair do |name, column|
+            value = row[column - 1].to_s
+            import_row[name] = value
+          end
+          data_count += 1
+          yield import_row, @converter, row_number, @estimated_rows
+        end
+      end
+      puts "Imported #{data_count} rows".cyan
+    end
+  end
+end

data/lib/active_import/import_excel.rb

@@ -0,0 +1,110 @@
+module ActiveImport
+  require "roo"
+  class ImportExcel
+    attr_reader :data_file, :converter, :estimated_rows
+
+    def initialize(model_converter, data_file)
+      @converter = model_converter
+      @data_file = data_file
+    end
+
+    def all_headers_found(headers)
+      mappings = @converter.columns
+
+      @missing_headers_mandatory = []
+      @missing_headers_optional = []
+      found_at_least_one = false
+
+      mappings.each_pair do |column_name, mapping|
+        if headers[column_name].nil?
+          if mapping[:mandatory]
+            @missing_headers_mandatory << column_name
+          else
+            @missing_headers_optional << column_name
+          end
+        else
+          found_at_least_one = true
+        end
+      end
+      if found_at_least_one
+        @missing_headers_optional.each { |field| puts "Missing optional column #{field.to_s.yellow}" }
+        @missing_headers_mandatory.each { |field| puts "Missing mandatory column #{field.to_s.red}" }
+      end
+      return false unless @missing_headers_mandatory.empty?
+      true
+    end
+
+    def find_excel_header_row(e)
+      column_mappings = @converter.columns
+
+      e.sheets.each do |sheet|
+        puts "Looking for the header row in sheet #{sheet}".cyan
+        e.default_sheet = sheet
+        e.first_row.upto(e.last_row) do |row|
+          headers = {}
+          (e.first_column..e.last_column).each do |column|
+            column_mappings.each do |column_name, mapping|
+              match = mapping[:match] || mapping[:header]
+              if /#{match}/.match(e.cell(row, column).to_s)
+                puts "Found header for #{column_name.to_s.green} at column #{column} at row #{row}"
+                if (headers[column_name].nil?)
+                  headers[column_name] = column
+                else
+                  puts "Found duplicate header '#{column_name.to_s.red}' on columns #{column} and #{headers[column_name]}."
+                end
+              end
+            end
+          end
+
+          if all_headers_found(headers)
+            puts "All headers found on row #{row}".green
+            return {:row => row, :sheet => sheet, :headers => headers}
+          end
+        end
+      end
+      return nil
+    end
+
+    def parse(&block)
+      column_mappings = @converter.columns
+
+      excelx = false
+      case File.extname(data_file).downcase
+        when ".xls"
+          e = Excel.new(data_file)
+        when ".xlsx"
+          excelx = true
+          e = Excelx.new(data_file)
+      end
+
+      result = find_excel_header_row(e)
+
+      if result.nil?
+        puts "Could not find header row.".red
+        return
+      end
+
+      e.default_sheet = result[:sheet]
+      header_row = result[:row]
+      headers = result[:headers]
+
+      # Loop through the data
+      puts "Reading data from row #{header_row + 1} to #{e.last_row}"
+      @estimated_rows = e.last_row - header_row;
+      row_number = 0
+      (header_row + 1).upto(e.last_row) do |row|
+        row_number += 1
+        import_row = {}
+        headers.each_pair do |name, column|
+          if excelx
+            value = e.cell(row, column).to_s
+          else
+            value = e.cell(row, column).to_s
+          end
+          import_row[name] = value
+        end
+        yield import_row, @converter, row_number, @estimated_rows
+      end
+    end
+  end
+end

data/lib/active_import/model_converter.rb

@@ -0,0 +1,119 @@
+module ActiveImport
+  class ModelConverter
+    attr_accessor :options, :columns
+    # TODO: columns should be a reader only once old code has been fixed up
+    attr_reader :converted_values, :raw_values
+
+    def initialize
+      @columns = {}
+      setup
+    end
+
+    def setup
+    end
+
+    def add_column column_name, options
+      @columns[column_name] = options
+    end
+
+    def before
+    end
+
+    def after
+    end
+
+    def print_columns
+      @columns.each_pair do |name, column|
+        puts name
+      end
+    end
+
+    def csv_headers()
+      selected_columns = @columns
+
+      [].tap do |o|
+        selected_columns.each_value do |column|
+          o << (column[:header] || column[:match])
+        end
+      end.to_csv.html_safe
+    end
+
+    def csv_values(values)
+      selected_columns = @columns
+
+      [].tap do |o|
+        selected_columns.each_key do |column|
+          o << values[column]
+        end
+      end.to_csv.html_safe
+    end
+
+    def process_values(values)
+      @raw_values = values
+      @converted_values = convert_attributes(values)
+    end
+
+    def remove_nil_from_converted_values
+      @converted_values.delete_if { |k, v| v.nil? }
+    end
+
+    def remove_blank_from_converted_values
+      @converted_values.delete_if { |k, v| v.to_s.blank? }
+    end
+
+    def convert_attributes(values)
+      cv = {}
+      @columns.each_pair do |name, column|
+        cv[name] = convert_attribute(name, values)
+      end
+      cv
+    end
+
+    def convert_string(value)
+      value = value.to_i if (value.to_i == value.to_f) if /^\s*[\d]+(\.0+){0,1}\s*$/.match(value.to_s)
+      return nil if value.to_s.blank? || value.to_s.nil?
+      value.to_s
+    end
+
+    def convert_clean_string(value)
+      value = value.to_i if (value.to_i == value.to_f) if /^\s*[\d]+(\.0+){0,1}\s*$/.match(value.to_s)
+      value = value.gsub(/[^A-Za-z0-9 \.,\?'""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~]/, '').strip if value.is_a?(String)
+      return nil if value.to_s.blank? || value.to_s.nil?
+      value.to_s
+    end
+
+    def convert_boolean value
+      /^y|t/.match(value.strip.downcase) ? true : false
+    end
+
+    def convert_date s
+      return nil if (s.nil? || s.blank?)
+      return Date.strptime(s, "%d/%m/%y") if /^[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2}$/.match(s)
+      begin
+      result = Date.parse(s)
+      rescue
+        puts "Could not parse date ".red + "'#{s}'"
+      end
+
+      return result
+    end
+
+    def report
+      ""
+    end
+
+    private
+
+    def convert_attribute(attribute, values)
+      return nil if values[attribute].nil?
+      conversion_function = "convert_#{columns[attribute][:type].to_s}"
+      value = values[attribute]
+      if self.respond_to? conversion_function
+        value = eval("#{conversion_function} value")
+      end
+      value
+    end
+
+
+  end
+end