active_import 0.0.2

data/lib/active_import/version.rb

@@ -0,0 +1,3 @@
+module ActiveImport
+  VERSION = "0.0.2"
+end

data/lib/generators/active_import/model_converter_generator.rb

@@ -0,0 +1,25 @@
+require 'rails/generators'
+
+module ActiveImport
+  module Generators
+    class ModelConverterGenerator < Rails::Generators::NamedBase
+      class_option :converter_name, :type => :string, :default => nil, :desc => "Names the converter file, defaults to the model name"
+
+      def self.source_root
+        @source_root ||= File.join(File.dirname(__FILE__), 'templates')
+      end
+
+      def create_files
+        Dir.mkdir("app/model_converters") unless File.directory?("app/model_converters")
+        @model_name = file_name
+        @class_name = class_name
+        @model =  eval(@class_name)
+        @attributes = @model.attribute_names
+        @converter_name = options.converter_name || @class_name
+        template 'model_converter.rb.erb', "app/model_converters/#{@converter_name.underscore}_converter.rb"
+        Dir.mkdir("db/active_import") unless File.directory?("db/active_import")
+        template 'data.csv.erb', "db/active_import/#{@converter_name.underscore}.csv"
+      end
+    end
+  end
+end

data/lib/generators/active_import/templates/data.csv.erb

@@ -0,0 +1 @@
+<%= CSV.generate { |csv| csv << eval("#{@class_name}.attribute_names") - ["id", "created_at", "updated_at"] } %>

data/lib/generators/active_import/templates/model_converter.rb.erb

@@ -0,0 +1,45 @@
+# ImportConverter created by ActiveImport
+
+class <%= @converter_name.camelize %>Converter < ActiveImport::ModelConverter
+    def setup 
+<% @attributes.each do |attribute| -%>
+<% unless ["id", "created_at", "updated_at"].include? attribute -%>
+        add_column :<%= attribute %>, :match => "<%= attribute %>", :type => :<%= @model.columns_hash[attribute].type %>, :mandatory => true 
+<% end -%>
+<% end -%>
+    end
+
+    def before
+    end
+
+    def <%= @model_name %>_attributes
+        attributes = {}
+
+        [<%= [].tap {|list| (@attributes - ["id", "created_at", "updated_at"]).each {|attr| list << ":#{attr}"}}.join(", ") %>].each do |attribute|
+            attributes[attribute] = converted_values[attribute]
+        end
+
+        return attributes
+    end
+
+    def save
+        # The following line prevents nil values coming from the import overwriting the model attributes
+        remove_nil_from_converted_values
+
+        # For create only
+        <%= @model_name %> = <%= @class_name %>.new
+        # For create or update, use the following instead and change the attribute name as required
+        # <%= @model_name %> = <%= @class_name %>.find_or_create_by_id(converted_values[:id])
+
+        <%= @model_name %>.update_attributes <%= @model_name %>_attributes
+
+        return <%= @model_name %>
+    end
+
+    def after
+    end
+
+    def report
+        ""
+    end
+end

data/lib/tasks/active_import_tasks.rake

@@ -0,0 +1,139 @@
+require "active_import"
+require "colorize"
+
+def import_file_with_path file
+  f = file.strip
+  unless /^\//.match(f)
+    f = File.join(Rails.root, "db", "active_import", f)
+  end
+  f
+end
+
+def process_converter_options options_string
+  converter_options = {}
+  if options_string
+    options = options_string.split(";")
+    options.each do |option|
+      s = option.split("=")
+      converter_options[s[0].strip] = s[1].strip
+    end
+  end
+  converter_options
+end
+
+def process_import_options
+  converter = ENV["converter"] || ENV["CONVERTER"]
+  converter_model = "#{converter.camelize}Converter"
+  options = ENV["converter_options"] || ENV["CONVERTER_OPTIONS"]
+  converter_options = process_converter_options options
+
+  puts "Converter Model: #{converter_model.cyan}"
+  puts "Converter Options: #{converter_options.to_s.cyan}"
+  file = ENV["file"] || ENV["FILE"]
+  file ||= "#{converter.underscore}.csv"
+  data_file = import_file_with_path file
+
+  if File.exists?(data_file)
+    puts "Found import file: #{data_file}".green
+  else
+    puts "File doesn't exist': #{data_file}".red
+  end
+  model_converter = eval("#{converter_model}.new")
+  model_converter.options = converter_options
+
+  return {:data_file => data_file, :model_converter => model_converter}
+end
+
+def parse(import, model_converter)
+  model_converter.before
+  import.parse do |import_row, converter, row_number, total_rows|
+    model_converter.process_values import_row
+    model = model_converter.save
+    if model.respond_to?("errors")
+      print "#{row_number.to_s.cyan}: "
+      if model.errors.empty?
+        puts "OK".green
+      else
+        model.errors.full_messages.each { |message| puts message.red }
+      end
+    end
+  end
+  model_converter.after
+  puts model_converter.report
+end
+
+def seed_from_file(seed_file_name)
+  seed_file = File.open(seed_file_name, "r")
+  seed_file.each_line do |seed_line|
+    unless (/^#/.match(seed_line))
+      seed_info = seed_line.split("|")
+      data_file_name = seed_info[0]
+      converter_name = seed_info[1]
+
+      unless data_file_name.nil? || converter_name.nil?
+        data_file_name = import_file_with_path data_file_name
+
+        if (File.exists? data_file_name)
+          converter_name.strip!
+          converter_options = process_converter_options seed_info[2]
+
+          puts "Importing from: #{data_file_name.cyan}"
+          puts "Using converter: #{converter_name.cyan}"
+          puts "Options: #{converter_options.to_s.cyan}"
+
+          model_converter = eval("#{converter_name.camelize}Converter.new")
+          model_converter.options = converter_options
+
+          import = nil
+          extension = File.extname(data_file_name).downcase.strip
+          case extension
+            when ".xls"
+              puts "Excel file detected".yellow
+              import = ActiveImport::ImportExcel.new(model_converter, data_file_name)
+            when ".xlsx"
+              puts "Excel(X) file detected".yellow
+              import = ActiveImport::ImportExcel.new(model_converter, data_file_name)
+            when ".csv"
+              puts "CSV file detected".yellow
+              import = ActiveImport::ImportCsv.new(model_converter, data_file_name)
+            else
+              puts "File type cannot be processed".red
+          end
+          parse(import, model_converter) unless import.nil?
+        else
+          puts "Skipping non existing file: #{data_file_name}".red
+        end
+      end
+    end
+  end
+end
+
+namespace :active_import do
+  desc "Load csv file into a model using a model converter"
+  task :csv => :environment do
+    result = process_import_options
+    unless result[:data_file].nil?
+      import = ActiveImport::ImportCsv.new(result[:model_converter], result[:data_file])
+      parse(import, result[:model_converter])
+    end
+  end
+  desc "Load excel file into a model using a model converter"
+  task :excel => :environment do
+    result = process_import_options
+    unless result[:data_file].nil?
+      import = ActiveImport::ImportExcel.new(result[:model_converter], result[:data_file])
+      parse(import, result[:model_converter])
+    end
+  end
+  desc "Seed a list of import files"
+  task :seed => :environment do
+    set = ENV["set"] || ENV["SET"] || ::Rails.env
+    seed_file_name = File.join(Rails.root, "db", "active_import", "#{set}.seed")
+    if File.exists?(seed_file_name)
+      puts "Seeding from file: '#{seed_file_name.yellow}'"
+      seed_from_file seed_file_name
+    else
+      puts "Cannot find seed file: '#{seed_file_name}'".red
+    end
+  end
+end

data/test/active_import_test.rb

@@ -0,0 +1,7 @@
+require 'test_helper'
+
+class ActiveImportTest < ActiveSupport::TestCase
+  test "truth" do
+    assert_kind_of Module, ActiveImport
+  end
+end

data/test/dummy/Rakefile

@@ -0,0 +1,7 @@
+#!/usr/bin/env rake
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Dummy::Application.load_tasks

data/test/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,9 @@
+// This is a manifest file that'll be compiled into including all the files listed below.
+// Add new JavaScript/Coffee code in separate files in this directory and they'll automatically
+// be included in the compiled file accessible from http://example.com/assets/application.js
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// the compiled file.
+//
+//= require jquery
+//= require jquery_ujs
+//= require_tree .

data/test/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,7 @@
+/*
+ * This is a manifest file that'll automatically include all the stylesheets available in this directory
+ * and any sub-directories. You're free to add application-wide styles to this file and they'll appear at
+ * the top of the compiled file, but it's generally better to create a new file per style scope.
+ *= require_self
+ *= require_tree . 
+*/

data/test/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,3 @@
+class ApplicationController < ActionController::Base
+  protect_from_forgery
+end

data/test/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/test/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Dummy</title>
+  <%= stylesheet_link_tag    "application" %>
+  <%= javascript_include_tag "application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/test/dummy/config.ru

@@ -0,0 +1,4 @@
+# This file is used by Rack-based servers to start the application.
+
+require ::File.expand_path('../config/environment',  __FILE__)
+run Dummy::Application

data/test/dummy/config/application.rb

@@ -0,0 +1,45 @@
+require File.expand_path('../boot', __FILE__)
+
+require 'rails/all'
+
+Bundler.require
+require "active_import"
+
+module Dummy
+  class Application < Rails::Application
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration should go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded.
+
+    # Custom directories with classes and modules you want to be autoloadable.
+    # config.autoload_paths += %W(#{config.root}/extras)
+
+    # Only load the plugins named here, in the order given (default is alphabetical).
+    # :all can be used as a placeholder for all plugins not explicitly named.
+    # config.plugins = [ :exception_notification, :ssl_requirement, :all ]
+
+    # Activate observers that should always be running.
+    # config.active_record.observers = :cacher, :garbage_collector, :forum_observer
+
+    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
+    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
+    # config.time_zone = 'Central Time (US & Canada)'
+
+    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
+    # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
+    # config.i18n.default_locale = :de
+
+    # Configure the default encoding used in templates for Ruby 1.9.
+    config.encoding = "utf-8"
+
+    # Configure sensitive parameters which will be filtered from the log file.
+    config.filter_parameters += [:password]
+
+    # Enable the asset pipeline
+    config.assets.enabled = true
+
+    # Version of your assets, change this if you want to expire all your assets
+    config.assets.version = '1.0'
+  end
+end
+

data/test/dummy/config/boot.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+gemfile = File.expand_path('../../../../Gemfile', __FILE__)
+
+if File.exist?(gemfile)
+  ENV['BUNDLE_GEMFILE'] = gemfile
+  require 'bundler'
+  Bundler.setup
+end
+
+$:.unshift File.expand_path('../../../../lib', __FILE__)

data/test/dummy/config/database.yml

@@ -0,0 +1,25 @@
+# SQLite version 3.x
+#   gem install sqlite3
+#
+#   Ensure the SQLite 3 gem is defined in your Gemfile
+#   gem 'sqlite3'
+development:
+  adapter: sqlite3
+  database: db/development.sqlite3
+  pool: 5
+  timeout: 5000
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  adapter: sqlite3
+  database: db/test.sqlite3
+  pool: 5
+  timeout: 5000
+
+production:
+  adapter: sqlite3
+  database: db/production.sqlite3
+  pool: 5
+  timeout: 5000

data/test/dummy/config/environment.rb

@@ -0,0 +1,5 @@
+# Load the rails application
+require File.expand_path('../application', __FILE__)
+
+# Initialize the rails application
+Dummy::Application.initialize!

data/test/dummy/config/environments/development.rb

@@ -0,0 +1,30 @@
+Dummy::Application.configure do
+  # Settings specified here will take precedence over those in config/application.rb
+
+  # In the development environment your application's code is reloaded on
+  # every request.  This slows down response time but is perfect for development
+  # since you don't have to restart the web server when you make code changes.
+  config.cache_classes = false
+
+  # Log error messages when you accidentally call methods on nil.
+  config.whiny_nils = true
+
+  # Show full error reports and disable caching
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+
+  # Don't care if the mailer can't send
+  config.action_mailer.raise_delivery_errors = false
+
+  # Print deprecation notices to the Rails logger
+  config.active_support.deprecation = :log
+
+  # Only use best-standards-support built into browsers
+  config.action_dispatch.best_standards_support = :builtin
+
+  # Do not compress assets
+  config.assets.compress = false
+
+  # Expands the lines which load the assets
+  config.assets.debug = true
+end

data/test/dummy/config/environments/production.rb

@@ -0,0 +1,60 @@
+Dummy::Application.configure do
+  # Settings specified here will take precedence over those in config/application.rb
+
+  # Code is not reloaded between requests
+  config.cache_classes = true
+
+  # Full error reports are disabled and caching is turned on
+  config.consider_all_requests_local       = false
+  config.action_controller.perform_caching = true
+
+  # Disable Rails's static asset server (Apache or nginx will already do this)
+  config.serve_static_assets = false
+
+  # Compress JavaScripts and CSS
+  config.assets.compress = true
+
+  # Don't fallback to assets pipeline if a precompiled asset is missed
+  config.assets.compile = false
+
+  # Generate digests for assets URLs
+  config.assets.digest = true
+
+  # Defaults to Rails.root.join("public/assets")
+  # config.assets.manifest = YOUR_PATH
+
+  # Specifies the header that your server uses for sending files
+  # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache
+  # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx
+
+  # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
+  # config.force_ssl = true
+
+  # See everything in the log (default is :info)
+  # config.log_level = :debug
+
+  # Use a different logger for distributed setups
+  # config.logger = SyslogLogger.new
+
+  # Use a different cache store in production
+  # config.cache_store = :mem_cache_store
+
+  # Enable serving of images, stylesheets, and JavaScripts from an asset server
+  # config.action_controller.asset_host = "http://assets.example.com"
+
+  # Precompile additional assets (application.js, application.css, and all non-JS/CSS are already added)
+  # config.assets.precompile += %w( search.js )
+
+  # Disable delivery errors, bad email addresses will be ignored
+  # config.action_mailer.raise_delivery_errors = false
+
+  # Enable threaded mode
+  # config.threadsafe!
+
+  # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
+  # the I18n.default_locale when a translation can not be found)
+  config.i18n.fallbacks = true
+
+  # Send deprecation notices to registered listeners
+  config.active_support.deprecation = :notify
+end