active_admin_csv_import 1.1.0 → 1.1.1

data/Rakefile

@@ -1,40 +1,15 @@
-#!/usr/bin/env rake
-begin
-  require 'bundler/setup'
-rescue LoadError
-  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
-end
-begin
-  require 'rdoc/task'
-rescue LoadError
-  require 'rdoc/rdoc'
-  require 'rake/rdoctask'
-  RDoc::Task = Rake::RDocTask
-end
-
-RDoc::Task.new(:rdoc) do |rdoc|
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title    = 'ActiveAdminCsvImport'
-  rdoc.options << '--line-numbers'
-  rdoc.rdoc_files.include('README.rdoc')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
-
-APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
-load 'rails/tasks/engine.rake'
-
-
-
+require "bundler"
+require 'rake'
+Bundler.setup
 Bundler::GemHelper.install_tasks
 
-require 'rake/testtask'
 
-Rake::TestTask.new(:test) do |t|
-  t.libs << 'lib'
-  t.libs << 'test'
-  t.pattern = 'test/**/*_test.rb'
-  t.verbose = false
+def cmd(command)
+  puts command
+  raise unless system command
 end
 
+# require File.expand_path('../spec/support/detect_rails_version', __FILE__)
 
-task :default => :test
+# Import all our rake tasks
+FileList['tasks/**/*.rake'].each { |task| import task }

data/app/assets/javascripts/active_admin_csv_import/import_csv.js

@@ -20,8 +20,12 @@ $(document).ready(function() {
     $($file).wrap('<form>').closest('form').get(0).reset();
     $($file).unwrap();
 
+    // Clear progress
     var progress = $("#csv-import-progress");
     progress.text("");
+
+    // Clear validation errors
+    $("#csv-import-errors").html("");
   };
 
   // listen for the file to be submitted
@@ -33,6 +37,7 @@ $(document).ready(function() {
     // create the dataset in the usual way but specifying file attribute
     var dataset = new recline.Model.Dataset({
       file: $file.files[0],
+      delimiter: import_csv_delimiter,
       backend: 'csv'
     });
 
@@ -44,19 +49,22 @@ $(document).ready(function() {
         return;
       }
 
-      // Re-query to get all records. Otherwise recline.js defaults to just 100.
-      dataset.query({size: dataset.recordCount});
+      // Re-query to just one record so we can check the headers.
+      dataset.query({
+        size: 1
+      });
 
       // Check whether the CSV's columns match up with our data model.
       // import_csv_fields is passed in from Rails in import_csv.html.erb
-      var wanted_columns = import_csv_fields;
+      var required_columns = import_csv_required_columns;
+      var all_columns = import_csv_columns;
       var csv_columns = _.pluck(data.records.first().fields.models, "id");
       var normalised_csv_columns = _.map(csv_columns, function(name) {
         return _.underscored(name);
       });
 
       // Check we have all the columns we want.
-      var missing_columns = _.difference(wanted_columns, normalised_csv_columns);
+      var missing_columns = _.difference(required_columns, normalised_csv_columns);
       var missing_columns_humanized = _.map(missing_columns, function(name) {
         return _.humanize(name);
       });
@@ -71,48 +79,90 @@ $(document).ready(function() {
         var succeeded = 0;
         var i = 0;
 
-        _.each(data.records.models, function(record) {
+        // Batch rows into 50s to send to the server
+        // var n = 50;
+        // var batchedModels = _.groupBy(data.records.models, function(a, b) {
+        //   return Math.floor(b / n);
+        // });
+
+        var rowIndex = 0;
+
+        var postRows = function(dataset, index) {
 
-          // Add a gap between each post to give the server
-          // room to breathe
-          setTimeout(function () {
+          // Query the data set for the next batch of rows.
+          dataset.query({
+            size: 100,
+            from: 100 * index
+          });
+          var currentBatch = data.records.models;
+
+          var records_data = [];
+
+          // Construct the payload for each row
+          _.each(currentBatch, function(record, i) {
 
             // Filter only the attributes we want, and normalise column names.
             var record_data = {};
-            record_data[import_csv_resource_name] = {};
+            record_data["_row"] = rowIndex;
 
+            // Construct the resource params with underscored keys
             _.each(_.pairs(record.attributes), function(attr) {
               var underscored_name = _.underscored(attr[0]);
-              if (_.contains(wanted_columns, underscored_name)) {
-                record_data[import_csv_resource_name][underscored_name] = attr[1];
+              if (_.contains(all_columns, underscored_name)) {
+
+                var value = attr[1];
+
+                // Prevent null values coming through as string 'null' so allow_blank works on validations.
+                if (value === null) {
+                  value = '';
+                }
+
+                record_data[underscored_name] = value;
               }
             });
 
-            $.post(
+            records_data.push(record_data);
+            rowIndex = rowIndex + 1;
+          });
+
+          var payload = {};
+          payload[import_csv_resource_name] = records_data;
+
+          // Send this batch to the server.
+          $.post(
             import_csv_path,
-            record_data,
-            function(data) {
-              succeeded = succeeded + 1;
-            }).always(function() {
-              loaded = loaded + 1;
+            payload,
+            null,
+            'json')
+            .always(function(xhr) {
+              loaded = loaded + currentBatch.length;
               progress.text("Progress: " + loaded + " of " + total);
 
-              if (loaded == total) {
-                progress.html("Done. Imported " + total + " records, " + succeeded + " succeeded.");
-                if (redirect_path) {
-                  progress.html(progress.text() + " <a href='"+redirect_path +"'>Click to continue.</a>");
+              // Show validation errors for any failed rows.
+              $("#csv-import-errors").append(xhr.responseText);
+
+              if (xhr.status == 200) {
+                if (loaded == total) {
+                  progress.html("Done. Imported " + total + " records.");
+                  if (redirect_path) {
+                    progress.html(progress.text() + " <a href='" + redirect_path + "'>Click to continue.</a>");
+                  }
+                } else {
+                  // Send the next batch!
+                  postRows(dataset, index + 1);
                 }
+              } else {
+                alert("Import interrupted. The server could not be reached or encountered an error.");
               }
-            });
 
-          }, 100 * i);
+            });
+        };
 
-          i++;
 
-        });
+        postRows(dataset, 0);
       }
 
       clearFileInput();
     });
   });
-});
+});

data/app/views/admin/csv/_import_csv_failed_row.html.erb

@@ -0,0 +1,5 @@
+<% @failures.each do |fail| %>
+  <div>
+    Row <%= (fail[:row_number].to_i + 2).to_s %>: <%= fail[:resource].errors.full_messages.to_sentence %>
+  </div>
+<% end %>

data/app/views/admin/csv/_instructions.html.erb

@@ -0,0 +1,9 @@
+<h3>
+  <%= t('.import_from_csv_file',
+        resource_name: active_admin_config.resource_name.to_s.pluralize.humanize,
+        default: 'Import %{resource_name} from a CSV File' ) %>
+</h3>
+<ul>
+  <li><%= t('.file_type', default: "Save a CSV as 'Windows Comma Separated' from Excel.") %></li>
+  <li><%= t('.must_have_headings', columns: @required_columns.to_sentence, default: "Your CSV should have the following column headings: %{columns}. The order doesn't matter.") %></li>
+</ul>

data/app/views/admin/csv/import_csv.html.erb

@@ -1,18 +1,17 @@
 <%= javascript_include_tag "active_admin_csv_import/import_csv" %>
 
-<h3>Import <%= active_admin_config.resource_name.pluralize.humanize %> from a CSV File<h3>
-<ul>
-  <li>Save a CSV as 'Windows Comma Separated' from Excel.</li>
-  <li>Your CSV should have the following column headings: <%= @fields.map(&:to_s).map(&:humanize).to_sentence %>. The order doesn't matter.</li>
-  <li>If a record already exists a duplicate will be created.</li>
-</ul>
+<%= render partial: 'admin/csv/instructions' %>
+
 <input id="csv-file-input" type="file">
 
 <div id="csv-import-progress"></div>
+<div id="csv-import-errors"></div>
 
 <script type="text/javascript">
-  var import_csv_fields = <%= @fields.to_json.html_safe %>;
+  var import_csv_columns = <%= @columns.to_json.html_safe %>;
+  var import_csv_required_columns = <%= @required_columns.to_json.html_safe %>;
   var import_csv_path = <%= @post_path.to_json.html_safe %>;
-  var import_csv_resource_name = <%= active_admin_config.resource_name.underscore.to_json.html_safe %>;
+  var import_csv_resource_name = <%= active_admin_config.resource_name.to_s.underscore.to_json.html_safe %>;
   var redirect_path = <%= @redirect_path.to_json.html_safe %>;
-</script>
+  var import_csv_delimiter = <%= @delimiter ? @delimiter.to_json.html_safe : "null" %>;
+</script>

data/lib/active_admin_csv_import/dsl.rb

@@ -2,23 +2,83 @@ module ActiveAdminCsvImport
   module DSL
 
     def csv_importable(options={})
+
       action_item :only => :index do
         link_to "Import #{active_admin_config.resource_name.to_s.pluralize}", :action => 'import_csv'
       end
 
+      # Shows the form and JS which accepts a CSV file, parses it and posts each row to the server.
       collection_action :import_csv do
-        @fields     = options[:columns] ||= active_admin_config.resource_class.columns.map(&:name) - ["id", "updated_at", "created_at"]
+        @columns           = options[:columns] ||= active_admin_config.resource_class.columns.map(&:name) - ["id", "updated_at", "created_at"]
+        @required_columns  = options[:required_columns] ||= @columns
 
         @post_path  = options[:path].try(:call)
-        @post_path ||= collection_path
+        @post_path ||= collection_path + "/import_rows"
 
         @redirect_path = options[:redirect_path].try(:call)
         @redirect_path ||= collection_path
 
+        @delimiter = options[:delimiter]
+
         render "admin/csv/import_csv"
       end
 
+      # Receives each row and saves it
+      collection_action :import_rows, :method => :post do
+
+        @failures = []
+
+        csv_resource_params.values.each do |row_params|
+          row_params = row_params.with_indifferent_access
+          row_number = row_params.delete('_row')
+
+          resource = existing_row_resource(options[:import_unique_key], row_params)
+          resource ||= active_admin_config.resource_class.new()
+
+          if not update_row_resource(resource, row_params)
+            @failures << {
+              row_number: row_number,
+              resource: resource
+            }
+          end
+        end
+
+        render :partial => "admin/csv/import_csv_failed_row", :status => 200
+      end
+
+      # Rails 4 Strong Parameters compatibility and backwards compatibility.
+      controller do
+        def csv_resource_params
+          # I don't think this will work any more.
+          if respond_to?(:permitted_params)
+            permitted_params[active_admin_config.resource_class.name.underscore]
+          else
+            params[active_admin_config.resource_class.name.underscore]
+          end
+        end
+
+        # Updates a resource with the CSV data and saves it.
+        #
+        # @param resource [Object] the object to save
+        # @param params [Hash] the CSV row
+        # @return [Boolean] Success
+        def update_row_resource(resource, params)
+          resource.attributes = params
+          resource.save
+        end
+
+        def existing_row_resource(lookup_column, params)
+          return unless lookup_column
+
+          finder_method = "find_by_#{lookup_column}".to_sym
+          value = params[lookup_column]
+          return unless value.present?
+
+          return active_admin_config.resource_class.send(finder_method, value)
+        end
+      end
+
     end
 
   end
-end
+end

data/lib/active_admin_csv_import/version.rb

@@ -1,3 +1,3 @@
 module ActiveAdminCsvImport
-  VERSION = "1.1.0"
+  VERSION = "1.1.1"
 end

data/vendor/assets/javascripts/recline/backend.csv.js

@@ -27,7 +27,7 @@ this.recline.Backend.CSV = this.recline.Backend.CSV || {};
           useMemoryStore: true
         });
       };
-      reader.onerror = function (e) {
+      reader.onerror = function(e) {
         alert('Failed to load file. Code: ' + e.target.error.code);
       };
       reader.readAsText(dataset.file, encoding);
@@ -56,14 +56,14 @@ this.recline.Backend.CSV = this.recline.Backend.CSV || {};
   //
   // @return The CSV parsed as an array
   // @type Array
-  // 
+  //
   // @param {String} s The string to convert
   // @param {Object} options Options for loading CSV including
   // 	@param {Boolean} [trim=false] If set to True leading and trailing whitespace is stripped off of each non-quoted field as it is imported
   //	@param {String} [separator=','] Separator for CSV file
   // Heavily based on uselesscode's JS CSV parser (MIT Licensed):
   // http://www.uselesscode.org/javascript/csv/
-  my.parseCSV= function(s, options) {
+  my.parseCSV = function(s, options) {
     // Get rid of any trailing \n
     s = chomp(s);
 
@@ -71,7 +71,6 @@ this.recline.Backend.CSV = this.recline.Backend.CSV || {};
     var trm = (options.trim === false) ? false : true;
     var separator = options.separator || ',';
     var delimiter = options.delimiter || '"';
-
     var cur = '', // The character we are currently processing.
       inQuote = false,
       fieldQuoted = false,
@@ -81,12 +80,12 @@ this.recline.Backend.CSV = this.recline.Backend.CSV || {};
       i,
       processField;
 
-    processField = function (field) {
+    processField = function(field) {
       if (fieldQuoted !== true) {
         // If field is empty set to null
         if (field === '') {
           field = null;
-        // If the field was not quoted and we are trimming fields, trim it
+          // If the field was not quoted and we are trimming fields, trim it
         } else if (trm === true) {
           field = trim(field);
         }
@@ -106,7 +105,7 @@ this.recline.Backend.CSV = this.recline.Backend.CSV || {};
 
       // If we are at a EOF or EOR
       if (inQuote === false && (cur === separator || cur === "\n")) {
-	field = processField(field);
+        field = processField(field);
         // Add the current field to the current row
         row.push(field);
         // If this is EOR append row to output and flush row
@@ -155,14 +154,14 @@ this.recline.Backend.CSV = this.recline.Backend.CSV || {};
     // contains a comma double quote or a newline
     // it needs to be quoted in CSV output
     rxNeedsQuoting = /^\s|\s$|,|"|\n/,
-    trim = (function () {
+    trim = (function() {
       // Fx 3.1 has a native trim function, it's about 10x faster, use it if it exists
       if (String.prototype.trim) {
-        return function (s) {
+        return function(s) {
           return s.trim();
         };
       } else {
-        return function (s) {
+        return function(s) {
           return s.replace(/^\s*/, '').replace(/\s*$/, '');
         };
       }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_admin_csv_import
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-16 00:00:00.000000000 Z
+date: 2013-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
-  requirement: &70194897983060 !ruby/object:Gem::Requirement
+  requirement: &70147043295120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '3.1'
   type: :runtime
   prerelease: false
-  version_requirements: *70194897983060
+  version_requirements: *70147043295120
 - !ruby/object:Gem::Dependency
   name: railties
-  requirement: &70194897981980 !ruby/object:Gem::Requirement
+  requirement: &70147043307680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: '3.1'
   type: :runtime
   prerelease: false
-  version_requirements: *70194897981980
+  version_requirements: *70147043307680
 description: CSV import for Active Admin capable of handling CSV files too large to
   import via direct file upload to Heroku
 email:
@@ -42,6 +42,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - app/assets/javascripts/active_admin_csv_import/import_csv.js
+- app/views/admin/csv/_import_csv_failed_row.html.erb
+- app/views/admin/csv/_instructions.html.erb
 - app/views/admin/csv/import_csv.html.erb
 - lib/active_admin_csv_import/dsl.rb
 - lib/active_admin_csv_import/engine.rb
@@ -83,3 +85,4 @@ signing_key:
 specification_version: 3
 summary: Add CSV import to Active Admin
 test_files: []
+has_rdoc: