csv-importer 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 452e11ff0f75dce583a2034fde100790e0327508
-  data.tar.gz: 09230194e20902779a053687122249ea7fd612ef
+  metadata.gz: b856aa990331713775dbd29f7323111622f05a3e
+  data.tar.gz: 5629137af1753b7d58df2f2e3b21ac3e8d3c075a
 SHA512:
-  metadata.gz: 0960dc9c1f6a8880c7ef7bf6ce83f2f38bd0c297be95890a22982fd0e6789f6fbcb4925f078538162c5cfcef0efdcc89bd06b7ff8f74c6e5e40b7b8c6bdb050d
-  data.tar.gz: 419168d5231b7a08a827090fdbed9c0bfe95df8d5d6e5900f024d7e3f736e1671803913ef22ffef3fbba9b8e409bb66852136c70da76b13b1908f179a5d6b6ee
+  metadata.gz: 2882b69b3801caa070731985fc856b0c8d3716102b61e5b1a5f0c5a54411cb1c07b9dbaa963ac07c1f8ed54700e3066131e776459900778eb3fa7f7c8d18da7f
+  data.tar.gz: e02004d36c285f1510ba45b68a83c0d9ed72ffd6ff2f7e8f5905196a4377d501b73f0fc9d166fe13ae8a3be30b40b92e78572e3ac50e389cd89b63cedbca18a6

data/README.md

@@ -3,7 +3,7 @@
 Importing a CSV file is easy to code until real users attempt to import
 real data.
 
-CSVImporter aims to handle validations, column mapping, actual import
+CSVImporter aims to handle validations, column mapping, import
 and reporting.
 
 [![Build
@@ -12,10 +12,28 @@ Status](https://travis-ci.org/BrewhouseTeam/csv-importer.svg)](https://travis-ci
 Climate](https://codeclimate.com/github/BrewhouseTeam/csv-importer/badges/gpa.svg)](https://codeclimate.com/github/BrewhouseTeam/csv-importer)
 [![Test
 Coverage](https://codeclimate.com/github/BrewhouseTeam/csv-importer/badges/coverage.svg)](https://codeclimate.com/github/BrewhouseTeam/csv-importer/coverage)
+[![Gem
+Version](https://badge.fury.io/rb/csv-importer.svg)](http://badge.fury.io/rb/csv-importer)
 
-## Usage
+## Rationale
+
+Importing CSV files seems easy until you deal with *real* users uploading
+their *real* CSV file. You then have to deal with ASCII-8BIT formats,
+missing columns, empty rows, malformed headers, wild separators, etc.
+Reporting progress and errors to the end-user is also key for a good
+experience.
+
+I went through this many times so I decided to build CSV Importer to
+save us a lot of trouble.
+
+
+CSV Importer provides:
+
+* a DSL to define the mapping between CSV columns and your model
+* good reporting to the end user
+* support for wild encodings and CSV formats.
 
-**This is still a work in progress**
+## Usage tldr;
 
 Define your CSVImporter:
 
@@ -23,12 +41,12 @@ Define your CSVImporter:
 class ImportUserCSV
   include CSVImporter
 
-  model User
+  model User # an active record like model
 
   column :email, to: ->(email) { email.downcase }, required: true
   column :first_name, as: [ /first.?name/i, /pr(é|e)nom/i ]
-  column :last_name,  to: :l_name
-  column :published,  to: ->(published, model) { model.published_at = published ? Time.now : nil }
+  column :last_name,  as: [ /last.?name/i, "nom" ]
+  column :published,  to: ->(published, user) { user.published_at = published ? Time.now : nil }
 
   identifier :email # will find_or_update via :email
 
@@ -36,71 +54,273 @@ class ImportUserCSV
 end
 ```
 
-Let's run an new import:
+Run the import:
 
 ```ruby
-# Import a file (IOStream or file path) and from CSV content
+import = ImportUserCSV.new(file: my_file)
 
-import = ImportUserCSV.new(file: InputStream)
-import = ImportUserCSV.new(path: String)
-import = ImportUserCSV.new(content: String)
+import.valid_header?  # => false
+import.report.message # => "The following columns are required: email"
 
-# Validate header
+# Assuming the header was valid, let's run the import!
 
-import.header.valid?
-  # => true if header is valid
+import.run!
+import.report.success? # => true
+import.report.message  # => "Import completed. 4 created, 2 updated, 1 failed to update"
+```
 
-import.header
-  # => returns an instance of `CSVImporter::Header`
+## Installation
 
-import.header.missing_required_columns # => ["email"]
-import.header.missing_columns          # => ["email", "first_name"]
-import.header.extra_columns            # => ["zip_code"]
-import.header.columns                  # => ["last_name", "zip_code"]
+Add this line to your application's Gemfile:
 
-# Manipulate rows
+```ruby
+gem 'csv-importer'
+```
+
+And then execute:
+
+    $ bundle
 
-import.rows
-  # => return a (lazy?) Array of Rows
-row = rows.first
+Or install it yourself as:
 
-row.raw_string       # => "bob@example.com,bob,,extra"
-row.raw_array        # => [ "bob@example.com", "bob", "", "extra" ]
-row.csv_attributes   # => { email: "bob@example.com", first_name: "bob" }
-row.model            # => User<email: "bob@example.com", f_name: "bob", id: nil>
-row.valid?           # delegate to model.valid?
+    $ gem install csv-importer
 
-# Time to run the import!
+## Usage
 
-report = import.run!
+### Create an Importer
 
-# The following methods return arrays of `Row`
-report.valid_rows
-report.invalid_rows
-report.created_rows
-report.updated_rows
-report.failed_to_create_rows
-report.failed_to_update_rows
+Create a class and include `CSVImporter`.
 
-report.success? # => true
-report.message # => "Import completed. 4 created, 2 updated, 1 failed to update"
+```ruby
+class ImportUserCSV
+  include CSVImporter
+end
 ```
 
-## Installation
+### Associate an active record model
 
-Add this line to your application's Gemfile:
+The `model` is likely to be an active record model.
 
 ```ruby
-gem 'csv-importer'
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+end
 ```
 
-And then execute:
+It can also be a relation which is handy to preset attributes:
 
-    $ bundle
+```ruby
+class User
+  scope :pending, -> { where(status: 'pending') }
+end
 
-Or install it yourself as:
+class ImportUserCSV
+  include CSVImporter
 
-    $ gem install csv-importer
+  model User.pending
+end
+```
+
+You can change the configuration at runtime to import associated records.
+
+```ruby
+class Team
+  has_many :users
+end
+
+team = Team.find(1)
+
+ImportUserCSV.new(path: "tmp/my_file.csv", model: team.users)
+```
+
+### Define columns and their mapping
+
+This is where the fun begins.
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+
+  column :email
+end
+```
+
+This will map the column named email to the email attribute. By default,
+we downcase and strip the columns so it will work with a column spelled " EMail ".
+
+Now, email could also be spelled "e-mail", or "mail", or even "courriel"
+(oh, canada). Let's give it a couple of aliases then:
+
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+
+  column :email, as: [/e.?mail/i, "courriel"]
+end
+```
+
+Nice, emails should be downcased though, so let's do this.
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+
+  column :email, as: [/e.?mail/i, "courriel"], to: ->(email) { email.downcase if email }
+end
+```
+
+If you need to do more advanced stuff, you've got access to the model:
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+
+  column :email, as: [/e.?mail/i, "courriel"], to: ->(email, user) { user.email = email.downcase; model.super_user! if email[/@brewhouse.io\z/] }
+end
+```
+
+Now, what if the user does not provide the email column? It's not worth
+running the import, we should just reject the CSV file right away.
+That's easy:
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+
+  column :email, required: true
+end
+
+import = ImportUserCSV.new(content: "name\nbob")
+import.valid_header? # => false
+import.report.status # => :invalid_header
+import.report.message # => "The following columns are required: 'email'"
+```
+
+
+### Update or Create
+
+You often want to find-and-update-or-create when importing a CSV file.
+Just provide an identifier, and we'll do the hard work for you.
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+
+  column :email, to: ->(email) { email.downcase if email }
+
+  identifier :email
+end
+```
+
+And yes, we'll look for an existing record using the downcased email. :)
+
+### Skip or Abort on error
+
+By default, we skip invalid records and report errors back to the user.
+There are times where you want your import to be an all or nothing. The
+`on_error` option is here for you.
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+
+  column :email, to: ->(email) { email.downcase if email }
+
+  on_error :abort
+end
+
+import = ImportUserCSV.new(content: "email\nbob@example.com\nINVALID_EMAIL")
+import.valid_header? # => true
+import.run!
+import.success? # => false
+import.report.status # => :aborted
+import.report.message # => "Import aborted"
+```
+
+You are now done defining your importer, let's run it!
+
+### Import from a file, path or string
+
+You can import from a file, path or just the CSV content. Please note
+that we currently load the entire file in memory. Feel free to
+contribute if you need to support CSV files with millions of lines! :)
+
+```ruby
+import = ImportUserCSV.new(file: my_file)
+import = ImportUserCSV.new(path: "tmp/new_users.csv")
+import = ImportUserCSV.new(content: "email,name\nbob@example.com,bob")
+```
+
+### Overwrite configuration at runtime
+
+It is often needed to change the configuration at runtime, that's quite
+easy:
+
+```ruby
+team = Team.find(1)
+import = ImportUserCSV.new(file: my_file, model: team.users)
+```
+
+### Validate the header
+
+On a web application, as soon as a CSV file is uploaded, you can check
+if it has the required columns. This is handy to fail early an provide
+the user with a meaningful error message right away.
+
+```ruby
+import = ImportUserCSV.new(file: params[:csv_file])
+import.valid_header? # => false
+import.report.message # => "The following columns are required: "email""
+```
+
+### Run the import and provide feedback to the user
+
+```ruby
+import = ImportUserCSV.new(file: params[:csv_file])
+import.run!
+import.report.message  # => "Import completed. 4 created, 2 updated, 1 failed to update"
+```
+
+You can get your hands dirty and fetch the errored rows and the
+associated error message:
+
+```ruby
+import.report.invalid_rows.map { |row| [row.model.email, row.errors] }
+  # => [ [ "INVALID_EMAIL", { "email" => "is invalid" } ] ]
+```
+
+We do our best to map the errors back to the original column name. So
+with the following definition:
+
+```ruby
+  column :email, as: /e.?mail/i
+```
+
+and csv:
+
+```csv
+E-Mail,name
+INVALID_EMAIL,bob
+```
+
+The error returned should be: `{ "E-Mail" => "is invalid" }`
 
 ## Development
 

data/lib/csv_importer.rb

@@ -49,6 +49,7 @@ module CSVImporter
     @csv = CSVReader.new(*args)
     @config = self.class.csv_importer_config.dup
     @config.attributes = args.last
+    @report = Report.new
   end
 
   attr_reader :csv, :report, :config
@@ -64,14 +65,23 @@ module CSVImporter
                                        model_klass: config.model, identifier: config.identifier) }
   end
 
+  def valid_header?
+    if @report.pending?
+      if header.valid?
+        @report = Report.new(status: :pending, extra_columns: header.extra_columns)
+      else
+        @report = Report.new(status: :invalid_header, missing_columns: header.missing_required_columns, extra_columns: header.extra_columns)
+      end
+    end
+
+    header.valid?
+  end
+
   # Run the import. Return a Report.
   def run!
-    if header.valid?
+    if valid_header?
       @report = Runner.call(rows: rows, when_invalid: config.when_invalid)
-    else
-      @report = Report.new(status: :invalid_header, missing_columns: header.missing_required_columns)
     end
-
   rescue CSV::MalformedCSVError => e
     @report = Report.new(status: :invalid_csv_file, parser_error: e.message)
   end

data/lib/csv_importer/report.rb

@@ -13,6 +13,7 @@ module CSVImporter
     attribute :status, Symbol, default: proc { :pending }
 
     attribute :missing_columns, Array[Symbol], default: proc { [] }
+    attribute :extra_columns, Array[Symbol], default: proc { [] }
 
     attribute :parser_error, String
 

data/lib/csv_importer/version.rb

@@ -1,3 +1,3 @@
 module CSVImporter
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv-importer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Philippe Creux
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-06-12 00:00:00.000000000 Z
+date: 2015-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: virtus