csv-importer 0.4.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: cfff4468934d3d7fd8e13b81d618bd2691f3fac7
-  data.tar.gz: f8980b3710130e74d400c9f649da525da7d46145
+SHA256:
+  metadata.gz: fa181c8425255dfe4e9a9fe01fc6c8ee3bd312e701e835999893ef4e9d379de5
+  data.tar.gz: b2a627e75724210a76fae4fd47eefd4a853006fec1f09d955c3c11d7c2a66120
 SHA512:
-  metadata.gz: cd7068f3d2a32f40b58234fc0805883891e5212aeed10dab1c42c1b3198475080e18a88107fd095d02cda7002740b05c1d1436ec0a6e034899d9c0b0a12c93dc
-  data.tar.gz: aa78cf760944b7db3204e746b480e01a161ab4a834ddb79aae65a4d33d7b41f8b03a34d68e898add41a7b209d033c3613b06f264fdfcbb380d092ce69e89ea7d
+  metadata.gz: 2b0cfce7e92e5036ef3cf26c4b874d95b2f531b96a46f38c275d0156206628b9b223b4900d13cd809e30611f8bd883c54c1c3f76075eb247c9783c8da9063bc1
+  data.tar.gz: 1fbc086129f7b54be4ec7036523d56ab799e8efaecb843d2fb22a115d6e60c5a1fbc02c0a6ca5b945c22cd453b24b9b22b63de2259cbb95dfeafc2342c74d2df

data/.travis.yml

@@ -1,6 +1,7 @@
 language: ruby
 rvm:
   - 2.3.0
+  - 2.7
 addons:
   code_climate:
     repo_token: bcecbf1b229a2ddd666a2c3830f26a0113fd56ae1586d30d2d3fb1af837bf0e4

data/CHANGELOG.md

@@ -2,11 +2,80 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.8.1] - 2021-02-21
+
+* Rails 6.2 compability. [#99][] by [@stevenou][]
+
+## [0.8.0] - 2020-09-02
+
+### Fixed
+
+* Ruby 2.7 compatibility. [#93][] by [@perezperret][]
+
+## [0.7.0] - 2019-11-29
+
+### Added
+
+* Improve algorithm to detect separators. [#88][] by [@brain-geek][]
+* `to:` accepts anything that responds to `#call` (lambda, procs,
+  classes etc). [#72][] by [@pcreux][] inspired by [#71][] by [@macfanatic][].
+
+### Fixed
+
+* `valid_header?` returns `false` when the CSV is malformed instead of
+  raising an exception. [#85][] by [@mltsy][]
+* Header infos (ex: `extra_columns`) aren't discarded after running an
+  import. [#83][] by [@mltsy][]
+
+
+## [0.6.0] - 2018-05-22
+
+### Added
+
+* We now pass the `column` object as the third parameter of the `column`
+  block for advanced usage. [#73][] by [@stas][].
+
+```ruby
+  column :extra, as: [/extra/], to: ->(value, user, column) do
+    attribute = column.name.sub(/^extra /, '')
+    user[attribute] = value
+  end
+```
+
+## [0.5.0] - 2018-01-13
+
+### Added
+
+* after_save supports block with arity of 2 for access to raw
+  attributes. [#68][] by [@macfanatic][].
+
+```ruby
+class Importer
+  model Task
+
+  column :assignee, to: ->(name) { User.active.find_by(name: name) }
+
+  after_save do |task, attributes|
+    if task.errors[:assignee].present? && attributes['Assignee'].present?
+      task.errors.add(:assignee, "'#{ attributes['Assignee'] }' is not part of this project."
+    end
+  end
+end
+```
+
+* support Proc identifiers. [#69][] by [@danielweinmann][]
+
+```ruby
+class Importer
+  identifier ->(user) { user.email.present? ? :email : [:company_id, :employee_id] }
+end
+```
+
 ## [0.4.0] - 2017-08-10
 
 ### Added
 
-* Rows are now aware of their line number.
+* Rows are now aware of their line number. [#63][] by [@paulodeleo][]
 
 ```ruby
 import.report.invalid_rows.map { |row| [row.line_number, row.model.email, row.errors] }
@@ -123,11 +192,31 @@ report object instead of raising an exception.
 * Initial Release
 
 <!--- The following link definition list is generated by PimpMyChangelog --->
-[#26]: https://github.com/BrewhouseTeam/csv-importer/issues/26
-[#38]: https://github.com/BrewhouseTeam/csv-importer/issues/38
-[#47]: https://github.com/BrewhouseTeam/csv-importer/issues/47
-[#52]: https://github.com/BrewhouseTeam/csv-importer/issues/52
+[#26]: https://github.com/pcreux/csv-importer/issues/26
+[#38]: https://github.com/pcreux/csv-importer/issues/38
+[#47]: https://github.com/pcreux/csv-importer/issues/47
+[#52]: https://github.com/pcreux/csv-importer/issues/52
+[#63]: https://github.com/pcreux/csv-importer/issues/63
+[#68]: https://github.com/pcreux/csv-importer/issues/68
+[#69]: https://github.com/pcreux/csv-importer/issues/69
+[#71]: https://github.com/pcreux/csv-importer/issues/71
+[#72]: https://github.com/pcreux/csv-importer/issues/72
+[#73]: https://github.com/pcreux/csv-importer/issues/73
+[#83]: https://github.com/pcreux/csv-importer/issues/83
+[#85]: https://github.com/pcreux/csv-importer/issues/85
+[#88]: https://github.com/pcreux/csv-importer/issues/88
+[#93]: https://github.com/pcreux/csv-importer/issues/93
+[#99]: https://github.com/pcreux/csv-importer/issues/99
+[@brain-geek]: https://github.com/brain-geek
+[@danielweinmann]: https://github.com/danielweinmann
 [@egg-chicken]: https://github.com/egg-chicken
-[@pnomolos]: https://github.com/pnomolos
 [@fxgallego]: https://github.com/fxgallego
+[@macfanatic]: https://github.com/macfanatic
+[@mltsy]: https://github.com/mltsy
+[@paulodeleo]: https://github.com/paulodeleo
+[@pcreux]: https://github.com/pcreux
+[@perezperret]: https://github.com/perezperret
+[@pnomolos]: https://github.com/pnomolos
 [@shvetsovdm]: https://github.com/shvetsovdm
+[@stas]: https://github.com/stas
+[@stevenou]: https://github.com/stevenou

data/Gemfile

@@ -2,9 +2,3 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in csv-importer.gemspec
 gemspec
-
-gem 'rspec', '~> 3.3.0'
-gem 'guard-rspec'
-gem 'activemodel'
-gem 'simplecov', require: nil
-gem "codeclimate-test-reporter", require: nil

data/README.md

@@ -178,6 +178,36 @@ If you need to do more advanced stuff, you've got access to the model:
   column :email, as: [/e.?mail/i, "courriel"], to: ->(email, user) { user.email = email.downcase; model.super_user! if email[/@brewhouse.io\z/] }
 ```
 
+Like very advanced stuff? We grant you access to the [`column`](https://github.com/pcreux/csv-importer/blob/master/lib/csv_importer/column.rb) object itself which contains the column name – quite handy if you want to support arbitrary columns.
+
+```ruby
+  column :extra, as: [/extra/], to: ->(value, user, column) do
+    attribute = column.name.sub(/^extra /, '')
+    user[attribute] = value
+  end
+```
+
+Note that `to:` accepts anything that responds to call and take 1, 2 or
+3 arguments.
+
+```ruby
+class ImportUserCSV
+  include CSVImporter
+
+  model User
+
+  column :birth_date, to: DateTransformer
+  column :renewal_date, to: DateTransformer
+  column :next_renewal_at, to: ->(value) { Time.at(value.to_i) }
+end
+
+class DateTransformer
+  def self.call(date)
+    Date.strptime(date, '%m/%d/%y')
+  end
+end
+```
+
 Now, what if the user does not provide the email column? It's not worth
 running the import, we should just reject the CSV file right away.
 That's easy:
@@ -197,7 +227,6 @@ import.report.status # => :invalid_header
 import.report.message # => "The following columns are required: 'email'"
 ```
 
-
 ### Update or Create
 
 You often want to find-and-update-or-create when importing a CSV file.
@@ -224,6 +253,14 @@ You can also define a composite identifier:
   identifier :company_id, :employee_id
 ```
 
+Or a Proc:
+
+```ruby
+  # Update records with email if email is present
+  # Update records matching company_id AND employee_id if email is not present
+  identifier ->(user) { user.email.empty? ? [:company_id, :employee_id] : :email }
+```
+
 ### Skip or Abort on error
 
 By default, we skip invalid records and report errors back to the user.

data/csv-importer.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["pcreux@gmail.com"]
 
   spec.summary       = %q{CSV Import for humans}
-  spec.homepage      = "https://github.com/BrewhouseTeam/csv-importer"
+  spec.homepage      = "https://github.com/pcreux/csv-importer"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
@@ -20,6 +20,10 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "virtus"
 
-  spec.add_development_dependency "bundler", "~> 1.8"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.3.0"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "guard-rspec"
+  spec.add_development_dependency "activemodel", "~> 5"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "codeclimate-test-reporter"
 end

data/lib/csv_importer.rb

@@ -85,13 +85,16 @@ module CSVImporter
     end
 
     header.valid?
+  rescue CSV::MalformedCSVError => e
+    @report = Report.new(status: :invalid_csv_file, parser_error: e.message)
+    false
   end
 
   # Run the import. Return a Report.
   def run!
     if valid_header?
       @report = Runner.call(rows: rows, when_invalid: config.when_invalid,
-                            after_save_blocks: config.after_save_blocks)
+                            after_save_blocks: config.after_save_blocks, report: @report)
     else
       @report
     end

data/lib/csv_importer/column_definition.rb

@@ -29,7 +29,7 @@ module CSVImporter
     attribute :name, Symbol
     attribute :to # Symbol or Proc
     attribute :as # Symbol, String, Regexp, Array
-    attribute :required, Boolean
+    attribute :required, Virtus::Attribute::Boolean
 
     # The model attribute that this column targets
     def attribute

data/lib/csv_importer/config.rb

@@ -5,7 +5,7 @@ module CSVImporter
 
     attribute :model
     attribute :column_definitions, Array[ColumnDefinition], default: proc { [] }
-    attribute :identifiers, Array[Symbol], default: []
+    attribute :identifiers # Array[Symbol] or Proc
     attribute :when_invalid, Symbol, default: proc { :skip }
     attribute :after_build_blocks, Array[Proc], default: []
     attribute :after_save_blocks, Array[Proc], default: []
@@ -27,4 +27,3 @@ module CSVImporter
     end
   end
 end
-

data/lib/csv_importer/csv_reader.rb

@@ -14,8 +14,12 @@ module CSVImporter
       @csv_rows ||= begin
         sane_content = sanitize_content(read_content)
         separator = detect_separator(sane_content)
-        cells = CSV.parse(sane_content, col_sep: separator, quote_char: quote_char, skip_blanks: true, encoding: encoding)
-        sanitize_cells(cells)
+        cells = CSV.parse(
+          sane_content,
+          col_sep: separator, quote_char: quote_char, skip_blanks: true,
+          external_encoding: source_encoding
+        )
+        sanitize_cells(encode_cells(cells))
       end
     end
 
@@ -44,16 +48,26 @@ module CSVImporter
     end
 
     def sanitize_content(csv_content)
-      internal_encoding = encoding.split(':').last
       csv_content
-        .encode(Encoding.find(internal_encoding), {invalid: :replace, undef: :replace, replace: ''}) # Remove invalid byte sequences
+        .encode(Encoding.find(source_encoding), invalid: :replace, undef: :replace, replace: '') # Remove invalid byte sequences
         .gsub(/\r\r?\n?/, "\n") # Replaces windows line separators with "\n"
     end
 
     SEPARATORS = [",", ";", "\t"]
 
     def detect_separator(csv_content)
-      SEPARATORS.sort_by { |separator| csv_content.count(separator) }.last
+      SEPARATORS.min_by do |separator|
+        csv_content.count(separator)
+
+        all_lines = csv_content.lines
+        base_number = all_lines.first.count(separator)
+
+        if base_number.zero?
+          Float::MAX
+        else
+          all_lines.map{|line| line.count(separator) - base_number }.map(&:abs).inject(0) { |sum, i| sum + i }
+        end
+      end
     end
 
     # Remove trailing white spaces and ensure we always return a string
@@ -64,5 +78,21 @@ module CSVImporter
         end
       end
     end
+
+    def encode_cells(rows)
+      rows.map do |cells|
+        cells.map do |cell|
+          cell ? cell.encode(target_encoding) : ""
+        end
+      end
+    end
+
+    def source_encoding
+      encoding.split(':').first || 'UTF-8'
+    end
+
+    def target_encoding
+      encoding.split(':').last || 'UTF-8'
+    end
   end
 end

data/lib/csv_importer/dsl.rb

@@ -10,8 +10,8 @@ module CSVImporter
       config.column_definitions << options.merge(name: name)
     end
 
-    def identifier(*identifiers)
-      config.identifiers = identifiers
+    def identifier(*params)
+      config.identifiers = params.first.is_a?(Proc) ? params.first : params
     end
 
     alias_method :identifiers, :identifier

data/lib/csv_importer/report.rb

@@ -12,8 +12,8 @@ module CSVImporter
 
     attribute :status, Symbol, default: proc { :pending }
 
-    attribute :missing_columns, Array[Symbol], default: proc { [] }
-    attribute :extra_columns, Array[Symbol], default: proc { [] }
+    attribute :missing_columns, Array[String], default: proc { [] }
+    attribute :extra_columns, Array[String], default: proc { [] }
 
     attribute :parser_error, String
 

data/lib/csv_importer/row.rb

@@ -10,9 +10,9 @@ module CSVImporter
     attribute :line_number, Integer
     attribute :row_array, Array[String]
     attribute :model_klass
-    attribute :identifiers, Array[Symbol]
+    attribute :identifiers # Array[Symbol] or Proc
     attribute :after_build_blocks, Array[Proc], default: []
-    attribute :skip, Boolean, default: false
+    attribute :skip, Virtus::Attribute::Boolean, default: false
 
     # The model to be persisted
     def model
@@ -41,27 +41,30 @@ module CSVImporter
           # can't dup Symbols, Integer etc...
         end
 
-        column_definition = column.definition
-        next if column_definition.nil?
+        next if column.definition.nil?
 
-        set_attribute(model, column_definition, value)
+        set_attribute(model, column, value)
       end
 
       model
     end
 
     # Set the attribute using the column_definition and the csv_value
-    def set_attribute(model, column_definition, csv_value)
-      if column_definition.to && column_definition.to.is_a?(Proc)
-        to_proc = column_definition.to
+    def set_attribute(model, column, csv_value)
+      column_definition = column.definition
+      transformer = column_definition.to
+      if transformer.respond_to?(:call)
+        arity = transformer.is_a?(Proc) ? transformer.arity : transformer.method(:call).arity
 
-        case to_proc.arity
+        case arity
         when 1 # to: ->(email) { email.downcase }
-          model.public_send("#{column_definition.name}=", to_proc.call(csv_value))
+          model.public_send("#{column_definition.name}=", transformer.call(csv_value))
         when 2 # to: ->(published, post) { post.published_at = Time.now if published == "true" }
-          to_proc.call(csv_value, model)
+          transformer.call(csv_value, model)
+        when 3 # to: ->(field_value, post, column) { post.hash_field[column.name] = field_value }
+          transformer.call(csv_value, model, column)
         else
-          raise ArgumentError, "`to` proc can only have 1 or 2 arguments"
+          raise ArgumentError, "arity: #{transformer.arity.inspect} - `to` can only have 1, 2 or 3 arguments"
         end
       else
         attribute = column_definition.attribute
@@ -74,11 +77,11 @@ module CSVImporter
     # Error from the model mapped back to the CSV header if we can
     def errors
       Hash[
-        model.errors.map do |attribute, errors|
+        model.errors.to_hash.map do |attribute, errors|
           if column_name = header.column_name_for_model_attribute(attribute)
-            [column_name, errors]
+            [column_name, errors.last]
           else
-            [attribute, errors]
+            [attribute, errors.last]
           end
         end
       ]
@@ -89,10 +92,14 @@ module CSVImporter
     end
 
     def find_model
-      return nil if identifiers.empty?
+      return nil if identifiers.nil?
 
       model = build_model
       set_attributes(model)
+
+      identifiers = model_identifiers(model)
+      return nil if identifiers.empty?
+
       query = Hash[
         identifiers.map { |identifier| [ identifier, model.public_send(identifier) ] }
       ]
@@ -106,5 +113,15 @@ module CSVImporter
     def skip!
       self.skip = true
     end
+
+    private
+
+    def model_identifiers(model)
+      if identifiers.is_a?(Proc)
+        [identifiers.call(model)].flatten
+      else
+        identifiers
+      end
+    end
   end
 end

data/lib/csv_importer/runner.rb

@@ -63,7 +63,16 @@ module CSVImporter
           end
 
           add_to_report(row, tags)
-          after_save_blocks.each { |block| block.call(row.model) }
+
+          after_save_blocks.each do |block|
+            case block.arity
+            when 0 then block.call
+            when 1 then block.call(row.model)
+            when 2 then block.call(row.model, row.csv_attributes)
+            else
+              raise ArgumentError, "after_save block of arity #{ block.arity } is not supported"
+            end
+          end
         end
       end
     end

data/lib/csv_importer/version.rb

@@ -1,3 +1,3 @@
 module CSVImporter
-  VERSION = "0.4.0"
+  VERSION = "0.8.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv-importer
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Philippe Creux
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2017-08-10 00:00:00.000000000 Z
+date: 2021-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: virtus
@@ -25,34 +25,90 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: 3.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: 3.3.0
 - !ruby/object:Gem::Dependency
   name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activemodel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
-description: 
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
 email:
 - pcreux@gmail.com
 executables: []
@@ -85,11 +141,11 @@ files:
 - lib/csv_importer/row.rb
 - lib/csv_importer/runner.rb
 - lib/csv_importer/version.rb
-homepage: https://github.com/BrewhouseTeam/csv-importer
+homepage: https://github.com/pcreux/csv-importer
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -104,9 +160,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5.1
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: CSV Import for humans
 test_files: []