csv-importer 0.3.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.travis.yml +4 -1
- data/CHANGELOG.md +99 -5
- data/Gemfile +0 -6
- data/README.md +44 -7
- data/csv-importer.gemspec +7 -3
- data/lib/csv_importer.rb +6 -3
- data/lib/csv_importer/column_definition.rb +1 -1
- data/lib/csv_importer/config.rb +1 -2
- data/lib/csv_importer/csv_reader.rb +35 -5
- data/lib/csv_importer/dsl.rb +2 -2
- data/lib/csv_importer/report.rb +2 -2
- data/lib/csv_importer/row.rb +31 -13
- data/lib/csv_importer/runner.rb +10 -1
- data/lib/csv_importer/version.rb +1 -1
- metadata +69 -14
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 | 
            -
             | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 2 | 
            +
            SHA256:
         | 
| 3 | 
            +
              metadata.gz: 803f33e5d2db4d5331bb2f6ef1c5fd6cc9ea19d1d8350e1c16a2041f2ac2176c
         | 
| 4 | 
            +
              data.tar.gz: 822e5caa1ae8d3e73909fcf42a9e57e6cff00dc6f2f2bb900b25e010bed72f19
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 0edf3c998a6b506a4778d0e36fddc828339cf8900653de0b41494489e4de57358df23a34ed30ad03f8583837ec0ca9c29d4166982ab80e17018481c00baad84d
         | 
| 7 | 
            +
              data.tar.gz: c11177ab4e4fb02751a8e1be12ded7c41360bd62f2f72156c75b8a90819d47f861fbff94b1ed1c8fa29940c5db4484ccb15341c8baaf2bfae256234fb2183c35
         | 
    
        data/.travis.yml
    CHANGED
    
    | @@ -1,6 +1,9 @@ | |
| 1 1 | 
             
            language: ruby
         | 
| 2 2 | 
             
            rvm:
         | 
| 3 3 | 
             
              - 2.3.0
         | 
| 4 | 
            +
              - 2.7
         | 
| 4 5 | 
             
            addons:
         | 
| 5 6 | 
             
              code_climate:
         | 
| 6 | 
            -
                repo_token:  | 
| 7 | 
            +
                repo_token: bcecbf1b229a2ddd666a2c3830f26a0113fd56ae1586d30d2d3fb1af837bf0e4
         | 
| 8 | 
            +
            after_success:
         | 
| 9 | 
            +
              - bundle exec codeclimate-test-reporter
         | 
    
        data/CHANGELOG.md
    CHANGED
    
    | @@ -2,6 +2,82 @@ | |
| 2 2 |  | 
| 3 3 | 
             
            All notable changes to this project will be documented in this file.
         | 
| 4 4 |  | 
| 5 | 
            +
            ## [0.8.0] - 2020-09-02
         | 
| 6 | 
            +
             | 
| 7 | 
            +
            ### Fixed
         | 
| 8 | 
            +
             | 
| 9 | 
            +
            * Ruby 2.7 compatibility. [#93][] by [@perezperret][]
         | 
| 10 | 
            +
             | 
| 11 | 
            +
            ## [0.7.0] - 2019-11-29
         | 
| 12 | 
            +
             | 
| 13 | 
            +
            ### Added
         | 
| 14 | 
            +
             | 
| 15 | 
            +
            * Improve algorithm to detect separators. [#88][] by [@brain-geek][]
         | 
| 16 | 
            +
            * `to:` accepts anything that responds to `#call` (lambda, procs,
         | 
| 17 | 
            +
              classes etc). [#72][] by [@pcreux][] inspired by [#71][] by [@macfanatic][].
         | 
| 18 | 
            +
             | 
| 19 | 
            +
            ### Fixed
         | 
| 20 | 
            +
             | 
| 21 | 
            +
            * `valid_header?` returns `false` when the CSV is malformed instead of
         | 
| 22 | 
            +
              raising an exception. [#85][] by [@mltsy][]
         | 
| 23 | 
            +
            * Header infos (ex: `extra_columns`) aren't discarded after running an
         | 
| 24 | 
            +
              import. [#83][] by [@mltsy][]
         | 
| 25 | 
            +
             | 
| 26 | 
            +
             | 
| 27 | 
            +
            ## [0.6.0] - 2018-05-22
         | 
| 28 | 
            +
             | 
| 29 | 
            +
            ### Added
         | 
| 30 | 
            +
             | 
| 31 | 
            +
            * We now pass the `column` object as the third parameter of the `column`
         | 
| 32 | 
            +
              block for advanced usage. [#73][] by [@stas][].
         | 
| 33 | 
            +
             | 
| 34 | 
            +
            ```ruby
         | 
| 35 | 
            +
              column :extra, as: [/extra/], to: ->(value, user, column) do
         | 
| 36 | 
            +
                attribute = column.name.sub(/^extra /, '')
         | 
| 37 | 
            +
                user[attribute] = value
         | 
| 38 | 
            +
              end
         | 
| 39 | 
            +
            ```
         | 
| 40 | 
            +
             | 
| 41 | 
            +
            ## [0.5.0] - 2018-01-13
         | 
| 42 | 
            +
             | 
| 43 | 
            +
            ### Added
         | 
| 44 | 
            +
             | 
| 45 | 
            +
            * after_save supports block with arity of 2 for access to raw
         | 
| 46 | 
            +
              attributes. [#68][] by [@macfanatic][].
         | 
| 47 | 
            +
             | 
| 48 | 
            +
            ```ruby
         | 
| 49 | 
            +
            class Importer
         | 
| 50 | 
            +
              model Task
         | 
| 51 | 
            +
             | 
| 52 | 
            +
              column :assignee, to: ->(name) { User.active.find_by(name: name) }
         | 
| 53 | 
            +
             | 
| 54 | 
            +
              after_save do |task, attributes|
         | 
| 55 | 
            +
                if task.errors[:assignee].present? && attributes['Assignee'].present?
         | 
| 56 | 
            +
                  task.errors.add(:assignee, "'#{ attributes['Assignee'] }' is not part of this project."
         | 
| 57 | 
            +
                end
         | 
| 58 | 
            +
              end
         | 
| 59 | 
            +
            end
         | 
| 60 | 
            +
            ```
         | 
| 61 | 
            +
             | 
| 62 | 
            +
            * support Proc identifiers. [#69][] by [@danielweinmann][]
         | 
| 63 | 
            +
             | 
| 64 | 
            +
            ```ruby
         | 
| 65 | 
            +
            class Importer
         | 
| 66 | 
            +
              identifier ->(user) { user.email.present? ? :email : [:company_id, :employee_id] }
         | 
| 67 | 
            +
            end
         | 
| 68 | 
            +
            ```
         | 
| 69 | 
            +
             | 
| 70 | 
            +
            ## [0.4.0] - 2017-08-10
         | 
| 71 | 
            +
             | 
| 72 | 
            +
            ### Added
         | 
| 73 | 
            +
             | 
| 74 | 
            +
            * Rows are now aware of their line number. [#63][] by [@paulodeleo][]
         | 
| 75 | 
            +
             | 
| 76 | 
            +
            ```ruby
         | 
| 77 | 
            +
            import.report.invalid_rows.map { |row| [row.line_number, row.model.email, row.errors] }
         | 
| 78 | 
            +
              # => [ [2, "INVALID_EMAIL", { "email" => "is invalid" } ] ]
         | 
| 79 | 
            +
            ```
         | 
| 80 | 
            +
             | 
| 5 81 | 
             
            ## [0.3.2] - 2017-01-06
         | 
| 6 82 |  | 
| 7 83 | 
             
            ### Fixed
         | 
| @@ -112,11 +188,29 @@ report object instead of raising an exception. | |
| 112 188 | 
             
            * Initial Release
         | 
| 113 189 |  | 
| 114 190 | 
             
            <!--- The following link definition list is generated by PimpMyChangelog --->
         | 
| 115 | 
            -
            [#26]: https://github.com/ | 
| 116 | 
            -
            [#38]: https://github.com/ | 
| 117 | 
            -
            [#47]: https://github.com/ | 
| 118 | 
            -
            [#52]: https://github.com/ | 
| 191 | 
            +
            [#26]: https://github.com/pcreux/csv-importer/issues/26
         | 
| 192 | 
            +
            [#38]: https://github.com/pcreux/csv-importer/issues/38
         | 
| 193 | 
            +
            [#47]: https://github.com/pcreux/csv-importer/issues/47
         | 
| 194 | 
            +
            [#52]: https://github.com/pcreux/csv-importer/issues/52
         | 
| 195 | 
            +
            [#63]: https://github.com/pcreux/csv-importer/issues/63
         | 
| 196 | 
            +
            [#68]: https://github.com/pcreux/csv-importer/issues/68
         | 
| 197 | 
            +
            [#69]: https://github.com/pcreux/csv-importer/issues/69
         | 
| 198 | 
            +
            [#71]: https://github.com/pcreux/csv-importer/issues/71
         | 
| 199 | 
            +
            [#72]: https://github.com/pcreux/csv-importer/issues/72
         | 
| 200 | 
            +
            [#73]: https://github.com/pcreux/csv-importer/issues/73
         | 
| 201 | 
            +
            [#83]: https://github.com/pcreux/csv-importer/issues/83
         | 
| 202 | 
            +
            [#85]: https://github.com/pcreux/csv-importer/issues/85
         | 
| 203 | 
            +
            [#88]: https://github.com/pcreux/csv-importer/issues/88
         | 
| 204 | 
            +
            [#93]: https://github.com/pcreux/csv-importer/issues/93
         | 
| 205 | 
            +
            [@brain-geek]: https://github.com/brain-geek
         | 
| 206 | 
            +
            [@danielweinmann]: https://github.com/danielweinmann
         | 
| 119 207 | 
             
            [@egg-chicken]: https://github.com/egg-chicken
         | 
| 120 | 
            -
            [@pnomolos]: https://github.com/pnomolos
         | 
| 121 208 | 
             
            [@fxgallego]: https://github.com/fxgallego
         | 
| 209 | 
            +
            [@macfanatic]: https://github.com/macfanatic
         | 
| 210 | 
            +
            [@mltsy]: https://github.com/mltsy
         | 
| 211 | 
            +
            [@paulodeleo]: https://github.com/paulodeleo
         | 
| 212 | 
            +
            [@pcreux]: https://github.com/pcreux
         | 
| 213 | 
            +
            [@perezperret]: https://github.com/perezperret
         | 
| 214 | 
            +
            [@pnomolos]: https://github.com/pnomolos
         | 
| 122 215 | 
             
            [@shvetsovdm]: https://github.com/shvetsovdm
         | 
| 216 | 
            +
            [@stas]: https://github.com/stas
         | 
    
        data/Gemfile
    CHANGED
    
    
    
        data/README.md
    CHANGED
    
    | @@ -7,11 +7,11 @@ CSVImporter aims to handle validations, column mapping, import | |
| 7 7 | 
             
            and reporting.
         | 
| 8 8 |  | 
| 9 9 | 
             
            [](https://travis-ci.org/pcreux/csv-importer)
         | 
| 11 11 | 
             
            [](https://codeclimate.com/github/pcreux/csv-importer)
         | 
| 13 13 | 
             
            [](https://codeclimate.com/github/pcreux/csv-importer/coverage)
         | 
| 15 15 | 
             
            [](http://badge.fury.io/rb/csv-importer)
         | 
| 17 17 |  | 
| @@ -178,6 +178,36 @@ If you need to do more advanced stuff, you've got access to the model: | |
| 178 178 | 
             
              column :email, as: [/e.?mail/i, "courriel"], to: ->(email, user) { user.email = email.downcase; model.super_user! if email[/@brewhouse.io\z/] }
         | 
| 179 179 | 
             
            ```
         | 
| 180 180 |  | 
| 181 | 
            +
            Like very advanced stuff? We grant you access to the [`column`](https://github.com/pcreux/csv-importer/blob/master/lib/csv_importer/column.rb) object itself which contains the column name – quite handy if you want to support arbitrary columns.
         | 
| 182 | 
            +
             | 
| 183 | 
            +
            ```ruby
         | 
| 184 | 
            +
              column :extra, as: [/extra/], to: ->(value, user, column) do
         | 
| 185 | 
            +
                attribute = column.name.sub(/^extra /, '')
         | 
| 186 | 
            +
                user[attribute] = value
         | 
| 187 | 
            +
              end
         | 
| 188 | 
            +
            ```
         | 
| 189 | 
            +
             | 
| 190 | 
            +
            Note that `to:` accepts anything that responds to call and take 1, 2 or
         | 
| 191 | 
            +
            3 arguments.
         | 
| 192 | 
            +
             | 
| 193 | 
            +
            ```ruby
         | 
| 194 | 
            +
            class ImportUserCSV
         | 
| 195 | 
            +
              include CSVImporter
         | 
| 196 | 
            +
             | 
| 197 | 
            +
              model User
         | 
| 198 | 
            +
             | 
| 199 | 
            +
              column :birth_date, to: DateTransformer
         | 
| 200 | 
            +
              column :renewal_date, to: DateTransformer
         | 
| 201 | 
            +
              column :next_renewal_at, to: ->(value) { Time.at(value.to_i) }
         | 
| 202 | 
            +
            end
         | 
| 203 | 
            +
             | 
| 204 | 
            +
            class DateTransformer
         | 
| 205 | 
            +
              def self.call(date)
         | 
| 206 | 
            +
                Date.strptime(date, '%m/%d/%y')
         | 
| 207 | 
            +
              end
         | 
| 208 | 
            +
            end
         | 
| 209 | 
            +
            ```
         | 
| 210 | 
            +
             | 
| 181 211 | 
             
            Now, what if the user does not provide the email column? It's not worth
         | 
| 182 212 | 
             
            running the import, we should just reject the CSV file right away.
         | 
| 183 213 | 
             
            That's easy:
         | 
| @@ -197,7 +227,6 @@ import.report.status # => :invalid_header | |
| 197 227 | 
             
            import.report.message # => "The following columns are required: 'email'"
         | 
| 198 228 | 
             
            ```
         | 
| 199 229 |  | 
| 200 | 
            -
             | 
| 201 230 | 
             
            ### Update or Create
         | 
| 202 231 |  | 
| 203 232 | 
             
            You often want to find-and-update-or-create when importing a CSV file.
         | 
| @@ -224,6 +253,14 @@ You can also define a composite identifier: | |
| 224 253 | 
             
              identifier :company_id, :employee_id
         | 
| 225 254 | 
             
            ```
         | 
| 226 255 |  | 
| 256 | 
            +
            Or a Proc:
         | 
| 257 | 
            +
             | 
| 258 | 
            +
            ```ruby
         | 
| 259 | 
            +
              # Update records with email if email is present
         | 
| 260 | 
            +
              # Update records matching company_id AND employee_id if email is not present
         | 
| 261 | 
            +
              identifier ->(user) { user.email.empty? ? [:company_id, :employee_id] : :email }
         | 
| 262 | 
            +
            ```
         | 
| 263 | 
            +
             | 
| 227 264 | 
             
            ### Skip or Abort on error
         | 
| 228 265 |  | 
| 229 266 | 
             
            By default, we skip invalid records and report errors back to the user.
         | 
| @@ -353,8 +390,8 @@ You can get your hands dirty and fetch the errored rows and the | |
| 353 390 | 
             
            associated error message:
         | 
| 354 391 |  | 
| 355 392 | 
             
            ```ruby
         | 
| 356 | 
            -
            import.report.invalid_rows.map { |row| [row.model.email, row.errors] }
         | 
| 357 | 
            -
              # => [ [ "INVALID_EMAIL", { "email" => "is invalid" } ] ]
         | 
| 393 | 
            +
            import.report.invalid_rows.map { |row| [row.line_number, row.model.email, row.errors] }
         | 
| 394 | 
            +
              # => [ [2, "INVALID_EMAIL", { "email" => "is invalid" } ] ]
         | 
| 358 395 | 
             
            ```
         | 
| 359 396 |  | 
| 360 397 | 
             
            We do our best to map the errors back to the original column name. So
         | 
| @@ -415,7 +452,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To | |
| 415 452 |  | 
| 416 453 | 
             
            ## Contributing
         | 
| 417 454 |  | 
| 418 | 
            -
            1. Fork it ( https://github.com/ | 
| 455 | 
            +
            1. Fork it ( https://github.com/pcreux/csv-importer/fork )
         | 
| 419 456 | 
             
            2. Create your feature branch (`git checkout -b my-new-feature`)
         | 
| 420 457 | 
             
            3. Commit your changes (`git commit -am 'Add some feature'`)
         | 
| 421 458 | 
             
            4. Push to the branch (`git push origin my-new-feature`)
         | 
    
        data/csv-importer.gemspec
    CHANGED
    
    | @@ -10,7 +10,7 @@ Gem::Specification.new do |spec| | |
| 10 10 | 
             
              spec.email         = ["pcreux@gmail.com"]
         | 
| 11 11 |  | 
| 12 12 | 
             
              spec.summary       = %q{CSV Import for humans}
         | 
| 13 | 
            -
              spec.homepage      = "https://github.com/ | 
| 13 | 
            +
              spec.homepage      = "https://github.com/pcreux/csv-importer"
         | 
| 14 14 | 
             
              spec.license       = "MIT"
         | 
| 15 15 |  | 
| 16 16 | 
             
              spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
         | 
| @@ -20,6 +20,10 @@ Gem::Specification.new do |spec| | |
| 20 20 |  | 
| 21 21 | 
             
              spec.add_dependency "virtus"
         | 
| 22 22 |  | 
| 23 | 
            -
              spec.add_development_dependency " | 
| 24 | 
            -
              spec.add_development_dependency "rake" | 
| 23 | 
            +
              spec.add_development_dependency "rspec", "~> 3.3.0"
         | 
| 24 | 
            +
              spec.add_development_dependency "rake"
         | 
| 25 | 
            +
              spec.add_development_dependency "guard-rspec"
         | 
| 26 | 
            +
              spec.add_development_dependency "activemodel", "~> 5"
         | 
| 27 | 
            +
              spec.add_development_dependency "simplecov"
         | 
| 28 | 
            +
              spec.add_development_dependency "codeclimate-test-reporter"
         | 
| 25 29 | 
             
            end
         | 
    
        data/lib/csv_importer.rb
    CHANGED
    
    | @@ -69,8 +69,8 @@ module CSVImporter | |
| 69 69 |  | 
| 70 70 | 
             
              # Initialize and return the `Row`s for the current CSV file
         | 
| 71 71 | 
             
              def rows
         | 
| 72 | 
            -
                csv.rows.map do |row_array|
         | 
| 73 | 
            -
                  Row.new(header: header, row_array: row_array, model_klass: config.model,
         | 
| 72 | 
            +
                csv.rows.map.with_index(2) do |row_array, line_number|
         | 
| 73 | 
            +
                  Row.new(header: header, line_number: line_number, row_array: row_array, model_klass: config.model,
         | 
| 74 74 | 
             
                          identifiers: config.identifiers, after_build_blocks: config.after_build_blocks)
         | 
| 75 75 | 
             
                end
         | 
| 76 76 | 
             
              end
         | 
| @@ -85,13 +85,16 @@ module CSVImporter | |
| 85 85 | 
             
                end
         | 
| 86 86 |  | 
| 87 87 | 
             
                header.valid?
         | 
| 88 | 
            +
              rescue CSV::MalformedCSVError => e
         | 
| 89 | 
            +
                @report = Report.new(status: :invalid_csv_file, parser_error: e.message)
         | 
| 90 | 
            +
                false
         | 
| 88 91 | 
             
              end
         | 
| 89 92 |  | 
| 90 93 | 
             
              # Run the import. Return a Report.
         | 
| 91 94 | 
             
              def run!
         | 
| 92 95 | 
             
                if valid_header?
         | 
| 93 96 | 
             
                  @report = Runner.call(rows: rows, when_invalid: config.when_invalid,
         | 
| 94 | 
            -
                                        after_save_blocks: config.after_save_blocks)
         | 
| 97 | 
            +
                                        after_save_blocks: config.after_save_blocks, report: @report)
         | 
| 95 98 | 
             
                else
         | 
| 96 99 | 
             
                  @report
         | 
| 97 100 | 
             
                end
         | 
| @@ -29,7 +29,7 @@ module CSVImporter | |
| 29 29 | 
             
                attribute :name, Symbol
         | 
| 30 30 | 
             
                attribute :to # Symbol or Proc
         | 
| 31 31 | 
             
                attribute :as # Symbol, String, Regexp, Array
         | 
| 32 | 
            -
                attribute :required, Boolean
         | 
| 32 | 
            +
                attribute :required, Virtus::Attribute::Boolean
         | 
| 33 33 |  | 
| 34 34 | 
             
                # The model attribute that this column targets
         | 
| 35 35 | 
             
                def attribute
         | 
    
        data/lib/csv_importer/config.rb
    CHANGED
    
    | @@ -5,7 +5,7 @@ module CSVImporter | |
| 5 5 |  | 
| 6 6 | 
             
                attribute :model
         | 
| 7 7 | 
             
                attribute :column_definitions, Array[ColumnDefinition], default: proc { [] }
         | 
| 8 | 
            -
                attribute :identifiers | 
| 8 | 
            +
                attribute :identifiers # Array[Symbol] or Proc
         | 
| 9 9 | 
             
                attribute :when_invalid, Symbol, default: proc { :skip }
         | 
| 10 10 | 
             
                attribute :after_build_blocks, Array[Proc], default: []
         | 
| 11 11 | 
             
                attribute :after_save_blocks, Array[Proc], default: []
         | 
| @@ -27,4 +27,3 @@ module CSVImporter | |
| 27 27 | 
             
                end
         | 
| 28 28 | 
             
              end
         | 
| 29 29 | 
             
            end
         | 
| 30 | 
            -
             | 
| @@ -14,8 +14,12 @@ module CSVImporter | |
| 14 14 | 
             
                  @csv_rows ||= begin
         | 
| 15 15 | 
             
                    sane_content = sanitize_content(read_content)
         | 
| 16 16 | 
             
                    separator = detect_separator(sane_content)
         | 
| 17 | 
            -
                    cells = CSV.parse( | 
| 18 | 
            -
             | 
| 17 | 
            +
                    cells = CSV.parse(
         | 
| 18 | 
            +
                      sane_content,
         | 
| 19 | 
            +
                      col_sep: separator, quote_char: quote_char, skip_blanks: true,
         | 
| 20 | 
            +
                      external_encoding: source_encoding
         | 
| 21 | 
            +
                    )
         | 
| 22 | 
            +
                    sanitize_cells(encode_cells(cells))
         | 
| 19 23 | 
             
                  end
         | 
| 20 24 | 
             
                end
         | 
| 21 25 |  | 
| @@ -44,16 +48,26 @@ module CSVImporter | |
| 44 48 | 
             
                end
         | 
| 45 49 |  | 
| 46 50 | 
             
                def sanitize_content(csv_content)
         | 
| 47 | 
            -
                  internal_encoding = encoding.split(':').last
         | 
| 48 51 | 
             
                  csv_content
         | 
| 49 | 
            -
                    .encode(Encoding.find( | 
| 52 | 
            +
                    .encode(Encoding.find(source_encoding), invalid: :replace, undef: :replace, replace: '') # Remove invalid byte sequences
         | 
| 50 53 | 
             
                    .gsub(/\r\r?\n?/, "\n") # Replaces windows line separators with "\n"
         | 
| 51 54 | 
             
                end
         | 
| 52 55 |  | 
| 53 56 | 
             
                SEPARATORS = [",", ";", "\t"]
         | 
| 54 57 |  | 
| 55 58 | 
             
                def detect_separator(csv_content)
         | 
| 56 | 
            -
                  SEPARATORS. | 
| 59 | 
            +
                  SEPARATORS.min_by do |separator|
         | 
| 60 | 
            +
                    csv_content.count(separator)
         | 
| 61 | 
            +
             | 
| 62 | 
            +
                    all_lines = csv_content.lines
         | 
| 63 | 
            +
                    base_number = all_lines.first.count(separator)
         | 
| 64 | 
            +
             | 
| 65 | 
            +
                    if base_number.zero?
         | 
| 66 | 
            +
                      Float::MAX
         | 
| 67 | 
            +
                    else
         | 
| 68 | 
            +
                      all_lines.map{|line| line.count(separator) - base_number }.map(&:abs).inject(0) { |sum, i| sum + i }
         | 
| 69 | 
            +
                    end
         | 
| 70 | 
            +
                  end
         | 
| 57 71 | 
             
                end
         | 
| 58 72 |  | 
| 59 73 | 
             
                # Remove trailing white spaces and ensure we always return a string
         | 
| @@ -64,5 +78,21 @@ module CSVImporter | |
| 64 78 | 
             
                    end
         | 
| 65 79 | 
             
                  end
         | 
| 66 80 | 
             
                end
         | 
| 81 | 
            +
             | 
| 82 | 
            +
                def encode_cells(rows)
         | 
| 83 | 
            +
                  rows.map do |cells|
         | 
| 84 | 
            +
                    cells.map do |cell|
         | 
| 85 | 
            +
                      cell ? cell.encode(target_encoding) : ""
         | 
| 86 | 
            +
                    end
         | 
| 87 | 
            +
                  end
         | 
| 88 | 
            +
                end
         | 
| 89 | 
            +
             | 
| 90 | 
            +
                def source_encoding
         | 
| 91 | 
            +
                  encoding.split(':').first || 'UTF-8'
         | 
| 92 | 
            +
                end
         | 
| 93 | 
            +
             | 
| 94 | 
            +
                def target_encoding
         | 
| 95 | 
            +
                  encoding.split(':').last || 'UTF-8'
         | 
| 96 | 
            +
                end
         | 
| 67 97 | 
             
              end
         | 
| 68 98 | 
             
            end
         | 
    
        data/lib/csv_importer/dsl.rb
    CHANGED
    
    | @@ -10,8 +10,8 @@ module CSVImporter | |
| 10 10 | 
             
                  config.column_definitions << options.merge(name: name)
         | 
| 11 11 | 
             
                end
         | 
| 12 12 |  | 
| 13 | 
            -
                def identifier(* | 
| 14 | 
            -
                  config.identifiers =  | 
| 13 | 
            +
                def identifier(*params)
         | 
| 14 | 
            +
                  config.identifiers = params.first.is_a?(Proc) ? params.first : params
         | 
| 15 15 | 
             
                end
         | 
| 16 16 |  | 
| 17 17 | 
             
                alias_method :identifiers, :identifier
         | 
    
        data/lib/csv_importer/report.rb
    CHANGED
    
    | @@ -12,8 +12,8 @@ module CSVImporter | |
| 12 12 |  | 
| 13 13 | 
             
                attribute :status, Symbol, default: proc { :pending }
         | 
| 14 14 |  | 
| 15 | 
            -
                attribute :missing_columns, Array[ | 
| 16 | 
            -
                attribute :extra_columns, Array[ | 
| 15 | 
            +
                attribute :missing_columns, Array[String], default: proc { [] }
         | 
| 16 | 
            +
                attribute :extra_columns, Array[String], default: proc { [] }
         | 
| 17 17 |  | 
| 18 18 | 
             
                attribute :parser_error, String
         | 
| 19 19 |  | 
    
        data/lib/csv_importer/row.rb
    CHANGED
    
    | @@ -7,11 +7,12 @@ module CSVImporter | |
| 7 7 | 
             
                include Virtus.model
         | 
| 8 8 |  | 
| 9 9 | 
             
                attribute :header, Header
         | 
| 10 | 
            +
                attribute :line_number, Integer
         | 
| 10 11 | 
             
                attribute :row_array, Array[String]
         | 
| 11 12 | 
             
                attribute :model_klass
         | 
| 12 | 
            -
                attribute :identifiers | 
| 13 | 
            +
                attribute :identifiers # Array[Symbol] or Proc
         | 
| 13 14 | 
             
                attribute :after_build_blocks, Array[Proc], default: []
         | 
| 14 | 
            -
                attribute :skip, Boolean, default: false
         | 
| 15 | 
            +
                attribute :skip, Virtus::Attribute::Boolean, default: false
         | 
| 15 16 |  | 
| 16 17 | 
             
                # The model to be persisted
         | 
| 17 18 | 
             
                def model
         | 
| @@ -40,27 +41,30 @@ module CSVImporter | |
| 40 41 | 
             
                      # can't dup Symbols, Integer etc...
         | 
| 41 42 | 
             
                    end
         | 
| 42 43 |  | 
| 43 | 
            -
                     | 
| 44 | 
            -
                    next if column_definition.nil?
         | 
| 44 | 
            +
                    next if column.definition.nil?
         | 
| 45 45 |  | 
| 46 | 
            -
                    set_attribute(model,  | 
| 46 | 
            +
                    set_attribute(model, column, value)
         | 
| 47 47 | 
             
                  end
         | 
| 48 48 |  | 
| 49 49 | 
             
                  model
         | 
| 50 50 | 
             
                end
         | 
| 51 51 |  | 
| 52 52 | 
             
                # Set the attribute using the column_definition and the csv_value
         | 
| 53 | 
            -
                def set_attribute(model,  | 
| 54 | 
            -
                   | 
| 55 | 
            -
             | 
| 53 | 
            +
                def set_attribute(model, column, csv_value)
         | 
| 54 | 
            +
                  column_definition = column.definition
         | 
| 55 | 
            +
                  transformer = column_definition.to
         | 
| 56 | 
            +
                  if transformer.respond_to?(:call)
         | 
| 57 | 
            +
                    arity = transformer.is_a?(Proc) ? transformer.arity : transformer.method(:call).arity
         | 
| 56 58 |  | 
| 57 | 
            -
                    case  | 
| 59 | 
            +
                    case arity
         | 
| 58 60 | 
             
                    when 1 # to: ->(email) { email.downcase }
         | 
| 59 | 
            -
                      model.public_send("#{column_definition.name}=",  | 
| 61 | 
            +
                      model.public_send("#{column_definition.name}=", transformer.call(csv_value))
         | 
| 60 62 | 
             
                    when 2 # to: ->(published, post) { post.published_at = Time.now if published == "true" }
         | 
| 61 | 
            -
                       | 
| 63 | 
            +
                      transformer.call(csv_value, model)
         | 
| 64 | 
            +
                    when 3 # to: ->(field_value, post, column) { post.hash_field[column.name] = field_value }
         | 
| 65 | 
            +
                      transformer.call(csv_value, model, column)
         | 
| 62 66 | 
             
                    else
         | 
| 63 | 
            -
                      raise ArgumentError, "`to`  | 
| 67 | 
            +
                      raise ArgumentError, "arity: #{transformer.arity.inspect} - `to` can only have 1, 2 or 3 arguments"
         | 
| 64 68 | 
             
                    end
         | 
| 65 69 | 
             
                  else
         | 
| 66 70 | 
             
                    attribute = column_definition.attribute
         | 
| @@ -88,10 +92,14 @@ module CSVImporter | |
| 88 92 | 
             
                end
         | 
| 89 93 |  | 
| 90 94 | 
             
                def find_model
         | 
| 91 | 
            -
                  return nil if identifiers. | 
| 95 | 
            +
                  return nil if identifiers.nil?
         | 
| 92 96 |  | 
| 93 97 | 
             
                  model = build_model
         | 
| 94 98 | 
             
                  set_attributes(model)
         | 
| 99 | 
            +
             | 
| 100 | 
            +
                  identifiers = model_identifiers(model)
         | 
| 101 | 
            +
                  return nil if identifiers.empty?
         | 
| 102 | 
            +
             | 
| 95 103 | 
             
                  query = Hash[
         | 
| 96 104 | 
             
                    identifiers.map { |identifier| [ identifier, model.public_send(identifier) ] }
         | 
| 97 105 | 
             
                  ]
         | 
| @@ -105,5 +113,15 @@ module CSVImporter | |
| 105 113 | 
             
                def skip!
         | 
| 106 114 | 
             
                  self.skip = true
         | 
| 107 115 | 
             
                end
         | 
| 116 | 
            +
             | 
| 117 | 
            +
                private
         | 
| 118 | 
            +
             | 
| 119 | 
            +
                def model_identifiers(model)
         | 
| 120 | 
            +
                  if identifiers.is_a?(Proc)
         | 
| 121 | 
            +
                    [identifiers.call(model)].flatten
         | 
| 122 | 
            +
                  else
         | 
| 123 | 
            +
                    identifiers
         | 
| 124 | 
            +
                  end
         | 
| 125 | 
            +
                end
         | 
| 108 126 | 
             
              end
         | 
| 109 127 | 
             
            end
         | 
    
        data/lib/csv_importer/runner.rb
    CHANGED
    
    | @@ -63,7 +63,16 @@ module CSVImporter | |
| 63 63 | 
             
                      end
         | 
| 64 64 |  | 
| 65 65 | 
             
                      add_to_report(row, tags)
         | 
| 66 | 
            -
             | 
| 66 | 
            +
             | 
| 67 | 
            +
                      after_save_blocks.each do |block|
         | 
| 68 | 
            +
                        case block.arity
         | 
| 69 | 
            +
                        when 0 then block.call
         | 
| 70 | 
            +
                        when 1 then block.call(row.model)
         | 
| 71 | 
            +
                        when 2 then block.call(row.model, row.csv_attributes)
         | 
| 72 | 
            +
                        else
         | 
| 73 | 
            +
                          raise ArgumentError, "after_save block of arity #{ block.arity } is not supported"
         | 
| 74 | 
            +
                        end
         | 
| 75 | 
            +
                      end
         | 
| 67 76 | 
             
                    end
         | 
| 68 77 | 
             
                  end
         | 
| 69 78 | 
             
                end
         | 
    
        data/lib/csv_importer/version.rb
    CHANGED
    
    
    
        metadata
    CHANGED
    
    | @@ -1,14 +1,14 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: csv-importer
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0. | 
| 4 | 
            +
              version: 0.8.0
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Philippe Creux
         | 
| 8 | 
            -
            autorequire: | 
| 8 | 
            +
            autorequire:
         | 
| 9 9 | 
             
            bindir: exe
         | 
| 10 10 | 
             
            cert_chain: []
         | 
| 11 | 
            -
            date:  | 
| 11 | 
            +
            date: 2020-09-03 00:00:00.000000000 Z
         | 
| 12 12 | 
             
            dependencies:
         | 
| 13 13 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 14 14 | 
             
              name: virtus
         | 
| @@ -25,34 +25,90 @@ dependencies: | |
| 25 25 | 
             
                  - !ruby/object:Gem::Version
         | 
| 26 26 | 
             
                    version: '0'
         | 
| 27 27 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 28 | 
            -
              name:  | 
| 28 | 
            +
              name: rspec
         | 
| 29 29 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| 30 30 | 
             
                requirements:
         | 
| 31 31 | 
             
                - - "~>"
         | 
| 32 32 | 
             
                  - !ruby/object:Gem::Version
         | 
| 33 | 
            -
                    version:  | 
| 33 | 
            +
                    version: 3.3.0
         | 
| 34 34 | 
             
              type: :development
         | 
| 35 35 | 
             
              prerelease: false
         | 
| 36 36 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 37 37 | 
             
                requirements:
         | 
| 38 38 | 
             
                - - "~>"
         | 
| 39 39 | 
             
                  - !ruby/object:Gem::Version
         | 
| 40 | 
            -
                    version:  | 
| 40 | 
            +
                    version: 3.3.0
         | 
| 41 41 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 42 42 | 
             
              name: rake
         | 
| 43 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 44 | 
            +
                requirements:
         | 
| 45 | 
            +
                - - ">="
         | 
| 46 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 47 | 
            +
                    version: '0'
         | 
| 48 | 
            +
              type: :development
         | 
| 49 | 
            +
              prerelease: false
         | 
| 50 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 51 | 
            +
                requirements:
         | 
| 52 | 
            +
                - - ">="
         | 
| 53 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 54 | 
            +
                    version: '0'
         | 
| 55 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 56 | 
            +
              name: guard-rspec
         | 
| 57 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 58 | 
            +
                requirements:
         | 
| 59 | 
            +
                - - ">="
         | 
| 60 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 61 | 
            +
                    version: '0'
         | 
| 62 | 
            +
              type: :development
         | 
| 63 | 
            +
              prerelease: false
         | 
| 64 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 65 | 
            +
                requirements:
         | 
| 66 | 
            +
                - - ">="
         | 
| 67 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 68 | 
            +
                    version: '0'
         | 
| 69 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 70 | 
            +
              name: activemodel
         | 
| 43 71 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| 44 72 | 
             
                requirements:
         | 
| 45 73 | 
             
                - - "~>"
         | 
| 46 74 | 
             
                  - !ruby/object:Gem::Version
         | 
| 47 | 
            -
                    version: ' | 
| 75 | 
            +
                    version: '5'
         | 
| 48 76 | 
             
              type: :development
         | 
| 49 77 | 
             
              prerelease: false
         | 
| 50 78 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 51 79 | 
             
                requirements:
         | 
| 52 80 | 
             
                - - "~>"
         | 
| 53 81 | 
             
                  - !ruby/object:Gem::Version
         | 
| 54 | 
            -
                    version: ' | 
| 55 | 
            -
             | 
| 82 | 
            +
                    version: '5'
         | 
| 83 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 84 | 
            +
              name: simplecov
         | 
| 85 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 86 | 
            +
                requirements:
         | 
| 87 | 
            +
                - - ">="
         | 
| 88 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 89 | 
            +
                    version: '0'
         | 
| 90 | 
            +
              type: :development
         | 
| 91 | 
            +
              prerelease: false
         | 
| 92 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 93 | 
            +
                requirements:
         | 
| 94 | 
            +
                - - ">="
         | 
| 95 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 96 | 
            +
                    version: '0'
         | 
| 97 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 98 | 
            +
              name: codeclimate-test-reporter
         | 
| 99 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 100 | 
            +
                requirements:
         | 
| 101 | 
            +
                - - ">="
         | 
| 102 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 103 | 
            +
                    version: '0'
         | 
| 104 | 
            +
              type: :development
         | 
| 105 | 
            +
              prerelease: false
         | 
| 106 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 107 | 
            +
                requirements:
         | 
| 108 | 
            +
                - - ">="
         | 
| 109 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 110 | 
            +
                    version: '0'
         | 
| 111 | 
            +
            description:
         | 
| 56 112 | 
             
            email:
         | 
| 57 113 | 
             
            - pcreux@gmail.com
         | 
| 58 114 | 
             
            executables: []
         | 
| @@ -85,11 +141,11 @@ files: | |
| 85 141 | 
             
            - lib/csv_importer/row.rb
         | 
| 86 142 | 
             
            - lib/csv_importer/runner.rb
         | 
| 87 143 | 
             
            - lib/csv_importer/version.rb
         | 
| 88 | 
            -
            homepage: https://github.com/ | 
| 144 | 
            +
            homepage: https://github.com/pcreux/csv-importer
         | 
| 89 145 | 
             
            licenses:
         | 
| 90 146 | 
             
            - MIT
         | 
| 91 147 | 
             
            metadata: {}
         | 
| 92 | 
            -
            post_install_message: | 
| 148 | 
            +
            post_install_message:
         | 
| 93 149 | 
             
            rdoc_options: []
         | 
| 94 150 | 
             
            require_paths:
         | 
| 95 151 | 
             
            - lib
         | 
| @@ -104,9 +160,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement | |
| 104 160 | 
             
                - !ruby/object:Gem::Version
         | 
| 105 161 | 
             
                  version: '0'
         | 
| 106 162 | 
             
            requirements: []
         | 
| 107 | 
            -
             | 
| 108 | 
            -
             | 
| 109 | 
            -
            signing_key: 
         | 
| 163 | 
            +
            rubygems_version: 3.0.3
         | 
| 164 | 
            +
            signing_key:
         | 
| 110 165 | 
             
            specification_version: 4
         | 
| 111 166 | 
             
            summary: CSV Import for humans
         | 
| 112 167 | 
             
            test_files: []
         |