csvbuilder-importer 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46f2394667af938e31e1d8741500adab3b9137ebc62266130fc4e9d51e3ffba9
-  data.tar.gz: 1fe82f6065b3a555b1f65379e06aa8dcf1af20f52176c5c1e8a5c4c41afd7b8a
+  metadata.gz: b630db32ac5aff6725cbdb9f66414aed575f0b7e5529dd982af31f2002c0ca88
+  data.tar.gz: d474369caa5696069c0273ca414e27e7f4aeb281e20312682fd99ec96d5880cc
 SHA512:
-  metadata.gz: 66aba6fed78608302a0d7a6ec0fb51677b1343e221fe2c0ad6875750965acd5622147cee35755810b901ca51fd7c304ef9519dc7ed22d269987a06f44a2e1a7a
-  data.tar.gz: eecc211fe2ab65813f26bbc2d8852e19f2dd56f1c8f5d150bea728372920c97189a032f4e03270e77320aa0332c9cbd050ca7c427b4f58e63ab7276e4d883bc9
+  metadata.gz: d188fde51bb5aafa126c10b0493f49ea085141d752cce03a4ed34c33ecbbc21712d3a06a3cb3232d2c56f9c40bd3b400224015ac42139e502844bdca497b1c06
+  data.tar.gz: c58f400d80291c30e4b9198381240b7a7ecfa4ddb0db4266b4fcd6eb0a744a0d7a7902a3f448ba8b73f8438a0c913c7195794750d2388aecee868fdccf8a3ba6

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 ## [Unreleased]
 
+## [Released]
+
+## [0.1.5] - 2023-07-26
+
+- Add a way to abort at the importer level, handy to handle wrong headers check https://github.com/joel/csvbuilder-importer/pull/12
+- Using Less Memory And Quicker Line Counter https://github.com/joel/csvbuilder-importer/pull/11
+
+## [0.1.4] - 2023-04-21
+
+- Potential Security Fix
+
+https://github.com/joel/csvbuilder-importer/compare/v0.1.2...v0.1.4
+
 ## [0.1.0] - 2022-12-16
 
 - Initial release

data/Gemfile.lock

@@ -0,0 +1,89 @@
+PATH
+  remote: .
+  specs:
+    csvbuilder-importer (0.1.5)
+      activemodel (>= 5.2)
+      activesupport (>= 5.2)
+      csvbuilder-core
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (7.0.4.2)
+      activesupport (= 7.0.4.2)
+    activesupport (7.0.4.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    ast (2.4.2)
+    concurrent-ruby (1.2.2)
+    csvbuilder-core (0.1.3)
+      activesupport (>= 5.2, < 8)
+    diff-lcs (1.5.0)
+    i18n (1.12.0)
+      concurrent-ruby (~> 1.0)
+    json (2.6.3)
+    minitest (5.17.0)
+    parallel (1.23.0)
+    parser (3.2.2.0)
+      ast (~> 2.4.1)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.8.0)
+    rexml (3.2.5)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.1)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    rubocop (1.50.2)
+      json (~> 2.3)
+      parallel (~> 1.10)
+      parser (>= 3.2.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.28.0)
+      parser (>= 3.2.1.0)
+    rubocop-capybara (2.18.0)
+      rubocop (~> 1.41)
+    rubocop-performance (1.17.1)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    rubocop-rspec (2.20.0)
+      rubocop (~> 1.33)
+      rubocop-capybara (~> 2.17)
+    ruby-progressbar (1.13.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.4.2)
+
+PLATFORMS
+  arm64-darwin-22
+  x86_64-linux
+
+DEPENDENCIES
+  csvbuilder-importer!
+  rake
+  rspec
+  rubocop
+  rubocop-performance
+  rubocop-rake
+  rubocop-rspec
+
+BUNDLED WITH
+   2.4.7

data/README.md

@@ -1,8 +1,8 @@
 # Csvbuilder::Importer
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/csvbuilder/importer`. To experiment with that code, run `bin/console` for an interactive prompt.
+[Csvbuilder::Importer](https://github.com/joel/csvbuilder-importer) is part of the [csvbuilder-collection](https://github.com/joel/csvbuilder)
 
-TODO: Delete this and the text above, and describe your gem
+The importer contains the implementation for importing data from a CSV file.
 
 ## Installation
 
@@ -14,9 +14,211 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
     $ gem install csvbuilder-importer
 
-## Usage
+# Usage
 
-TODO: Write usage instructions here
+Importing data from a CSV is critical and requires two validation layers. First, you need to ensure data from the CSV are correct, and Second, you need to check that data respects the business logic before inserting it into the system.
+
+To do that, `Csvbuilder::Import` use `ActiveModel::Validations` so you can write your validations in the `CsvImportModel`.
+
+```ruby
+class UserCsvImportModel
+  include Csvbuilder::Model
+  include Csvbuilder::Import
+
+  column :first_name
+  column :last_name
+
+  validates :first_name, presence: true, length: { minimum: 2 }
+
+  def abort?
+    "#{first_name} #{last_name}" == "Bill Gates"
+  end
+end
+```
+
+The import takes the CSV file and the Import class.
+
+```ruby
+rows = Csvbuilder::Import::File.new(file.path, UserCsvImportModel).each
+row_enumerator = rows.each # It's essential to go through the Enumerator to benefit from the callbacks. See References[^1]
+row_model_instance = row_enumerator.next
+```
+
+`Csvbuilder::Import` implement two essential methods:
+
+1. skip?
+2. abort? # NOTE: abort can be trigger at the importer level too.
+
+You have to provide your implementation of the method `abort?`. If the method `abort?` returns true, the iteration will stop.
+
+By default, `skip?` return true if the `CsvImportClass` is invalid, but it is safe to override. This means the previous iteration will not return any invalid row.
+
+# Integration
+
+Let's say we want to do something useful and add users if those users are valid.
+
+Let's extract the `CsvRowModel` for more clarity:
+
+```ruby
+class UserCsvRowModel
+  include Csvbuilder::Model
+
+  column :first_name
+  column :last_name
+end
+```
+
+```ruby
+class UserCsvImportModel < UserCsvRowModel
+  include Csvbuilder::Import
+
+  validates :first_name, presence: true, length: { minimum: 2 }
+  validates :last_name, presence: true, length: { minimum: 2 }
+
+  def user
+    User.new(first_name: first_name, last_name: last_name)
+  end
+
+  # Skip if the row is not valid,
+  # the user is not valid or
+  # the user already exists
+  def skip?
+    super || !user.valid? || user.exists?
+  end
+end
+```
+
+Now, we can safely import our users.
+
+```ruby
+[
+  ["First name", "Last name"],
+  ["John"      , "Doe"      ],
+]
+
+Csvbuilder::Import::File.new(file.path, UserCsvImportModel).each do |row_model|
+  row_model.user.save
+end
+```
+
+# Advance Integration
+
+`Csvbuilder::Import::File` implement callbacks. It provides the following:
+
+1.  before_each_iteration
+2.  around_each_iteration
+3.  after_each_iteration
+4.  before_next
+5.  around_next
+6.  after_next
+7.  before_abort
+8.  before_skip
+
+Let's extend `Csvbuilder::Import::File`
+
+```ruby
+class Importer < Csvbuilder::Import::File
+  attr_reader :row_in_errors
+
+  def initialize(*args)
+    super
+    @row_in_errors = RowErrors.new
+  end
+
+  after_next do
+    next true unless current_row_model # End of File
+    next true if current_row_model.valid? # No Errors To Collect
+
+    row_in_errors.append_errors(current_row_model)
+  end
+end
+```
+
+Now the importer can report the errors encountered instead of ignoring them.
+
+For documentation purposes, here is a possible implementation of the errors collector:
+
+```ruby
+class RowErrors
+  attr_reader :headers, :errors
+
+  def initialize
+    @errors = []
+  end
+
+  def append_errors(row_model)
+    @headers ||= begin
+      errors << row_model.class.headers
+      row_model.class.headers
+    end
+
+    row_in_error = []
+    row_model.source_attributes.map do |key, value|
+      row_in_error << if row_model.errors.messages[key].present?
+                        "Initial Value: [#{value}] - Errors: #{row_model.errors.messages[key].join(", ")}"
+                      else
+                        value
+                      end
+    end
+    errors << row_in_error
+  end
+end
+```
+
+Now we can nicely show the errors which occur over the import.
+
+```ruby
+[
+  ["First name", "Last name"],
+  ["J", "Doe"]
+]
+
+importer.row_in_errors.errors
+# => [
+# =>   ["First Name", "Last Name"],
+# =>   ["Initial Value: [J] - Errors: is too short (minimum is 2 characters)", "Doe"]
+# => ]
+```
+
+Thanks to the callback mechanism, the opportunities to interact with the import are immense. For instance, you can show the errors on a Web Form and offer the chance to the user to change the data and re-attempt.
+
+For long imports, you can show a progress bar to help customers cope with the import time; as you know, if errors have occurred, you can change the colour of the progress bar accordingly and offer the possibility to stop the import earlier.
+
+## Aborting an import
+
+There is a design challenge to handling an import line-by-line. If it makes the code more efficient and decoupled, we might have cases when we want to check something shared with all lines. The obvious ones are the headers. Let's say we want to check them and abort all imports if something wrong is detected. We probably don't want to add the abort conditioning on every line (Csvbuilder::Model or, more precisely, its extension Csvbuilder::Import). We would rather have it in the Importer itself. In that case, we can stop the Importer from invoking "abort!". Let's consider the following example:
+
+```ruby
+class Importer < Csvbuilder::Import::File
+
+  after_next do
+    if HeaderChecker.new(current_row_model).invalid?
+      abort!
+      next true # Keep going into #each and hit the callbacks
+    end
+  end
+
+end
+```
+
+```ruby
+context "with incorrect headers" do
+
+  should "not import data" do
+    importer = Importer.new(@file.path, @importer_class, @context)
+
+    row_enumerator = importer.each
+
+    assert_raises StopIteration do
+      row_enumerator.next
+    end
+  end
+end
+```
+
+## References
+
+- [^1] Csvbuilder::Import::File#each https://github.com/joel/csvbuilder-importer/blob/e8e6633a03dda4ae0e5d6775ec9d395dec553fbe/lib/csvbuilder/importer/public/import/file.rb#L66-L68
 
 ## Development
 
@@ -26,7 +228,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/csvbuilder-importer. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/csvbuilder-importer/blob/main/CODE_OF_CONDUCT.md).
+Bug reports and pull requests are welcome on GitHub at https://github.com/joel/csvbuilder-importer. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/csvbuilder-importer/blob/main/CODE_OF_CONDUCT.md).
 
 ## License
 

data/gemfiles/rails_edge.gemfile

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Runtime dependencies
+gem "activemodel", github: "rails/rails", branch: "main"
+gem "activesupport", github: "rails/rails", branch: "main"
+
+gem "csvbuilder-core"
+
+# Development dependencies
+gem "rake"
+gem "rspec"
+gem "rubocop"
+gem "rubocop-performance"
+gem "rubocop-rake"
+gem "rubocop-rspec"

data/lib/csvbuilder/importer/internal/import/csv.rb

@@ -20,10 +20,10 @@ module Csvbuilder
         reset
       end
 
-      # http://stackoverflow.com/questions/2650517/count-the-number-of-lines-in-a-file-without-reading-entire-file-into-memory
+      # https://gist.github.com/guilhermesimoes/d69e547884e556c3dc95?permalink_comment_id=4502636#gistcomment-4502636
       # @return [Integer] the number of rows in the file, including empty new lines
       def size
-        @size ||= `wc -l #{file_path}`.split[0].to_i + 1
+        @size ||= ::File.read(file_path).count($/)
       end
 
       # If the current position is at the headers, skip it and return it. Otherwise, only return false.

data/lib/csvbuilder/importer/public/import/file.rb

@@ -9,7 +9,7 @@ module Csvbuilder
       extend ActiveModel::Callbacks
       include ActiveModel::Validations
 
-      attr_reader :csv, :row_model_class, :index, :current_row_model, :previous_row_model, :context # -1 = start of file, 0 to infinity = index of row_model, nil = end of file, no row_model
+      attr_reader :interrupt, :csv, :row_model_class, :index, :current_row_model, :previous_row_model, :context # -1 = start of file, 0 to infinity = index of row_model, nil = end of file, no row_model
 
       delegate :size, :end_of_file?, :line_number, to: :csv
 
@@ -26,6 +26,7 @@ module Csvbuilder
         @csv             = ::Csvbuilder::Import::Csv.new(file_path) # Full namespace provided to avoid confusion with Ruby CSV class.
         @row_model_class = row_model_class
         @context         = context.to_h.symbolize_keys
+        @interrupt = false
         reset
       end
 
@@ -39,6 +40,7 @@ module Csvbuilder
         csv.reset
         @index = -1
         @current_row_model = nil
+        @interrupt = false
       end
 
       # Gets the next row model based on the context
@@ -74,7 +76,13 @@ module Csvbuilder
 
       # @return [Boolean] returns true, if the file should abort reading
       def abort?
-        !valid? || !!current_row_model.try(:abort?)
+        interrupt || !valid? || !!current_row_model.try(:abort?)
+      end
+
+      def abort!
+        @interrupt = true
+
+        nil
       end
 
       # @return [Boolean] returns true, if the file should skip `current_row_model`

data/lib/csvbuilder/importer/version.rb

@@ -2,6 +2,6 @@
 
 module Csvbuilder
   module Importer
-    VERSION = "0.1.3"
+    VERSION = "0.1.5"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csvbuilder-importer
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Joel Azemar
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-12-20 00:00:00.000000000 Z
+date: 2023-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -17,9 +17,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -27,9 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '8'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -37,9 +31,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -47,9 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '8'
 - !ruby/object:Gem::Dependency
   name: csvbuilder-core
   requirement: !ruby/object:Gem::Requirement
@@ -77,12 +65,14 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - gemfiles/Gemfile.5.2._gemfile
 - gemfiles/Gemfile.6.1._gemfile
 - gemfiles/Gemfile.7.0._gemfile
+- gemfiles/rails_edge.gemfile
 - lib/csvbuilder/importer.rb
 - lib/csvbuilder/importer/concerns/import/attributes.rb
 - lib/csvbuilder/importer/concerns/import/base.rb
@@ -115,7 +105,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.22
+rubygems_version: 3.4.17
 signing_key:
 specification_version: 4
 summary: Csvbuilder Importer contain the components to handle CSV importing