csv_step_importer 0.9.2 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d2d8c24884eb47655167c0b233d9506f73924a9e2c7834f989e615cb8ba7c40
-  data.tar.gz: 66246011d61cd911ae2de6c011696b3d36180aac06f84e6d056445a0efeeb863
+  metadata.gz: 930cc2407877056e544604dab4c86d885349db9206e217e6518ccddda4969a06
+  data.tar.gz: 2e2a217f8832430d9365018bfb0c9b99fa6f28787a25230748c74890d0efa5b3
 SHA512:
-  metadata.gz: d5200e7097b5342b90ce63febbabe657352368142faec7815256624f3fb5632fde121173467ca8c2f8f4275148658f505b4f3b60831e4c1d7e62c97d3017027e
-  data.tar.gz: 431414d5a60f2047bd17a5207cfa80be826e3af1865f39271d52d266354f68da9bd60aa0153acf808be9746b88f3babc19fc7cb6e4bc5d0cbe59c959d311a494
+  metadata.gz: 596bd1bce8110e37d1ab496cb75a2469d2e0bc438ecf2389764b28f82e940003cf10a4e296c28ef6e4a36d30dbf835023d23793b70fdea59a64b1d2d4e774a0a
+  data.tar.gz: 1e296ae4f9c321f90f129eedaa4a26a55e8b5d472239d2d9061a639bec184a39162e5bf9277be66f8f754fb0e8a6e5e0d6853efe7ce555f43744b58436ae6135

data/.ruby-version

@@ -0,0 +1 @@
+2.5.1

data/CHANGELOG.md

@@ -57,3 +57,32 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
       " A z " becomes :"A z"
       "A\"z" becomes :"A\"z"
 ```
+
+## 2018-09-11 Version 0.10.0
+### Added
+- Added a simple wrapper class for File and Chunk called Loader to use the same interface for importing a plain array of hashes as well as CSV files.
+
+Usage:
+
+```ruby
+CSVStepImporter::Loader.new(rows: data, processor_classes: [Author::ImportableModel]).save!
+```
+
+See lib/csv_step_importer/chunk.rb for more options
+
+or
+
+```ruby
+CSVStepImporter::Loader.new(
+  path: 'authors.csv',
+  processor_classes: [Author::ImportableModel],
+  csv_options: {file_encoding: "UTF-8"}
+).save!
+```
+
+See lib/csv_step_importer/file.rb for more options
+
+### Changed
+
+- Changed README, especially a note to include smarter_csv 2.0.0.pre1 into your project
+- Chunks default for the first row index is now 0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    csv_step_importer (0.9.2)
+    csv_step_importer (0.10.0)
       activemodel
       activerecord-import
       activesupport
@@ -87,7 +87,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    rubocop (0.58.2)
+    rubocop (0.59.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
       parser (>= 2.5, != 2.5.1.1)
@@ -101,7 +101,7 @@ GEM
     rubocop-rails_config (0.2.3)
       railties (>= 3.0)
       rubocop (~> 0.56)
-    rubocop-rspec (1.28.0)
+    rubocop-rspec (1.29.1)
       rubocop (>= 0.58.0)
     ruby-progressbar (1.10.0)
     smarter_csv (1.2.4)
@@ -124,4 +124,4 @@ DEPENDENCIES
   rubocop-rspec
 
 BUNDLED WITH
-   1.16.2
+   1.16.4

data/README.md

@@ -1,6 +1,6 @@
 # csv_step_importer
 
-A library to validate, speed up and organize bulk insertion of complex CSV data, including multi-table data.
+A library to validate, speed up and organize bulk insertion of complex CSV data into multiple tables.
 
 It depends on
 
@@ -12,7 +12,9 @@ It depends on
 Add this line to your application's Gemfile:
 
 ```ruby
+# Quicker CSV processing
 gem 'csv_step_importer'
+gem 'smarter_csv', github: 'tilo/smarter_csv'
 ```
 
 NOTE: you might need to add `gem 'smarter_csv', github: 'tilo/smarter_csv'` if you encounter problems building rows
@@ -27,62 +29,125 @@ Or install it yourself as:
 
 ## Usage
 
-### Hello world
+### Hello world setup (super simple sample, single table, no user-defined row, no user-defined dao)
 
-Create a new rails application:
+First let's create a basic rails application to play around with
 
 ```shell
-rails new currency_wiki --database=mysql
-cd currency_wiki
-echo "gem 'csv_step_importer'" >> Gemfile
+rails new bookshop --database=mysql
+cd bookshop
+echo "gem 'csv_step_importer'
+gem 'smarter_csv', github: 'tilo/smarter_csv'" >> Gemfile
 bundle install
-rails g model currency code:string:uniq name:string
+rails g model author name:string:uniq email:string
+rails g model book author:references name:string:uniq
 rails db:create db:migrate
 ```
 
 Then edit the model like this:
 
-/app/models/currency.rb
+app/models/author.rb
 
 ```ruby
-class Currency < ApplicationRecord
+class Author < ApplicationRecord
   class ImportableModel < CSVStepImporter::Model::ImportableModel
     # The model to be updated
     def model_class
-      ::Currency
+      puts Module.nesting.inspect
+      Module.nesting[1]
     end
 
     def columns
-      [:code, :name, :created_at, :updated_at]
+      [:name, :email, :created_at, :updated_at]
     end
 
     def on_duplicate_key_update
-      [:name, :updated_at]
+      [:email, :updated_at]
     end
   end
 end
 ```
 
+### Simple upload of a single row
 
-Create a test CSV file and upload it
+```shell
+rails c
+```
+
+```ruby
+irb(main)> data = [{ name: 'Milan Kundera', email: 'milan.kundera@example.com' }]
+irb(main)> importer = CSVStepImporter::Loader.new(rows: data, processor_classes: [Author::ImportableModel])
+irb(main)> importer.valid?
+=> true
+irb(main)> importer.save!
+   (1.1ms)  SET NAMES utf8,  @@SESSION.sql_mode = CONCAT(CONCAT(@@sql_mode, ',STRICT_ALL_TABLES'), ',NO_AUTO_VALUE_ON_ZERO'),  @@SESSION.sql_auto_is_null = 0, @@SESSION.wait_timeout = 2147483
+   (0.4ms)  BEGIN
+  [Author::ImportableModel, Author(id: integer, name: string, email: string, created_at: datetime, updated_at: datetime)]
+   (13.3ms)  SHOW VARIABLES like 'max_allowed_packet';
+  Author Create Many Without Validations Or Callbacks (9.2ms)  INSERT INTO `authors` (`name`,`email`,`created_at`,`updated_at`) VALUES ('Milan Kundera','milan.kundera@example.com','2018-09-11 11:33:07','2018-09-11 11:33:07') ON DUPLICATE KEY UPDATE `authors`.`email`=VALUES(`email`),`authors`.`updated_at`=VALUES(`updated_at`)
+   (154.6ms)  COMMIT
+  => true
+
+irb(main)> puts JSON.parse(Author.all.to_json).to_yaml # Isn't there an easy way to get clean yaml...
+
+   ---
+   - id: 7
+     name: Milan Kundera
+     email: milan.kundera@example.com
+     created_at: '2018-09-11T11:42:15.000Z'
+     updated_at: '2018-09-11T11:42:15.000Z'
+
+irb(main)> data = [{ name: 'Milan Kundera', email: 'milan.kundera2@example.com' }, { name: 'Immanuel Kant', email: 'immanuel.kant@example.com' }]
+irb(main)> CSVStepImporter::Loader.new(rows: data, processor_classes: [Author::ImportableModel]).save!
+=> true
+irb(main)> puts JSON.parse(Author.all.to_json).to_yaml # Isn't there an easy way to get clean yaml...
+
+# NOTE: updated_at changed, but the id did not
+---
+- id: 7
+  name: Milan Kundera
+  email: milan.kundera2@example.com
+  created_at: '2018-09-11T11:42:15.000Z'
+  updated_at: '2018-09-11T12:19:17.000Z'
+- id: 9
+  name: Immanuel Kant
+  email: immanuel.kant@example.com
+  created_at: '2018-09-11T12:19:17.000Z'
+  updated_at: '2018-09-11T12:19:17.000Z'
+```
+
+### File import using [tilo/smarter_csv](https://github.com/tilo/smarter_csv)
 
 ```shell
 rails c
 ```
 
 ```ruby
-File.open("currencies.csv", "w") do |file|
+irb(main)> File.open("authors.csv", "w") do |file|
   file.write(<<~CSV)
-    Name,Code
-    Euro,EUR
-    United States dollar,USD
-    Japanese Yen,JPY
+    Name,Email
+    Milan Kundera,milan.kundera@example.com
+    Immanuel Kant,immanuel.kant@example.com
   CSV
 end
 
-CSVStepImporter::File.new(path: 'currencies.csv', processor_classes: [Currency::ImportableModel]).save
-
-puts Currency.all.to_yaml
+irb(main)> CSVStepImporter::Loader.new(path: 'authors.csv', processor_classes: [Author::ImportableModel], csv_options: {file_encoding: "UTF-8"}).save
+=> true
+
+irb(main)> puts JSON.parse(Author.all.to_json).to_yaml
+
+# NOTE: The email and updated_at is updated as specified in on_duplicate_key_update
+---
+- id: 7
+  name: Milan Kundera
+  email: milan.kundera@example.com
+  created_at: '2018-09-11T11:42:15.000Z'
+  updated_at: '2018-09-11T12:24:13.000Z'
+- id: 9
+  name: Immanuel Kant
+  email: immanuel.kant@example.com
+  created_at: '2018-09-11T12:19:17.000Z'
+  updated_at: '2018-09-11T12:24:13.000Z'
 ```
 
 ### Simple model
@@ -110,7 +175,7 @@ class SimpleModel < CSVStepImporter::Model::Model
   end
 end
 
-CSVStepImporter::File.new(path: 'currencies.csv', processor_classes: [SimpleModel]).save
+CSVStepImporter::Loader.new(path: 'currencies.csv', processor_classes: [SimpleModel]).save
 ```
 
 ## Development

data/lib/csv_step_importer/chunk.rb

@@ -4,7 +4,7 @@ module CSVStepImporter
   class Chunk < CSVStepImporter::Node
     attr_accessor :cache, :rows, :first_row
 
-    def initialize(rows: [], row_class: CSVStepImporter::Row, processor_classes: nil, first_row: 2, **attributes)
+    def initialize(rows: [], row_class: CSVStepImporter::Row, processor_classes: nil, first_row: 0, **attributes)
       super **attributes
 
       self.cache = {}

data/lib/csv_step_importer/file.rb

@@ -14,12 +14,12 @@ module CSVStepImporter
     validates :row_class, presence: true
     validate :validate_csv_load_error
 
-    def initialize(path:, chunk_class: nil, row_class: nil, csv_options: {}, processor_classes: nil, **attributes)
+    def initialize(path:, chunk_class: CSVStepImporter::Chunk, row_class: CSVStepImporter::Row, csv_options: {}, processor_classes: nil, **attributes)
       super **attributes
 
-      self.chunk_class = chunk_class || CSVStepImporter::Chunk
+      self.chunk_class = chunk_class
       self.path = path
-      self.row_class = row_class || CSVStepImporter::Row
+      self.row_class = row_class
       self.processor_classes = processor_classes
 
       self.csv_options = csv_options

data/lib/csv_step_importer/loader.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "smarter_csv"
+
+module CSVStepImporter
+  class Loader < CSVStepImporter::Node
+    def initialize(file_class: CSVStepImporter::File, chunk_class: CSVStepImporter::Chunk, **attributes)
+      super **attributes.slice(:parent, :children, :env)
+      add_children attributes[:path] ? file_class.new( **attributes.merge!(chunk_class: chunk_class) ) : chunk_class.new( **attributes )
+    end
+  end
+end

data/lib/csv_step_importer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CSVStepImporter
-  VERSION = "0.9.2"
+  VERSION = "0.10.0"
 end

data/lib/csv_step_importer.rb

@@ -7,6 +7,7 @@ require "csv_step_importer/node"
 require "csv_step_importer/row"
 require "csv_step_importer/file"
 require "csv_step_importer/chunk"
+require "csv_step_importer/loader"
 require "csv_step_importer/model/dao"
 require "csv_step_importer/model/model"
 require "csv_step_importer/model/reflector"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv_step_importer
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.10.0
 platform: ruby
 authors:
 - Christian-Manuel Butzke
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-22 00:00:00.000000000 Z
+date: 2018-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -160,6 +160,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
+- ".ruby-version"
 - ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -175,6 +176,7 @@ files:
 - lib/csv_step_importer/base.rb
 - lib/csv_step_importer/chunk.rb
 - lib/csv_step_importer/file.rb
+- lib/csv_step_importer/loader.rb
 - lib/csv_step_importer/model/dao.rb
 - lib/csv_step_importer/model/importable_model.rb
 - lib/csv_step_importer/model/importer.rb
@@ -203,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Import your CSV files in multiple steps