csv_step_importer 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 930cc2407877056e544604dab4c86d885349db9206e217e6518ccddda4969a06
-  data.tar.gz: 2e2a217f8832430d9365018bfb0c9b99fa6f28787a25230748c74890d0efa5b3
+  metadata.gz: a36a92f9240a933980983aff9d079724cc44882cd914ed2d6c2b51c1fbb47cca
+  data.tar.gz: 57dbea48c65e11867605da9621029b5da65d6700ffda0f092dfbe0c00e5b2858
 SHA512:
-  metadata.gz: 596bd1bce8110e37d1ab496cb75a2469d2e0bc438ecf2389764b28f82e940003cf10a4e296c28ef6e4a36d30dbf835023d23793b70fdea59a64b1d2d4e774a0a
-  data.tar.gz: 1e296ae4f9c321f90f129eedaa4a26a55e8b5d472239d2d9061a639bec184a39162e5bf9277be66f8f754fb0e8a6e5e0d6853efe7ce555f43744b58436ae6135
+  metadata.gz: 15157867c54bdd4d7868901c353667775f24a9b9ae5a34cc98659184bb52d0119f923cfa4577a4e6385b440da810c2fca4ba03ae2c1bce35cdf07d70045c6b98
+  data.tar.gz: 94f5eebbdc2fac03edcf0a18d3727bc0681f062a8f2cc631f201a8c2e94a2b97460fdb8370615f8dc262ae7e152de58dec77772a13a36ebe4685fd1cd02a0793

data/CHANGELOG.md

@@ -86,3 +86,46 @@ See lib/csv_step_importer/file.rb for more options
 
 - Changed README, especially a note to include smarter_csv 2.0.0.pre1 into your project
 - Chunks default for the first row index is now 0
+
+## 2018-09-11 Version 0.11.0
+### Added
+- Added methods to DAO
+  - dao_for(model:, pluralize: false)
+    retrieve a dao for a different model using the same CSV row. This is useful e.g. if you use the reflector to get ids of related data
+  - link!
+    link this dao to a row
+  - unlink!
+    unlink this dao from the row and replace it with a different dao
+- Model allows now to specify a unique composite key `composite_key_columns` to avoid duplicated daos.
+
+  Usage:
+
+  ```ruby
+  class Author < ApplicationRecord
+    class ImportableModel < CSVStepImporter::Model::ImportableModel
+      def composite_key_columns
+        [:name]
+      end
+    end
+  end
+  ```
+
+  And a CSV which contains the same name twice or more, like this:
+
+  ```csv
+  Author,Book
+  A1,B1
+  A1,B2
+  A2,B3
+  ```
+
+  If you do NOT specify `composite_key_columns` you will get three DAOs for A1, A1 and A2.
+  If you specify `composite_key_columns` non unique daos will be removed and you only will get A1 and A2.
+
+- The `Model`'s `cache_key` method allows now a `pluralize` option.
+- The `Model`'s `cache_key` method is now available in the instance as well.
+
+### Changed
+
+- ImportableModel's finder_keys method now defaults to composite_key_columns
+- ImportableModel's Importer (uses ActiveRecord::Import) now raises an exception if the import fails

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    csv_step_importer (0.10.0)
+    csv_step_importer (0.11.0)
       activemodel
       activerecord-import
       activesupport

data/README.md

@@ -53,14 +53,23 @@ class Author < ApplicationRecord
   class ImportableModel < CSVStepImporter::Model::ImportableModel
     # The model to be updated
     def model_class
-      puts Module.nesting.inspect
       Module.nesting[1]
     end
 
+    # return CSVStepImporter::Model::Reflector in order to enable reflections (e.g. get ids of all rows)
+    # disabled by default
+    def reflector_class
+      CSVStepImporter::Model::Reflector
+    end
+
     def columns
       [:name, :email, :created_at, :updated_at]
     end
 
+    def composite_key_columns
+      [:name]
+    end
+
     def on_duplicate_key_update
       [:email, :updated_at]
     end
@@ -68,7 +77,7 @@ class Author < ApplicationRecord
 end
 ```
 
-### Simple upload of a single row
+### Import a single data row
 
 ```shell
 rails c
@@ -116,7 +125,7 @@ irb(main)> puts JSON.parse(Author.all.to_json).to_yaml # Isn't there an easy way
   updated_at: '2018-09-11T12:19:17.000Z'
 ```
 
-### File import using [tilo/smarter_csv](https://github.com/tilo/smarter_csv)
+### Import a CSV data file
 
 ```shell
 rails c
@@ -153,7 +162,7 @@ irb(main)> puts JSON.parse(Author.all.to_json).to_yaml
 ### Simple model
 
 By default, for each row read from the CSV file, a DAO belonging to a model will be created.
-These models will be validated and saved in the order specified by the processor_classes option.
+These models will be validated and saved in the order specified by the `processor_classes` option.
 
 The simplest model is one, which simply calls `save` on all DAOs, which calls internally `create_or_update`.
 `create_or_update` is customizable.
@@ -165,7 +174,7 @@ This example will call `find_or_create_by` for each row after all validations ha
 ```ruby
 class SimpleDAO < CSVStepImporter::Model::DAO
   def create_or_update
-    Currency.find_or_create_by( name: row.name, code: row.code )
+    Currency.find_or_create_by(name: row.name, code: row.code)
   end
 end
 
@@ -182,7 +191,7 @@ CSVStepImporter::Loader.new(path: 'currencies.csv', processor_classes: [SimpleMo
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `rake install`. To release a new version, update the version number in `version.rb`, and then run `rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 

data/lib/csv_step_importer/model/dao.rb

@@ -20,12 +20,14 @@ module CSVStepImporter
         parent.parent
       end
 
+      # returns an array of all column values, used for batch importing
       def value
         @value ||= columns.each_with_object({}) do |key, values|
           values[key] = value_for_key key
         end
       end
 
+      # retrieve a value for a key from the dao or row
       def value_for_key(key)
         if respond_to?(key)
           send key
@@ -41,7 +43,7 @@ module CSVStepImporter
       end
 
       def create_or_update
-        # Daoの保存処理は基本的にstepsで行います
+        # DAOs are usually processed in batches by the model and not saved one by one
         true
       end
 
@@ -56,6 +58,32 @@ module CSVStepImporter
       def updated_at
         current_timestamp
       end
+
+      # retrieve a dao for a different model using the same CSV row. This is useful e.g. if you use the reflector to get ids of related data
+      def dao_for model:, pluralize: false
+        row.cache[model.cache_key(pluralize: pluralize)]
+      end
+
+      # link this dao to a row
+      def link!
+        # add to cache with pluralized key
+        (row.cache[model.cache_key(pluralize: true)] ||= []) << self
+
+        # add to cache with singular key (for convenience)
+        row.cache[model.cache_key(pluralize: false)] = self
+      end
+
+      # unlink this dao from the row and replace it with a different dao
+      def unlink! replace_with: nil
+        cached_daos = row.cache[model.cache_key(pluralize: true)]
+
+        # remove from cache with pluralized key
+        cached_daos.delete self
+        cached_daos << replace_with
+
+        # set any dao to cache with singular key (for convenience)
+        row.cache[model.cache_key(pluralize: false)] = cached_daos.first
+      end
     end
   end
 end

data/lib/csv_step_importer/model/importable_model.rb

@@ -32,7 +32,7 @@ module CSVStepImporter
       # NOTE: required only when reflector class is set
       # example: [:email]
       def finder_keys
-        raise "please extend and implement"
+        composite_key_columns || raise("please extend and implement")
       end
 
       # NOTE: required only when importer class is set

data/lib/csv_step_importer/model/importer.rb

@@ -14,7 +14,7 @@ module CSVStepImporter
       delegate :dao_values, to: :parent
 
       def create_or_update
-        model_class.import(
+        model_class.import!(
           columns,
           dao_values,
           on_duplicate_key_ignore: on_duplicate_key_ignore,

data/lib/csv_step_importer/model/model.rb

@@ -7,11 +7,13 @@ module CSVStepImporter
     class Model < CSVStepImporter::Node
       attr_accessor :dao_values
       delegate :rows, :cache, to: :parent
+      delegate :cache_key, to: :class
 
       def initialize(**attributes)
         super **attributes
 
         add_daos
+        filter_daos! if composite_key_columns
         add_model_children
       end
 
@@ -19,8 +21,9 @@ module CSVStepImporter
       # Configuration
       #########################################################
 
-      def self.cache_key
-        name.underscore.split("/").last.to_sym
+      def self.cache_key pluralize: false
+        key = name.underscore.gsub('/', '_')
+        (pluralize ? key.pluralize : key.singularize).to_sym
       end
 
       # example: [:email, :updated_at, :created_at]
@@ -32,6 +35,11 @@ module CSVStepImporter
         CSVStepImporter::Model::DAO
       end
 
+      # specify to an array of columns in order filter duplicates from daos
+      def composite_key_columns
+        nil
+      end
+
       #########################################################
       # Logic
       #########################################################
@@ -67,15 +75,24 @@ module CSVStepImporter
         dao_class.new parent: dao_node, row: row
       end
 
-      def link_rows_to_daos(daos:)
-        daos.each do |dao|
-          # add to cache with pluralized key
-          (dao.row.cache[self.class.cache_key.to_s.pluralize.to_sym] ||= []) << dao
+      # TODO a possible feature would be to add validation errors if the duplicates do not match in all columns (columns other than the composite key)
+      def filter_daos!
+        unique_daos = {}
+
+        daos.delete_if do |dao|
+          hash = dao.value.slice(composite_key_columns).hash
+          should_delete = (unique_daos[hash] ||= dao) == dao
 
-          # add dao to cache
-          dao.row.cache[self.class.cache_key] = dao
+          # unlink to be deleted dao and add a link to
+          dao.unlink! replace_with: unique_daos[hash] unless should_delete
+
+          should_delete
         end
       end
+
+      def link_rows_to_daos(daos:)
+        daos.each(&:link!)
+      end
     end
   end
 end

data/lib/csv_step_importer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CSVStepImporter
-  VERSION = "0.10.0"
+  VERSION = "0.11.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv_step_importer
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Christian-Manuel Butzke
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-11 00:00:00.000000000 Z
+date: 2018-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler