dumped_railers 0.1.4 → 0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e4a93dbe684079409978553480483dbd5d2727f1f694470a8b62e89e0715fbe8
-  data.tar.gz: 266714eaca3eb9311effa69a5a6a1d4d784c758f04ea50574d24a44fcbe6a210
+  metadata.gz: 5025d07432c51021973eb7064be24d2a1cacbff2a31919c87341bc2f6a71693d
+  data.tar.gz: bbd212538475cda7a08278da201e151acbd13bfe8f05dd3558d257f485217ad2
 SHA512:
-  metadata.gz: f858057d3a62e389a01580a42fc926491a6c55e1f351a27a1d631fffba06727ccfecf6109848e1d547897218924398f1dc1176d9733205d8164cb7332987dae3
-  data.tar.gz: 4ee34e3d764eb00f8b244fdacce154f94cae38a6839c71bd7b640907fea235c84d8eaf83b1519fc6872fe305383d031253a55b5f91b79d39b5c96facb821c4f8
+  metadata.gz: 02f2ac6903602017cf440ba1c7431922dc600880665d4768da022031b62806673bae06d4a66bd6d134cab7a3e002734b453c76f838a0f256454094de3166eed4
+  data.tar.gz: 9d6d3ea246624629e0b1a24bd42ab301623e1fefe6699cb009b1558fed0e878c47fd41288c31ef5ca6e62fbb51e254cc57b71a0d139aad2f631063d6c541fe09

data/.gitignore

@@ -14,3 +14,5 @@ Gemfile.lock
 
 # rspec failure tracking
 .rspec_status
+
+.ruby-version

data/.travis.yml

@@ -1,12 +1,17 @@
 ---
 language: ruby
 cache: bundler
+before_install: gem install bundler
 rvm:
+  - 3.0.0
   - 2.7.2
   - 2.6.6
   - 2.5.8
 gemfile:
-  - gemfiles/Gemfile.rails_6.1.0
-  - gemfiles/Gemfile.rails_6.0.3.4
-  - gemfiles/Gemfile.rails_5.2.4
-before_install: gem install bundler -v 2.1.4
+  - gemfiles/Gemfile.rails_6.1
+  - gemfiles/Gemfile.rails_6.0
+  - gemfiles/Gemfile.rails_5.2
+jobs:
+  exclude:
+    - rvm: 3.0.0
+      gemfile: gemfiles/Gemfile.rails_5.2

data/CHANGELOG.md

@@ -26,3 +26,31 @@
 ### Changed
 - Update documents not to eagerload DumpedRailers to prevent accidental data breakage / leakage.
   To activate, it is preferable to require explicitly where necessary.
+
+## [0.1.5]
+### Added
+- Supported in-memopry fixtures. Now users can dump into and import from in-memory fixture object without saving files.
+
+## [0.2.0]
+### Added
+- Provide options to limit models to import, so that users can prohibit modification to arbitrary models.
+- Support for Ruby 3.0.0 (requires Rails >= 6.0)
+
+### Changed
+- Accept both global configuration as well as runtime (one-off) settings for all the available options.
+  Now all the configured settings will be overridden at runtime when the settings are provided with arguments.
+
+## [0.3.0]
+### Added
+- Support `before_save`/`after_save` callbacks with import! method. The callbacks are invoked just before (or after) each table's records are saved.
+
+## [0.3.1]
+### Added
+- Accept multiple (array) callbacks for `before_save` / `after_save` arguments with DumpedRailers.import!.
+
+## [0.4.0]
+### Changed
+- **BREAKING** preprocessor interface has changed so as that its API have consistency with that of callbacks.
+  - Preprocessors now require arguments (`model`, `attributes`) in this order. Before version < 0.4, it was (`attributes`, `model`)
+  - Attributes needs to be updated destructively within preprocessors. Return values are no longer required reflect the changes.
+

data/README.md

@@ -67,24 +67,35 @@ DumpedRailers.import!('tmp/fixtures/users.yml', 'tmp/fixtures/items.yml')
 
 NOTE: you at least have to provide all the dependent records, so that DumpedRailers can resolve dependencies among the fixtures provided.
 
-### Ignored Columns
+### Using In-Memory Fixtures
+
+DumpedRailers.dump! also returns an object, which can be imported directly as in-memory fixture.
+
+```ruby
+fixtures = DumpedRailers.dump!(User, Item, Tag)
+DumpedRailers.import!(fixtures)
+```
+
+DumpedRailers does not save the fixtures when `base_dir` keyword argument is not specified.
+
+### Ignoring Certain Columns
 
 * By default, DumpedRailers ignore three columns - `id`, `created_at`, `updated_at`. You can always update/change this settings as follows.
 
 ```ruby
 DumpedRailers.configure do |config|
-  config.ignorable_columns += [:published_on] # published_on will be ignored on top of default settings.
+  config.ignorable_columns += [:published_on] # :published_on will be ignored *on top of* default settings.
 end
 ```
 
 * of course you can totally replace the settings with your own.
 ```ruby
 DumpedRailers.configure do |config|
-  config.ignorable_columns = %i[uuid created_on updated_on] # uuid and created_on will be ignored instead of id, created_at, updated_at
+  config.ignorable_columns = %i[uuid created_on updated_on] # :uuid and :created_on will be ignored *instead of* :id, :created_at, :updated_at
 end
 ```
 
-### Masking, filtering
+### Masking, Filtering
 
 * you can pass `preprocessors` to DumpedRailers before it starts dump. All the attributes are filtered through preprocessors in order of registration.
 
@@ -92,15 +103,15 @@ end
 DumpedRailers.dump!(User, Item, base_dir: 'tmp/', preprocessors: [MaskingPreprocessor.new])
 ```
 
-* "Preprocessors" can be lambda, or module, or any objects that can repond to #call(atrs, model).
+* "Preprocessors" can be lambda, or module, or any objects that can repond to #call(model, attributes).
 
 
 ```ruby
 class MaskingPreprocessor
-  def call(attrs, model)
-    attrs.map { |col, value|
-      col.match?(/password/) ? [col, '<MASKED>'] : [col, value]
-    }.to_h
+  def call(model, attrs)
+    attrs.each { |col, _value|
+      attrs[col] = '<MASKED>' if col.match?(/password/)
+    }
   end
 end
 ```
@@ -108,12 +119,82 @@ end
 * a lambda object can be accepted as well
 
 ```ruby
-masking_preprocessor = -> (attrs, model) { attrs.transform_values(&:upcase) }
+masking_preprocessor = -> (model, attrs) { attrs.transform_values!(&:upcase) }
+```
+
+NOTE:
+* In order to reflect changes to the output, **preprocessors must change the attributes destructively**.
+* If you set multiple preprocessors, each preprocessor will be invoked sequentially from left to right, which means your second preprocessor receives attributes only after your first preprocessor update them.
+
+### Limiting Import with Authorized Models Only
+
+* In case you don't want to accept arbitrary fixtures to import, you can limit model access as follows:
+
+```ruby
+DumpedRailers.import!(fixtures, authorized_models: [Item, Price])
 ```
 
-NOTE: The proprocessors must return attributes in the same format `{ attributes_name: value }` so that preprocessors and dump handlers can preprocessors in nested manner.
+This would allow us to import fixtures for items and prices, but reject modification on User, Purchase, Admin data.
+
+NOTE: Only DumpedRailers.import! is affected by this option. DumpedRailers.dump! can't be scoped (at least in the current version).
+
+
+### Setting Callbacks
+
+* You can set `before_save` / `after_save` callbacks for import! method.
+The callbacks are invoked just before/after each table's records are saved.
+
+```ruby
+before_callback = -> (model, records) {
+  if model == User
+    # set random initial passwords before saving
+    records.each do |user|
+      user.password = user.password_confirmation = SecureRandom.hex(12)
+    end
+  end
+}
+
+after_callback1 = -> (model, records) {
+  if model == User
+    records.each do |user|
+      user.confirm!
+    end
+  end
+}
+
+after_callback2 = -> (model, records) {
+  if model == Admin
+    records.each |admin|
+      notify_to_slack(admin.email, admin.name)
+    end
+  end
+}
+
+DumpedRailers.import!(fixture_path, before_save: before_callback, after_save: [after_callback1, after_callback2])
+```
+
+`before_save` /  `after_save` can accept both single and multiple (array) arguments.
+
+### Configuration
+
+* All the settings can be configured by either configuration (global) or arguments (at runtime).
+* When you have duplicated setting, arguments are respected: you can always override configured settings by arguments.
+
+```ruby
+DumpedRailers.configure do |config|
+  config.ignorable_columns = [:archived_at]
+  config.preprocessors = [FooPreprocessor, BarPreprocessor]
+end
+
+DumpedRailers.dump!(Item, ignorable_columns: [:id], preprocessors: [BazPreprocessor], base_dir: 'tmp/')
+# this would ignore `id` column, and apply BazPreprocessor only
+
+DumpedRailers.dump!(Price, base_dir: 'tmp/')
+# this would ignore `archived_at`, applies FooPreprocessor and BazPreprocessor
+# (settings provided with arguments are considered as one-off, and don't survive globally)
+```
 
-### pseudo multi-tenancy (such as ActsAsTenant)
+### Dump/Import under default_scope (e.g. ActsAsTenant)
 
 * Such library builds multi-tenancy environment on one single database, using default_scope to switch over database access rights between tenants. You can incorporate data from Tenant A to Tenant B as follows. let's say we use [ActsAsTenant](https://github.com/ErwinM/acts_as_tenant)
 
@@ -170,7 +251,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[fursich]/dumped_railers.
+Bug reports and pull requests are welcome on GitHub at https://github.com/fursich/dumped_railers.
 
 
 ## License

data/dumped_railers.gemspec

@@ -27,9 +27,9 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'rake', '~> 12.3.3'
-  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec', '~> 3.10'
   spec.add_development_dependency 'sqlite3'
-  spec.add_development_dependency 'activerecord', '~> 5.2'
+  spec.add_development_dependency 'activerecord', '>= 5.2'
   spec.add_development_dependency 'database_cleaner-active_record', '~> 1.8'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'pry-byebug'

data/gemfiles/{Gemfile.rails_6.0.3.4 → Gemfile.rails_5.2}

@@ -1,6 +1,6 @@
 source "https://rubygems.org"
 
 # Specify your gem's dependencies in dumped_railers.gemspec
-gem 'activerecord', '6.1.0'
+gem 'activerecord', '5.2.4.4'
 
 gemspec path: '../'

data/gemfiles/{Gemfile.rails_5.2.4 → Gemfile.rails_6.1}

@@ -1,6 +1,6 @@
 source "https://rubygems.org"
 
 # Specify your gem's dependencies in dumped_railers.gemspec
-gem 'activerecord', '5.2.4'
+gem 'activerecord', '6.1.1'
 
 gemspec path: '../'

data/lib/dumped_railers.rb

@@ -4,43 +4,58 @@ require 'dumped_railers/version'
 require 'dumped_railers/file_helper.rb'
 require 'dumped_railers/dump'
 require 'dumped_railers/import'
+require 'dumped_railers/configuration'
 
 module DumpedRailers
-  class << self
-
-    def dump!(*models, base_dir: './', preprocessors: nil)
-      preprocessors = [Preprocessor::StripIgnorables.new, *preprocessors].compact.uniq
+  extend Configuration
 
-      fixture_handler = Dump.new(*models, preprocessors: preprocessors)
-      fixture_handler.build_fixtures!
+  class << self
+    def dump!(*models, base_dir: nil, preprocessors: nil, ignorable_columns: nil)
+      # override global config settings when options are specified
+      runtime_options = { preprocessors: preprocessors.presence, ignorable_columns: ignorable_columns.presence }.compact.reverse_merge(dump_options.deep_dup)
+      runtime_options[:preprocessors].unshift(
+        default_preprocessor(runtime_options[:ignorable_columns])
+      )
+
+      fixture_handler = Dump.new(*models, preprocessors: runtime_options[:preprocessors])
+      fixtures = fixture_handler.build_fixtures!
       fixture_handler.persist_all!(base_dir)
+
+      fixtures
     end
 
-    def import!(*paths)
+    def import!(*paths, authorized_models: nil, before_save: nil, after_save: nil)
       # make sure class-baseed caches starts with clean state
       DumpedRailers::RecordBuilder::FixtureRow::RecordStore.clear!
       DumpedRailers::RecordBuilder::DependencyTracker.clear!
 
-      fixture_handler = Import.new(*paths)
+      # override global config settings when options are specified
+      runtime_options = { authorized_models: authorized_models.presence }.compact.reverse_merge(import_options)
+
+      before_save = Array(before_save).compact
+      after_save  = Array(after_save).compact
+
+      fixture_handler = Import.new(
+        *paths,
+        authorized_models: runtime_options[:authorized_models],
+        before_save: before_save,
+        after_save:  after_save,
+      )
       fixture_handler.import_all!
     end
 
-    class Configuration < ::OpenStruct; end
+    private
 
-    def config
-      @_config ||= Configuration.new
+    def default_preprocessor(ignorable_columns)
+      Preprocessor::StripIgnorables.new(*ignorable_columns)
     end
 
-    def configure
-      yield config
+    def dump_options
+      options.slice(:ignorable_columns, :preprocessors)
     end
 
-    # FIXME: make it minimum
-    IGNORABLE_COLUMNS = %w[id created_at updated_at]
-    def configure_defaults!
-      configure do |config|
-        config.ignorable_columns = IGNORABLE_COLUMNS
-      end
+    def import_options
+      options.slice(:authorized_models)
     end
   end
 

data/lib/dumped_railers/configuration.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module DumpedRailers
+  module Configuration
+    extend Forwardable
+    def_delegators :@_config, :preprocessors, :ignorable_columns, :authorized_models
+
+    def configure
+      yield config
+    end
+
+    def options
+      config.to_h
+    end
+
+    IGNORABLE_COLUMNS = %w[id created_at updated_at]
+    def configure_defaults!
+      clear_configuration!(
+        ignorable_columns: IGNORABLE_COLUMNS,
+        preprocessors:     [],
+        authorized_models: :any,
+      )
+    end
+
+    def config
+      @_config ||= OpenStruct.new
+    end
+    private :config
+
+    def clear_configuration!(**attrs)
+      @_config = OpenStruct.new(attrs)
+    end
+    private :clear_configuration!
+  end
+end

data/lib/dumped_railers/dump.rb

@@ -7,7 +7,7 @@ module DumpedRailers
   class Dump
     def initialize(*models, preprocessors: [])
       @fixture_tables = models.map { |model|
-        FixtureBuilder::Model.new(model, preprocessors: preprocessors)
+        FixtureBuilder::Model.new(model,  preprocessors: preprocessors)
       }
     end
 
@@ -16,8 +16,10 @@ module DumpedRailers
     end
 
     def persist_all!(base_dir)
-      FileUtils.mkdir_p(base_dir)
-      FileHelper.write(*@fixtures, base_dir: base_dir)
+      if base_dir
+        FileUtils.mkdir_p(base_dir)
+        FileHelper.write(*@fixtures, base_dir: base_dir)
+      end
     end
   end
 end

data/lib/dumped_railers/fixture_builder/record.rb

@@ -11,11 +11,11 @@ module DumpedRailers
 
       def build!
         id = @record.id
-        attributes = 
-          @preprocessors.inject(@record.attributes) { |attrs, preprocessor|
-            preprocessor.call(attrs, @model)
-          }
-  
+        attributes = @record.attributes.deep_dup
+        @preprocessors.each do |preprocessor|
+          preprocessor.call(@model, attributes)
+        end
+
         # convert "belong_to association" foreign keys into record-unique labels
         @model.reflect_on_all_associations.select(&:belongs_to?).each do |rel|
           # skip ignorables
@@ -39,15 +39,15 @@ module DumpedRailers
 
         [record_label_for(@model.name, id), attributes]
       end
-    
+
       private
-  
+
       def record_label_for(class_name, id, type=nil)
         return nil unless id
 
         identifier = "#{class_name.to_s.underscore}_#{id}"
         type_specifier = "(#{type})" if type
-  
+
         "__#{identifier}#{type_specifier}"
       end
     end

data/lib/dumped_railers/import.rb

@@ -6,22 +6,39 @@ module DumpedRailers
   class Import
     attr_reader :fixture_set
 
-    def initialize(*paths)
-      @raw_fixtures = FileHelper.read_fixtures(*paths)
-      @fixture_set = RecordBuilder::FixtureSet.new(@raw_fixtures)
+    def initialize(*paths, authorized_models: [], before_save: [], after_save: [])
+      @before_save =  before_save
+      @after_save  =  after_save
+
+      if (paths.first.is_a? Hash)
+        @raw_fixtures = paths.first.values
+      else
+        @raw_fixtures = FileHelper.read_fixtures(*paths)
+      end
+
+      @fixture_set = RecordBuilder::FixtureSet.new(@raw_fixtures, authorized_models: authorized_models)
     end
 
-    def import_all!
+    def import_all!(&block)
+      fixture_set.authorize_models!
       fixture_set.sort_by_table_dependencies!
       @record_sets = fixture_set.build_record_sets!
 
       ActiveRecord::Base.transaction(joinable: false, requires_new: true) do
         # models have to be persisted one-by-one so that dependent models are able to 
         # resolve "belongs_to" (parent) association
-        @record_sets.each do |_model, records|
+        @record_sets.each do |model, records|
+          @before_save.each do |callback|
+            callback.call(model, records)
+          end
+
           # FIXME: faster implementation wanted, parhaps with activerocord-import
           # (objects needs to be reloaded somehow when using buik insert)
           records.each(&:save!)
+
+          @after_save.each do |callback|
+            callback.call(model, records)
+          end
         end
       end
     end

data/lib/dumped_railers/preprocessor/strip_ignorables.rb

@@ -3,9 +3,13 @@
 module DumpedRailers
   module Preprocessor
     class StripIgnorables
-      def call(attributes, _model)
-        attributes.reject { |column_name, _v|
-          DumpedRailers.config.ignorable_columns.map(&:to_s).include?(column_name)
+      def initialize(*ignorable_columns)
+        @ignorable_columns = ignorable_columns.compact.map(&:to_s)
+      end
+
+      def call(_model, attributes)
+        attributes.delete_if { |column_name, _v|
+          @ignorable_columns.include?(column_name)
         }
       end
     end

data/lib/dumped_railers/record_builder/fixture_set.rb

@@ -8,35 +8,57 @@ module DumpedRailers
     class FixtureSet
       include TSort
       attr_reader :fixture_tables, :record_sets
-  
-      def initialize(raw_fixtures)
+
+      def initialize(raw_fixtures, authorized_models: [])
         @fixture_tables = raw_fixtures.map { |raw_records| build_fixture_table(raw_records) }
+        @authorized_models = Array(authorized_models)
       end
-  
+
       def sort_by_table_dependencies!
         @fixture_tables.each(&:analyze_metadata_dependencies!)
         # dependency are sorted in topological order using Active Record reflection
         @fixture_tables = tsort
-  
+
         self
       end
-  
+
+      def authorize_models!
+        return if @authorized_models.include?(:any)
+
+        unauthorized_models = fixture_models.reject { |model|
+          @authorized_models.include? model
+        }
+        return if unauthorized_models.empty?
+
+        raise RuntimeError, <<~"ERROR_MESSAGE"
+           You are trying to import data into unauthorized models.
+           Make sure that the fixture contains records for authorized models only.
+
+           Models that are forbidden to access: #{unauthorized_models.map(&:name).join(', ')}
+
+        ERROR_MESSAGE
+      end
+
       def build_record_sets!
         @record_sets = @fixture_tables.map { |table|
           [table.model, table.build_records!]
         }.to_h
       end
-  
+
       private
-  
+
       def build_fixture_table(raw_records)
         FixtureTable.new(raw_records)
       end
-  
+
+      def fixture_models
+        @fixture_tables.map(&:model)
+      end
+
       def tsort_each_node(&block)
         @fixture_tables.each { |table| block.call(table) }
       end
-  
+
       def tsort_each_child(node, &block)
         dependent_nodes = @fixture_tables.select { |table| node.dependencies.include? table.model_name }
         dependent_nodes.each &block

data/lib/dumped_railers/version.rb

@@ -1,3 +1,3 @@
 module DumpedRailers
-  VERSION = "0.1.4"
+  VERSION = '0.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dumped_railers
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: '0.4'
 platform: ruby
 authors:
 - Koji Onishi
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-12-11 00:00:00.000000000 Z
+date: 2021-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -70,14 +70,14 @@ dependencies:
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
 - !ruby/object:Gem::Dependency
@@ -156,10 +156,11 @@ files:
 - bin/console
 - bin/setup
 - dumped_railers.gemspec
-- gemfiles/Gemfile.rails_5.2.4
-- gemfiles/Gemfile.rails_6.0.3.4
-- gemfiles/Gemfile.rails_6.1.0
+- gemfiles/Gemfile.rails_5.2
+- gemfiles/Gemfile.rails_6.0
+- gemfiles/Gemfile.rails_6.1
 - lib/dumped_railers.rb
+- lib/dumped_railers/configuration.rb
 - lib/dumped_railers/dump.rb
 - lib/dumped_railers/file_helper.rb
 - lib/dumped_railers/fixture_builder/model.rb
@@ -193,7 +194,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.0
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: A flexible fixture importer/exporter, that can transport ActiveRecord data