dumped_railers 0.1.3 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 440f4c535538d4dc81d6b117d26c27a9639146776b11fa5358ee90bbd8528b8b
-  data.tar.gz: ea53e8c0ca3a4220432bfcf40895e79a5bff18b616965b2627db7f78b73b39b0
+  metadata.gz: cbb1709558461827585966be0f48b9741555fc9d70cd22a6c7970cb22260e6a6
+  data.tar.gz: f4db2cebc825ff52aeda68f2e06e466dd16663172e4b76289347374e35b046d4
 SHA512:
-  metadata.gz: 847f16c2456ddd871963feee72a95e57bffac72a63c7c67f714b93dfb7c95698f0ee44f368b0f2166a1a1bd1cc4bd9a4bc3d084fd0fc556c055d233bcc9f67f9
-  data.tar.gz: 11e9d92b573fa02f1602af0035d5834648d6893b0b04c6ad64729c6b19504b99ba3ce1bdea7ee9c04a02e2833164a5c1e10e809badd8eacb09f06c35f4eb9c43
+  metadata.gz: 51ddf56172e72afb4977cb899be013ac6678627905df7e90b6dc147b2afcfd0b2b6c9cd010e8a9a500dd9aa48ccc2a0ff70cf82fc02778fa7ef505f0d8df7f52
+  data.tar.gz: 24ee1c963fb89cb99dfcb2064c3d77e0e626525a14452e362aa12aeb9dd9f5f6409feb5c44a8f53ecb91d1eacd92f4793c8ab440714b1177a24d4d573048e7f0

data/.gitignore

@@ -14,3 +14,5 @@ Gemfile.lock
 
 # rspec failure tracking
 .rspec_status
+
+.ruby-version

data/.travis.yml

@@ -1,12 +1,17 @@
 ---
 language: ruby
 cache: bundler
+before_install: gem install bundler
 rvm:
+  - 3.0.0
   - 2.7.2
   - 2.6.6
   - 2.5.8
 gemfile:
-  - gemfiles/Gemfile.rails_6.1.0
-  - gemfiles/Gemfile.rails_6.0.3.4
-  - gemfiles/Gemfile.rails_5.2.4
-before_install: gem install bundler -v 2.1.4
+  - gemfiles/Gemfile.rails_6.1
+  - gemfiles/Gemfile.rails_6.0
+  - gemfiles/Gemfile.rails_5.2
+jobs:
+  exclude:
+    - rvm: 3.0.0
+      gemfile: gemfiles/Gemfile.rails_5.2

data/CHANGELOG.md

@@ -22,4 +22,28 @@
 - Cope with relations that has different name with the actual class name
   (e.g. associations defined using `:class_name` and `:foreign_key` options)
 
+## [0.1.4]
+### Changed
+- Update documents not to eagerload DumpedRailers to prevent accidental data breakage / leakage.
+  To activate, it is preferable to require explicitly where necessary.
+
+## [0.1.5]
+### Added
+- Supported in-memopry fixtures. Now users can dump into and import from in-memory fixture object without saving files.
+
+## [0.2.0]
+### Added
+- Provide options to limit models to import, so that users can prohibit modification to arbitrary models.
+- Support for Ruby 3.0.0 (requires Rails >= 6.0)
 
+### Changed
+- Accept both global configuration as well as runtime (one-off) settings for all the available options.
+  Now all the configured settings will be overridden at runtime when the settings are provided with arguments.
+
+## [0.3.0]
+### Added
+- Support `before_save`/`after_save` callbacks with import! method. The callbacks are invoked just before (or after) each table's records are saved.
+
+## [0.3.0]
+### Added
+- Accept multiple (array) callbacks for `before_save` / `after_save` arguments with DumpedRailers.import!.

data/README.md

@@ -22,7 +22,10 @@ This feature can particularily help you in the following senarios:
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'dumped_railers'
+# if you use this gem in production, `require: false` would be recommended
+# to prevent accidental database leakage/breakage.
+
+gem 'dumped_railers', require: false
 ```
 
 And then execute:
@@ -37,7 +40,13 @@ Or install it yourself as:
 
 ### Getting Started
 
-* if you want to dump (let's say) User, Item, and Tag models, just run the following.
+* Require dumped_railers where necessary (e.g. in your rake tasks, or script files)
+
+```ruby
+require 'dumped_railers'
+```
+
+* if you want to dump records from (let's say) User, Item, and Tag, just run the following.
 
 ```ruby
 DumpedRailers.dump!(User, Item, Tag, base_dir: 'tmp/fixtures/')
@@ -58,24 +67,35 @@ DumpedRailers.import!('tmp/fixtures/users.yml', 'tmp/fixtures/items.yml')
 
 NOTE: you at least have to provide all the dependent records, so that DumpedRailers can resolve dependencies among the fixtures provided.
 
-### Ignored Columns
+### Using In-Memory Fixtures
+
+DumpedRailers.dump! also returns an object, which can be imported directly as in-memory fixture.
+
+```ruby
+fixtures = DumpedRailers.dump!(User, Item, Tag)
+DumpedRailers.import!(fixtures)
+```
+
+DumpedRailers does not save the fixtures when `base_dir` keyword argument is not specified.
+
+### Ignoring Certain Columns
 
 * By default, DumpedRailers ignore three columns - `id`, `created_at`, `updated_at`. You can always update/change this settings as follows.
 
 ```ruby
 DumpedRailers.configure do |config|
-  config.ignorable_columns += [:published_on] # published_on will be ignored on top of default settings.
+  config.ignorable_columns += [:published_on] # :published_on will be ignored *on top of* default settings.
 end
 ```
 
 * of course you can totally replace the settings with your own.
 ```ruby
 DumpedRailers.configure do |config|
-  config.ignorable_columns = %i[uuid created_on updated_on] # uuid and created_on will be ignored instead of id, created_at, updated_at
+  config.ignorable_columns = %i[uuid created_on updated_on] # :uuid and :created_on will be ignored *instead of* :id, :created_at, :updated_at
 end
 ```
 
-### Masking, filtering
+### Masking, Filtering
 
 * you can pass `preprocessors` to DumpedRailers before it starts dump. All the attributes are filtered through preprocessors in order of registration.
 
@@ -104,7 +124,75 @@ masking_preprocessor = -> (attrs, model) { attrs.transform_values(&:upcase) }
 
 NOTE: The proprocessors must return attributes in the same format `{ attributes_name: value }` so that preprocessors and dump handlers can preprocessors in nested manner.
 
-### pseudo multi-tenancy (such as ActsAsTenant)
+### Limiting Import with Authorized Models Only
+
+* In case you don't want to accept arbitrary fixtures to import, you can limit model access as follows:
+
+```ruby
+DumpedRailers.import!(fixtures, authorized_models: [Item, Price])
+```
+
+This would allow us to import fixtures for items and prices, but reject modification on User, Purchase, Admin data.
+
+NOTE: Only DumpedRailers.import! is affected by this option. DumpedRailers.dump! can't be scoped (at least in the current version).
+
+
+### Setting Callbacks
+
+* You can set `before_save` / `after_save` callbacks for import! method.
+The callbacks are invoked just before/after each table's records are saved.
+
+```ruby
+before_callback = -> (model, records) {
+  if model == User
+    # set random initial passwords before saving
+    records.each do |user|
+      user.password = user.password_confirmation = SecureRandom.hex(12)
+    end
+  end
+}
+
+after_callback1 = -> (model, records) {
+  if model == User
+    records.each do |user|
+      user.confirm!
+    end
+  end
+}
+
+after_callback2 = -> (model, records) {
+  if model == Admin
+    records.each |admin|
+      notify_to_slack(admin.email, admin.name)
+    end
+  end
+}
+
+DumpedRailers.import!(fixture_path, before_save: before_callback, after_save: [after_callback1, after_callback2])
+```
+
+`before_save` /  `after_save` can accept both single and multiple (array) arguments.
+
+### Configuration
+
+* All the settings can be configured by either configuration (global) or arguments (at runtime).
+* When you have duplicated setting, arguments are respected: you can always override configured settings by arguments.
+
+```ruby
+DumpedRailers.configure do |config|
+  config.ignorable_columns = [:archived_at]
+  config.preprocessors = [FooPreprocessor, BarPreprocessor]
+end
+
+DumpedRailers.dump!(Item, ignorable_columns: [:id], preprocessors: [BazPreprocessor], base_dir: 'tmp/')
+# this would ignore `id` column, and apply BazPreprocessor only
+
+DumpedRailers.dump!(Price, base_dir: 'tmp/')
+# this would ignore `archived_at`, applies FooPreprocessor and BazPreprocessor
+# (settings provided with arguments are considered as one-off, and don't survive globally)
+```
+
+### Dump/Import under default_scope (e.g. ActsAsTenant)
 
 * Such library builds multi-tenancy environment on one single database, using default_scope to switch over database access rights between tenants. You can incorporate data from Tenant A to Tenant B as follows. let's say we use [ActsAsTenant](https://github.com/ErwinM/acts_as_tenant)
 
@@ -146,9 +234,13 @@ class Egg < ActiveRecord::Base
 end
 ```
 
-* When exception raised, checking your log might give you a good hint (desperately staring at the backtrace won't give much information)
+* For the same reason, self-associated model cannot be imported currently. (might be updated in the future version)
+
+* When an exception was raised, checking your log might give you a good hint (desperately staring at the backtrace won't give much information)
   consider displaying `tail -f logs/development.log` while executing your script.
 
+* DumpedRailers is tested against various association types, as long as target models and underlying tables have 1-on-1 mapping (straightforward ActiveRecord pattern). Currently we haven't test this against STI, CTI, as such. Applying this gem to such models might lead to some unexpected results.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -157,7 +249,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[fursich]/dumped_railers.
+Bug reports and pull requests are welcome on GitHub at https://github.com/fursich/dumped_railers.
 
 
 ## License

data/dumped_railers.gemspec

@@ -27,9 +27,9 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'rake', '~> 12.3.3'
-  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec', '~> 3.10'
   spec.add_development_dependency 'sqlite3'
-  spec.add_development_dependency 'activerecord', '~> 5.2'
+  spec.add_development_dependency 'activerecord', '>= 5.2'
   spec.add_development_dependency 'database_cleaner-active_record', '~> 1.8'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'pry-byebug'

data/gemfiles/{Gemfile.rails_5.2.4 → Gemfile.rails_5.2}

@@ -1,6 +1,6 @@
 source "https://rubygems.org"
 
 # Specify your gem's dependencies in dumped_railers.gemspec
-gem 'activerecord', '5.2.4'
+gem 'activerecord', '5.2.4.4'
 
 gemspec path: '../'

data/lib/dumped_railers.rb

@@ -4,43 +4,58 @@ require 'dumped_railers/version'
 require 'dumped_railers/file_helper.rb'
 require 'dumped_railers/dump'
 require 'dumped_railers/import'
+require 'dumped_railers/configuration'
 
 module DumpedRailers
-  class << self
-
-    def dump!(*models, base_dir: './', preprocessors: nil)
-      preprocessors = [Preprocessor::StripIgnorables.new, *preprocessors].compact.uniq
+  extend Configuration
 
-      fixture_handler = Dump.new(*models, preprocessors: preprocessors)
-      fixture_handler.build_fixtures!
+  class << self
+    def dump!(*models, base_dir: nil, preprocessors: nil, ignorable_columns: nil)
+      # override global config settings when options are specified
+      runtime_options = { preprocessors: preprocessors.presence, ignorable_columns: ignorable_columns.presence }.compact.reverse_merge(dump_options.deep_dup)
+      runtime_options[:preprocessors].unshift(
+        default_preprocessor(runtime_options[:ignorable_columns])
+      )
+
+      fixture_handler = Dump.new(*models, preprocessors: runtime_options[:preprocessors])
+      fixtures = fixture_handler.build_fixtures!
       fixture_handler.persist_all!(base_dir)
+
+      fixtures
     end
 
-    def import!(*paths)
+    def import!(*paths, authorized_models: nil, before_save: nil, after_save: nil)
       # make sure class-baseed caches starts with clean state
       DumpedRailers::RecordBuilder::FixtureRow::RecordStore.clear!
       DumpedRailers::RecordBuilder::DependencyTracker.clear!
 
-      fixture_handler = Import.new(*paths)
+      # override global config settings when options are specified
+      runtime_options = { authorized_models: authorized_models.presence }.compact.reverse_merge(import_options)
+
+      before_save = Array(before_save).compact
+      after_save  = Array(after_save).compact
+
+      fixture_handler = Import.new(
+        *paths,
+        authorized_models: runtime_options[:authorized_models],
+        before_save: before_save,
+        after_save:  after_save,
+      )
       fixture_handler.import_all!
     end
 
-    class Configuration < ::OpenStruct; end
+    private
 
-    def config
-      @_config ||= Configuration.new
+    def default_preprocessor(ignorable_columns)
+      Preprocessor::StripIgnorables.new(*ignorable_columns)
     end
 
-    def configure
-      yield config
+    def dump_options
+      options.slice(:ignorable_columns, :preprocessors)
     end
 
-    # FIXME: make it minimum
-    IGNORABLE_COLUMNS = %w[id created_at updated_at]
-    def configure_defaults!
-      configure do |config|
-        config.ignorable_columns = IGNORABLE_COLUMNS
-      end
+    def import_options
+      options.slice(:authorized_models)
     end
   end
 

data/lib/dumped_railers/configuration.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module DumpedRailers
+  module Configuration
+    extend Forwardable
+    def_delegators :@_config, :preprocessors, :ignorable_columns, :authorized_models
+
+    def configure
+      yield config
+    end
+
+    def options
+      config.to_h
+    end
+
+    IGNORABLE_COLUMNS = %w[id created_at updated_at]
+    def configure_defaults!
+      clear_configuration!(
+        ignorable_columns: IGNORABLE_COLUMNS,
+        preprocessors:     [],
+        authorized_models: :any,
+      )
+    end
+
+    def config
+      @_config ||= OpenStruct.new
+    end
+    private :config
+
+    def clear_configuration!(**attrs)
+      @_config = OpenStruct.new(attrs)
+    end
+    private :clear_configuration!
+  end
+end

data/lib/dumped_railers/dump.rb

@@ -7,7 +7,7 @@ module DumpedRailers
   class Dump
     def initialize(*models, preprocessors: [])
       @fixture_tables = models.map { |model|
-        FixtureBuilder::Model.new(model, preprocessors: preprocessors)
+        FixtureBuilder::Model.new(model,  preprocessors: preprocessors)
       }
     end
 
@@ -16,8 +16,10 @@ module DumpedRailers
     end
 
     def persist_all!(base_dir)
-      FileUtils.mkdir_p(base_dir)
-      FileHelper.write(*@fixtures, base_dir: base_dir)
+      if base_dir
+        FileUtils.mkdir_p(base_dir)
+        FileHelper.write(*@fixtures, base_dir: base_dir)
+      end
     end
   end
 end

data/lib/dumped_railers/import.rb

@@ -6,22 +6,39 @@ module DumpedRailers
   class Import
     attr_reader :fixture_set
 
-    def initialize(*paths)
-      @raw_fixtures = FileHelper.read_fixtures(*paths)
-      @fixture_set = RecordBuilder::FixtureSet.new(@raw_fixtures)
+    def initialize(*paths, authorized_models: [], before_save: [], after_save: [])
+      @before_save =  before_save
+      @after_save  =  after_save
+
+      if (paths.first.is_a? Hash)
+        @raw_fixtures = paths.first.values
+      else
+        @raw_fixtures = FileHelper.read_fixtures(*paths)
+      end
+
+      @fixture_set = RecordBuilder::FixtureSet.new(@raw_fixtures, authorized_models: authorized_models)
     end
 
-    def import_all!
+    def import_all!(&block)
+      fixture_set.authorize_models!
       fixture_set.sort_by_table_dependencies!
       @record_sets = fixture_set.build_record_sets!
 
       ActiveRecord::Base.transaction(joinable: false, requires_new: true) do
         # models have to be persisted one-by-one so that dependent models are able to 
         # resolve "belongs_to" (parent) association
-        @record_sets.each do |_model, records|
+        @record_sets.each do |model, records|
+          @before_save.each do |callback|
+            callback.call(model, records)
+          end
+
           # FIXME: faster implementation wanted, parhaps with activerocord-import
           # (objects needs to be reloaded somehow when using buik insert)
           records.each(&:save!)
+
+          @after_save.each do |callback|
+            callback.call(model, records)
+          end
         end
       end
     end

data/lib/dumped_railers/preprocessor/strip_ignorables.rb

@@ -3,9 +3,13 @@
 module DumpedRailers
   module Preprocessor
     class StripIgnorables
+      def initialize(*ignorable_columns)
+        @ignorable_columns = ignorable_columns.compact.map(&:to_s)
+      end
+
       def call(attributes, _model)
         attributes.reject { |column_name, _v|
-          DumpedRailers.config.ignorable_columns.map(&:to_s).include?(column_name)
+          @ignorable_columns.include?(column_name)
         }
       end
     end

data/lib/dumped_railers/record_builder/fixture_set.rb

@@ -8,35 +8,57 @@ module DumpedRailers
     class FixtureSet
       include TSort
       attr_reader :fixture_tables, :record_sets
-  
-      def initialize(raw_fixtures)
+
+      def initialize(raw_fixtures, authorized_models: [])
         @fixture_tables = raw_fixtures.map { |raw_records| build_fixture_table(raw_records) }
+        @authorized_models = Array(authorized_models)
       end
-  
+
       def sort_by_table_dependencies!
         @fixture_tables.each(&:analyze_metadata_dependencies!)
         # dependency are sorted in topological order using Active Record reflection
         @fixture_tables = tsort
-  
+
         self
       end
-  
+
+      def authorize_models!
+        return if @authorized_models.include?(:any)
+
+        unauthorized_models = fixture_models.reject { |model|
+          @authorized_models.include? model
+        }
+        return if unauthorized_models.empty?
+
+        raise RuntimeError, <<~"ERROR_MESSAGE"
+           You are trying to import data into unauthorized models.
+           Make sure that the fixture contains records for authorized models only.
+
+           Models that are forbidden to access: #{unauthorized_models.map(&:name).join(', ')}
+
+        ERROR_MESSAGE
+      end
+
       def build_record_sets!
         @record_sets = @fixture_tables.map { |table|
           [table.model, table.build_records!]
         }.to_h
       end
-  
+
       private
-  
+
       def build_fixture_table(raw_records)
         FixtureTable.new(raw_records)
       end
-  
+
+      def fixture_models
+        @fixture_tables.map(&:model)
+      end
+
       def tsort_each_node(&block)
         @fixture_tables.each { |table| block.call(table) }
       end
-  
+
       def tsort_each_child(node, &block)
         dependent_nodes = @fixture_tables.select { |table| node.dependencies.include? table.model_name }
         dependent_nodes.each &block

data/lib/dumped_railers/version.rb

@@ -1,3 +1,3 @@
 module DumpedRailers
-  VERSION = "0.1.3"
+  VERSION = '0.3.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dumped_railers
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.3.1
 platform: ruby
 authors:
 - Koji Onishi
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-12-11 00:00:00.000000000 Z
+date: 2021-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -70,14 +70,14 @@ dependencies:
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
 - !ruby/object:Gem::Dependency
@@ -156,10 +156,11 @@ files:
 - bin/console
 - bin/setup
 - dumped_railers.gemspec
-- gemfiles/Gemfile.rails_5.2.4
-- gemfiles/Gemfile.rails_6.0.3.4
-- gemfiles/Gemfile.rails_6.1.0
+- gemfiles/Gemfile.rails_5.2
+- gemfiles/Gemfile.rails_6.0
+- gemfiles/Gemfile.rails_6.1
 - lib/dumped_railers.rb
+- lib/dumped_railers/configuration.rb
 - lib/dumped_railers/dump.rb
 - lib/dumped_railers/file_helper.rb
 - lib/dumped_railers/fixture_builder/model.rb
@@ -193,7 +194,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: A flexible fixture importer/exporter, that can transport ActiveRecord data