dumped_railers 0.1.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7408bd3c1d02ff10b02e387037453b1a92ee3ea34e788e1a71eb8cc525cb8cb3
-  data.tar.gz: 20591a7ec81c99031e1a02f0d92226f14098d0b6fe42162609cc14bcd2241bd5
+  metadata.gz: fed65c606bfe60816965d6ded3909c05b3c70e1deaa177607a4ad34adf70b4aa
+  data.tar.gz: fab8b30addebfd9d26efefe35a922cac885797c5371ed1a7c4cfa5d573381b72
 SHA512:
-  metadata.gz: 3b1fe003568e67b17de96dd5f48de7bcf2ae5c1535325e0fb1756deaa6892af1eeead3e96d63e39fba68e2ec591c08305296b77f5a0d4ab3bec1e8fa209ca84d
-  data.tar.gz: 8bb87ed990555e17e3177d3555634e621352e3af67147fa82e32790068380b6fd0fb7525b3b09c94efce231d23cc9e173a7e911656248f90a58cf2172184ed8c
+  metadata.gz: ef030d65359058b04c74e91f6219bffede2dc2737ab1d3189ee6a1028a64f1f925c128df2e6d438742d9dca1277dace81082aec3d4763791c550d43467b1c50a
+  data.tar.gz: 4584cfc5ee262e40d1a6c432960b17b616c886f92fcdbd0c6a976e0e0379797e2b79d8fd8840f10c5d742661ced0dd13c373fe734bc1842b6e42a49721ba6a57

data/.gitignore

@@ -14,3 +14,5 @@ Gemfile.lock
 
 # rspec failure tracking
 .rspec_status
+
+.ruby-version

data/.travis.yml

@@ -1,12 +1,17 @@
 ---
 language: ruby
 cache: bundler
+before_install: gem install bundler
 rvm:
+  - 3.0.0
   - 2.7.2
   - 2.6.6
   - 2.5.8
 gemfile:
-  - gemfiles/Gemfile.rails_6.1.0
-  - gemfiles/Gemfile.rails_6.0.3.4
-  - gemfiles/Gemfile.rails_5.2.4
-before_install: gem install bundler -v 2.1.4
+  - gemfiles/Gemfile.rails_6.1
+  - gemfiles/Gemfile.rails_6.0
+  - gemfiles/Gemfile.rails_5.2
+jobs:
+  exclude:
+    - rvm: 3.0.0
+      gemfile: gemfiles/Gemfile.rails_5.2

data/CHANGELOG.md

@@ -8,3 +8,38 @@
 - Add config options to be able to ignore specific columns
 - Accept preprocessors to make filtering behavior pluggable and customizable
 - Add Readme
+
+## [0.1.1]
+### Fixed
+- Fix mis-handling of namespaced class names (separated by `::`)
+
+## [0.1.2]
+### Changed
+- Accept wider types of reference labels, allowing spaces in-between (tentative)
+
+## [0.1.3]
+### Fixed
+- Cope with relations that has different name with the actual class name
+  (e.g. associations defined using `:class_name` and `:foreign_key` options)
+
+## [0.1.4]
+### Changed
+- Update documents not to eagerload DumpedRailers to prevent accidental data breakage / leakage.
+  To activate, it is preferable to require explicitly where necessary.
+
+## [0.1.5]
+### Added
+- Supported in-memopry fixtures. Now users can dump into and import from in-memory fixture object without saving files.
+
+## [0.2.0]
+### Added
+- Provide options to limit models to import, so that users can prohibit modification to arbitrary models.
+- Support for Ruby 3.0.0 (requires Rails >= 6.0)
+
+### Changed
+- Accept both global configuration as well as runtime (one-off) settings for all the available options.
+  Now all the configured settings will be overridden at runtime when the settings are provided with arguments.
+
+## [0.3.0]
+### Added
+- Support `before_save`/`after_save` callbacks with import! method. The callbacks are invoked just before (or after) each table's records are saved.

data/README.md

@@ -22,7 +22,10 @@ This feature can particularily help you in the following senarios:
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'dumped_railers'
+# if you use this gem in production, `require: false` would be recommended
+# to prevent accidental database leakage/breakage.
+
+gem 'dumped_railers', require: false
 ```
 
 And then execute:
@@ -37,7 +40,13 @@ Or install it yourself as:
 
 ### Getting Started
 
-* if you want to dump (let's say) User, Item, and Tag models, just run the following.
+* Require dumped_railers where necessary (e.g. in your rake tasks, or script files)
+
+```ruby
+require 'dumped_railers'
+```
+
+* if you want to dump records from (let's say) User, Item, and Tag, just run the following.
 
 ```ruby
 DumpedRailers.dump!(User, Item, Tag, base_dir: 'tmp/fixtures/')
@@ -58,24 +67,35 @@ DumpedRailers.import!('tmp/fixtures/users.yml', 'tmp/fixtures/items.yml')
 
 NOTE: you at least have to provide all the dependent records, so that DumpedRailers can resolve dependencies among the fixtures provided.
 
-### Ignored Columns
+### Using In-Memory Fixtures
+
+DumpedRailers.dump! also returns an object, which can be imported directly as in-memory fixture.
+
+```ruby
+fixtures = DumpedRailers.dump!(User, Item, Tag)
+DumpedRailers.import!(fixtures)
+```
+
+DumpedRailers does not save the fixtures when `base_dir` keyword argument is not specified.
+
+### Ignoring Certain Columns
 
 * By default, DumpedRailers ignore three columns - `id`, `created_at`, `updated_at`. You can always update/change this settings as follows.
 
 ```ruby
 DumpedRailers.configure do |config|
-  config.ignorable_columns += [:published_on] # published_on will be ignored on top of default settings.
+  config.ignorable_columns += [:published_on] # :published_on will be ignored *on top of* default settings.
 end
 ```
 
 * of course you can totally replace the settings with your own.
 ```ruby
 DumpedRailers.configure do |config|
-  config.ignorable_columns = %i[uuid created_on updated_on] # uuid and created_on will be ignored instead of id, created_at, updated_at
+  config.ignorable_columns = %i[uuid created_on updated_on] # :uuid and :created_on will be ignored *instead of* :id, :created_at, :updated_at
 end
 ```
 
-### Masking, filtering
+### Masking, Filtering
 
 * you can pass `preprocessors` to DumpedRailers before it starts dump. All the attributes are filtered through preprocessors in order of registration.
 
@@ -104,7 +124,65 @@ masking_preprocessor = -> (attrs, model) { attrs.transform_values(&:upcase) }
 
 NOTE: The proprocessors must return attributes in the same format `{ attributes_name: value }` so that preprocessors and dump handlers can preprocessors in nested manner.
 
-### pseudo multi-tenancy (such as ActsAsTenant)
+### Limiting Import with Authorized Models Only
+
+* In case you don't want to accept arbitrary fixtures to import, you can limit model access as follows:
+
+```ruby
+DumpedRailers.import!(fixtures, authorized_models: [Item, Price])
+```
+
+This would allow us to import fixtures for items and prices, but reject modification on User, Purchase, Admin data.
+
+NOTE: Only DumpedRailers.import! is affected by this option. DumpedRailers.dump! can't be scoped (at least in the current version).
+
+
+### Setting Callbacks
+
+* DumpedRailers can accept a `before_save` / `after_save` callbacks with import! method.
+The callback is invoked just before/after each table's records are saved.
+
+```ruby
+before_callback = -> (model, records) {
+  if model == User
+    # set random initial passwords before saving
+    records.each do |user|
+      user.password = user.password_confirmation = SecureRandom.hex(12)
+    end
+  end
+}
+
+after_callback = -> (model, records) {
+  if model == User
+    records.each do |user|
+      user.confirm!
+    end
+  end
+}
+
+DumpedRailers.import!(fixture_path, before_save: before_callback, after_save: after_callback)
+```
+
+### Configuration
+
+* All the settings can be configured by either configuration (global) or arguments (at runtime).
+* When you have duplicated setting, arguments are respected: you can always override configured settings by arguments.
+
+```ruby
+DumpedRailers.configure do |config|
+  config.ignorable_columns = [:archived_at]
+  config.preprocessors = [FooPreprocessor, BarPreprocessor]
+end
+
+DumpedRailers.dump!(Item, ignorable_columns: [:id], preprocessors: [BazPreprocessor], base_dir: 'tmp/')
+# this would ignore `id` column, and apply BazPreprocessor only
+
+DumpedRailers.dump!(Price, base_dir: 'tmp/')
+# this would ignore `archived_at`, applies FooPreprocessor and BazPreprocessor
+# (settings provided with arguments are considered as one-off, and don't survive globally)
+```
+
+### Dump/Import under default_scope (e.g. ActsAsTenant)
 
 * Such library builds multi-tenancy environment on one single database, using default_scope to switch over database access rights between tenants. You can incorporate data from Tenant A to Tenant B as follows. let's say we use [ActsAsTenant](https://github.com/ErwinM/acts_as_tenant)
 
@@ -146,9 +224,13 @@ class Egg < ActiveRecord::Base
 end
 ```
 
-* When exception raised, checking your log might give you a good hint (desperately staring at the backtrace won't give much information)
+* For the same reason, self-associated model cannot be imported currently. (might be updated in the future version)
+
+* When an exception was raised, checking your log might give you a good hint (desperately staring at the backtrace won't give much information)
   consider displaying `tail -f logs/development.log` while executing your script.
 
+* DumpedRailers is tested against various association types, as long as target models and underlying tables have 1-on-1 mapping (straightforward ActiveRecord pattern). Currently we haven't test this against STI, CTI, as such. Applying this gem to such models might lead to some unexpected results.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -157,7 +239,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[fursich]/dumped_railers.
+Bug reports and pull requests are welcome on GitHub at https://github.com/fursich/dumped_railers.
 
 
 ## License

data/dumped_railers.gemspec

@@ -27,9 +27,9 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'rake', '~> 12.3.3'
-  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec', '~> 3.10'
   spec.add_development_dependency 'sqlite3'
-  spec.add_development_dependency 'activerecord', '~> 5.2'
+  spec.add_development_dependency 'activerecord', '>= 5.2'
   spec.add_development_dependency 'database_cleaner-active_record', '~> 1.8'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'pry-byebug'

data/gemfiles/{Gemfile.rails_5.2.4 → Gemfile.rails_5.2}

@@ -1,6 +1,6 @@
 source "https://rubygems.org"
 
 # Specify your gem's dependencies in dumped_railers.gemspec
-gem 'activerecord', '5.2.4'
+gem 'activerecord', '5.2.4.4'
 
 gemspec path: '../'

data/lib/dumped_railers.rb

@@ -4,43 +4,55 @@ require 'dumped_railers/version'
 require 'dumped_railers/file_helper.rb'
 require 'dumped_railers/dump'
 require 'dumped_railers/import'
+require 'dumped_railers/configuration'
 
 module DumpedRailers
-  class << self
-
-    def dump!(*models, base_dir: './', preprocessors: nil)
-      preprocessors = [Preprocessor::StripIgnorables.new, *preprocessors].compact.uniq
+  extend Configuration
 
-      fixture_handler = Dump.new(*models, preprocessors: preprocessors)
-      fixture_handler.build_fixtures!
+  class << self
+    def dump!(*models, base_dir: nil, preprocessors: nil, ignorable_columns: nil)
+      # override global config settings when options are specified
+      runtime_options = { preprocessors: preprocessors.presence, ignorable_columns: ignorable_columns.presence }.compact.reverse_merge(dump_options.deep_dup)
+      runtime_options[:preprocessors].unshift(
+        default_preprocessor(runtime_options[:ignorable_columns])
+      )
+
+      fixture_handler = Dump.new(*models, preprocessors: runtime_options[:preprocessors])
+      fixtures = fixture_handler.build_fixtures!
       fixture_handler.persist_all!(base_dir)
+
+      fixtures
     end
 
-    def import!(*paths)
+    def import!(*paths, authorized_models: nil, before_save: nil, after_save: nil)
       # make sure class-baseed caches starts with clean state
       DumpedRailers::RecordBuilder::FixtureRow::RecordStore.clear!
       DumpedRailers::RecordBuilder::DependencyTracker.clear!
 
-      fixture_handler = Import.new(*paths)
+      # override global config settings when options are specified
+      runtime_options = { authorized_models: authorized_models.presence }.compact.reverse_merge(import_options)
+
+      fixture_handler = Import.new(
+        *paths,
+        authorized_models: runtime_options[:authorized_models],
+        before_save: before_save,
+        after_save:  after_save,
+      )
       fixture_handler.import_all!
     end
 
-    class Configuration < ::OpenStruct; end
+    private
 
-    def config
-      @_config ||= Configuration.new
+    def default_preprocessor(ignorable_columns)
+      Preprocessor::StripIgnorables.new(*ignorable_columns)
     end
 
-    def configure
-      yield config
+    def dump_options
+      options.slice(:ignorable_columns, :preprocessors)
     end
 
-    # FIXME: make it minimum
-    IGNORABLE_COLUMNS = %w[id created_at updated_at]
-    def configure_defaults!
-      configure do |config|
-        config.ignorable_columns = IGNORABLE_COLUMNS
-      end
+    def import_options
+      options.slice(:authorized_models)
     end
   end
 

data/lib/dumped_railers/configuration.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module DumpedRailers
+  module Configuration
+    extend Forwardable
+    def_delegators :@_config, :preprocessors, :ignorable_columns, :authorized_models
+
+    def configure
+      yield config
+    end
+
+    def options
+      config.to_h
+    end
+
+    IGNORABLE_COLUMNS = %w[id created_at updated_at]
+    def configure_defaults!
+      clear_configuration!(
+        ignorable_columns: IGNORABLE_COLUMNS,
+        preprocessors:     [],
+        authorized_models: :any,
+      )
+    end
+
+    def config
+      @_config ||= OpenStruct.new
+    end
+    private :config
+
+    def clear_configuration!(**attrs)
+      @_config = OpenStruct.new(attrs)
+    end
+    private :clear_configuration!
+  end
+end

data/lib/dumped_railers/dump.rb

@@ -7,7 +7,7 @@ module DumpedRailers
   class Dump
     def initialize(*models, preprocessors: [])
       @fixture_tables = models.map { |model|
-        FixtureBuilder::Model.new(model, preprocessors: preprocessors)
+        FixtureBuilder::Model.new(model,  preprocessors: preprocessors)
       }
     end
 
@@ -16,8 +16,10 @@ module DumpedRailers
     end
 
     def persist_all!(base_dir)
-      FileUtils.mkdir_p(base_dir)
-      FileHelper.write(*@fixtures, base_dir: base_dir)
+      if base_dir
+        FileUtils.mkdir_p(base_dir)
+        FileHelper.write(*@fixtures, base_dir: base_dir)
+      end
     end
   end
 end

data/lib/dumped_railers/fixture_builder/record.rb

@@ -20,18 +20,18 @@ module DumpedRailers
         @model.reflect_on_all_associations.select(&:belongs_to?).each do |rel|
           # skip ignorables
           next unless attributes.has_key? rel.foreign_key.to_s
-  
+
           if rel.polymorphic?
-            model_name = attributes[rel.foreign_type.to_s]
-  
+            class_name = attributes[rel.foreign_type.to_s]
+
             attributes[rel.name.to_s] = record_label_for(
-              model_name,
+              class_name,
               attributes.delete(rel.foreign_key.to_s),
               attributes.delete(rel.foreign_type.to_s)
             )
           else
             attributes[rel.name.to_s] = record_label_for(
-              rel.name,
+              rel.class_name,
               attributes.delete(rel.foreign_key.to_s)
             )
           end
@@ -42,10 +42,10 @@ module DumpedRailers
     
       private
   
-      def record_label_for(model_name, id, type=nil)
+      def record_label_for(class_name, id, type=nil)
         return nil unless id
 
-        identifier = "#{model_name.to_s.underscore}_#{id}"
+        identifier = "#{class_name.to_s.underscore}_#{id}"
         type_specifier = "(#{type})" if type
   
         "__#{identifier}#{type_specifier}"

data/lib/dumped_railers/import.rb

@@ -6,22 +6,33 @@ module DumpedRailers
   class Import
     attr_reader :fixture_set
 
-    def initialize(*paths)
-      @raw_fixtures = FileHelper.read_fixtures(*paths)
-      @fixture_set = RecordBuilder::FixtureSet.new(@raw_fixtures)
+    def initialize(*paths, authorized_models: [], before_save: nil, after_save: nil)
+      @before_save =  before_save
+      @after_save  =  after_save
+
+      if (paths.first.is_a? Hash)
+        @raw_fixtures = paths.first.values
+      else
+        @raw_fixtures = FileHelper.read_fixtures(*paths)
+      end
+
+      @fixture_set = RecordBuilder::FixtureSet.new(@raw_fixtures, authorized_models: authorized_models)
     end
 
-    def import_all!
+    def import_all!(&block)
+      fixture_set.authorize_models!
       fixture_set.sort_by_table_dependencies!
       @record_sets = fixture_set.build_record_sets!
 
       ActiveRecord::Base.transaction(joinable: false, requires_new: true) do
         # models have to be persisted one-by-one so that dependent models are able to 
         # resolve "belongs_to" (parent) association
-        @record_sets.each do |_model, records|
+        @record_sets.each do |model, records|
+          @before_save.call(model, records) if @before_save
           # FIXME: faster implementation wanted, parhaps with activerocord-import
           # (objects needs to be reloaded somehow when using buik insert)
           records.each(&:save!)
+          @after_save.call(model, records) if @after_save
         end
       end
     end

data/lib/dumped_railers/preprocessor/strip_ignorables.rb

@@ -3,9 +3,13 @@
 module DumpedRailers
   module Preprocessor
     class StripIgnorables
+      def initialize(*ignorable_columns)
+        @ignorable_columns = ignorable_columns.compact.map(&:to_s)
+      end
+
       def call(attributes, _model)
         attributes.reject { |column_name, _v|
-          DumpedRailers.config.ignorable_columns.map(&:to_s).include?(column_name)
+          @ignorable_columns.include?(column_name)
         }
       end
     end

data/lib/dumped_railers/record_builder/fixture_set.rb

@@ -8,35 +8,57 @@ module DumpedRailers
     class FixtureSet
       include TSort
       attr_reader :fixture_tables, :record_sets
-  
-      def initialize(raw_fixtures)
+
+      def initialize(raw_fixtures, authorized_models: [])
         @fixture_tables = raw_fixtures.map { |raw_records| build_fixture_table(raw_records) }
+        @authorized_models = Array(authorized_models)
       end
-  
+
       def sort_by_table_dependencies!
         @fixture_tables.each(&:analyze_metadata_dependencies!)
         # dependency are sorted in topological order using Active Record reflection
         @fixture_tables = tsort
-  
+
         self
       end
-  
+
+      def authorize_models!
+        return if @authorized_models.include?(:any)
+
+        unauthorized_models = fixture_models.reject { |model|
+          @authorized_models.include? model
+        }
+        return if unauthorized_models.empty?
+
+        raise RuntimeError, <<~"ERROR_MESSAGE"
+           You are trying to import data into unauthorized models.
+           Make sure that the fixture contains records for authorized models only.
+
+           Models that are forbidden to access: #{unauthorized_models.map(&:name).join(', ')}
+
+        ERROR_MESSAGE
+      end
+
       def build_record_sets!
         @record_sets = @fixture_tables.map { |table|
           [table.model, table.build_records!]
         }.to_h
       end
-  
+
       private
-  
+
       def build_fixture_table(raw_records)
         FixtureTable.new(raw_records)
       end
-  
+
+      def fixture_models
+        @fixture_tables.map(&:model)
+      end
+
       def tsort_each_node(&block)
         @fixture_tables.each { |table| block.call(table) }
       end
-  
+
       def tsort_each_child(node, &block)
         dependent_nodes = @fixture_tables.select { |table| node.dependencies.include? table.model_name }
         dependent_nodes.each &block

data/lib/dumped_railers/version.rb

@@ -1,3 +1,3 @@
 module DumpedRailers
-  VERSION = "0.1.2"
+  VERSION = '0.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dumped_railers
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Koji Onishi
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-12-10 00:00:00.000000000 Z
+date: 2021-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -70,14 +70,14 @@ dependencies:
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
 - !ruby/object:Gem::Dependency
@@ -156,10 +156,11 @@ files:
 - bin/console
 - bin/setup
 - dumped_railers.gemspec
-- gemfiles/Gemfile.rails_5.2.4
-- gemfiles/Gemfile.rails_6.0.3.4
-- gemfiles/Gemfile.rails_6.1.0
+- gemfiles/Gemfile.rails_5.2
+- gemfiles/Gemfile.rails_6.0
+- gemfiles/Gemfile.rails_6.1
 - lib/dumped_railers.rb
+- lib/dumped_railers/configuration.rb
 - lib/dumped_railers/dump.rb
 - lib/dumped_railers/file_helper.rb
 - lib/dumped_railers/fixture_builder/model.rb
@@ -193,7 +194,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: A flexible fixture importer/exporter, that can transport ActiveRecord data