seed-do 3.2.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f41a0a75dff7954f90a7ed6cafe60eef73f0c20490ff9e42952b34e2efe896bc
-  data.tar.gz: dd04a94f944a7a290d211b16cec2537a3fa34bc2816cee6c1b594d22a91a907c
+  metadata.gz: c5486ba5dbc636b111be4e69d838b7f317b233ea9cf4b516d611fdf5070f0780
+  data.tar.gz: a04c1e585a72163d13184a317aecf059eb4c21f24d9b3e51fd208debf452a574
 SHA512:
-  metadata.gz: ee3af13c18e0354bc04a378db5d0fc6ef344e2af3c617cc5405153200dd37d316be5e31d07234e077719e875c0daa6098878907146c27924ccb2021c9ba36725
-  data.tar.gz: 36367e0c2457b3d18781824ef255095545b7c441c37468c28c2448413877656c5180edca15da8f8d7309cc2bda78d4d9e2e0e7066afdfe607ec0ad71dc89fd89
+  metadata.gz: a5d0cbf12b3790a33f0c362967c4a7b20a85ef7cdb3e9a0c15b47111d0a4fab0ae5912728c1fbab4be32697669f4ed672d221badc00ea5abca06999117e3a4fb
+  data.tar.gz: 0011f72cf7c27fcfef5d8dad96ca74525582eaf9ddc01e66dd0b1b575e255d28c72c49db6b81f0523704991ab12545d7a4ebfc75c83fe46ef8ede8a71448d423

data/README.md

@@ -142,6 +142,30 @@ Seed files can be run automatically using `rake db:seed_do`. There are two optio
 
 You can also do a similar thing in your code by calling `SeedDo.seed(fixture_paths, filter)`.
 
+## Bulk upsert
+
+If you load a large amount of seed data, you can enable bulk upsert mode:
+
+```ruby
+SeedDo.seed(bulk: true)
+```
+
+In bulk mode, SeedDo buffers seed data for each file and writes it with `upsert_all`, which can significantly reduce the number of queries.
+
+You can also control the batch size per `upsert_all` call. The default is `1000`.
+
+```ruby
+SeedDo.seed(bulk: { batch_size: 100 })
+```
+
+### Requirements and caveats
+
+- Bulk mode relies on Rails `upsert_all`
+- The seed constraints are passed to `unique_by`, so the related columns must have a unique index
+- Bulk mode is supported only on databases that support conflict targets, such as PostgreSQL and SQLite
+- Bulk mode is not available on MySQL
+- `SeedDo.seed` and `SeedDo.seed(bulk: true)` may not always produce the same record order, so verify the behavior carefully before enabling it in an existing production environment
+
 ## Disable output
 
 To disable output from Seed Do, set `SeedDo.quiet = true`.
@@ -200,6 +224,22 @@ DB=postgresql bundle exec rspec
 
 The connection paramaters for each of these are specified in spec/connections/, which you can edit if necessary (for example to change the username/password).
 
+### Dev Container
+
+This repository includes a Dev Container setup for running the test suite locally with Ruby, SQLite, MySQL, and PostgreSQL ready to use.
+
+1. Open the repository in a Dev Container.
+2. Wait for `bundle install` to finish.
+3. Run the specs you want:
+
+```
+bundle exec rspec
+DB=mysql2 bundle exec rspec
+DB=postgresql bundle exec rspec
+```
+
+The Dev Container config also sets `MYSQL_HOST=mysql` and `POSTGRES_HOST=postgres` automatically, so the database-backed specs work without editing local machine settings.
+
 ## Original Author
 
 [Michael Bleigh](http://www.mbleigh.com/) is the original author

data/lib/seed-do/active_record_extension.rb

@@ -29,7 +29,7 @@ module SeedDo
     #       { :x => 5, :y => 9,  :name => "Office" }
     #     )
     def seed(*args, &block)
-      SeedDo::Seeder.new(self, *parse_seed_do_args(args, block)).seed
+      SeedDo.current_seeder.seed(self, *parse_seed_do_args(args, block))
     end
 
     # Has the same syntax as {#seed}, but if a record already exists with the same values for
@@ -40,8 +40,7 @@ module SeedDo
     #   Person.seed(:id, :id => 1, :name => "Bob") # => Name changed
     #   Person.seed_once(:id, :id => 1, :name => "Harry") # => Name *not* changed
     def seed_once(*args, &block)
-      constraints, data = parse_seed_do_args(args, block)
-      SeedDo::Seeder.new(self, constraints, data, insert_only: true).seed
+      SeedDo.current_seeder.seed_once(self, *parse_seed_do_args(args, block))
     end
 
     private

data/lib/seed-do/bulk_seeder.rb

@@ -0,0 +1,66 @@
+module SeedDo
+  # Buffers seeds during a file run and flushes them in bulk.
+  class BulkSeeder
+    attr_reader :buffer
+
+    def initialize(batch_size: 1000)
+      @batch_size = batch_size
+      @buffer = nil
+
+      validate_support!
+    end
+
+    def seed(model, constraints, data)
+      puts " - #{model} #{data.inspect}" unless SeedDo.quiet
+      buffer[:seed] << { model: model, constraints: constraints, data: data }
+    end
+
+    def seed_once(model, constraints, data)
+      puts " - #{model} #{data.inspect}" unless SeedDo.quiet
+      buffer[:seed_once] << { model: model, constraints: constraints, data: data }
+    end
+
+    def with_seed_file
+      @buffer = { seed: [], seed_once: [] }
+      yield
+      process_buffer
+    ensure
+      @buffer = nil
+    end
+
+    private
+
+    def validate_support!
+      return if ActiveRecord::Base.connection.supports_insert_conflict_target?
+
+      raise ArgumentError,
+            "Bulk mode is not supported for #{ActiveRecord::Base.connection.adapter_name}. " \
+            'The database does not support upsert operations with conflict targets. ' \
+            'Please use SeedDo.seed without the bulk option.'
+    end
+
+    def process_buffer
+      return unless buffer
+
+      buffer.each do |type, operations|
+        next if operations.empty?
+
+        operations.chunk { |operation| [operation[:model], operation[:constraints]] }
+                  .each do |(model, constraints), chunk|
+          flush_chunk(model, constraints, type, chunk)
+        end
+      end
+    end
+
+    def flush_chunk(model, constraints, type, chunk)
+      all_data = chunk.flat_map { |operation| operation[:data] }
+
+      options = { unique_by: constraints }
+      options[:on_duplicate] = :skip if type == :seed_once
+
+      all_data.each_slice(@batch_size) do |batch|
+        model.upsert_all(batch, **options)
+      end
+    end
+  end
+end

data/lib/seed-do/runner.rb

@@ -3,9 +3,7 @@ require 'active_support/core_ext/array/wrap'
 
 module SeedDo
   # Runs seed files.
-  #
-  # It is not recommended to use this class directly. Instead, use {SeedDo.seed SeedDo.seed}, which creates
-  # an instead of {Runner} and calls {#run #run}.
+  # It is not recommended to use this class directly. Instead, use {SeedDo.seed SeedDo.seed}, which creates an instead of {Runner} and calls {#run #run}.
   #
   # @see SeedDo.seed SeedDo.seed
   class Runner
@@ -13,18 +11,31 @@ module SeedDo
     #   `SeedDo.fixture_paths` if {nil}. If the argument is not an array, it will be wrapped by one.
     # @param [Regexp] filter If given, only seed files with a file name matching this pattern will
     #   be used
-    def initialize(fixture_paths = nil, filter = nil)
+    # @param [Boolean, Hash] bulk If true, use upsert_all to insert/update records in bulk.
+    #   If a hash, can include :batch_size (default: 1000) to control the number of records
+    #   per upsert_all call.
+    def initialize(fixture_paths = nil, filter = nil, bulk: false)
       @fixture_paths = Array.wrap(fixture_paths || SeedDo.fixture_paths)
       @filter        = filter
+      @seeder        = build_seeder(bulk)
     end
 
     # Run the seed files.
     def run
+      SeedDo.current_seeder = @seeder
       puts "\n== Filtering seed files against regexp: #{@filter.inspect}" if @filter && !SeedDo.quiet
 
-      filenames.each do |filename|
-        run_file(filename)
-      end
+      filenames.each { |filename| run_file(filename) }
+    ensure
+      SeedDo.current_seeder = nil
+    end
+
+    def seed(model, constraints, data)
+      @seeder.seed(model, constraints, data)
+    end
+
+    def seed_once(model, constraints, data)
+      @seeder.seed_once(model, constraints, data)
     end
 
     private
@@ -33,18 +44,28 @@ module SeedDo
       puts "\n== Seed from #{filename}" unless SeedDo.quiet
 
       ActiveRecord::Base.transaction do
-        open(filename) do |file|
-          chunked_ruby = +''
-          file.each_line do |line|
-            if line == "# BREAK EVAL\n"
-              eval(chunked_ruby)
-              chunked_ruby = +''
-            else
-              chunked_ruby << line
-            end
+        @seeder.with_seed_file { _run_file(filename) }
+      end
+    end
+
+    def build_seeder(bulk)
+      return SeedDo::BulkSeeder.new(batch_size: bulk.fetch(:batch_size, 1000)) if bulk.is_a?(Hash)
+
+      bulk ? SeedDo::BulkSeeder.new : SeedDo::Seeder.new
+    end
+
+    def _run_file(filename)
+      open(filename) do |file|
+        chunked_ruby = +''
+        file.each_line do |line|
+          if line == "# BREAK EVAL\n"
+            eval(chunked_ruby)
+            chunked_ruby = +''
+          else
+            chunked_ruby << line
           end
-          eval(chunked_ruby) unless chunked_ruby == ''
         end
+        eval(chunked_ruby) unless chunked_ruby == ''
       end
     end
 

data/lib/seed-do/seeder.rb

@@ -8,30 +8,31 @@ module SeedDo
   #
   # @see ActiveRecordExtension
   class Seeder
-    # @param [ActiveRecord::Base] model_class The model to be seeded
-    # @param [Array<Symbol>] constraints A list of attributes which identify a particular seed. If
-    #   a record with these attributes already exists then it will be updated rather than created.
-    # @param [Array<Hash>] data Each item in this array is a hash containing attributes for a
-    #   particular record.
-    # @param [Hash] options
-    # @option options [Boolean] :quiet (SeedDo.quiet) If true, output will be silenced
-    # @option options [Boolean] :insert_only (false) If true then existing records which match the
-    #   constraints will not be updated, even if the seed data has changed
-    def initialize(model_class, constraints, data, options = {})
+    def seed(model_class, constraints, data)
+      seed_records(model_class, constraints, data)
+    end
+
+    def seed_once(model_class, constraints, data)
+      seed_records(model_class, constraints, data, insert_only: true)
+    end
+
+    def with_seed_file
+      yield
+    end
+
+    private
+
+    # Insert/update the records as appropriate. Validation is skipped while saving.
+    # @return [Array<ActiveRecord::Base>] The records which have been seeded
+    def seed_records(model_class, constraints, data, options = {})
       @model_class = model_class
       @constraints = constraints.to_a.empty? ? [:id] : constraints
       @data        = data.to_a || []
       @options     = options.symbolize_keys
 
-      @options[:quiet] ||= SeedDo.quiet
-
       validate_constraints!
       validate_data!
-    end
 
-    # Insert/update the records as appropriate. Validation is skipped while saving.
-    # @return [Array<ActiveRecord::Base>] The records which have been seeded
-    def seed
       records = @model_class.transaction do
         @data.map { |record_data| seed_record(record_data.symbolize_keys) }
       end
@@ -39,8 +40,6 @@ module SeedDo
       records
     end
 
-    private
-
     def validate_constraints!
       unknown_columns = @constraints.map(&:to_s) - @model_class.column_names
       return if unknown_columns.empty?
@@ -62,7 +61,7 @@ module SeedDo
       record = find_or_initialize_record(data)
       return if @options[:insert_only] && !record.new_record?
 
-      puts " - #{@model_class} #{data.inspect}" unless @options[:quiet]
+      puts " - #{@model_class} #{data.inspect}" unless SeedDo.quiet
 
       record.assign_attributes(data)
       record.save(validate: false) || raise(ActiveRecord::RecordNotSaved, 'Record not saved!')

data/lib/seed-do/version.rb

@@ -1,3 +1,3 @@
 module SeedDo
-  VERSION = '3.2.0'
+  VERSION = '4.0.0'
 end

data/lib/seed-do.rb

@@ -7,6 +7,7 @@ module SeedDo
   autoload :Seeder,                'seed-do/seeder'
   autoload :ActiveRecordExtension, 'seed-do/active_record_extension'
   autoload :BlockHash,             'seed-do/block_hash'
+  autoload :BulkSeeder,            'seed-do/bulk_seeder'
   autoload :Runner,                'seed-do/runner'
   autoload :Writer,                'seed-do/writer'
 
@@ -17,12 +18,20 @@ module SeedDo
   # plugin, SeedDo will set it to contain `Rails.root/db/fixtures` and
   # `Rails.root/db/fixtures/Rails.env`
   mattr_accessor :fixture_paths, default: ['db/fixtures']
-
   # Load seed data from files
   # @param [Array] fixture_paths The paths to look for seed files in
   # @param [Regexp] filter If given, only filenames matching this expression will be loaded
-  def self.seed(fixture_paths = SeedDo.fixture_paths, filter = nil)
-    Runner.new(fixture_paths, filter).run
+  # @param [Boolean] bulk If true, bulk insert/upsert will be used
+  def self.seed(fixture_paths = SeedDo.fixture_paths, filter = nil, bulk: false)
+    Runner.new(fixture_paths, filter, bulk: bulk).run
+  end
+
+  def self.current_seeder
+    @current_seeder || Seeder.new
+  end
+
+  def self.current_seeder=(seeder)
+    @current_seeder = seeder
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: seed-do
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Shinichi Maeshima
@@ -66,6 +66,7 @@ files:
 - lib/seed-do.rb
 - lib/seed-do/active_record_extension.rb
 - lib/seed-do/block_hash.rb
+- lib/seed-do/bulk_seeder.rb
 - lib/seed-do/capistrano.rb
 - lib/seed-do/capistrano3.rb
 - lib/seed-do/railtie.rb
@@ -94,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Easily manage seed data in your Active Record application
 test_files: []