db_seed_dump 1.0.0

data/lib/seed_dump/dump_methods.rb

@@ -0,0 +1,173 @@
+class SeedDump
+  module DumpMethods
+    APPEND_FILE_MODE = 'a+'.freeze
+    OVERWRITE_FILE_MODE = 'w+'.freeze
+    include Enumeration
+
+    def dump(records, options = {})
+      return nil if records.count.zero?
+
+      io = open_io(options)
+
+      write_records_to_io(records, io, options)
+    ensure
+      io.close if io.present?
+    end
+
+    private
+
+    def dump_record(record, options)
+      attribute_strings = []
+
+      # We select only string attribute names to avoid conflict
+      # with the composite_primary_keys gem (it returns composite
+      # primary key attribute names as hashes).
+      record.attributes.select { |key| key.is_a?(String) || key.is_a?(Symbol) }.each do |attribute, value|
+        unless options[:exclude].include?(attribute.to_sym)
+          attribute_strings << dump_attribute_new(attribute, value,
+                                                  options)
+        end
+      end
+
+      open_character, close_character = options[:import] ? ['[', ']'] : ['{', '}']
+
+      "#{open_character}#{attribute_strings.join(', ')}#{close_character}"
+    end
+
+    def dump_attribute_new(attribute, value, options)
+      options[:import] ? value_to_s(value) : "#{attribute}: #{value_to_s(value)}"
+    end
+
+    def value_to_s(value)
+      value = case value
+              when BigDecimal, IPAddr
+                value.to_s
+              when Date, Time, DateTime
+                value.to_s(:db)
+              when Range
+                range_to_string(value)
+              when ->(v) { v.class.ancestors.map(&:to_s).include?('RGeo::Feature::Instance') }
+                value.to_s
+              else
+                value
+              end
+
+      value.inspect
+    end
+
+    def range_to_string(object)
+      from = object.begin.respond_to?(:infinite?) && object.begin.infinite? ? '' : object.begin
+      to   = object.end.respond_to?(:infinite?) && object.end.infinite? ? '' : object.end
+      "[#{from},#{to}#{object.exclude_end? ? ')' : ']'}"
+    end
+
+    def open_io(options)
+      if options[:file].present?
+        mode = options[:append] ? APPEND_FILE_MODE : OVERWRITE_FILE_MODE
+        file_path = if options[:file_split_limit]
+                      file_path_with_file_index(options)
+                    else
+                      options[:file]
+                    end
+
+        File.open(file_path, mode)
+      else
+        StringIO.new('', OVERWRITE_FILE_MODE)
+      end
+    end
+
+    def file_path_with_file_index(options)
+      base_name = File.basename(options[:file], '.*')
+      options[:file].reverse.sub(
+        base_name.reverse,
+        [
+          base_name,
+          (options[:current_file_index]&.to_i || 1)
+        ].join('_').reverse
+      ).reverse
+    end
+
+    def write_records_to_io(records, io, options)
+      options[:exclude] ||= %i[id created_at updated_at]
+
+      setup_io(io, options, records)
+
+      enumeration_method = if records.is_a?(ActiveRecord::Relation) || records.is_a?(Class)
+                             :active_record_enumeration
+                           else
+                             :enumerable_enumeration
+                           end
+
+      send(enumeration_method, records, io, options) do |record_strings, last_batch, file_split_required|
+        io.write(record_strings.join(",\n  "))
+
+        io.write(",\n  ") unless last_batch
+
+        if options[:file].present? && file_split_required
+          options[:current_file_index] = ((options[:current_file_index]&.to_i || 1) + 1)
+          io.write("\n]#{active_record_import_options(options)})\n")
+          io = open_io(options)
+          setup_io(io, options, records)
+        end
+      end
+
+      io.write("\n]#{active_record_import_options(options)})\n")
+
+      if options[:file].present?
+        nil
+      else
+        io.rewind
+        io.read
+      end
+    end
+
+    def setup_io(io, options, records)
+      method = chosen_creation_method(options)
+      io.write("#{model_for(records)}.#{method}(")
+      if options[:import]
+        io.write("[#{attribute_names(records, options).map { |name| name.to_sym.inspect }.join(', ')}], ")
+      end
+      io.write("[\n  ")
+    end
+
+    def chosen_creation_method(options)
+      if options[:import]
+        'import'
+      elsif options[:insert_all]
+        'insert_all'
+      else
+        'create!'
+      end
+    end
+
+    def active_record_import_options(options)
+      return unless options[:import].is_a?(Hash) || options[:import_options].present?
+
+      if options[:import].is_a?(Hash)
+        ', ' + options[:import].map { |key, value| "#{key}: #{value}" }.join(', ')
+      else
+        ', ' + options[:import_options]
+      end
+    end
+
+    def attribute_names(records, options)
+      attribute_names = if records.is_a?(ActiveRecord::Relation) || records.is_a?(Class)
+                          records.attribute_names
+                        else
+                          records[0].attribute_names
+                        end
+
+      attribute_names.reject { |name| options[:exclude].include?(name.to_sym) }
+    end
+
+    def model_for(records)
+      if records.is_a?(Class)
+        records
+      elsif records.respond_to?(:model)
+        records.model
+      else
+        records[0].class
+      end
+    end
+  end
+end

data/lib/seed_dump/environment.rb

@@ -0,0 +1,182 @@
+class SeedDump
+  module Environment
+    def dump_using_environment(env = {})
+      Rails.application.eager_load!
+
+      models = retrieve_models(env) - retrieve_models_exclude(env)
+
+      current_file_index = 1
+      limit = retrieve_limit_value(env)
+      append = retrieve_append_value(env)
+
+      # Eliminate HABTM models that have the same underlying table; otherwise
+      # they'll be dumped twice, once in each direction. Probably should apply
+      # to all models, but it's possible there are edge cases in which this
+      # is not the right behavior.
+
+      habtm, non_habtm = models.partition { |m| m.name =~ /^HABTM_/ }
+      models = non_habtm + habtm.uniq(&:table_name)
+
+      resolved_dependencies = DependencyUnwrangler.new(models).evaluation_order - retrieve_models_exclude(env)
+      models = resolved_dependencies unless resolved_dependencies.empty?
+      models.each do |model|
+        model = model.limit(limit) if limit.present?
+        options = {
+          append: append,
+          batch_size: retrieve_batch_size_value(env),
+          exclude: retrieve_dump_all_value(env) ? [] : retrieve_exclude_value(env),
+          insert_all: retrieve_insert_all_value(env),
+          file_split_limit: retreive_file_split_limit_value(env),
+          file: retrieve_file_value(env),
+          import: retrieve_import_value(env),
+          current_file_index: current_file_index,
+          import_options: retrieve_import_options(env)
+        }
+
+        SeedDump.dump(model, options)
+
+        append = true # Always append for every model after the first
+        # (append for the first model is determined by
+        # the APPEND environment variable).
+        current_file_index = options[:current_file_index]
+      end
+    end
+
+    private
+
+    # Internal: Array of Strings corresponding to Active Record model class names
+    # that should be excluded from the dump.
+    ACTIVE_RECORD_INTERNAL_MODELS = ['ActiveRecord::SchemaMigration',
+                                     'ActiveRecord::InternalMetadata'].freeze
+
+    # Internal: Retrieves an Array of Active Record model class constants to be
+    # dumped.
+    #
+    # If a "MODEL" or "MODELS" environment variable is specified, there will be
+    # an attempt to parse the environment variable String by splitting it on
+    # commmas and then converting it to constant.
+    #
+    # Model classes that do not have corresponding database tables or database
+    # records will be filtered out, as will model classes internal to Active
+    # Record.
+    #
+    # env - Hash of environment variables from which to parse Active Record
+    #       model classes. The Hash is not optional but the "MODEL" and "MODELS"
+    #       keys are optional.
+    #
+    # Returns the Array of Active Record model classes to be dumped.
+    def retrieve_models(env)
+      # Parse either the "MODEL" environment variable or the "MODELS"
+      # environment variable, with "MODEL" taking precedence.
+      models_env = env['MODEL'] || env['MODELS']
+
+      # If there was a use models environment variable, split it and
+      # convert the given model string (e.g. "User") to an actual
+      # model constant (e.g. User).
+      #
+      # If a models environment variable was not given, use descendants of
+      # ActiveRecord::Base as the target set of models. This should be all
+      # model classes in the project.
+      models = if models_env
+                 models_env.split(',')
+                           .collect { |x| x.strip.underscore.singularize.camelize.constantize }
+               else
+                 ActiveRecord::Base.descendants
+               end
+
+      # Filter the set of models to exclude:
+      #   - The ActiveRecord::SchemaMigration model which is internal to Rails
+      #     and should not be part of the dumped data.
+      #   - Models that don't have a corresponding table in the database.
+      #   - Models whose corresponding database tables are empty.
+      filtered_models = models.select do |model|
+        !ACTIVE_RECORD_INTERNAL_MODELS.include?(model.to_s) && \
+          model.table_exists? && \
+          model.exists?
+      end
+    end
+
+    # Internal: Returns a Boolean indicating whether the value for the "APPEND"
+    # key in the given Hash is equal to the String "true" (ignoring case),
+    # false if no value exists.
+    def retrieve_append_value(env)
+      parse_boolean_value(env['APPEND'])
+    end
+
+    # Internal: Returns a Boolean indicating whether the value for the "FILE_SPLIT_COUNT"
+    # key in the given Hash is equal to the String "true" (ignoring case),
+    # false if no value exists.
+    def retreive_file_split_limit_value(env)
+      retrieve_integer_value('FILE_SPLIT_LIMIT', env)
+    end
+
+    # Internal: Returns a Boolean indicating whether the value for the "IMPORT"
+    # key in the given Hash is equal to the String "true" (ignoring case),
+    # false if  no value exists.
+    def retrieve_import_value(env)
+      parse_boolean_value(env['IMPORT'])
+    end
+
+    def retrieve_import_options(env)
+      env['IMPORT_OPTIONS']
+    end
+
+    # Internal: Returns a Boolean indicating whether the value for the "INSERT_ALL"
+    # key in the given Hash is equal to the String "true" (ignoring case),
+    # false if  no value exists.
+    def retrieve_insert_all_value(env)
+      parse_boolean_value(env['INSERT_ALL'])
+    end
+
+    # Internal: Returns a Boolean indicating whether the value for the "DUMP_ALL"
+    # key in the given Hash is equal to the String "true" (ignoring case),
+    # false if no value exists.
+    def retrieve_dump_all_value(env)
+      parse_boolean_value(env['DUMP_ALL'])
+    end
+
+    # Internal: Retrieves an Array of Class constants parsed from the value for
+    # the "MODELS_EXCLUDE" key in the given Hash, and an empty Array if such
+    # key exists.
+    def retrieve_models_exclude(env)
+      env['MODELS_EXCLUDE'].to_s
+                           .split(',')
+                           .collect { |x| x.strip.underscore.singularize.camelize.constantize }
+    end
+
+    # Internal: Retrieves an Integer from the value for the "LIMIT" key in the
+    # given Hash, and nil if no such key exists.
+    def retrieve_limit_value(env)
+      retrieve_integer_value('LIMIT', env)
+    end
+
+    # Internal: Retrieves an Array of Symbols from the value for the "EXCLUDE"
+    # key from the given Hash, and nil if no such key exists.
+    def retrieve_exclude_value(env)
+      env['EXCLUDE'] ? env['EXCLUDE'].split(',').map { |e| e.strip.to_sym } : nil
+    end
+
+    # Internal: Retrieves the value for the "FILE" key from the given Hash, and
+    # 'db/seeds.rb' if no such key exists.
+    def retrieve_file_value(env)
+      env['FILE'] || 'db/seeds.rb'
+    end
+
+    # Internal: Retrieves an Integer from the value for the "BATCH_SIZE" key in
+    # the given Hash, and nil if no such key exists.
+    def retrieve_batch_size_value(env)
+      retrieve_integer_value('BATCH_SIZE', env)
+    end
+
+    # Internal: Retrieves an Integer from the value for the given key in
+    # the given Hash, and nil if no such key exists.
+    def retrieve_integer_value(key, hash)
+      hash[key]&.to_i
+    end
+
+    # Internal: Parses a Boolean from the given value.
+    def parse_boolean_value(value)
+      value.to_s.casecmp('true').zero?
+    end
+  end
+end

data/lib/seed_dump/railtie.rb

@@ -0,0 +1,7 @@
+class SeedDump
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load 'tasks/seed_dump.rake'
+    end
+  end
+end

data/lib/seed_dump.rb

@@ -0,0 +1,15 @@
+require 'ipaddr'
+require 'rgl/adjacency'
+require 'rgl/topsort'
+
+require 'seed_dump/dump_methods/enumeration'
+require 'seed_dump/dump_methods'
+require 'seed_dump/environment'
+require 'seed_dump/dependency_unwrangler'
+
+class SeedDump
+  extend Environment
+  extend DumpMethods
+
+  require 'seed_dump/railtie' if defined?(Rails)
+end

data/lib/tasks/seed_dump.rake

@@ -0,0 +1,8 @@
+namespace :db do
+  namespace :seed do
+    desc 'Dump records from the database into db/seeds.rb'
+    task dump: :environment do
+      SeedDump.dump_using_environment(ENV)
+    end
+  end
+end

data/spec/dump_methods_spec.rb

@@ -0,0 +1,200 @@
+require 'spec_helper'
+
+describe SeedDump do
+  def expected_output(operation: 'create!', include_id: false, id_offset: 0)
+    output = "Sample.#{operation}([\n  "
+
+    data = []
+    ((1 + id_offset)..(3 + id_offset)).each do |i|
+      data << "{#{include_id ? "id: #{i}, " : ''}string: \"string\", text: \"text\", integer: 42, float: 3.14, decimal: \"2.72\", datetime: \"1776-07-04 19:14:00\", time: \"2000-01-01 03:15:00\", date: \"1863-11-19\", binary: \"binary\", boolean: false}"
+    end
+
+    "#{output}#{data.join(",\n  ")}\n])\n"
+  end
+
+  describe '.dump' do
+    before do
+      Rails.application.eager_load!
+
+      create_db
+
+      FactoryBot.create_list(:sample, 3)
+    end
+
+    context 'without file option' do
+      it 'returns the dump of the models passed in' do
+        described_class.dump(Sample).should eq(expected_output)
+      end
+    end
+
+    context 'with file option' do
+      before do
+        @filename = Tempfile.new(File.join(Dir.tmpdir, 'foo'), nil)
+      end
+
+      after do
+        File.unlink(@filename)
+      end
+
+      it 'dumps the models to the specified file' do
+        described_class.dump(Sample, file: @filename)
+
+        File.open(@filename) { |file| file.read.should eq(expected_output) }
+      end
+
+      context 'with append option' do
+        it 'appends to the file rather than overwriting it' do
+          described_class.dump(Sample, file: @filename)
+          described_class.dump(Sample, file: @filename, append: true)
+
+          File.open(@filename) { |file| file.read.should eq(expected_output + expected_output) }
+        end
+      end
+    end
+
+    context 'with file option and file split option' do
+      let(:file_path) { Tempfile.new('./foo').path }
+      let(:result_file_path) { [file_path, '1'].join('_') }
+
+      after do
+        File.unlink(result_file_path)
+      end
+
+      it 'stores the information in file_path with file index' do
+        described_class.dump(Sample, file: file_path, file_split_limit: 5)
+
+        File.open(result_file_path) { |file| expect(file.read).to eq(expected_output) }
+      end
+    end
+
+    context 'ActiveRecord relation' do
+      it 'returns nil if the count is 0' do
+        described_class.dump(EmptyModel).should be(nil)
+      end
+
+      context 'with an order parameter' do
+        it 'dumps the models in the specified order' do
+          Sample.delete_all
+          samples = 3.times { |i| FactoryBot.create(:sample, integer: i) }
+
+          described_class.dump(Sample.order('integer DESC')).should eq("Sample.create!([\n  {string: \"string\", text: \"text\", integer: 2, float: 3.14, decimal: \"2.72\", datetime: \"1776-07-04 19:14:00\", time: \"2000-01-01 03:15:00\", date: \"1863-11-19\", binary: \"binary\", boolean: false},\n  {string: \"string\", text: \"text\", integer: 1, float: 3.14, decimal: \"2.72\", datetime: \"1776-07-04 19:14:00\", time: \"2000-01-01 03:15:00\", date: \"1863-11-19\", binary: \"binary\", boolean: false},\n  {string: \"string\", text: \"text\", integer: 0, float: 3.14, decimal: \"2.72\", datetime: \"1776-07-04 19:14:00\", time: \"2000-01-01 03:15:00\", date: \"1863-11-19\", binary: \"binary\", boolean: false}\n])\n")
+        end
+      end
+
+      context 'without an order parameter' do
+        it 'dumps the models sorted by primary key ascending' do
+          described_class.dump(Sample).should eq(expected_output)
+        end
+      end
+
+      context 'with a limit parameter' do
+        it 'dumps the number of models specified by the limit when the limit is smaller than the batch size' do
+          expected_output = "Sample.create!([\n  {string: \"string\", text: \"text\", integer: 42, float: 3.14, decimal: \"2.72\", datetime: \"1776-07-04 19:14:00\", time: \"2000-01-01 03:15:00\", date: \"1863-11-19\", binary: \"binary\", boolean: false}\n])\n"
+
+          described_class.dump(Sample.limit(1)).should eq(expected_output)
+        end
+
+        it 'dumps the number of models specified by the limit when the limit is larger than the batch size but not a multiple of the batch size' do
+          Sample.delete_all
+          FactoryBot.create_list(:sample, 4)
+
+          described_class.dump(Sample.limit(3), batch_size: 2).should eq(
+            expected_output(include_id: false,
+                            id_offset: 3)
+          )
+        end
+      end
+    end
+
+    context 'with a batch_size parameter' do
+      it 'does not raise an exception' do
+        described_class.dump(Sample, batch_size: 100)
+      end
+
+      it 'does not cause records to not be dumped' do
+        described_class.dump(Sample, batch_size: 2).should eq(expected_output)
+
+        described_class.dump(Sample, batch_size: 1).should eq(expected_output)
+      end
+    end
+
+    context 'Array' do
+      it 'returns the dump of the models passed in' do
+        described_class.dump(Sample.all.to_a, batch_size: 2).should eq(expected_output)
+      end
+
+      it 'returns nil if the array is empty' do
+        described_class.dump([]).should be(nil)
+      end
+    end
+
+    context 'with an exclude parameter' do
+      it 'excludes the specified attributes from the dump' do
+        expected_output = "Sample.create!([\n  {text: \"text\", integer: 42, decimal: \"2.72\", time: \"2000-01-01 03:15:00\", date: \"1863-11-19\", binary: \"binary\", boolean: false},\n  {text: \"text\", integer: 42, decimal: \"2.72\", time: \"2000-01-01 03:15:00\", date: \"1863-11-19\", binary: \"binary\", boolean: false},\n  {text: \"text\", integer: 42, decimal: \"2.72\", time: \"2000-01-01 03:15:00\", date: \"1863-11-19\", binary: \"binary\", boolean: false}\n])\n"
+
+        described_class.dump(Sample,
+                             exclude: %i[id created_at updated_at string float datetime]).should eq(expected_output)
+      end
+    end
+
+    context 'Range' do
+      it 'dumps a class with ranges' do
+        expected_output = "RangeSample.create!([\n  {range_with_end_included: \"[1,3]\", range_with_end_excluded: \"[1,3)\", positive_infinite_range: \"[1,]\", negative_infinite_range: \"[,1]\", infinite_range: \"[,]\"}\n])\n"
+
+        described_class.dump([RangeSample.new]).should eq(expected_output)
+      end
+    end
+
+    context 'activerecord-insert-all' do
+      it 'dumps in the activerecord-insert-all format when insert-all is true' do
+        described_class.dump(Sample, insert_all: true).should eq(expected_output(operation: 'insert_all'))
+      end
+    end
+
+    context 'activerecord-import' do
+      it 'dumps in the activerecord-import format when import is true' do
+        described_class.dump(Sample, import: true, exclude: []).should eq <<~RUBY
+          Sample.import([:id, :string, :text, :integer, :float, :decimal, :datetime, :time, :date, :binary, :boolean, :created_at, :updated_at], [
+            [1, "string", "text", 42, 3.14, "2.72", "1776-07-04 19:14:00", "2000-01-01 03:15:00", "1863-11-19", "binary", false, "1969-07-20 20:18:00", "1989-11-10 04:20:00"],
+            [2, "string", "text", 42, 3.14, "2.72", "1776-07-04 19:14:00", "2000-01-01 03:15:00", "1863-11-19", "binary", false, "1969-07-20 20:18:00", "1989-11-10 04:20:00"],
+            [3, "string", "text", 42, 3.14, "2.72", "1776-07-04 19:14:00", "2000-01-01 03:15:00", "1863-11-19", "binary", false, "1969-07-20 20:18:00", "1989-11-10 04:20:00"]
+          ])
+        RUBY
+      end
+
+      it 'omits excluded columns if they are specified' do
+        described_class.dump(Sample, import: true, exclude: %i[id created_at updated_at]).should eq <<~RUBY
+          Sample.import([:string, :text, :integer, :float, :decimal, :datetime, :time, :date, :binary, :boolean], [
+            ["string", "text", 42, 3.14, "2.72", "1776-07-04 19:14:00", "2000-01-01 03:15:00", "1863-11-19", "binary", false],
+            ["string", "text", 42, 3.14, "2.72", "1776-07-04 19:14:00", "2000-01-01 03:15:00", "1863-11-19", "binary", false],
+            ["string", "text", 42, 3.14, "2.72", "1776-07-04 19:14:00", "2000-01-01 03:15:00", "1863-11-19", "binary", false]
+          ])
+        RUBY
+      end
+
+      context 'should add the params to the output if they are specified' do
+        it 'dumps in the activerecord-import format when import is true' do
+          described_class.dump(Sample, import: { validate: false }, exclude: []).should eq <<~RUBY
+            Sample.import([:id, :string, :text, :integer, :float, :decimal, :datetime, :time, :date, :binary, :boolean, :created_at, :updated_at], [
+              [1, "string", "text", 42, 3.14, "2.72", "1776-07-04 19:14:00", "2000-01-01 03:15:00", "1863-11-19", "binary", false, "1969-07-20 20:18:00", "1989-11-10 04:20:00"],
+              [2, "string", "text", 42, 3.14, "2.72", "1776-07-04 19:14:00", "2000-01-01 03:15:00", "1863-11-19", "binary", false, "1969-07-20 20:18:00", "1989-11-10 04:20:00"],
+              [3, "string", "text", 42, 3.14, "2.72", "1776-07-04 19:14:00", "2000-01-01 03:15:00", "1863-11-19", "binary", false, "1969-07-20 20:18:00", "1989-11-10 04:20:00"]
+            ], validate: false)
+          RUBY
+        end
+      end
+    end
+  end
+end
+
+class RangeSample
+  def attributes
+    {
+      'range_with_end_included' => (1..3),
+      'range_with_end_excluded' => (1...3),
+      'positive_infinite_range' => (1..Float::INFINITY),
+      'negative_infinite_range' => (-Float::INFINITY..1),
+      'infinite_range' => (-Float::INFINITY..Float::INFINITY)
+    }
+  end
+end