live_fixtures 0.3.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a728fd952006fc7d03f41ba8a0e7680005937f08
-  data.tar.gz: 169ea945ee73cc0d6183ddd590b72f520d851186
+SHA256:
+  metadata.gz: 9414f9b5a408acaff1c2749f0f3127412f23f6d364988fcf1f75ce9a15b0cbe4
+  data.tar.gz: f6304bcef06ba8b4e153d7ff19744e9af837592ea0aa72a758a5446e3933055c
 SHA512:
-  metadata.gz: e7b69490985e19b66e30726fda1d0a1c41f031a6f166512f1b1c929658453259ed7945237b926cd73991233284f35717dcb9a9076a5e307d80c8abed30b11a9f
-  data.tar.gz: 2fc58a3b1b222bea1a9869269a5241cf269dec3447ad00c17dcec0a9f6a2ffa0e356795c84a88abfab3f793c6ddde7dcec9d4dec446c48eff4f62371e1eee2af
+  metadata.gz: 348ebda33ef0e11062c823ab14d64dc5b5c5947686f91d2a094836b7a1499061ff7e306a1e943824a27e958dad4e8ed4581b5626b835c191a5409a2d735f557b
+  data.tar.gz: 03eb2d921c1de9d75f144c99dd5166298d2ed67d79844bb5b2aac1b44acb31a00a2076ef4ec27e2a50a72b5f41de68d6168bb517786fb1fef7aadec8d610199f

data/CHANGELOG.md

@@ -2,6 +2,44 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.1.0] - 2020-12-09
+### Added
+support for selectively import tables with custom callbacks per table (#34)
+
+## [2.0.0] - 2020-11-11
+### Breaking changes
+This is a breaking change because LiveFixtures::Import.new now needs to receive the class_names Hash to be able to correctly compute the insert_order in case there are some unconventional class names associations. And the class_names argument is removed from import_all. But this is the only change.
+
+### Added
+  - compute insert_order automatically (#33)
+
+## [1.0.1] - 2019-04-10
+### Fixed
+  - fixed incompatibility with mysql
+
+### Added
+  - mysql regression test, confirmation that this gem doesn't work with psotgres
+
+## [1.0.0] - 2019-02-15
+### Breaking changes
+ - drop support for rails 4.2, ruby < 2.3
+
+### Fixed
+ - support for rails 5
+
+### Added
+ - None
+
+## [0.3.1] - 2018-03-28
+### Breaking changes
+ - None
+
+### Fixed
+ - None
+
+### Added
+ - It is now possible to export an attribute named "id" when it is included among the [additional attributes](https://github.com/NoRedInk/live_fixtures/tree/3868aaddbeb1c0174261673855610c4f8d9e7842#additional-attributes). #25
+
 ## [0.3.0] - 2017-08-10
 ### Breaking changes
  - Imports now raise an error when unable to find a referenced model.

data/README.md

@@ -13,6 +13,9 @@ LiveFixtures uses a different strategy that means it is safer to use in a live e
 
 For more information, see [the motivation section below](#motivation).
 
+## Compatibility
+LiveFixtures is tested against Sqlite3 & mysql. It is known to be incompatible with postgres.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -73,7 +76,7 @@ The `LiveFixtures::Export` module is meant to be included into your export class
 
 ### Importing
 
-The `LiveFixtures::Import` class allows you to specify the location of your fixtures and the order in which to import them. Once you've done that, you can import them directly to your database.
+The `LiveFixtures::Import` class allows you to specify the location of your fixtures and, optionally, the order in which to import them. If you don't specify an order, the order will be computed from the ActiveRecord models associations. Once you've done that, you can import them directly to your database.
 
 
     module Seed::User

data/lib/live_fixtures.rb

@@ -1,9 +1,11 @@
 require "live_fixtures/version"
 require "live_fixtures/import"
 require "live_fixtures/import/fixtures"
+require "live_fixtures/import/insertion_order_computer"
 require "live_fixtures/export"
 require "live_fixtures/export/fixture"
 require "ruby-progressbar"
+require "yaml"
 
 module LiveFixtures
   module_function

data/lib/live_fixtures/export.rb

@@ -55,10 +55,15 @@ module LiveFixtures::Export
   # @yieldparam model [ActiveRecord::Base] each successive model.
   # @yieldreturn [Hash{String => Object}] a hash of attributes to be merged and saved with the model's attributes.
   def export_fixtures(models, with_references = [])
-    return unless models.present?
+    return [] unless models.present?
 
-    table_name = models.first.class.table_name
-    File.open(File.join(@dir, table_name + '.yml'), 'w') do |file|
+    model_class = models.first.class
+
+    File.open(File.join(@dir, model_class.table_name + '.yml'), 'w') do |file|
+      file.write <<~PRELUDE
+        _fixture:
+          model_class: #{model_class.name}
+      PRELUDE
 
       iterator = export_options[:show_progress] ? ProgressBarIterator : SimpleIterator
 
@@ -66,7 +71,6 @@ module LiveFixtures::Export
         more_attributes = block_given? ? yield(model) : {}
         file.write Fixture.to_yaml(model, with_references, more_attributes)
       end
-
     end
   end
 

data/lib/live_fixtures/export/fixture.rb

@@ -36,8 +36,7 @@ module LiveFixtures::Export::Fixture
   end
 
   private_class_method def yml_attributes(model, more_attributes)
-    model.attributes.merge(more_attributes).map do |name, value|
-      next if %w{id}.include? name
+    model.attributes.except("id").merge(more_attributes).map do |name, value|
       next if value.nil?
 
       serialize_attribute(model, name, value)
@@ -48,7 +47,7 @@ module LiveFixtures::Export::Fixture
     attribute_type = model.class.type_for_attribute(name)
 
     if attribute_type.is_a?(ActiveRecord::Type::Serialized)
-      value = attribute_type.type_cast_for_database(value) unless value.is_a?(String)
+      value = attribute_type.serialize(value) unless value.is_a?(String)
 
       "#{name}: |-\n#{value.to_s.indent(4)}" unless value.nil?
     elsif value.is_a? LiveFixtures::Export::Reference

data/lib/live_fixtures/import.rb

@@ -1,36 +1,71 @@
+require 'benchmark'
+
 # An object that facilitates the import of fixtures into a database.
 class LiveFixtures::Import
   NO_LABEL = nil
 
+  # Returns the insert order that was specified in the constructor or
+  # the inferred one if none was specified.
+  attr_reader :insert_order
+
+  # Map of table_name to import routine
+  # @return [Hash<String => Proc>]
+  attr_reader :alternate_imports
+
+  # Accessor for string label for a given fixture mapping to it's db id
+  attr_reader :label_to_id
+
   # Instantiate a new Import with the directory containing your fixtures, and
   # the order in which to import them. The order should ensure fixtures
   # containing references to another fixture are imported AFTER the referenced
   # fixture.
   # @raise [ArgumentError] raises an argument error if not every element in the insert_order has a corresponding yml file.
   # @param root_path [String] path to the directory containing the yml files to import.
-  # @param insert_order [Array<String>] a list of yml files (without .yml extension) in the order they should be imported.
+  # @param insert_order [Array<String> | Nil] a list of yml files (without .yml extension) in the order they should be imported, or `nil` if these order is to be inferred by this class.
+  # @param class_names [Hash{Symbol => String}] a mapping table name => Model class, for any that don't follow convention.
   # @param [Hash] opts export configuration options
   # @option opts [Boolean] show_progress whether or not to show the progress bar
   # @option opts [Boolean] skip_missing_tables when false, an error will be raised if a yaml file isn't found for each table in insert_order
   # @option opts [Boolean] skip_missing_refs when false, an error will be raised if an ID isn't found for a label.
+  # @option opts [Boolean] use_insert_order_as_table_names when true, table names will be those passed in insert_order, not read from yaml files
   # @return [LiveFixtures::Import] an importer
   # @see LiveFixtures::Export::Reference
-  def initialize(root_path, insert_order, **opts)
-    defaut_options = { show_progress: true, skip_missing_tables: false, skip_missing_refs: false }
+  def initialize(root_path, insert_order = nil, class_names = {}, **opts)
+    defaut_options = {
+      show_progress: true,
+      skip_missing_tables: false,
+      skip_missing_refs: false,
+      use_insert_order_as_table_names: false,
+    }
     @options = defaut_options.merge(opts)
     @root_path = root_path
-    @table_names = Dir.glob(File.join(@root_path, '{*,**}/*.yml')).map do |filepath|
-      File.basename filepath, ".yml"
+
+    if insert_order && @options[:use_insert_order_as_table_names]
+      @table_names = insert_order
+    else
+      @table_names = Dir.glob(File.join(@root_path, '{*,**}/*.yml')).map do |filepath|
+        File.basename filepath, ".yml"
+      end
     end
-    @table_names = insert_order.select {|table_name| @table_names.include? table_name}
-    if @table_names.size < insert_order.size && !@options[:skip_missing_tables]
-      raise ArgumentError, "table(s) mentioned in `insert_order` which has no yml file to import: #{insert_order - @table_names}"
+
+    @class_names = class_names
+    @table_names.each { |n|
+      @class_names[n.tr('/', '_').to_sym] ||= n.classify if n.include?('/')
+    }
+
+    @insert_order = insert_order
+    @insert_order ||= InsertionOrderComputer.compute(@table_names, @class_names, compute_polymorphic_associations)
+
+    @table_names = @insert_order.select {|table_name| @table_names.include? table_name}
+    if @table_names.size < @insert_order.size && !@options[:skip_missing_tables]
+      raise ArgumentError, "table(s) mentioned in `insert_order` which has no yml file to import: #{@insert_order - @table_names}"
     end
+
     @label_to_id = {}
+    @alternate_imports = {}
   end
 
   # Within a transaction, import all the fixtures into the database.
-  # @param class_names [Hash{Symbol => String}] a mapping table name => Model class, for any that don't follow convention.
   #
   # The very similar method: ActiveRecord::FixtureSet.create_fixtures has the
   # unfortunate side effect of truncating each table!!
@@ -39,20 +74,25 @@ class LiveFixtures::Import
   # with calling {LiveFixtures::Import::Fixtures#each_table_row_with_label} instead of
   # `AR::Fixtures#table_rows`, and using those labels to populate `@label_to_id`.
   # @see https://github.com/rails/rails/blob/4-2-stable/activerecord/lib/active_record/fixtures.rb#L496
-  def import_all(class_names = {})
-    @table_names.each { |n|
-      class_names[n.tr('/', '_').to_sym] ||= n.classify if n.include?('/')
-    }
-
+  def import_all
     connection = ActiveRecord::Base.connection
+    show_progress = @options[:show_progress]
 
+    # TODO: should be additive with alternate_imports so we can delete the fixture file
     files_to_read = @table_names
 
-    unless files_to_read.empty?
-      connection.transaction(requires_new: true) do
-        files_to_read.each do |path|
-          table_name = path.tr '/', '_'
-          class_name = class_names[table_name.to_sym] || table_name.classify
+    return if files_to_read.empty?
+
+    connection.transaction(requires_new: true) do
+      files_to_read.each do |path|
+        table_name = path.tr '/', '_'
+        if alternate = @alternate_imports[table_name]
+          time = Benchmark.ms do
+            alternate.call(@label_to_id)
+          end
+          puts "Imported %s in %.0fms" % [table_name, time] if show_progress
+        else
+          class_name = @class_names[table_name.to_sym] || table_name.classify
 
           ff = Fixtures.new(connection,
                             table_name,
@@ -62,21 +102,76 @@ class LiveFixtures::Import
                             skip_missing_refs: @options[:skip_missing_refs])
 
           conn = ff.model_connection || connection
-          iterator = @options[:show_progress] ? ProgressBarIterator : SimpleIterator
-          iterator.new(ff).each do |table_name, label, row|
-            conn.insert_fixture(row, table_name)
-            @label_to_id[label] = conn.last_inserted_id(table_name) unless label == NO_LABEL
+
+          iterator = show_progress ? ProgressBarIterator : SimpleIterator
+          iterator.new(ff).each do |tname, label, row|
+            conn.insert_fixture(row, tname)
+            @label_to_id[label] = conn.send(:last_inserted_id, tname) unless label == NO_LABEL
           end
         end
       end
     end
   end
 
+  # Override import of table using a callable object
+  # @param table_name [String] table to use callable instead of fixture file
+  # @param callable [Proc] Proc/lambda that will be called with @label_to_id
+  def override(table_name, callable)
+    @alternate_imports[table_name] = callable
+    self
+  end
+
+  private
+
+  # Here we go through each of the fixture YAML files to see what polymorphic
+  # dependencies exist for each of the models.
+  # We do this by inspecting the value of any field that ends with `_type`,
+  # for example `author_type`, `assignment_type`, etc.
+  # Becuase we can't know all the possible types of a polymorphic association
+  # we compute them from the YAML file contents.
+  # Returns a Hash[Class => Set[Class]]
+  def compute_polymorphic_associations
+    polymorphic_associations = Hash.new { |h, k| h[k] = Set.new }
+
+    connection = ActiveRecord::Base.connection
+    files_to_read = @table_names
+
+    files_to_read.each do |path|
+      table_name = path.tr '/', '_'
+      class_name = @class_names[table_name.to_sym] || table_name.classify
+
+      # Here we use the yaml file and YAML.load instead of ActiveRecord::FixtureSet.new
+      # because it's faster and we can also check whether we actually need to
+      # load the file: only if it includes "_type" in it, otherwise there will be
+      # no polymorphic types in there.
+
+      filename = ::File.join(@root_path, "#{path}.yml")
+      file = File.read(filename)
+      next unless file =~ /_type/
+
+      yaml = YAML.load(file)
+      yaml.each do |key, object|
+        object.each do |field, value|
+          next unless  field.ends_with?("_type")
+
+          begin
+            polymorphic_associations[class_name.constantize] << value.constantize
+          rescue NameError
+            # It might be the case that the `..._type` field doesn't actually
+            # refer to a type name, so we just ignore it.
+          end
+        end
+      end
+    end
+
+    polymorphic_associations
+  end
+
   class ProgressBarIterator
     def initialize(ff)
       @ff = ff
       @bar = LiveFixtures.get_progress_bar(
-        total:ff.fixtures.size,
+        total: ff.fixtures.size,
         title: ff.model_class.name
       )
     end

data/lib/live_fixtures/import/insertion_order_computer.rb

@@ -0,0 +1,127 @@
+class LiveFixtures::Import
+  # :nodoc:
+  class InsertionOrderComputer
+    # :nodoc:
+    class Node
+      attr_reader :path
+      attr_reader :class_name
+      attr_reader :klass
+
+      # The classes this node depends on
+      attr_reader :dependencies
+
+      def initialize(path, class_name, klass)
+        @path = path
+        @class_name = class_name
+        @klass = klass
+        @dependencies = Set.new
+      end
+    end
+
+    def self.compute(table_names, class_names = {}, polymorphic_associations = {})
+      new(table_names, class_names, polymorphic_associations).compute
+    end
+
+    def initialize(table_names, class_names = {}, polymorphic_associations = {})
+      @table_names = table_names
+      @class_names = class_names
+      @polymorphic_associations = polymorphic_associations
+    end
+
+    def compute
+      nodes = build_nodes
+      compute_insert_order(nodes)
+    end
+
+    private
+
+    # Builds an Array of Nodes, each containing dependencies to other nodes
+    # using their class names.
+    def build_nodes
+      # Create a Hash[Class => Node] for each table/class
+      nodes = {}
+      @table_names.each do |path|
+        table_name = path.tr "/", "_"
+        class_name = @class_names[table_name.to_sym] || table_name.classify
+        klass = class_name.constantize
+        nodes[klass] = Node.new(path, class_name, klass)
+      end
+
+      # First iniitalize dependencies from polymorphic associations that we
+      # explicitly found in the yaml files.
+      @polymorphic_associations.each do |klass, associations|
+        associations.each do |association|
+          node = nodes[klass]
+          next unless node
+          next unless nodes.key?(association)
+
+          node.dependencies << association
+        end
+      end
+
+      # Compute dependencies between nodes/classes by reflecting on their
+      # ActiveRecord associations.
+      nodes.each do |_, node|
+        klass = node.klass
+        klass.reflect_on_all_associations.each do |assoc|
+          # We can't handle polymorphic associations, but the concrete types
+          # should have been deduced from the yaml files contents
+          next if assoc.polymorphic?
+
+          # Don't add a dependency if the class is not in the given table names
+          next unless nodes.key?(assoc.klass)
+
+          # A class might depend on itself, but we don't add it as a dependency
+          # because otherwise we'll never make it (the class can probably be created
+          # just fine and these dependencies are optional/nilable)
+          next if klass == assoc.klass
+
+          case assoc.macro
+          when :belongs_to
+            node.dependencies << assoc.klass
+          when :has_one, :has_many
+            # Skip `through` association becuase it will be already computed
+            # for the related `has_one`/`has_many` association
+            next if assoc.options[:through]
+
+            nodes[assoc.klass].dependencies << klass
+          end
+        end
+      end
+
+      # Finally sort all values by name for consistent results
+      nodes.values.sort_by { |node| node.klass.name }
+    end
+
+    def compute_insert_order(nodes)
+      insert_order = []
+
+      until nodes.empty?
+        # Pick a node that has no dependencies
+        free_node = nodes.find { |node| node.dependencies.empty? }
+
+        if free_node.nil?
+          msg = "Can't compute an insert order.\n\n"
+          msg << "These models seem to depend on each other:\n"
+          nodes.each do |node|
+            msg << "  #{node.klass.name}\n"
+            msg << "   - depends on: #{node.dependencies.map(&:name).join(", ")}\n"
+          end
+          raise msg
+        end
+
+        insert_order << free_node.path
+
+        # Delete this node from the other nodes' dependencies
+        nodes.each do |node|
+          node.dependencies.delete(free_node.klass)
+        end
+
+        # And delete this node because we are done with it
+        nodes.delete(free_node)
+      end
+
+      insert_order
+    end
+  end
+end

data/lib/live_fixtures/version.rb

@@ -1,3 +1,3 @@
 module LiveFixtures
-  VERSION = "0.3.0"
+  VERSION = "2.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: live_fixtures
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 2.1.0
 platform: ruby
 authors:
 - jleven
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2017-08-10 00:00:00.000000000 Z
+date: 1970-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '5.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '5.2'
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -44,56 +44,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.8'
 - !ruby/object:Gem::Dependency
   name: temping
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -110,6 +110,34 @@ dependencies:
         version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.13
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.13
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -150,7 +178,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - josh@noredink.com
 executables: []
@@ -165,12 +193,13 @@ files:
 - lib/live_fixtures/export/fixture.rb
 - lib/live_fixtures/import.rb
 - lib/live_fixtures/import/fixtures.rb
+- lib/live_fixtures/import/insertion_order_computer.rb
 - lib/live_fixtures/version.rb
-homepage: 
+homepage:
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -185,9 +214,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.8
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Tools for exporting and importing between databases managed by ActiveRecord.
 test_files: []