live_fixtures 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8395ce611ed8a861cdc45201f736748f1553170b
+  data.tar.gz: 2f2366f259279f43a1df43df05b6d42ab16dfca4
+SHA512:
+  metadata.gz: d8967a938a1ce69b465e64c2d82949624274cc69fe82f4fa46a3d4b11d055ce7ddb7edec0227b8c15ec60dd1cd9968e9d3a979de7a74d14feb47d091a8ee75c2
+  data.tar.gz: 74867a18f1f3996c48ced1cfa65498b7b30fa08a4511650766706291d6d4bf960491b0b073b9fbac747e93dc85a30036822fc95947df8cb41b6dd8faf75170be

data/CHANGELOG.md

@@ -0,0 +1,13 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+This project adheres to [Semantic Versioning](http://semver.org/).
+
+## [0.1.1] - 2016-06-30
+### Fixed
+- [live fixtures works better with slow TTYs & zeus](https://github.com/NoRedInk/live_fixtures/pull/4)
+
+## 0.1.0 - 2016-06-13
+### Added
+- initial release
+
+[0.1.1]: https://github.com/NoRedInk/live_fixtures/compare/v0.1.0...v0.1.1

data/MIT-LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 raorao
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# LiveFixtures
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/live_fixtures`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'live_fixtures'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install live_fixtures
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/live_fixtures. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/lib/live_fixtures/export/fixture.rb

@@ -0,0 +1,56 @@
+module LiveFixtures::Export::Fixture
+  module_function
+  def to_yaml(model, references = [], more_attributes = {})
+    table_name = model.class.table_name
+
+    more_attributes.merge! attributes_from_references(model, references)
+
+    <<-YML
+#{table_name}_#{model.id || SecureRandom.uuid.underscore}:
+  #{yml_attributes(model, more_attributes)}
+
+    YML
+  end
+
+  private_class_method def attributes_from_references(model, references)
+    {}.tap do |options|
+      Array(references).each do |assoc_name|
+
+        if model.respond_to? assoc_name # need to check #respond_to? because some assoc only exist in certain subclasses
+          assoc_model = model.send assoc_name
+        end
+
+        if assoc_model.present?
+          options["#{assoc_name}_id"] = LiveFixtures::Export::Reference.new(assoc_name, assoc_model)
+        end
+      end
+    end
+  end
+
+  private_class_method def yml_attributes(model, more_attributes)
+    model.attributes.merge(more_attributes).map do |name, value|
+      next if %w{id}.include? name
+      next if value.nil?
+
+      yml_value ||= case value
+                      when Time, DateTime
+                        value.utc.to_s(:db)
+                      when Date
+                        value.to_s(:db)
+                      when Hash
+                        value.to_yaml.inspect
+                      when String
+                        value.inspect
+                      when LiveFixtures::Export::Template
+                        value.code
+                      when LiveFixtures::Export::Reference
+                        name, value = value.name, value.value
+                        "#{value.class.table_name}_#{value.id}"
+                      else
+                        value.to_s
+                    end
+
+      "#{name}: " + yml_value
+    end.compact.join("\n  ")
+  end
+end

data/lib/live_fixtures/export.rb

@@ -0,0 +1,57 @@
+# This module is meant to be `include`ed into your export class.
+#
+# 1. Call #set_export_dir to set the dir where files should be created.
+#    If the dir does not already exist, it will be created for you.
+#
+# 2. Then call #export_fixtures for each db table, which will produce
+#    one yml file for each db table. Do *not* call export_fixtures multiple
+#    times for the same db table - that will overwrite the file each time!
+
+module LiveFixtures::Export
+  Template  = Struct.new(:code)
+  Reference = Struct.new(:name, :value)
+
+  private
+
+  def set_export_dir(dir)
+    @dir = dir
+    FileUtils.mkdir_p(@dir) unless File.directory?(@dir)
+  end
+
+  ##
+  # Export models to a yml file named after the corresponding table.
+  #
+  # Takes an optional block that will be invoked for each model.
+  # The block should return a hash of attributes to be merged and
+  # saved with the model's attributes.
+  def export_fixtures(models, with_references = [])
+    return unless models.present?
+
+    table_name = models.first.class.table_name
+    File.open(File.join(@dir, table_name + '.yml'), 'w') do |file|
+
+      ProgressBarIterator.new(models).each do |model|
+        more_attributes = block_given? ? yield(model) : {}
+        file.write Fixture.to_yaml(model, with_references, more_attributes)
+      end
+
+    end
+  end
+
+  class ProgressBarIterator
+    def initialize(models)
+      @models = models
+      @bar = LiveFixtures.get_progress_bar(
+        total:models.size,
+        title: models.first.class.name
+      )
+    end
+
+    def each
+      @models.each do |model|
+        yield model
+        @bar.increment
+      end
+    end
+  end
+end

data/lib/live_fixtures/import/fixtures.rb

@@ -0,0 +1,103 @@
+require 'active_record/fixtures'
+
+#rubocop:disable Style/PerlBackrefs
+
+class LiveFixtures::Import
+  class Fixtures
+    delegate :model_class, :table_name, :fixtures, to: :ar_fixtures
+    attr_reader :ar_fixtures
+
+    def initialize(connection, table_name, class_name, filepath, label_to_id)
+      @ar_fixtures = ActiveRecord::Fixtures.new connection,
+                                                table_name,
+                                                class_name,
+                                                filepath
+      @label_to_id = label_to_id
+    end
+
+    # https://github.com/rails/rails/blob/3-2-stable/activerecord/lib/active_record/fixtures.rb#L569
+    # Rewritten to take advantage of @label_to_id instead of AR::Fixtures#identify,
+    # and to make an iterator.
+    #
+    # Iterator which yields [table_name, label, row] for each fixture
+    # (and for any implicit join table records)
+    def each_table_row_with_label
+      join_table_rows = Hash.new { |h,table| h[table] = [] }
+
+      fixtures.map do |label, fixture|
+        row = fixture.to_hash
+
+        reflection_class = reflection_class_for row
+
+        reflection_class.reflect_on_all_associations.each do |association|
+          next unless row[association.name.to_s]
+
+          case association.macro
+            when :belongs_to
+              maybe_convert_association_to_foreign_key row, association
+
+            when :has_and_belongs_to_many
+              join_table_name = association.options[:join_table]
+
+              targets = row.delete(association.name.to_s)
+              targets = targets.split(/\s*,\s*/) unless targets.is_a?(Array)
+
+              join_table_rows[join_table_name] << { targets: targets,
+                                                    association: association,
+                                                    label: label }
+          end
+        end
+
+        yield [table_name, label, row]
+      end
+
+      join_table_rows.each do |table_name, rows|
+        rows.each do |targets:, association:, label:|
+          targets.each do |target|
+            assoc_fk = @label_to_id[target] || target
+            row = { association.foreign_key             => @label_to_id[label],
+                    association.association_foreign_key => assoc_fk }
+            yield [table_name, NO_LABEL, row]
+          end
+        end
+      end
+    end
+
+    def model_connection
+      model_class.connection if model_class.respond_to? :connection
+    end
+
+    private
+
+    def inheritance_column_name
+      @inheritance_column_name ||= model_class && model_class.inheritance_column
+    end
+
+    # If STI is used, find the correct subclass for association reflection
+    def reflection_class_for(row)
+      return model_class unless row.include?(inheritance_column_name)
+
+      row[inheritance_column_name].constantize
+    rescue
+      model_class
+    end
+
+    def maybe_convert_association_to_foreign_key(row, association)
+      fk_name = (association.options[:foreign_key] || "#{association.name}_id").to_s
+
+      # Do not replace association name with association foreign key if they are named the same
+      return if association.name.to_s == fk_name
+
+      value = row.delete(association.name.to_s)
+
+      # support polymorphic belongs_to as "label (Type)"
+      if association.options[:polymorphic] && value.sub!(/\s*\(([^\)]*)\)\s*$/, "")
+        row[association.foreign_type] = $1
+      end
+
+      row[fk_name] = @label_to_id[value]
+    end
+
+    private :ar_fixtures
+  end
+end

data/lib/live_fixtures/import.rb

@@ -0,0 +1,123 @@
+# ActiveRecord::Fixtures are a powerful way of populating data in a db;
+# however, its strategy for handling primary keys and associations is
+# UNACCEPTABLE for use with a production db. LiveFixtures works around this.
+#
+#
+########### Here's how ActiveRecord::Fixtures work ###########################
+#
+# Each record is assigned a label in its yml file. Primary key values are
+# assigned using a guid algorithm that maps a label to a consistent integer
+# between 1 and 2^30-1. Primary keys can then be assigned before saving any
+# records to the db.
+#
+# Why would they do this? Because, this enables us to use labels in the
+# Fixture yml files to refer to associations. For example:
+#
+#       <users.yml>
+#       bob:
+#         username: thebob
+#
+#       <posts.yml>
+#       hello:
+#         message: Hello everyone!
+#         user: bob
+#
+# The ActiveRecord::Fixture system first converts every instance of `bob` and
+# `hello` into an integer using ActiveRecord::Fixture#identify, and then can
+# save the records IN ANY ORDER and know that all foreign keys will be valid.
+#
+# There is a big problem with this. In a test db, each table is empty and so the
+# odds of inserting a few dozen records causing a primary key collision is
+# very small. However, for a production table with a hundred million rows, this
+# is no longer the case! Collisions abound and db insertion fails.
+#
+# Also, autoincrement primary keys will continue from the LARGEST existing
+# primary key value. If we insert a record at 1,000,000,000 - we've reduced the
+# total number of records we can store in that table in half. Fine for a test db
+# but not ideal for production.
+#
+#
+########### LiveFixtures work differently ####################################
+#
+# Since we want to be able to take advantage of normal autoincrement behavior,
+# we cannot know the primary keys of each record before saving it to the db.
+# Instead, we save each record, and then maintain a mapping (`@label_to_id`)
+# from that record's label (`bob`), to its primary key (`213`). Later, when
+# another record (`hello`) references `bob`, we can use this mapping to look up
+# the primary key for `bob` before saving `hello`.
+#
+# This means that the order we insert records into the db matters: `bob` must
+# be inserted before `hello`! This order is defined in INSERT_ORDER, and
+# reflected in the order of the `@table_names` array.
+
+class LiveFixtures::Import
+  NO_LABEL = nil
+
+  def initialize(root_path, insert_order)
+    @root_path = root_path
+    @table_names = Dir.glob(File.join(@root_path, '{*,**}/*.yml')).map do |filepath|
+      File.basename filepath, ".yml"
+    end
+    @table_names = insert_order.select {|table_name| @table_names.include? table_name}
+    if @table_names.size < insert_order.size
+      raise ArgumentError, "table(s) mentioned in `insert_order` which has no yml file to import: #{insert_order - @table_names}"
+    end
+    @label_to_id = {}
+  end
+
+  # https://github.com/rails/rails/blob/3-2-stable/activerecord/lib/active_record/fixtures.rb#L462
+  # The very similar method: ActiveRecord::Fixtures.create_fixtures has the
+  # unfortunate side effect of truncating each table!!
+  #
+  # Therefore, we have reproduced the relevant sections here, without DELETEs,
+  # with calling `LF::Import::Fixtures#each_table_row_with_label` instead of
+  # `AR::Fixtures#table_rows`, and using those labels to populate `@label_to_id`.
+  def import_all(class_names = {})
+    @table_names.each { |n|
+      class_names[n.tr('/', '_').to_sym] ||= n.classify if n.include?('/')
+    }
+
+    connection = ActiveRecord::Base.connection
+
+    files_to_read = @table_names
+
+    unless files_to_read.empty?
+      connection.transaction(requires_new: true) do
+        files_to_read.each do |path|
+          table_name = path.tr '/', '_'
+          class_name = class_names[table_name.to_sym] || table_name.classify
+
+          ff = Fixtures.new(connection,
+                            table_name,
+                            class_name,
+                            ::File.join(@root_path, path),
+                            @label_to_id)
+
+          conn = ff.model_connection || connection
+          ProgressBarIterator.new(ff).each do |table_name, label, row|
+            conn.insert_fixture(row, table_name)
+            @label_to_id[label] = conn.last_inserted_id(table_name) unless label == NO_LABEL
+          end
+        end
+      end
+    end
+  end
+
+  class ProgressBarIterator
+    def initialize(ff)
+      @ff = ff
+      @bar = LiveFixtures.get_progress_bar(
+        total:ff.fixtures.size,
+        title: ff.model_class.name
+      )
+    end
+
+    def each
+      @ff.each_table_row_with_label do |*args|
+        yield(*args)
+        @bar.increment unless @bar.finished?
+      end
+      @bar.finish
+    end
+  end
+end

data/lib/live_fixtures/version.rb

@@ -0,0 +1,3 @@
+module LiveFixtures
+  VERSION = "0.1.1"
+end

data/lib/live_fixtures.rb

@@ -0,0 +1,18 @@
+require "live_fixtures/version"
+require "live_fixtures/import"
+require "live_fixtures/import/fixtures"
+require "live_fixtures/export"
+require "live_fixtures/export/fixture"
+require "ruby-progressbar"
+
+module LiveFixtures
+  module_function
+  def get_progress_bar total:, title:
+    ProgressBar.create(
+      total: total,
+      title: title,
+      format:'%t: |%B| %P% %E',
+      throttle_rate: 0.1
+    )
+  end
+end

metadata

@@ -0,0 +1,165 @@
+--- !ruby/object:Gem::Specification
+name: live_fixtures
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- jleven
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-05-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: temping
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- josh@noredink.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- MIT-LICENSE
+- README.md
+- lib/live_fixtures.rb
+- lib/live_fixtures/export.rb
+- lib/live_fixtures/export/fixture.rb
+- lib/live_fixtures/import.rb
+- lib/live_fixtures/import/fixtures.rb
+- lib/live_fixtures/version.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: Tools for exporting and importing between databases managed by ActiveRecord.
+test_files: []