clone_kit 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b62cef9da193908351d660a1089e8778553a963e
+  data.tar.gz: ff375e6bce0abff22230fb94ec0be5bd3e31adc0
+SHA512:
+  metadata.gz: 3417281d9a7f0c7d6f701f9185f187165d766263215b4faf5f8b9ca06916f29f8eadd61ae34724c83c6e75e28564aa62153d65dca0e940dafb7f3c2b6964109b
+  data.tar.gz: 9d34d0962ce7a93b60eb5e1688be2b8e07e39d7b0aa695eb70e2a974a57b4ed4a541cdb0151bf1017dd122298933d732076b0b5be1db142419d04ac3fb6f16fa

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+vendor/ruby

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.kapost.yml

@@ -0,0 +1,99 @@
+# Please don't modify this file, without first checking if it makes sense to
+# adopt for most other Kapost projects. Ideally, we would keep these files as
+# similar as possible to ease maintainability. Instead, open a PR in the
+# codeclimate-common repo at https://github.com/kapost/codeclimate-common
+
+Rails:
+  Enabled: true
+
+AllCops:
+  DisplayCopNames: true
+  DisplayStyleGuide: true
+  Include:
+    - "**/Rakefile"
+    - "**/config.ru"
+  Exclude:
+    - "vendor/**/*"
+    - "spec/fixtures/**/*"
+    - "bin/**/*"
+    - "script/**/*"
+
+Metrics/LineLength:
+  Max: 120
+Rails/Date:
+  Enabled: false
+Rails/TimeZone:
+  Enabled: false
+AllCops:
+  TargetRubyVersion: 2.3
+Style/AndOr:
+  EnforcedStyle: conditionals
+Style/CaseIndentation:
+  IndentOneStep: true
+Style/Documentation:
+  Enabled: false
+Style/EachWithObject:
+  Enabled: false
+Style/ExtraSpacing:
+  Exclude:
+    - "config/routes.rb"
+Style/HashSyntax:
+  Exclude:
+    - "lib/tasks/**/*"
+Style/MultilineOperationIndentation:
+  EnforcedStyle: indented
+Style/NumericLiterals:
+  Enabled: false
+Style/PercentLiteralDelimiters:
+  PreferredDelimiters:
+    "%w": "[]"
+    "%W": "[]"
+    "%i": "[]"
+    "%I": "[]"
+    "%r": "()"
+Style/SignalException:
+  EnforcedStyle: semantic
+Style/SingleLineBlockParams:
+  Enabled: false
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+Style/MultilineMethodCallIndentation:
+  Exclude:
+    - "spec/**/*.rb"
+
+# These are better handled by reek
+Metrics/MethodLength:
+  Enabled: false
+Metrics/ParameterLists:
+  Enabled: false
+
+# Rubocop's global exclude seems to fail to exclude the bin/ dir, so set all
+# the cops that are failing manually, since most of these files are
+# auto-generated anyways. Also, if the rules also appear above, we need to copy
+# the same attrs, because YAML won't merge, only overwrite.
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+  Exclude:
+    - "bin/**/*"
+Style/FrozenStringLiteralComment:
+  Exclude:
+    - "bin/**/*"
+Style/LeadingCommentSpace:
+  Exclude:
+    - "bin/**/*"
+Style/SpaceInsideParens:
+  Exclude:
+    - "bin/**/*"
+Style/AlignParameters:
+  Exclude:
+    - "bin/**/*"
+Style/ExtraSpacing:
+  Exclude:
+    - "bin/**/*"
+  Exclude:
+    - "config/routes.rb"
+Lint/PercentStringArray:
+  Exclude:
+    # SecureHeaders needs the single quotes in `%w[https: 'self']`
+    - config/initializers/secure_headers.rb

data/.rubocop.yml

@@ -0,0 +1,31 @@
+# Please don't modify this file, without first checking if it makes sense to
+# adopt for most other Kapost projects. Ideally, we would keep these files as
+# similar as possible to ease maintainability. Instead, open a PR in the
+# codeclimate-common repo at https://github.com/kapost/codeclimate-common
+
+# This file is where app-specific Rubocop configs go. I wish every tool supported inherited configs...
+
+inherit_from:
+  - .rubocop.kapost.yml
+
+# Frozen string break everything in napa. This is gonna be a big project to replace
+Style/FrozenStringLiteralComment:
+  Exclude:
+    - "**/*.rb"
+
+Rails/ActionFilter:
+  EnforcedStyle: filter
+
+Style/TrailingCommaInArguments:
+  Enabled: false
+
+Style/NumericPredicate:
+  Enabled: false
+
+Style/BlockDelimiters:
+  EnforcedStyle: braces_for_chaining
+
+# Disabled since it only applies to Rails 5. Reenable when we migrate our apps to Rails 5.
+# https://github.com/bbatsov/rubocop/issues/3629
+Rails/HttpPositionalArguments:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+2.3.1

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in clone_kit.gemspec
+gemspec

data/README.md

@@ -0,0 +1,79 @@
+# CloneKit!
+
+An ActiveRecord-ish toolkit library for building database record cloning without the business logic and executing cloning operations, especially for multi-tenant applications using Mongoid.
+
+## Why does cloning require a special toolkit?
+
+When operating a multi-tenant system, copying database records is fraught with perils that can wreak havoc on customer integrity. Failing to remap foreign ids does not usually trigger database integrity errors (especially in MongoDB :trollface:) but are even more insidious.
+
+There is likely an alternative business logic required when records are copied and merged. CloneKit can help you assemble that logic.
+
+## How do I clone?
+
+Let's pretend you have a account that you want to clone to a new account.
+
+```ruby
+class BlogPost
+  include Mongoid::Document
+
+  field :account_id, type: BSON::ObjectId
+  field :blog_type_id, type: BSON::ObjectId
+  field :body, type: String
+end
+```
+
+You can specify the dependency order of cloning, the scope of the operation, and the specific cloning behavior inside a specification:
+
+```ruby
+CloneKit::Specification.new(BlogPost) do |spec|
+spec.dependencies = %w(Account BlogType)                     # Helps derive the cloning order
+  spec.emitter = TenantEmitter.new(BlogPost)                 # The scope of the operation for this collection
+  spec.cloner = CloneKit::Cloners::MongoidRulesetCloner.new( # The cloning behavior
+    BlogPost,
+    rules: [
+      ReTenantRule.new,
+      CloneKit::Rules::Remap.new("BlogPost", "Account" => "account_id", "BlogType" => "blog_type_id")
+    ]
+  )
+end
+```
+
+## Writing an Emitter
+
+You have to write some emitters for your app. By default, CloneKit specifications utilize and empty emitter, making all clones no-operations.
+
+TODO
+
+## Writing a Cloner
+
+TODO
+
+## Extending the built-in Cloners
+
+TODO
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'clone_kit'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install clone_kit
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/clone_kit.

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "clone_kit"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/clone_kit.gemspec

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "clone_kit/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "clone_kit"
+  spec.version       = CloneKit::VERSION
+  spec.authors       = ["Brandon Croft"]
+  spec.email         = ["brandon@kapost.com"]
+
+  spec.summary       = "A toolkit to assist in complex cloning operations"
+  spec.description   = "Supports rules-based cloning, Mongoid, and distributed operations"
+  spec.homepage      = "https://github.com/kapost/clone_kit"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(spec)/})
+  end
+
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r(^exe/)) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "redis"
+  spec.add_runtime_dependency "activesupport", "> 3.0.0" # For core ext Array#wrap and Object#blank?
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "mongoid", "~> 4.0.2"
+  spec.add_development_dependency "database_cleaner", "1.5.3"
+  spec.add_development_dependency "rake", "~> 11.0"
+  spec.add_development_dependency "rspec", "~> 3.4"
+  spec.add_development_dependency "rspec-collection_matchers"
+  spec.add_development_dependency "pry-byebug"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "fakeredis"
+end

data/lib/clone_kit.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "clone_kit/version"
+require "clone_kit/graph"
+require "clone_kit/specification"
+require "clone_kit/operation"
+require "clone_kit/shared_id_map"
+require "clone_kit/rule"
+
+require "active_support/core_ext/array/wrap"
+require "active_support/core_ext/object/blank"
+
+module CloneKit
+  def self.graph
+    @graph ||= CloneKit::Graph.new
+  end
+
+  def self.load_rails_models!
+    Rails.application.eager_load! if defined?(Rails) && !defined?(@eager_loaded_once)
+    @eager_loaded_once = true
+  end
+
+  def self.spec
+    @spec ||= {}
+  end
+
+  def self.add_specification(specification)
+    spec[specification.model.name] = specification
+    refresh_specification(specification)
+  end
+
+  def self.refresh_specification(specification)
+    graph.add_vertex(specification.model.name, *specification.dependencies)
+  end
+
+  def self.cloneable_models(already_cloned)
+    result = []
+    graph.nodes.each do |model_name, deps|
+      next if already_cloned.include?(model_name)
+      next unless deps.all? { |dep| already_cloned.include?(dep) }
+
+      result << model_name
+    end
+    result
+  end
+end

data/lib/clone_kit/cloners/mongoid_merging_ruleset_cloner.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "clone_kit/cloners/mongoid_ruleset_cloner"
+
+module CloneKit
+  module Cloners
+    class MongoidMergingRulesetCloner < MongoidRulesetCloner
+      def initialize(model_klass, rules: [], merge_fields: ["name"])
+        super(model_klass, rules: rules)
+        self.merge_fields = merge_fields
+      end
+
+      def clone_ids(ids, operation)
+        @saved_id_map = {}
+        initialize_cloner(operation)
+        apply_rules_and_save(find_and_merge_existing_records(ids))
+      end
+
+      protected
+
+      # These methods are super simple and should usually be overridden to merge records in a more nuanced fashion
+
+      def compare(first, second)
+        merge_fields.all? { |name| first[name] == second[name] }
+      end
+
+      def merge(records)
+        result = nil
+        records.each do |rec|
+          clone_all_embedded_fields(rec)
+          result = if result.nil?
+                     rec.deep_dup
+                   else
+                     result.merge(rec)
+                   end
+        end
+
+        result
+      end
+
+      private
+
+      attr_accessor :merge_fields
+
+      def find_and_merge_existing_records(ids)
+        all_records = []
+        each_existing_record(ids) do |rec|
+          all_records << rec
+        end
+
+        result = []
+        skip = Set.new
+
+        all_records.each_with_index do |record, i|
+          next if skip.include?(record["_id"])
+          mergeable = [record]
+
+          all_records[i + 1..all_records.length].each do |other_record|
+            next unless compare(record, other_record)
+
+            mergeable << other_record
+            skip << other_record["_id"]
+          end
+
+          new_id = BSON::ObjectId.new
+          new_record = if mergeable.length == 1
+                         copy = clone(mergeable[0])
+                         @saved_id_map[copy["_id"]] = new_id
+                         copy
+                       else
+                         merged = merge(mergeable)
+
+                         mergeable.each do |m|
+                           @saved_id_map[m["_id"]] = new_id
+                         end
+                         merged
+                       end
+
+          new_record["_id"] = new_id
+          result << new_record
+        end
+
+        CloneKit::SharedIdMap.new(current_operation.id).insert_many(model_klass, @saved_id_map)
+
+        result
+      end
+
+      def apply_rules_and_save(records)
+        records.each do |attributes|
+          id = attributes["_id"]
+          rules.each do |rule|
+            begin
+              rule.fix(@saved_id_map.key(id), attributes)
+            rescue StandardError => e
+              id = attributes["_id"]
+              message = "Unhandled error when applying rule #{rule.class.name} to #{model_klass} #{id}: #{e.class}"
+              current_operation.error(message)
+            end
+          end
+
+          save_or_fail(attributes)
+        end
+      end
+    end
+  end
+end