xcopier 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 315ef998ff2ab861e209e027bc7abd3d59ce3f15cdbaf74e9f3d782db5750cb8
+  data.tar.gz: 251b908019418f61bbe4b3acd029b0bd247621c859bee85e586c7e92c55f5587
+SHA512:
+  metadata.gz: ae63e3be2c29c82b13c9702c744d9cbd1c2e3ffca0c815cb95261d6f3863854c1cbf8c092b1b133b2073b2e848b09b78f69229ffc0634cff65ea326fa3ea8ba2
+  data.tar.gz: 59b587c6d6897b75cf8d5623a3255aa0ac91d06120194b1e9a9279484b0d73c87e66febb76e80556772e9ba516af8597eaee32799b22a4573de72345bb45135d

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [1.0.0] - 2024-12-29
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Cristian Bica
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,144 @@
+# Xcopier
+
+Xcopier is a tool to copy data from one database to another. It is designed to be used in a development environment to copy data from a production database to a local database (e.g., to test a data migration or data fix) allowing you to override and/or anonymize the data.
+
+:warning: :warning: :warning:
+
+This is a "sharp knife" tool. It can be used to copy data from one database to another. Make sure you properly set the source and destination connections to avoid data loss or corruption.
+
+## Installation
+
+Install the gem and add it to the application's Gemfile by executing:
+
+```bash
+bundle add xcopier --group=development
+```
+
+## Usage
+
+Create a file (e.g., `app/libs/company_copier.rb`) and define a class that includes `Xcopier::DSL`.
+
+You could also use the generator provided by this gem:
+
+```bash
+bundle exec rails generate xcopier:copier company
+```
+
+```ruby
+class CompanyCopier
+  include Xcopier::DSL
+
+  # you can use here a symbol to reference a connection defined in database.yml
+  #  or a hash with connection details
+  #  or a string with a connection url
+  source :production
+  destination :development
+
+  argument :company_ids, :integer, list: true
+
+  copy :companies, scope: -> { Company.where(id: arguments[:company_ids]) }
+  copy :users, scope: -> { User.where(company_id: arguments[:company_ids]) }, chunk_size: 100
+end
+```
+
+Then run the copier:
+
+```bash
+bundle exec xcopier company --company-ids 1,2
+```
+
+The above will load your app, instantiate the `CompanyCopier` class, and run the `copy` method for the `companies` and `users` tables.
+
+You could also do this from a Rails console:
+
+```ruby
+CompanyCopier.new(company_ids: [1, 2]).run
+# or give the argument as a string and it will be parsed
+CompanyCopier.new(company_ids: "1,2").run
+```
+
+### Arguments
+
+The DSL includes an `argument` directive. Its purpose is to provide copier arguments to be used in queries to copy data. It supports typecasting for the following types: string, integer, time, date, boolean. You can also specify if the argument is a list by setting the `list` option to `true`.
+
+Example:
+
+```ruby
+argument :str, :string
+argument :str_list, :string, list: true
+argument :int, :integer
+argument :int_list, :integer, list: true
+argument :time, :time # it will parse the time using Time.parse
+argument :date, :date # it will parse the date using Date.parse
+argument :bool, :boolean # it will recognize as truthy the values: "1", "yes", "true", true
+
+copier.new(str: "string", str_list: "string1,string2", int: "1", int_list: "1,2", time: "2020-01-01 12:00", date: "2020-01-01", bool: "true")
+
+copier.arguments[:str] # => "string"
+copier.arguments[:str_list] # => ["string1", "string2"]
+copier.arguments[:int] # => 1
+copier.arguments[:int_list] # => [1, 2]
+copier.arguments[:time] # => Time.parse("2020-01-01 12:00")
+copier.arguments[:date] # => Date.parse("2020-01-01")
+copier.arguments[:bool] # => true
+```
+
+Example:
+
+```ruby
+copy :companies, anonymize: true
+
+copy :users,
+     model: User, # this is not actually needed, it will be inferred from the name
+     scope: -> { User.all }, # this is also not needed as .all on the model is the default
+     chunk_size: 500, # this is the default value
+     overrides: {
+       email: ->(email) { email.gsub(/@/, "+#{SecureRandom.hex(4)}@") },
+       password: "password",
+       last_login_at: -> (last_login_at, attributes) { attributes[:created_at] + 1.minute }
+     },
+     anonymize: %w(first_name last_name street_address)
+```
+
+### Copy Operations
+
+The `copy` directive is to instruct the copier what to copy. It accepts the following options:
+- `name`
+   - a name for the copy operation (usually the table name)
+   - will be used to determine the model if not given
+- `model`
+  - the model to use for the copy operation
+  - if not given it will be inferred from the `name`
+- `scope`
+  - a lambda that returns the records to be copied
+  - if not given it will copy all records using the `model.all`
+- `chunk_size`
+  - the number of records to copy at once
+  - default value is `500`
+- `overrides`
+  - rules to transform the data before writing
+  - it is a hash where the key is the column name and the replacement is the value
+  - the value can be a lambda that returns the new value
+  - the lambda can receive no arguments or a single argument with the original value or two arguments with the original value and a hash of the record
+- `anonymize`
+  - where to try to anonymize the data
+  - can be `true` to anonymize all columns or a list of columns to anonymize
+  - more on the anonymization in the next section
+  - anonymization is not done for columns that have an override
+  - anonymization is done in the [`Xcopier::Anonymizer`](https://github.com/cristianbica/xcopier/blob/master/lib/xcopier/anonymizer.rb) class, is based on the column name and uses the [faker](https://rubygems.org/gems/faker) gem
+  - :warning: anonymization is not guaranteed to be secure and has currently a limited implementation
+  - feel free to adjust it in your app (`Xcopier::Anonymizer::RULES` is a mutable hash where the key is a regex to match the column and the value is a lambda that returns the anonymized value) or contribute to this gem
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/cristianbica/xcopier.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "minitest/test_task"
+
+namespace :dummy do
+  require_relative "test/dummy/config/application"
+  Dummy::Application.load_tasks
+end
+
+Minitest::TestTask.create
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[test rubocop]

data/exe/xcopier

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+unless File.exist?("config/environment.rb")
+  warn "Expected to be ran from the root of a Rails project"
+  exit 1
+end
+
+require File.expand_path("config/environment")
+require "xcopier/cli"
+
+Xcopier::CLI.start(ARGV)

data/lib/generators/xcopier/copier/copier_generator.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module Xcopier
+  module Generators
+    class CopierGenerator < ::Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      def create_copier
+        template "copiator.rb.tt", File.join("app/libs", class_path, "#{file_name}_copier.rb")
+      end
+
+      def models
+        Rails.application.eager_load! unless Rails.application.config.eager_load
+        ApplicationRecord.subclasses
+      end
+    end
+  end
+end

data/lib/generators/xcopier/copier/templates/copiator.rb.tt

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+<% module_namespacing do -%>
+class <%= class_name %>Copier
+  include Xcopier::DSL
+
+  argument :tenant_ids, type: :integer, list: true
+<% models.each do |model| -%>
+
+  copy :<%= model.table_name %>,
+       model: -> { <%= model.name %> },
+       scope: -> { <%= model.name %>.where(tenant_id: arguments[:tenant_ids]) },
+       overrides: {
+         # field: "value"
+         # field: -> { "value" }
+         # field: ->(value) { value.upcase }
+         # field: ->(value, attributes) { attributes[:other_field] == "value" ? "value" : value }
+       }
+<% end -%>
+end
+<% end -%>

data/lib/xcopier/actor.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/module/delegation"
+
+module Xcopier
+  class Actor
+    class UnknownMessageError < StandardError; end
+
+    attr_reader :copier
+    attr_accessor :__queue, :thread, :parent, :result
+
+    delegate :logger, to: :copier
+    delegate :log, :info, :debug, :error, to: :logger, allow_nil: true
+
+    def self.spawn!(*args)
+      actor = new(*args)
+      actor.__queue = Thread::Queue.new
+      actor.thread = Thread.new do
+        Thread.current[:xcopier_actor] = actor
+        actor.__work__
+      end
+      actor.thread.name = name.demodulize.underscore
+      actor.thread.report_on_exception = false
+      actor
+    end
+
+    def initialize(copier)
+      @copier = copier
+      super()
+    end
+
+    def wait
+      thread.value
+    end
+
+    def __work__
+      while (message = __queue.pop)
+        begin
+          return result if message == :__terminate
+
+          on_message(message)
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          on_error(e)
+        end
+      end
+    end
+
+    def tell(message)
+      __queue.push(message)
+    end
+
+    def on_message(message)
+      raise NotImplementedError
+    end
+
+    def terminate!
+      debug "#{self.class.name.demodulize}: terminating"
+      __queue.clear
+      __queue.push(:__terminate)
+    end
+
+    def on_error(error); end
+  end
+end

data/lib/xcopier/anonymizer.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+begin
+  require "faker"
+rescue LoadError # rubocop:disable Lint/SuppressedException
+end
+
+module Xcopier
+  module Anonymizer
+    module_function
+
+    RULES = { # rubocop:disable Style/MutableConstant
+      /email/ => -> { Faker::Internet.email },
+      /first_?name/ => -> { Faker::Name.first_name },
+      /last_?name/ => -> { Faker::Name.last_name },
+      /name/ => -> { Faker::Name.name },
+      /phone/ => -> { Faker::PhoneNumber.phone_number },
+      /address/ => -> { Faker::Address.full_address },
+      /city/ => -> { Faker::Address.city },
+      /country/ => -> { Faker::Address.country },
+      /zip/ => -> { Faker::Address.zip_code },
+      /(company|organization)/ => -> { Faker::Company.name }
+    }
+
+    def anonymize(name, value)
+      unless defined?(Faker)
+        puts "[WARN] Please add the faker gem to your Gemfile to be able to automatic annonimize data"
+        return value
+      end
+      RULES.each do |rule, block|
+        return block.call if name.match?(rule)
+      end
+      value
+    end
+  end
+end

data/lib/xcopier/cli.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "active_support/core_ext/string/inflections"
+
+module Xcopier
+  class CLI
+    def self.start(args)
+      new(args).run
+    end
+
+    def initialize(args)
+      @args = args
+    end
+
+    def run
+      valid = parse
+      return unless valid
+
+      @options[:copier].new(**@options[:args]).run
+    end
+
+    def parse # rubocop:disable Metrics/CyclomaticComplexity,Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/MethodLength
+      @options = { copier_name: @args.shift, args: {} }
+      @options[:copier] = @options[:copier_name].classify.safe_constantize
+
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: xcopier copier_name [--arg1=value1 ...] [options]"
+
+        opts.separator "\nCOPIER ARGUMENTS" if @options[:copier]&._arguments&.any?
+        @options[:copier]&._arguments&.each do |arg|
+          @options[:args][arg[:name]] = nil
+          banner = "--#{arg[:name].to_s.gsub("_", "-").parameterize}"
+          banner += "=VALUE" if arg[:type] != :boolean
+          help = "#{arg[:name]} copier argument"
+          help << " (comma separated list)" if arg[:list]
+          opts.on(banner, help) do |value|
+            @options[:args][arg[:name]] = value
+          end
+        end
+
+        opts.separator "\nOPTIONS"
+        opts.on("-h", "--help", "Show help message") do
+          @options[:help] = true
+        end
+
+        opts.on("-v", "--verbose", "Run verbosely") do
+          @options[:verbose] = true
+        end
+      end
+      parser.parse(@args)
+
+      if @options[:help]
+        puts parser
+        return false
+      end
+
+      if @options[:copier].nil?
+        puts "ERROR: Copier not found\n\n"
+        puts parser
+        return false
+      end
+
+      if @options[:args].values.any?(&:nil?)
+        puts "ERROR: Missing argument values\n\n"
+        puts parser
+        return false
+      end
+
+      true
+    end
+  end
+end

data/lib/xcopier/dsl.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+require "active_support/core_ext/class/attribute"
+require "active_support/core_ext/enumerable"
+require "xcopier/version"
+require_relative "operation"
+require_relative "runner"
+
+module Xcopier
+  module DSL
+    extend ActiveSupport::Concern
+
+    BOOLS = ["1", "yes", "true", true].freeze
+
+    included do
+      attr_reader :arguments, :logger
+
+      class_attribute :source, default: nil
+      class_attribute :destination, default: :development
+      class_attribute :_arguments, instance_accessor: false, instance_predicate: false, default: []
+      class_attribute :_operations, instance_accessor: false, instance_predicate: false, default: []
+    end
+
+    class_methods do
+      def copy(name, **options)
+        _operations << { name: name, **options }
+      end
+
+      def argument(name, type = :string, **options)
+        _arguments << { name: name.to_sym, type: type, **options }
+      end
+    end
+
+    def initialize(**args)
+      validate_arguments(args)
+      parse_arguments(args)
+    end
+
+    def operations
+      @operations ||= self.class._operations.map { |operation| Operation.new(self, **operation) }
+    end
+
+    def run
+      setup
+      Runner.run(self)
+    ensure
+      teardown
+    end
+
+    private
+
+    def setup
+      ApplicationRecord.connects_to(
+        shards: {
+          xcopier: { reading: source, writing: destination }
+        }
+      )
+      @logger = Xcopier.logger
+    end
+
+    def teardown; end
+
+    def validate_arguments(args)
+      given_arguments = args.keys
+      expected_arguments = self.class._arguments.pluck(:name)
+
+      missing_arguments = expected_arguments - given_arguments
+      raise ArgumentError, "Missing arguments: #{missing_arguments}" if missing_arguments.any?
+
+      unknown_arguments = given_arguments - expected_arguments
+      raise ArgumentError, "Unknown arguments: #{unknown_arguments}" if unknown_arguments.any?
+    end
+
+    def parse_arguments(args)
+      @arguments = self.class._arguments.each_with_object({}) do |definition, hash|
+        name = definition[:name]
+
+        if definition[:list]
+          values = args[name].split(",").map(&:strip).compact
+          hash[name] = values.map { |v| typecast_argument(v, definition) }
+        else
+          hash[name] = typecast_argument(args[name], definition)
+        end
+      end
+    end
+
+    def typecast_argument(value, definition)
+      type = definition[:type]
+      return value if type == :string
+      return value.to_i if type == :integer
+      return Time.parse(value) if type == :time
+      return Date.parse(value) if type == :date
+      return Xcopier::DSL::BOOLS.include?(value) if type == :boolean
+
+      raise ArgumentError, "Unknown argument type: #{type}"
+    end
+  end
+end

data/lib/xcopier/operation.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/hash/indifferent_access"
+
+module Xcopier
+  class Operation
+    attr_reader :name, :model, :scope, :chunk_size, :overrides, :anonymize
+
+    def initialize(copier, name:, model: nil, scope: nil, chunk_size: 500, overrides: {}, anonymize: []) # rubocop:disable Metrics/ParameterLists
+      @name = name
+      @model = model
+      @scope = scope
+      @chunk_size = chunk_size
+      @overrides = overrides.is_a?(Hash) ? overrides.with_indifferent_access : overrides
+      @anonymize = anonymize.is_a?(Array) ? anonymize.map(&:to_s) : anonymize
+      prepare_model_and_scope(copier)
+    end
+
+    def inspect
+      "#<#{self.class.name} name: #{name}, model: #{model.name}, chunk_size: #{chunk_size}, overrides: #{overrides.inspect}, anonymize: #{anonymize.inspect}>"
+    end
+
+    private
+
+    def prepare_model_and_scope(copier)
+      @model = name.to_s.classify.constantize if model.nil?
+      @model = model.call if model.is_a?(Proc)
+      @scope = copier.instance_exec(&scope) if scope.is_a?(Proc)
+      @scope = model.all if scope.nil?
+    end
+  end
+end

data/lib/xcopier/railtie.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Xcopier
+  class Railtie < Rails::Railtie
+  end
+end

data/lib/xcopier/reader.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative "actor"
+
+module Xcopier
+  class Reader < Actor
+    attr_reader :queue, :operation
+
+    def initialize(queue, *rest)
+      @queue = queue
+      super(*rest)
+    end
+
+    def on_message(message)
+      case message
+      in [:read, Operation => operation]
+        debug "Reader#message: type=read operation=#{operation.inspect}"
+        process(operation)
+      else
+        debug "Reader#message: type=unknown message=#{message.inspect}"
+        raise UnknownMessageError, "Unknown message: #{message.inspect}"
+      end
+    end
+
+    def on_error(error)
+      debug "Reader#error: #{error.message}"
+      parent.tell([:error, error])
+    end
+
+    def process(operation)
+      setup(operation)
+      work
+    ensure
+      teardown
+    end
+
+    def work
+      each_chunk do |chunk|
+        queue.push(chunk)
+        debug "Reader: read and pushed #{chunk.size} records"
+      end
+      debug "Reader: done"
+      queue.push(:done)
+    end
+
+    def each_chunk
+      ApplicationRecord.connected_to(shard: :xcopier, role: :reading) do
+        operation.scope.in_batches(of: operation.chunk_size) do |relation|
+          yield operation.model.connection.exec_query(relation.to_sql).to_a
+        end
+      end
+    end
+
+    def setup(operation)
+      @operation = operation
+    end
+
+    def teardown
+      @operation = nil
+    end
+  end
+end

data/lib/xcopier/runner.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require_relative "actor"
+require_relative "reader"
+require_relative "transformer"
+require_relative "writer"
+
+module Xcopier
+  class Runner < Actor
+    attr_reader :source_queue, :destination_queue, :copier, :reader, :transformer, :writer, :promise
+    attr_accessor :index
+
+    def self.run(copier)
+      runner = spawn!(copier)
+      runner.tell(:run)
+      ret = runner.wait
+      raise ret if ret.is_a?(Exception)
+
+      ret
+    end
+
+    def initialize(copier)
+      @source_queue = Queue.new
+      @destination_queue = Queue.new
+      @reader = Reader.spawn!(source_queue, copier).tap { |actor| actor.parent = self }
+      @transformer = Transformer.spawn!(source_queue, destination_queue, copier).tap { |actor| actor.parent = self }
+      @writer = Writer.spawn!(destination_queue, copier).tap { |actor| actor.parent = self }
+      @index = -1
+      super
+    end
+
+    def on_message(message)
+      case message
+      in :run
+        debug "Runner#message: type=run"
+        process
+      in :done
+        process
+      in [:error, e]
+        debug "Runner#message: type=error error=#{e.message}"
+        finish(e)
+      else
+        debug "Runner#message: type=unknown message=#{message.inspect}"
+        raise UnknownMessageError, "Unknown message: #{message.inspect}"
+      end
+    end
+
+    def on_error(error)
+      debug "Runner#error: #{error.message}"
+      finish(error)
+    end
+
+    def process
+      self.index += 1
+      if current_operation.nil?
+        finish(true)
+      else
+        reader.tell([:read, current_operation])
+        transformer.tell([:transform, current_operation])
+        writer.tell([:write, current_operation])
+      end
+    end
+
+    def finish(message = nil)
+      debug "Runner#finish: message=#{message.inspect}"
+      self.result = message
+
+      source_queue.push(:done)
+      destination_queue.push(:done)
+
+      reader.terminate!
+      transformer.terminate!
+      writer.terminate!
+      terminate!
+    end
+
+    def current_operation
+      copier.operations[index]
+    end
+  end
+end

data/lib/xcopier/transformer.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require_relative "actor"
+require_relative "anonymizer"
+
+module Xcopier
+  class Transformer < Actor
+    attr_reader :input, :output, :operation
+
+    def initialize(input, output, *rest)
+      @input = input
+      @output = output
+      super(*rest)
+    end
+
+    def on_message(message)
+      case message
+      in [:transform, Operation => operation]
+        debug "Transformer#message: type=transform operation=#{operation.inspect}"
+        process(operation)
+      else
+        debug "Transformer#message: type=unknown message=#{message.inspect}"
+        raise UnknownMessageError, "Unknown message: #{message.inspect}"
+      end
+    end
+
+    def on_error(error)
+      debug "Transformer#error: #{error.message}"
+      parent.tell([:error, error])
+    end
+
+    def process(operation)
+      setup(operation)
+      work
+    ensure
+      teardown
+    end
+
+    def work
+      loop do
+        chunk = input.pop
+
+        if chunk == :done
+          debug "Transformer: done"
+          output.push(:done)
+          @operation = nil
+          break
+        end
+        debug "Transformer: transforming #{chunk.size} records"
+        transformed = transform(chunk)
+        output.push(transformed)
+      end
+    end
+
+    def transform(chunk)
+      chunk.map do |record|
+        transform_record(record)
+      end
+    end
+
+    def transform_record(record)
+      record.each_with_object({}) do |(key, value), hash|
+        value = apply_overrides(value, key, record)
+        value = apply_anonymization(value, key)
+        hash[key] = value
+      end
+    end
+
+    def setup(operation)
+      @operation = operation
+    end
+
+    def teardown
+      @operation = nil
+    end
+
+    private
+
+    def apply_overrides(value, key, record)
+      return value unless operation.overrides.key?(key)
+
+      new_value = operation.overrides[key]
+      return new_value unless new_value.respond_to?(:call)
+
+      new_value.call(*[value, record].first(new_value.arity))
+    end
+
+    def apply_anonymization(value, key)
+      return value if operation.overrides.key?(key)
+      return Anonymizer.anonymize(key, value) if operation.anonymize == true
+      return Anonymizer.anonymize(key, value) if operation.anonymize.is_a?(Array) && operation.anonymize.include?(key)
+
+      value
+    end
+  end
+end

data/lib/xcopier/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Xcopier
+  VERSION = "1.0.0"
+end

data/lib/xcopier/writer.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative "actor"
+
+module Xcopier
+  class Writer < Actor
+    attr_reader :queue, :operation
+
+    def initialize(queue, *rest)
+      @queue = queue
+      super(*rest)
+    end
+
+    def on_message(message)
+      case message
+      in [:write, Operation => operation]
+        debug "Writer#message: type=write operation=#{operation.inspect}"
+        process(operation)
+      else
+        debug "Writer#message: type=unknown message=#{message.inspect}"
+        raise UnknownMessageError, "Unknown message: #{message.inspect}"
+      end
+    end
+
+    def on_error(error)
+      debug "Writer#error: #{error.message}"
+      parent.tell([:error, error])
+    end
+
+    def process(operation)
+      setup(operation)
+      work
+    ensure
+      teardown
+    end
+
+    def work
+      loop do
+        chunk = queue.pop
+        if chunk == :done
+          debug "Writer: done"
+          @operation = nil
+          parent.tell(:done)
+          break
+        end
+        write(chunk)
+      end
+    end
+
+    def write(records)
+      ApplicationRecord.connected_to(shard: :xcopier, role: :writing) do
+        debug "Writer: writing #{records.size} records"
+        operation.model.insert_all(records)
+      end
+    end
+
+    def setup(operation)
+      @operation = operation
+    end
+
+    def teardown
+      @operation = nil
+    end
+  end
+end

data/lib/xcopier.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative "xcopier/version"
+require_relative "xcopier/dsl"
+require_relative "xcopier/railtie"
+
+require "active_support/core_ext/module/attribute_accessors"
+require "logger"
+
+module Xcopier
+  mattr_accessor :logger do
+    @logger ||= Logger.new($stdout, level: ENV.fetch("LOG_LEVEL", :info))
+  end
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: xcopier
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Cristian Bică
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-12-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Xcopier is a tool to copy data from one database to another. It is designed
+  to be used in a development environment to copy data from a production database
+  to a local database (e.g., to test a data migration or data fix) allowing you to
+  override and/or anonymize the data.
+email:
+- cristian.bica@gmail.com
+executables:
+- xcopier
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- exe/xcopier
+- lib/generators/xcopier/copier/copier_generator.rb
+- lib/generators/xcopier/copier/templates/copiator.rb.tt
+- lib/xcopier.rb
+- lib/xcopier/actor.rb
+- lib/xcopier/anonymizer.rb
+- lib/xcopier/cli.rb
+- lib/xcopier/dsl.rb
+- lib/xcopier/operation.rb
+- lib/xcopier/railtie.rb
+- lib/xcopier/reader.rb
+- lib/xcopier/runner.rb
+- lib/xcopier/transformer.rb
+- lib/xcopier/version.rb
+- lib/xcopier/writer.rb
+homepage: https://github.com/cristianbica/xcopier
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/cristianbica/xcopier
+  source_code_uri: https://github.com/cristianbica/xcopier
+  changelog_uri: https://github.com/cristianbica/xcopier/blob/master/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Xcopier is a tool to copy data from one database to another.
+test_files: []