fury_dumper 0.1.0

data/lib/fury_dumper/dumpers/model.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module FuryDumper
+  module Dumpers
+    class Model
+      attr_reader :source_model, :iteration, :warnings
+      attr_accessor :relation_items, :root_model
+
+      def initialize(source_model:, relation_items:, iteration: 0, root_model: nil)
+        raise ArgumentError unless source_model.is_a?(String)
+        raise ArgumentError unless relation_items.is_a?(RelationItems)
+        raise ArgumentError unless iteration.is_a?(Numeric)
+        raise ArgumentError unless root_model.is_a?(Model) || root_model.nil?
+
+        @source_model   = source_model
+        @relation_items = relation_items
+        @iteration      = iteration
+        @root_model     = root_model
+        @warnings       = []
+      end
+
+      def copy
+        self.class.new(source_model: source_model,
+                       relation_items: relation_items.copy,
+                       iteration: iteration,
+                       root_model: root_model)
+      end
+
+      delegate :table_name, to: :active_record_model
+
+      def column_names
+        active_record_model.columns.map(&:name)
+      end
+
+      def ==(other)
+        raise ArgumentError unless other.is_a?(Model)
+
+        @source_model == other.source_model &&
+          @relation_items.eql?(other.relation_items) &&
+          sub_path?(other)
+      end
+
+      def sub_path?(other_model)
+        raise ArgumentError unless other_model.is_a?(Model)
+
+        min_length = [root_path.length, other_model.root_path.length].min - 1
+        root_path[0..min_length] == other_model.root_path[0..min_length]
+      end
+
+      def root_path
+        return [nil] if @root_model.nil?
+
+        @root_model.root_path + [@root_model.source_model]
+      end
+
+      def to_full_str
+        buffer = "MODEL #{@source_model} by #{root_model&.source_model.presence || '-'} WHERE "
+
+        buffer += @relation_items.items.map do |item|
+          if item.complex
+            item.key
+          elsif item.values_for_key.count > 10
+            "#{item.key} = #{item.values_for_key[0..10]} and #{item.values_for_key.count - 10} elements"
+          else
+            "#{item.key} = #{item.values_for_key}"
+          end
+        end.join(' AND ')
+
+        buffer
+      end
+
+      def active_record_model
+        @source_model.constantize
+      end
+
+      delegate :primary_key, to: :active_record_model
+
+      def to_short_str
+        "#{@source_model}.#{@relation_items.keys.join(' & ')}"
+      end
+
+      def fetch_complex_items
+        @relation_items.complex_items.map(&:key).join(' AND ')
+      end
+
+      def fetch_equality_items_hash
+        values = {}
+        @relation_items.equality_items.each do |item|
+          if active_record_model.column_names.include?(item.key)
+            values[item.key] = item.values_for_key
+          else
+            @warnings << "Shit relation: #{@source_model}.#{item.key} does not exist"
+            next
+          end
+        end
+
+        values
+      end
+
+      def all_non_scoped_models
+        active_record_model.reflect_on_all_associations.select { |rr| rr.scope.nil? }.map do |rr|
+          rr.klass.to_s
+        rescue NameError, LoadError => e
+          @warnings << e
+          next
+        end.compact.uniq
+      end
+
+      def to_active_record_relation
+        return @active_record_relation if @active_record_relation
+
+        complex_values          = fetch_complex_items
+        equality_values         = fetch_equality_items_hash
+        @active_record_relation = active_record_model.where(equality_values)
+                                                     .where(complex_values)
+                                                     .limit(FuryDumper::Config.limit)
+
+        unless FuryDumper::Config.fast?
+          order_value = { primary_key => :desc }
+          @active_record_relation = @active_record_relation.order(order_value)
+        end
+
+        @active_record_relation
+      end
+
+      def next_iteration
+        @iteration + 1
+      end
+    end
+  end
+end

data/lib/fury_dumper/dumpers/model_queue.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module FuryDumper
+  module Dumpers
+    class ModelQueue
+      def initialize
+        @queue = []
+      end
+
+      def add_element(model:, dump_state:)
+        raise ArgumentError, "Expected model as Dumpers::Model, got: #{model.class}" unless model.is_a?(Model)
+
+        unless dump_state.is_a?(DumpState)
+          raise ArgumentError,
+                "Expected dump_state as Dumpers::DumpState, got: #{dump_state.class}"
+        end
+
+        @queue << [model, dump_state]
+      end
+
+      def empty?
+        @queue.empty?
+      end
+
+      def fetch_element
+        @queue.delete_at(0)
+      end
+
+      def count
+        @queue.count
+      end
+    end
+  end
+end

data/lib/fury_dumper/dumpers/relation_items.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module FuryDumper
+  module Dumpers
+    RelationItem = Struct.new(:key, :values_for_key, :complex, :additional) do
+      def initialize(key:, values_for_key:, complex: false, additional: false)
+        super(key, values_for_key, complex, additional)
+      end
+
+      def eql?(other)
+        key == other.key
+      end
+
+      def copy
+        self.class.new(key: key, values_for_key: values_for_key.dup, complex: complex, additional: additional)
+      end
+    end
+
+    class RelationItems
+      attr_accessor :items
+
+      def initialize(items: [])
+        raise ArgumentError unless items.is_a?(Array)
+        raise ArgumentError unless items.all? { |item| item.is_a?(RelationItem) }
+
+        @items = items
+      end
+
+      def self.new_with_key_value(item_key: 'id', item_values: [])
+        new(items: [RelationItem.new(key: item_key, values_for_key: item_values.compact)])
+      end
+
+      def self.new_with_items(items: [])
+        new(items: items)
+      end
+
+      def eql?(other)
+        raise ArgumentError unless other.is_a?(RelationItems)
+
+        other.items.reject(&:additional).all? do |other_item|
+          items.reject(&:additional).any? { |item| item.eql?(other_item) }
+        end
+      end
+
+      def equality_items
+        items.reject(&:complex)
+      end
+
+      def complex_items
+        items.select(&:complex)
+      end
+
+      def keys
+        items.map(&:key).sort
+      end
+
+      def values(key)
+        items.select { |item| item.key == key }.values_for_key
+      end
+
+      def copy
+        self.class.new(items: copy_items)
+      end
+
+      def copy_items
+        items.map(&:copy)
+      end
+
+      def copy_with_new_values(key, new_values)
+        new_items = copy_items
+        new_items.each do |item|
+          item.values_for_key = new_values.compact.dup if item.key == key
+        end
+
+        self.class.new_with_items(items: new_items)
+      end
+    end
+  end
+end

data/lib/fury_dumper/encrypter.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module FuryDumper
+  class Encrypter
+    KEY = "\xBE\nXx\xE2\xDB\x85\xBD\xE1j}qz?}\xB0j6\xA95\xBAy80\x95\xE6\xC1\x9D\x9F\x89\xA2t"
+
+    def self.encrypt(msg)
+      crypt = ActiveSupport::MessageEncryptor.new(KEY)
+      crypt.encrypt_and_sign(msg)
+    end
+
+    def self.decrypt(encrypted_data)
+      crypt = ActiveSupport::MessageEncryptor.new(KEY)
+      crypt.decrypt_and_verify(encrypted_data)
+    end
+  end
+end

data/lib/fury_dumper/engine.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module FuryDumper
+  class Engine < ::Rails::Engine
+    isolate_namespace FuryDumper
+  end
+end

data/lib/fury_dumper/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module FuryDumper
+  VERSION = '0.1.0'
+end

data/lib/fury_dumper.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'rails'
+require 'fury_dumper/version'
+require 'fury_dumper/config'
+require 'fury_dumper/engine'
+require 'fury_dumper/dumper'
+require 'fury_dumper/api'
+require 'fury_dumper/dumpers/relation_items'
+require 'fury_dumper/dumpers/dump_state'
+require 'fury_dumper/dumpers/model_queue'
+require 'fury_dumper/dumpers/model'
+require 'fury_dumper/encrypter'
+require 'httpclient'
+require 'highline/import'
+
+module FuryDumper
+  class Error < StandardError
+  end
+
+  def self.configuration
+    @configuration ||= Config.config
+  end
+
+  # Start dumping
+  #
+  # @param password[String]           - password for remote DB
+  # @param host[String]               - host for remote DB
+  # @param port[String]               - port for remote DB
+  # @param user[String]               - username for remote DB
+  # @param model_name[String]         - name of model for dump
+  # @param field_name[String]         - field name for model
+  # @param field_values[Array|Range]  - values of field_name
+  # @param database[String]           - DB remote name
+  # @param debug_mode[Symbol]         - debug mode (full - all msgs, short - part of msgs, none -  nothing)
+  # @param ask[Boolean]               - ask user for confirm different schema of target & remote DB
+  #
+  # @example FuryDumper.dump( password:     '12345',
+  #                             host:         'localhost',
+  #                             port:         '5432',
+  #                             user:         'username',
+  #                             model_name:   'User',
+  #                             field_name:   'admin_token',
+  #                             field_values: ['99999999-8888-4444-1212-111111111111'],
+  #                             database:     'staging',
+  #                             debug_mode:   :short)
+  def self.dump(password:,
+                host:,
+                port:,
+                user:,
+                field_values:, database:, model_name: 'Lead',
+                field_name: 'id',
+                debug_mode: :none,
+                ask: true)
+
+    check_type(model_name, String, 'model name')
+    check_type(field_name, String, 'field name')
+    check_type(field_values, [Array, Range], 'field values')
+
+    states = []
+    field_values.to_a.in_groups_of(FuryDumper::Config.batch_size) do |batch|
+      relation_items = FuryDumper::Dumpers::RelationItems.new_with_key_value(item_key: field_name, item_values: batch)
+
+      sync = Dumper.new \
+        password: password,
+        host: host,
+        port: port,
+        user: user,
+        database: database,
+        model: FuryDumper::Dumpers::Model.new(source_model: model_name, relation_items: relation_items),
+        debug_mode: debug_mode
+
+      if ask && !sync.equal_schemas?
+        confirm = ask('Are you sure to continue? [Y/N] ') { |yn| yn.limit = 1, yn.validate = /[yn]/i }
+        ask     = false
+        return unless confirm.downcase == 'y' # rubocop:disable Lint/NonLocalExitFromIterator
+      end
+
+      sync.sync_models
+      states << sync.dump_state
+    end
+
+    states.each_with_index do |state, index|
+      p "Batch ##{index}:"
+      state.print_statistic
+    end
+    true
+  end
+
+  def self.check_type(field, expected_type, field_name)
+    is_ok     = if expected_type.is_a?(Array)
+                  expected_type.any? do |type|
+                    field.is_a?(type)
+                  end
+                else
+                  field.is_a?(expected_type)
+                end
+    types_str = expected_type.is_a?(Array) ? expected_type.join(' or ') : expected_type
+
+    raise ArgumentError, "Expected #{field_name} as #{types_str}, got: #{field.class}" unless is_ok
+  end
+end

data/lib/generators/fury_dumper/config_generator.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module FuryDumper
+  module Generators
+    class ConfigGenerator < Rails::Generators::Base
+      def self.gem_root
+        File.expand_path('../../..', __dir__)
+      end
+
+      def self.source_root
+        # Use the templates from the 2.3.x generator
+        File.join(gem_root, 'rails_generators', 'fury_dumper_config', 'templates')
+      end
+
+      def generate
+        template 'fury_dumper.rb', File.join('config', 'initializers', 'fury_dumper.rb')
+        template 'fury_dumper.yml', File.join('config', 'fury_dumper.yml')
+      end
+    end
+  end
+end

data/rails_generators/fury_dumper_config/fury_dumper_config_generator.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class FuryDumperConfigGenerator < Rails::Generators::Base
+  def manifest
+    record do |m|
+      m.template('fury_dumper.yml', 'config/fury_dumper.yml')
+      m.template('fury_dumper.rb', 'config/fury_dumper.rb')
+    end
+  end
+end

data/rails_generators/fury_dumper_config/templates/fury_dumper.rb

@@ -0,0 +1 @@
+FuryDumper::Config.load("#{Rails.root}/config/fury_dumper.yml")

data/rails_generators/fury_dumper_config/templates/fury_dumper.yml

@@ -0,0 +1,44 @@
+exclude_relations: <class_name>.<relation_name>, <class_name>.<relation_name>
+batch_size: 100
+ratio_records_batches: 10
+fast: true
+mode: wide
+relative_services:
+  <service_name>:
+    database: <database_name>
+    host: <host>
+    port: <port>
+    user: <user>
+    password: <password>
+    tables:
+      <self_table_name>:
+        <remote_table_name>:
+          self_field_name: <self_field_name>
+          ms_model_name: <ms_model_name>
+          ms_field_name: <ms_field_name>
+        <other_remote_table_name>:
+          self_field_name: <self_field_name>
+          ms_model_name: <ms_model_name>
+          ms_field_name: <ms_field_name>
+      <other_self_table_name>:
+        <remote_table_name>:
+          self_field_name: <self_field_name>
+          ms_model_name: <ms_model_name>
+          ms_field_name: <ms_field_name>
+        <other_remote_table_name>:
+          self_field_name: <self_field_name>
+          ms_model_name: <ms_model_name>
+          ms_field_name: <ms_field_name>
+  <other_service_name>:
+    database: <database_name>
+    host: <host>
+    port: <port>
+    user: <user>
+    password: <password>
+    tables:
+      <self_table_name>:
+        <remote_table_name>:
+          self_field_name: <self_field_name>
+          ms_model_name: <ms_model_name>
+          ms_field_name: <ms_field_name>
+

metadata

@@ -0,0 +1,181 @@
+--- !ruby/object:Gem::Specification
+name: fury_dumper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Nastya Patutina
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-03-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.13
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.13
+description: Dump from remote DB by lead_ids interval
+email:
+- npatutina@gmail.con
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/build.yaml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- ".travis.yml"
+- Breadth-first.png
+- CODE_OF_CONDUCT.md
+- Depth-first.png
+- Gemfile
+- Gemfile.lock
+- README.md
+- README.ru.md
+- Rakefile
+- app/controllers/fury_dumper/dump_process_controller.rb
+- config/routes.rb
+- fury_dumper.gemspec
+- lib/fury_dumper.rb
+- lib/fury_dumper/api.rb
+- lib/fury_dumper/config.rb
+- lib/fury_dumper/dumper.rb
+- lib/fury_dumper/dumpers/dump_state.rb
+- lib/fury_dumper/dumpers/model.rb
+- lib/fury_dumper/dumpers/model_queue.rb
+- lib/fury_dumper/dumpers/relation_items.rb
+- lib/fury_dumper/encrypter.rb
+- lib/fury_dumper/engine.rb
+- lib/fury_dumper/version.rb
+- lib/generators/fury_dumper/config_generator.rb
+- rails_generators/fury_dumper_config/fury_dumper_config_generator.rb
+- rails_generators/fury_dumper_config/templates/fury_dumper.rb
+- rails_generators/fury_dumper_config/templates/fury_dumper.yml
+homepage: https://github.com/NastyaPatutina/fury_dumper
+licenses: []
+metadata:
+  homepage_uri: https://github.com/NastyaPatutina/fury_dumper
+  source_code_uri: https://github.com/NastyaPatutina/fury_dumper
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: Simple dump for main service and other microservices
+test_files: []