dump_cleaner 0.5.0

data/lib/dump_cleaner/log.rb

@@ -0,0 +1,42 @@
+module DumpCleaner
+  require "logger"
+
+  class Log < ::Logger
+    require "singleton"
+
+    include Singleton
+
+    attr_reader :logger
+
+    def initialize
+      super($stdout)
+
+      init_log_level
+      self.formatter = ->(severity, datetime, _progname, msg) { "#{datetime} #{severity}: #{msg}\n" }
+    end
+
+    def init_log_level
+      self.level = Logger::INFO
+    end
+
+    def self.debug(&block)
+      instance.debug(&block)
+    end
+
+    def self.info(&block)
+      instance.info(&block)
+    end
+
+    def self.warn(&block)
+      instance.warn(&block)
+    end
+
+    def self.error(&block)
+      instance.error(&block)
+    end
+
+    def self.fatal(&block)
+      instance.fatal(&block)
+    end
+  end
+end

data/lib/dump_cleaner/options.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module DumpCleaner
+  class Options
+    require "optparse"
+
+    DEFAULT_OPTIONS = {
+      config_file: "config/dump_cleaner.yml"
+    }.freeze
+
+    attr_accessor :source_dump_path, :destination_dump_path, :config_file
+
+    def initialize(argv)
+      DEFAULT_OPTIONS.each { |k, v| send(:"#{k}=", v) }
+      parse(argv)
+      validate
+    end
+
+    private
+
+    def parse(argv)
+      OptionParser.new do |parser|
+        parser.banner = "Usage: dump_cleaner -f source_dump -t cleaned_dump [options]"
+
+        parser.on("-f", "--from=SOURCE_DUMP_PATH",
+                  "File or directory of the original (source) dump") do |option|
+          self.source_dump_path = option
+        end
+        parser.on("-t", "--to=DESTINATION_DUMP_PATH",
+                  "File or directory of the cleaned (destination) dump") do |option|
+          self.destination_dump_path = option
+        end
+        parser.on("-c", "--config=CONFIG_FILE", "Configuration file path") do |option|
+          self.config_file = option
+        end
+      end.parse!(argv)
+    end
+
+    def validate
+      if !source_dump_path || !destination_dump_path # rubocop:disable Style/GuardClause
+        raise ArgumentError, "Missing source or destination dump file or directory,
+                              please use -f and -t options. Use -h for help.".gsub(/\s+/, " ")
+      end
+    end
+  end
+end

data/lib/dump_cleaner/processor.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module DumpCleaner
+  class Processor
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def run
+      start_time = Time.now
+
+      cleaner_class = case config.dump_format
+                      when "mysql_shell"
+                        Cleaners::MysqlShellDumpCleaner
+                      else
+                        raise Config::ConfigurationError, "Unsupported dump format #{config.dump_format}"
+                      end
+
+      Log.debug { "Starting cleanup with #{cleaner_class}…" }
+      cleaner = cleaner_class.new(config:, options:)
+      cleaner.pre_cleanup
+      cleaner.clean
+      cleaner.post_cleanup
+
+      diff = Time.now - start_time
+      Log.info { "Finished in #{diff.div(60)}m #{(diff % 60).to_i}s." }
+    end
+
+    private
+
+    def config
+      @config ||= Config.new(options.config_file)
+    end
+  end
+end

data/lib/dump_cleaner/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DumpCleaner
+  VERSION = "0.5.0"
+end

data/lib/dump_cleaner.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+
+module DumpCleaner
+  class Error < StandardError; end
+  # Your code goes here...
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: dump_cleaner
+version: !ruby/object:Gem::Version
+  version: 0.5.0
+platform: ruby
+authors:
+- Matouš Borák
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-06-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+description: Deterministically anonymizes data in logical database dumps. Useful for
+  importing (anonymized) production data into development environments.
+email:
+- matous.borak@nejremeslnici.cz
+executables:
+- dump_cleaner
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- doc/workflow_steps.md
+- dump_cleaner.gemspec
+- exe/dump_cleaner
+- lib/dump_cleaner.rb
+- lib/dump_cleaner/cleaners/base_cleaner.rb
+- lib/dump_cleaner/cleaners/mysql_shell_dump_cleaner.rb
+- lib/dump_cleaner/cleaners/mysql_shell_dump_helpers.rb
+- lib/dump_cleaner/cleaners/mysql_shell_table_cleaner.rb
+- lib/dump_cleaner/cleanup/bytesize_helpers.rb
+- lib/dump_cleaner/cleanup/cleaning.rb
+- lib/dump_cleaner/cleanup/cleaning_steps/add_repetition_suffix.rb
+- lib/dump_cleaner/cleanup/cleaning_steps/base.rb
+- lib/dump_cleaner/cleanup/cleaning_steps/fill_up_with_string.rb
+- lib/dump_cleaner/cleanup/cleaning_steps/generate_random_string.rb
+- lib/dump_cleaner/cleanup/cleaning_steps/inspect_context.rb
+- lib/dump_cleaner/cleanup/cleaning_steps/randomize_email.rb
+- lib/dump_cleaner/cleanup/cleaning_steps/randomize_formatted_number.rb
+- lib/dump_cleaner/cleanup/cleaning_steps/randomize_number.rb
+- lib/dump_cleaner/cleanup/cleaning_steps/select_data_by_bytesize.rb
+- lib/dump_cleaner/cleanup/cleaning_steps/select_data_by_pattern.rb
+- lib/dump_cleaner/cleanup/cleaning_steps/take_sample.rb
+- lib/dump_cleaner/cleanup/data_source.rb
+- lib/dump_cleaner/cleanup/data_source_steps/base.rb
+- lib/dump_cleaner/cleanup/data_source_steps/group_by_bytesize.rb
+- lib/dump_cleaner/cleanup/data_source_steps/inspect_context.rb
+- lib/dump_cleaner/cleanup/data_source_steps/load_yaml_file.rb
+- lib/dump_cleaner/cleanup/data_source_steps/remove_accents.rb
+- lib/dump_cleaner/cleanup/inspection.rb
+- lib/dump_cleaner/cleanup/step_context.rb
+- lib/dump_cleaner/cleanup/uniqueness.rb
+- lib/dump_cleaner/cleanup/workflow.rb
+- lib/dump_cleaner/conditions.rb
+- lib/dump_cleaner/config.rb
+- lib/dump_cleaner/log.rb
+- lib/dump_cleaner/options.rb
+- lib/dump_cleaner/processor.rb
+- lib/dump_cleaner/version.rb
+homepage: https://github.com/NejRemeslnici/dump-cleaner
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/NejRemeslnici/dump-cleaner
+  source_code_uri: https://github.com/NejRemeslnici/dump-cleaner
+  changelog_uri: https://github.com/NejRemeslnici/dump-cleaner/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.3
+signing_key:
+specification_version: 4
+summary: Anonymizes data in logical database dumps.
+test_files: []