data_taster 0.2.2

data/gemfiles/rails_7_0.gemfile.lock

@@ -0,0 +1,263 @@
+PATH
+  remote: ../../rubocop-powerhome
+  specs:
+    rubocop-powerhome (0.5.2)
+      rubocop (~> 1.52.0)
+      rubocop-performance
+      rubocop-rails
+      rubocop-rake
+      rubocop-rspec
+
+PATH
+  remote: ..
+  specs:
+    data_taster (0.2.2)
+      rails (>= 6.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (7.0.6)
+      actionpack (= 7.0.6)
+      activesupport (= 7.0.6)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+    actionmailbox (7.0.6)
+      actionpack (= 7.0.6)
+      activejob (= 7.0.6)
+      activerecord (= 7.0.6)
+      activestorage (= 7.0.6)
+      activesupport (= 7.0.6)
+      mail (>= 2.7.1)
+      net-imap
+      net-pop
+      net-smtp
+    actionmailer (7.0.6)
+      actionpack (= 7.0.6)
+      actionview (= 7.0.6)
+      activejob (= 7.0.6)
+      activesupport (= 7.0.6)
+      mail (~> 2.5, >= 2.5.4)
+      net-imap
+      net-pop
+      net-smtp
+      rails-dom-testing (~> 2.0)
+    actionpack (7.0.6)
+      actionview (= 7.0.6)
+      activesupport (= 7.0.6)
+      rack (~> 2.0, >= 2.2.4)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actiontext (7.0.6)
+      actionpack (= 7.0.6)
+      activerecord (= 7.0.6)
+      activestorage (= 7.0.6)
+      activesupport (= 7.0.6)
+      globalid (>= 0.6.0)
+      nokogiri (>= 1.8.5)
+    actionview (7.0.6)
+      activesupport (= 7.0.6)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activejob (7.0.6)
+      activesupport (= 7.0.6)
+      globalid (>= 0.3.6)
+    activemodel (7.0.6)
+      activesupport (= 7.0.6)
+    activerecord (7.0.6)
+      activemodel (= 7.0.6)
+      activesupport (= 7.0.6)
+    activestorage (7.0.6)
+      actionpack (= 7.0.6)
+      activejob (= 7.0.6)
+      activerecord (= 7.0.6)
+      activesupport (= 7.0.6)
+      marcel (~> 1.0)
+      mini_mime (>= 1.1.0)
+    activesupport (7.0.6)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    appraisal (2.5.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    ast (2.4.2)
+    builder (3.2.4)
+    concurrent-ruby (1.2.2)
+    crass (1.0.6)
+    date (3.3.4)
+    diff-lcs (1.5.0)
+    docile (1.1.5)
+    erubi (1.12.0)
+    globalid (1.2.1)
+      activesupport (>= 6.1)
+    i18n (1.14.1)
+      concurrent-ruby (~> 1.0)
+    json (2.7.1)
+    license_finder (7.1.0)
+      bundler
+      rubyzip (>= 1, < 3)
+      thor (~> 1.2)
+      tomlrb (>= 1.3, < 2.1)
+      with_env (= 1.1.0)
+      xml-simple (~> 1.1.9)
+    loofah (2.22.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.12.0)
+    mail (2.8.1)
+      mini_mime (>= 0.1.1)
+      net-imap
+      net-pop
+      net-smtp
+    marcel (1.0.2)
+    method_source (1.0.0)
+    mini_mime (1.1.5)
+    minitest (5.20.0)
+    net-imap (0.4.9.1)
+      date
+      net-protocol
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.2.2)
+      timeout
+    net-smtp (0.4.0.1)
+      net-protocol
+    nio4r (2.7.0)
+    nokogiri (1.15.5-arm64-darwin)
+      racc (~> 1.4)
+    parallel (1.24.0)
+    parser (3.2.2.4)
+      ast (~> 2.4.1)
+      racc
+    power_assert (2.0.3)
+    racc (1.7.3)
+    rack (2.2.8)
+    rack-test (2.1.0)
+      rack (>= 1.3)
+    rails (7.0.6)
+      actioncable (= 7.0.6)
+      actionmailbox (= 7.0.6)
+      actionmailer (= 7.0.6)
+      actionpack (= 7.0.6)
+      actiontext (= 7.0.6)
+      actionview (= 7.0.6)
+      activejob (= 7.0.6)
+      activemodel (= 7.0.6)
+      activerecord (= 7.0.6)
+      activestorage (= 7.0.6)
+      activesupport (= 7.0.6)
+      bundler (>= 1.15.0)
+      railties (= 7.0.6)
+    rails-dom-testing (2.2.0)
+      activesupport (>= 5.0.0)
+      minitest
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.6.0)
+      loofah (~> 2.21)
+      nokogiri (~> 1.14)
+    railties (7.0.6)
+      actionpack (= 7.0.6)
+      activesupport (= 7.0.6)
+      method_source
+      rake (>= 12.2)
+      thor (~> 1.0)
+      zeitwerk (~> 2.5)
+    rainbow (2.2.2)
+      rake
+    rake (13.1.0)
+    regexp_parser (2.8.3)
+    rexml (3.2.6)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
+    rubocop (1.52.1)
+      json (~> 2.3)
+      parallel (~> 1.10)
+      parser (>= 3.2.2.3)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.30.0)
+      parser (>= 3.2.1.0)
+    rubocop-capybara (2.20.0)
+      rubocop (~> 1.41)
+    rubocop-factory_bot (2.24.0)
+      rubocop (~> 1.33)
+    rubocop-performance (1.20.1)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
+    rubocop-rails (2.23.1)
+      activesupport (>= 4.2.0)
+      rack (>= 1.1)
+      rubocop (>= 1.33.0, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    rubocop-rspec (2.25.0)
+      rubocop (~> 1.40)
+      rubocop-capybara (~> 2.17)
+      rubocop-factory_bot (~> 2.22)
+    ruby-progressbar (1.13.0)
+    rubyzip (2.3.2)
+    simplecov (0.15.1)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    test-unit (3.1.5)
+      power_assert
+    thor (1.3.0)
+    timeout (0.4.1)
+    tomlrb (2.0.3)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.5.0)
+    websocket-driver (0.7.6)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    with_env (1.1.0)
+    xml-simple (1.1.9)
+      rexml
+    yard (0.9.34)
+    zeitwerk (2.6.12)
+
+PLATFORMS
+  arm64-darwin-22
+  arm64-darwin-23
+
+DEPENDENCIES
+  appraisal (= 2.5.0)
+  bundler (~> 2.1)
+  data_taster!
+  license_finder (~> 7.0)
+  nokogiri (< 1.16)
+  parser (>= 2.5, != 2.5.1.1)
+  rails (= 7.0.6)
+  rainbow (= 2.2.2)
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop-powerhome!
+  simplecov (= 0.15.1)
+  test-unit (= 3.1.5)
+  yard (= 0.9.34)
+
+BUNDLED WITH
+   2.4.22

data/lib/data_taster/collection.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module DataTaster
+  # Ingests the processed yml file and returns either
+  # an empty hash (for cases that are skippable)
+  # or a hash that contains:
+  # select:
+  # (whatever is configured to go in the SELECT .. WHERE clause)
+  # sanitize:
+  # the columns and values that need custom sanitization
+  class Collection
+    def initialize(table_name)
+      @table_name = table_name
+      @ingredients = DataTaster.confection[table_name]
+      @include_insert = DataTaster.config.include_insert
+    end
+
+    def assemble
+      DataTaster.logger.info("#{table_name}...")
+
+      if skippable?
+        DataTaster.logger.info("configured to skip both schema and data")
+        {}
+      else
+        { select: selection, sanitize: sanitization }
+      end
+    end
+
+  private
+
+    attr_reader :table_name, :ingredients, :include_insert
+
+    def skippable?
+      table_name.downcase.match(/^_/) ||
+        ingredients == DataTaster::SKIP_CODE
+    end
+
+    def selection
+      insert = include_insert ? "INSERT INTO #{working_db}.#{table_name}" : ""
+
+      sql = <<-SQL.squish
+        #{insert}
+        SELECT * FROM #{source_db}.#{table_name}
+        WHERE #{where_clause}
+      SQL
+
+      DataTaster.logger.info(sql)
+      sql
+    end
+
+    # The yml file allows you to define either a simple clause
+    # or some more fine-grained sanitization. If neither is
+    # defined, we pass a clause that selects nothing.
+    def where_clause
+      clause = ingredients.is_a?(Hash) ? ingredients["select"] : ingredients
+
+      clause || "1 = 0"
+    end
+
+    def sanitization
+      return unless ingredients.is_a?(Hash)
+
+      ingredients["sanitize"]
+    end
+
+    def source_db
+      @source_db ||= DataTaster.config.source_client.query_options[:database]
+    end
+
+    def working_db
+      @working_db ||= DataTaster.config.working_client.query_options[:database]
+    end
+  end
+end

data/lib/data_taster/confection.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "erb"
+require "socket"
+require "yaml"
+require "data_taster/flavors"
+
+module DataTaster
+  # Ingests the list of data_taster_export_tables.yml files
+  # and processes them through an erb template
+  # returns a ruby hash of the data
+  class Confection
+    def assemble
+      DataTaster.config.list.each_with_object(default_data) do |path, merged_list|
+        merged_list.merge!(load_yml(path.to_s))
+      end
+    end
+
+    def load_yml(filename)
+      return {} unless File.exist?(filename)
+
+      erb = ::ERB.new(File.read(filename))
+      erb.filename = filename
+      flavored_erb = erb.def_class(DataTaster::Flavors, "render()")
+      erb_result = flavored_erb.new.render
+
+      YAML.safe_load(erb_result.gsub(/((.|\n)*---)/, "\n---")) || {}
+    end
+
+  private
+
+    def default_data
+      {
+        "schema_migrations" => "1 = 1",
+      }
+    end
+  end
+end

data/lib/data_taster/detergent.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module DataTaster
+  # Returns SQL for given data, based on types. Used for sanitizing inputs.
+  class Detergent
+    SANITIZE_FUNCTIONS = [
+      /CONCAT/,
+      /DATE/,
+      /DAY/,
+      /FORMAT/,
+      /LOWER/,
+      /REPLACE/,
+      /TRIM/,
+      /UCASE/,
+      /UPPER/,
+    ].freeze
+
+    def initialize(table_name, column_name, given_value)
+      @table_name = table_name
+      @column_name = column_name
+      @value = parse_value(given_value)
+    end
+
+    def deliver
+      return value if value == DataTaster::SKIP_CODE
+
+      sql = sql_for(value)
+
+      DataTaster.logger.info("--> #{sql}")
+      sql
+    end
+
+  private
+
+    attr_reader :table_name, :column_name, :value
+
+    def parse_value(given_value)
+      # yml files can't hold custom-set dates, they have to be converted to strings
+      return given_value unless given_value.is_a?(String) && given_value.match?(/\d{4}-\d{2}-\d{2}/)
+
+      Date.parse(given_value)
+    end
+
+    def sql_for(value)
+      if value.is_a?(Date)
+        sql_for_date_value
+      elsif value.is_a?(Numeric) || sanitize_function?
+        sql_for_uncast_value
+      elsif value.blank?
+        sql_for_nil_value
+      else
+        sql_for_cast_value
+      end
+    end
+
+    def sanitize_function?
+      SANITIZE_FUNCTIONS.any? { |fun| value.to_s.match(fun) }
+    end
+
+    def sql_for_uncast_value
+      <<-SQL.squish
+        UPDATE #{working_db}.#{table_name}
+        SET #{column_name} = #{value}
+        WHERE #{column_name} IS NOT NULL
+        AND #{column_name} <> #{value}
+      SQL
+    end
+
+    def sql_for_date_value
+      <<-SQL.squish
+        UPDATE #{working_db}.#{table_name}
+        SET #{column_name} = '#{value}'
+        WHERE #{column_name} IS NOT NULL
+      SQL
+    end
+
+    def sql_for_nil_value
+      <<-SQL.squish
+        UPDATE #{working_db}.#{table_name}
+        SET #{column_name} = NULL
+        WHERE #{column_name} IS NOT NULL
+      SQL
+    end
+
+    def sql_for_cast_value
+      <<-SQL.squish
+        UPDATE #{working_db}.#{table_name}
+        SET #{column_name} = '#{value}'
+        WHERE #{column_name} IS NOT NULL
+        AND #{column_name} <> ''
+      SQL
+    end
+
+    def working_db
+      DataTaster.config.working_client.query_options[:database]
+    end
+  end
+end

data/lib/data_taster/flavors.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module DataTaster
+  # helper methods made to make data_taster_export_tables.yml
+  # files more user-friendly
+  class Flavors
+    include DataTaster::Helper
+
+    def current_date
+      @current_date ||= Date.current
+    end
+
+    def date
+      @date ||= if DataTaster.config.months
+                  (current_date - DataTaster.config.months.to_i.months).beginning_of_day.to_s(:db)
+                else
+                  (current_date - 1.week).beginning_of_day.to_s(:db)
+                end
+    end
+
+    # skips dumping both schema and data
+    def deprecated_table
+      DataTaster::SKIP_CODE
+    end
+
+    def skip_sanitization
+      DataTaster::SKIP_CODE
+    end
+
+    def encrypt(klass, column, value = nil)
+      value_to_encrypt = value || default_value_for(column)
+
+      klass.new.encrypt(column, value_to_encrypt)
+    end
+
+    def default_value_for(column)
+      case column
+      when /date_of_birth/, /dob/
+        (Date.current - 25.years).strftime("%m/%d/%Y")
+      when /ssn/, /license/
+        "111111111"
+      when /compensation/
+        1
+      else
+        "1"
+      end
+    end
+
+    def full_table_dump
+      "1 = 1"
+    end
+
+    def recent_table_updates
+      "created_at >= '#{date}' OR updated_at >= '#{date}'"
+    end
+
+    def recent_ids(table_name, col_name)
+      <<~SQL.squish
+        (SELECT DISTINCT(#{col_name})
+        FROM #{source_db}.#{table_name}
+        WHERE
+        created_at >= '#{date}'
+        OR
+        updated_at >= '#{date}')
+      SQL
+    end
+
+    def source_db
+      @source_db ||= db_config["database"]
+    end
+  end
+end

data/lib/data_taster/helper.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module DataTaster
+  # helpers used globally in DataTaster
+  module Helper
+    def sanitize_command(command, params = nil)
+      sanitized_command = command.gsub(Shellwords.escape(ENV.fetch("DEV_DUMP_USER", nil)), "<username>")
+                                 .gsub(Shellwords.escape(ENV.fetch("DEV_DUMP_PASSWORD", nil)), "<pwd>")
+                                 .gsub(ENV.fetch("DEV_DUMP_PASSWORD", nil), "<pwd>")
+
+      sanitized_command = sanitized_command.gsub(Shellwords.escape(params["password"]), "<pwd>") if params
+
+      sanitized_command
+    end
+
+    def db_yml
+      @db_yml ||= YAML.safe_load(ERB.new(Rails.root.join("config", "database.yml").read).result, aliases: true)
+    end
+
+    def db_config
+      @db_config ||= db_yml[Rails.env]
+    end
+
+    def logg(message)
+      DataTaster.logger.debug { "[#{Time.current}] #{message}" }
+    end
+  end
+end

data/lib/data_taster/sample.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module DataTaster
+  # Selects and processes tables from the source_db
+  # to insert (or query) into the working_db
+  class Sample
+    def initialize(table_name)
+      @table_name = table_name
+      @include_insert = DataTaster.config.include_insert
+      @collection = DataTaster::Collection.new(
+        table_name
+      ).assemble
+    end
+
+    def serve!
+      # Any table that does not return SQL is considered deprecated and we should fully skip it
+      if collection.empty? && include_insert
+        DataTaster.safe_execute("DROP TABLE IF EXISTS #{table_name}")
+      else
+        ensure_empty_table
+        process_select(collection[:select])
+        DataTaster::Sanitizer.new(table_name, collection[:sanitize]).clean!
+      end
+    end
+
+  private
+
+    attr_reader :table_name, :include_insert, :collection
+
+    def ensure_empty_table
+      DataTaster.safe_execute("TRUNCATE TABLE #{working_db}.#{table_name}")
+    end
+
+    def process_select(sql)
+      DataTaster.safe_execute(sql)
+    rescue => e
+      e.message << " executing SQL statement for #{table_name}: #{sql}"
+      raise e
+    end
+
+    def working_db
+      @working_db ||= DataTaster.config.working_client.query_options[:database]
+    end
+  end
+end