masker 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2d1fd50722f8660778d1e9675aa49e77d421ca86
+  data.tar.gz: 2a55877fa12a4458cd500a8319e2d24d868406a0
+SHA512:
+  metadata.gz: 0443e9b1df55fa2c2c424be37359936ac341bfacd9bff884daedaf22fde819464c340b3a8dbc9b3adaa62a3337b8a74b5808fa48ed22a111bb5bb0c70c489b55
+  data.tar.gz: 65ebaf8019304fc640c64cc8cb8a07546143fa981a7752103913cce1dd451bfea9fe20e45ed0aa7b16a829c9644d128591b66567454f1a485e1ef57d7469e7ac

data/lib/masker.rb

@@ -0,0 +1,20 @@
+require 'masker/adapters/postgres'
+require 'masker/configurations/postgres'
+require 'masker/null_object'
+require 'masker/configuration'
+require 'masker/data_generator'
+require 'pg'
+
+class Masker
+  def initialize(database_url:, config_path:, logger: NullObject.new, opts: {})
+    @adapter = Adapters::Postgres.new(database_url, config_path, logger, opts)
+  end
+
+  def mask
+    adapter.mask
+  end
+
+  private
+
+  attr_reader :adapter
+end

data/lib/masker/adapters/postgres.rb

@@ -0,0 +1,83 @@
+class Masker
+  module Adapters
+    class Postgres
+      def initialize(database_url, config_path, logger, opts = {})
+        @conn = PG.connect(database_url)
+        @config = Configurations::Postgres.new(conn, config_path, logger, opts)
+        @logger = logger
+      end
+
+      def mask
+        remove_temp_tables
+        config.remove_missing_tables
+        config.remove_missing_columns
+        create_temp_tables
+        insert_fake_data_into_temp_tables
+        merge_tables
+        truncate
+      ensure
+        remove_temp_tables
+      end
+
+      private
+
+      def remove_temp_tables
+        tables.keys.each do |table_name|
+          conn.exec("DROP TABLE IF EXISTS temp_#{table_name};")
+        end
+      end
+
+      def create_temp_tables
+        tables.keys.each do |table_name|
+          conn.exec("CREATE TABLE temp_#{table_name} AS SELECT * FROM #{table_name} LIMIT 0;")
+        end
+      end
+
+      def insert_fake_data_into_temp_tables
+        tables.each do |table, columns|
+          logger.info "Masking #{table}..."
+          conn.transaction do |conn|
+            config.ids_to_mask[table].each_slice(1000) do |ids|
+              fake_rows = create_fake_rows(ids, columns)
+              conn.exec("INSERT INTO temp_#{table} (id, #{columns.keys.join(", ")}) VALUES #{fake_rows};")
+            end
+          end
+        end
+      end
+
+      def create_fake_rows(ids, columns)
+        ids.map { |id| "(#{create_fake_row(id, columns)})" }.join(", ")
+      end
+
+      def create_fake_row(id, columns)
+        columns.map { |_, mask_type| stringify(DataGenerator.generate(mask_type)) }
+          .unshift(id)
+          .join(", ")
+      end
+
+      def stringify(value)
+        value.nil? ? 'NULL' : "'#{conn.escape_string(value.to_s)}'"
+      end
+
+      def merge_tables
+        tables.each do |table, columns|
+          set_statement = columns.keys.map { |column| "#{column} = temp_#{table}.#{column}" }.join(", ")
+          conn.exec("UPDATE #{table} SET #{set_statement} FROM temp_#{table} WHERE #{table}.id = temp_#{table}.id;")
+        end
+      end
+
+      def truncate
+        config.tables_to_truncate.each do |table|
+          logger.info "Truncating #{table}..."
+          conn.exec("TRUNCATE #{table};")
+        end
+      end
+
+      def tables
+        config.tables
+      end
+
+      attr_reader :logger, :config, :conn
+    end
+  end
+end

data/lib/masker/configuration.rb

@@ -0,0 +1,8 @@
+require 'yaml'
+
+module Configuration
+  def self.load(config_path)
+    fail "File not found: #{config_path}" unless File.exist?(config_path)
+    YAML.load_file(config_path)
+  end
+end

data/lib/masker/configurations/postgres.rb

@@ -0,0 +1,77 @@
+class Masker
+  module Configurations
+    class Postgres
+      attr_reader :tables
+
+      def initialize(conn, config_path, logger, opts = {})
+        @config = Configuration.load(config_path)
+        @conn = conn
+        @logger = logger
+        @opts = opts
+        @tables = config['mask']
+      end
+
+      def ids_to_mask
+        @ids_to_mask ||=
+          tables.keys.each_with_object(Hash.new { |k, v| k[v] = [] }) do |table, ids|
+            conn.exec("SELECT id FROM #{table};") do |result|
+              ids[table].concat(result.values.flatten - Array(opts.dig(:safe_ids, table.to_sym)).map(&:to_s))
+            end
+          end
+      end
+
+      def missing_tables
+        tables.keys.each_with_object([]) do |table_name, missing_tables|
+          conn.exec("SELECT EXISTS (SELECT 1 FROM pg_tables WHERE tablename = '#{table_name}');") do |result|
+            if result[0]['exists'] == 'f'
+              missing_tables << table_name
+              logger.warn "Table: #{table_name} exists in configuration but not in database."
+            end
+          end
+        end
+      end
+
+      def missing_columns
+        tables.each_with_object(Hash.new { |h, k| h[k] = [] }) do |(table_name, columns), missing_columns|
+          columns.keys.each do |column_name|
+            sql = <<~SQL
+              SELECT EXISTS (
+                SELECT 1 FROM information_schema.columns
+                WHERE table_name='#{table_name}'
+                AND column_name='#{column_name}'
+              );
+            SQL
+            conn.exec(sql) do |result|
+              if result[0]['exists'] == 'f'
+                missing_columns[table_name] << column_name
+                logger.warn "Column: #{table_name}:#{column_name} exists in configuration but not in database."
+              end
+            end
+          end
+        end
+      end
+
+      def remove_missing_tables
+        missing_tables.each do |table|
+          tables.delete(table)
+        end
+      end
+
+      def remove_missing_columns
+        missing_columns.each do |table, columns|
+          columns.each do |column|
+            tables[table].delete(column)
+          end
+        end
+      end
+
+      def tables_to_truncate
+        config['truncate']
+      end
+
+      private
+
+      attr_reader :config, :opts, :conn, :logger
+    end
+  end
+end

data/lib/masker/data_generator.rb

@@ -0,0 +1,50 @@
+require 'faker'
+
+module DataGenerator
+  class << self
+    def generate(type)
+      case type
+      when :name
+        Faker::Name.name
+      when :company_name
+        Faker::Company.name
+      when :first_name
+        Faker::Name.first_name
+      when :last_name
+        Faker::Name.last_name
+      when :email
+        "#{SecureRandom.hex(8).upcase}_#{Faker::Internet.email}"
+      when :text
+        Faker::Lorem.sentence
+      when :date
+        Faker::Date.forward(1000)
+      when :city
+        "#{Faker::Address.city}_#{SecureRandom.hex(8).upcase}"
+      when :domain_name
+        Faker::Internet.domain_name
+      when :country
+        "#{Faker::Address.country}_#{SecureRandom.hex(8).upcase}"
+      when :characters
+        Faker::Lorem.characters(10)
+      when :zip_code
+        Faker::Address.zip_code
+      when :year
+        Faker::Number.between(1900, 2020)
+      when :integer
+        Faker::Number.number(8)
+      when :low_integer
+        Faker::Number.between(1, 200)
+      when :float
+        Faker::Number.decimal(2, 2)
+      when :state
+        "#{Faker::Address.state}_#{SecureRandom.hex(8).upcase}"
+      when :phone
+        Faker::Number.number(10)
+      when :street_address
+        Faker::Address.street_address
+      else
+        type
+      end
+    end
+  end
+end

data/lib/masker/null_object.rb

@@ -0,0 +1,5 @@
+class NullObject
+  def method_missing(*args, &block)
+    self
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: masker
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Danny Park
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-10-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+- !ruby/object:Gem::Dependency
+  name: faker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+description: |-
+  Production databases contain sensitive information that should not be
+                       propagated down to other environments. This gem allows users to create
+                       masking strategies in a YML file that specify columns to mask and tables
+                       to truncate
+email:
+- dannypark92@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/masker.rb
+- lib/masker/adapters/postgres.rb
+- lib/masker/configuration.rb
+- lib/masker/configurations/postgres.rb
+- lib/masker/data_generator.rb
+- lib/masker/null_object.rb
+homepage: https://www.github.com/viewthespace/masker
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Database masking for sensitive information
+test_files: []