pg_dump_anonymize 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 903be54f2c5727d77dc205f98565863caa7977639c6302a47d9a52f7cdcac15e
+  data.tar.gz: 244fe8be50f46e717bddf6222f46360b2cff2be016285f101a0f9698a10bed6e
+SHA512:
+  metadata.gz: 1d4a8ef4dc0050fd1fd942e3c4925bfe0a1cbf5145b6b643fc1188e0479fcfb5262e04d90cfae9715ac6f0f3060757ae4da21eb0ea7c72cb6785f7b39290e183
+  data.tar.gz: 0c8f53ddcf4d46c5ea49426059419848aa97205579e99ab7a44eed5f6143bf603c994ccd52a58f3928b2171204e18e33648b95b0a9234e96dae9bebb02eaae78

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.7.1
+before_install: gem install bundler -v 2.1.4

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in pg_dump_anonymize.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "rspec", "~> 3.0"

data/Gemfile.lock

@@ -0,0 +1,34 @@
+PATH
+  remote: .
+  specs:
+    pg_dump_anonymize (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.4.4)
+    rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.3)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pg_dump_anonymize!
+  rake (~> 12.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Sean McCleary
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,84 @@
+# PgDumpAnonymize
+
+Anonymizing pg_dump data isn't always straight forward. This tool helps. It has no dependencies, other than ruby. Another good thing is that sensitive data doesn't need to ever be stored temporarily as this can be used with Unix pipes and the data is passed through as IO data.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pg_dump_anonymize'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install pg_dump_anonymize
+
+## Usage
+
+Example usage:
+
+```
+pg_dump --no-privileges --no-owner <database-name> | pg_dump_anonymize -d sample_definition.rb > anonymized_dump.sql
+```
+
+### Definition File
+
+The definition file is any ruby code you'd like, but it should return a hash with table names as the top level keys and attribute names nested under the table keys. Any faking ruby gems dependencies, and other gem dependencies you may choose to use in your definitions, are gems that will need to be available on the server this is ran on.
+
+Example:
+
+```ruby
+{
+  table_1: {
+    sensitive_field_1: 'some string value',
+    sensitive_field_2: -> { rand(100) },
+  },
+  table_2: {
+    sensitive_field_3: nil
+  }
+}
+```
+
+Here is a more concrete example using the [faker gem](https://github.com/faker-ruby/faker).
+
+```ruby
+require 'faker'
+
+{
+  users: {
+    first_name: -> { Faker::Name.first_name },
+    last_name: -> { Faker::Name.last_name },
+    email: -> { Faker::Internet.email },
+    city: -> 'Portland'
+  },
+  accounts: {
+    bank_name: -> { Faker::Bank.name },
+    account_num: -> { Faker::Bank.account_number },
+    routing_num: -> { Faker::Bank.routing_number }
+  }
+}
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Performance
+
+This has been tested with a 1GB dump file, and it took 11 seconds on a 2013 MacBook Pro to anonymize it. It largely depends on how much you're anonymizing, and the anonymizing definitions you're applying. So milage will vary. Still, this is plenty fast for most of my needs.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pg_dump_anonymize.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pg_dump_anonymize"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/pg_dump_anonymize

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.expand_path('../lib', __dir__)
+require 'pg_dump_anonymize'
+require 'optparse'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename(__FILE__)} [options]"
+
+  opts.on('-d', '--definition DEFINITION_FILE', 'Definition file to read. This is required') do |v|
+    options[:definition_file_path] = v
+  end
+
+  opts.on('-f', '--file OUTPUT_FILE', 'Output file') do |v|
+    options[:output_file_path] = v
+  end
+end.parse!
+
+def_file = options[:definition_file_path]
+
+unless def_file && File.exist?(def_file)
+  puts 'Definition file not found'
+  puts 'See usage with --help'
+  exit 1
+end
+
+output_file = options[:output_file_path]
+output = if output_file
+           File.open(output_file, 'w')
+         else
+           STDOUT
+         end
+
+PgDumpAnonymize.anonymize(def_file, STDIN, output)

data/lib/pg_dump_anonymize.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'pg_dump_anonymize/version'
+require 'pg_dump_anonymize/definition'
+
+module PgDumpAnonymize
+  class Error < StandardError; end
+
+  def self.anonymize(definitions_file_path, input_io, output_io)
+    definitions_hash = eval(File.open(definitions_file_path).read) # rubocop:disable Security/Eval
+    definitions = Definition.new(definitions_hash)
+
+    input_io.each_line do |line|
+      output_io.write definitions.process_line(line)
+    end
+  end
+end

data/lib/pg_dump_anonymize/definition.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module PgDumpAnonymize
+  # This is used to define rules and apply the rules when parsing the dump sql file
+  class Definition
+    def initialize(attribute_rules)
+      @attribute_rules = attribute_rules
+      @current_table = nil
+      @positional_substitutions = nil
+    end
+
+    def process_line(line)
+      if @current_table
+        if end_stdin?(line)
+          clear_current_table
+        else
+          line = anonymize_line(line)
+        end
+      else
+        process_copy_line(line)
+      end
+      line
+    end
+
+    private
+
+    # This assumes the line is a tab delimited data line
+    def anonymize_line(line)
+      values = line.split("\t")
+      @positional_substitutions.each do |index, val_def|
+        anonymous_value = val_def.is_a?(Proc) ? val_def.call : val_def
+        values[index] = anonymous_value
+      end
+      values.join("\t")
+    end
+
+    def process_copy_line(line)
+      match_data = line.match(line_regex)
+      return unless match_data
+
+      table = match_data[:table_name].to_sym
+      fields = match_data[:field_defs]
+
+      @current_table = table
+      @positional_substitutions = find_positions(fields, @attribute_rules[table])
+    end
+
+    # Finds the positional range of the attribute to be replaced
+    # returns an array of arrays. The inner array is [<field_index>, <anonymous_value>]
+    def find_positions(fields_str, rules)
+      fields = fields_str.gsub('"', '').split(', ')
+
+      rules.map do |target_field, val|
+        index = fields.index(target_field.to_s)
+        [index, val] if index
+      end.compact
+    end
+
+    def line_regex
+      @line_regex ||= /^COPY public\.(?<table_name>#{table_names.join('|')}) \((?<field_defs>.*)\) FROM stdin;$/
+    end
+
+    # stdin is escaped with a line that is just '\.'
+    def end_stdin?(line)
+      line =~ /^\\.$/
+    end
+
+    def table_names
+      @attribute_rules.keys
+    end
+
+    def clear_current_table
+      @current_table = nil
+      @positional_substitutions = nil
+    end
+  end
+end

data/lib/pg_dump_anonymize/version.rb

@@ -0,0 +1,3 @@
+module PgDumpAnonymize
+  VERSION = "0.1.0"
+end

data/pg_dump_anonymize.gemspec

@@ -0,0 +1,29 @@
+require_relative 'lib/pg_dump_anonymize/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pg_dump_anonymize"
+  spec.version       = PgDumpAnonymize::VERSION
+  spec.authors       = ["Sean McCleary"]
+  spec.email         = ["seanmcc@gmail.com"]
+
+  spec.summary       = %q{A tool to take pg_dump SQL and anonymize defined tables and field}
+  spec.description   = %q{Given the default pg_dump text SQL dump, this can take a simple definition of tables and fields to anonymize and efficiently anonymize the dump.}
+  spec.homepage      = "https://github.com/mrinterweb/pg_dump_anonymize"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+end

data/sample_definition.rb

@@ -0,0 +1,9 @@
+require 'faker'
+
+{
+  users: {
+    first_name: -> { Faker::Name.first_name },
+    last_name: -> { Faker::Name.last_name },
+    email: -> { Faker::Internet.email }
+  }
+}

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: pg_dump_anonymize
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sean McCleary
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2020-10-05 00:00:00.000000000 Z
+dependencies: []
+description: Given the default pg_dump text SQL dump, this can take a simple definition
+  of tables and fields to anonymize and efficiently anonymize the dump.
+email:
+- seanmcc@gmail.com
+executables:
+- pg_dump_anonymize
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/pg_dump_anonymize
+- lib/pg_dump_anonymize.rb
+- lib/pg_dump_anonymize/definition.rb
+- lib/pg_dump_anonymize/version.rb
+- pg_dump_anonymize.gemspec
+- sample_definition.rb
+homepage: https://github.com/mrinterweb/pg_dump_anonymize
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/mrinterweb/pg_dump_anonymize
+  source_code_uri: https://github.com/mrinterweb/pg_dump_anonymize
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: A tool to take pg_dump SQL and anonymize defined tables and field
+test_files: []