pgdump_scrambler 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '09085520af68283995aeb54b3ae886c303b2ec2a9123b7d76da9a201463bb0d9'
+  data.tar.gz: e7ee185201e95a18b84fa9b9eaafce6e98b0f0187125af647044ff4eb4789058
+SHA512:
+  metadata.gz: 45cf30192e04b4af8f30d90aa753cd52b410f9d51597fd9f9eefbf050a6b15f16abd5c2c6ee3241e8261a1f63c22b4207f88e064e240523c371bec9db2a638b2
+  data.tar.gz: b7a5383078df617610d10eec93049f68194eea2aae21970752059dd632ff5b67b94e7a713a3701a57db3a8886991f86ec1c7d2a085b5d1a0ba97753fea99b52a

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,2 @@
+inherit_from: .rubocop_todo.yml
+

data/.rubocop_todo.yml

@@ -0,0 +1,173 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2018-03-08 00:50:23 +0900 using RuboCop version 0.52.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: Include, TreatCommentsAsGroupSeparators.
+# Include: **/*.gemspec
+Gemspec/OrderedDependencies:
+  Exclude:
+    - 'pgdump_scrambler.gemspec'
+
+# Offense count: 5
+# Cop supports --auto-correct.
+Layout/EmptyLineAfterMagicComment:
+  Exclude:
+    - 'lib/pgdump_scrambler.rb'
+    - 'lib/pgdump_scrambler/railtie.rb'
+    - 'lib/pgdump_scrambler/table.rb'
+    - 'lib/pgdump_scrambler/version.rb'
+    - 'lib/tasks/pgdump_scrambler_tasks.rake'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
+Layout/ExtraSpacing:
+  Exclude:
+    - 'pgdump_scrambler.gemspec'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: auto_detection, squiggly, active_support, powerpack, unindent
+Layout/IndentHeredoc:
+  Exclude:
+    - 'spec/pgdump_scrambler_spec.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: AllowForAlignment.
+Layout/SpaceAroundOperators:
+  Exclude:
+    - 'pgdump_scrambler.gemspec'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SpaceBeforeBlockParameters.
+# SupportedStyles: space, no_space
+# SupportedStylesForEmptyBraces: space, no_space
+Layout/SpaceInsideBlockBraces:
+  Exclude:
+    - 'Gemfile'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: final_newline, final_blank_line
+Layout/TrailingBlankLines:
+  Exclude:
+    - 'lib/pgdump_scrambler/table.rb'
+    - 'lib/tasks/pgdump_scrambler_tasks.rake'
+
+# Offense count: 4
+# Cop supports --auto-correct.
+Layout/TrailingWhitespace:
+  Exclude:
+    - 'lib/pgdump_scrambler/table.rb'
+
+# Offense count: 1
+# Configuration parameters: AllowSafeAssignment.
+Lint/AssignmentInCondition:
+  Exclude:
+    - 'lib/pgdump_scrambler/table.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: IgnoreEmptyBlocks, AllowUnusedKeywordArguments.
+Lint/UnusedBlockArgument:
+  Exclude:
+    - 'lib/pgdump_scrambler/table.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Security/YAMLLoad:
+  Exclude:
+    - 'lib/pgdump_scrambler/table.rb'
+
+# Offense count: 5
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/pgdump_scrambler.rb'
+    - 'lib/pgdump_scrambler/railtie.rb'
+    - 'lib/pgdump_scrambler/table.rb'
+
+# Offense count: 6
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: when_needed, always, never
+Style/FrozenStringLiteralComment:
+  Exclude:
+    - 'Gemfile'
+    - 'Rakefile'
+    - 'bin/console'
+    - 'pgdump_scrambler.gemspec'
+    - 'spec/pgdump_scrambler_spec.rb'
+    - 'spec/spec_helper.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, UseHashRocketsWithSymbolValues, PreferHashRocketsForNonAlnumEndingSymbols.
+# SupportedStyles: ruby19, hash_rockets, no_mixed_keys, ruby19_no_mixed_keys
+Style/HashSyntax:
+  Exclude:
+    - 'Rakefile'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/IfUnlessModifier:
+  Exclude:
+    - 'lib/pgdump_scrambler.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: PreferredDelimiters.
+Style/PercentLiteralDelimiters:
+  Exclude:
+    - 'pgdump_scrambler.gemspec'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/RescueModifier:
+  Exclude:
+    - 'lib/pgdump_scrambler/table.rb'
+
+# Offense count: 32
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
+# SupportedStyles: single_quotes, double_quotes
+Style/StringLiterals:
+  Exclude:
+    - 'Gemfile'
+    - 'Rakefile'
+    - 'bin/console'
+    - 'lib/pgdump_scrambler.rb'
+    - 'lib/pgdump_scrambler/version.rb'
+    - 'pgdump_scrambler.gemspec'
+    - 'spec/spec_helper.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyleForMultiline.
+# SupportedStylesForMultiline: comma, consistent_comma, no_comma
+Style/TrailingCommaInLiteral:
+  Exclude:
+    - 'lib/pgdump_scrambler/table.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/UnneededPercentQ:
+  Exclude:
+    - 'pgdump_scrambler.gemspec'
+
+# Offense count: 11
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Metrics/LineLength:
+  Max: 96

data/.ruby-version

@@ -0,0 +1 @@
+3.1.3

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.5.0
+before_install: gem install bundler -v 1.16.1

data/.vscode/tasks.json

@@ -0,0 +1,16 @@
+{
+  // See https://go.microsoft.com/fwlink/?LinkId=733558
+  // for the documentation about the tasks.json format
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "rspec",
+      "type": "shell",
+      "command": "bundle exec rake",
+      "group": {
+        "kind": "build",
+        "isDefault": true
+      }
+    }
+  ]
+}

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+## [0.4.0] - 2023-10-18
+- Add scramble functions
+- Support ARM64 Linux/Mac
+
+## [0.3.0] - 2023-04-13
+- Prefer using YAML.safe_load over YAML.load
+- Test with Rails 7.0
+
+### Fixes
+- Fix JSON number format error

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in pgdump_scrambler.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,194 @@
+PATH
+  remote: .
+  specs:
+    pgdump_scrambler (0.4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+    actionmailbox (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      activejob (= 7.0.4.3)
+      activerecord (= 7.0.4.3)
+      activestorage (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+      mail (>= 2.7.1)
+      net-imap
+      net-pop
+      net-smtp
+    actionmailer (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      actionview (= 7.0.4.3)
+      activejob (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+      mail (~> 2.5, >= 2.5.4)
+      net-imap
+      net-pop
+      net-smtp
+      rails-dom-testing (~> 2.0)
+    actionpack (7.0.4.3)
+      actionview (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+      rack (~> 2.0, >= 2.2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actiontext (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      activerecord (= 7.0.4.3)
+      activestorage (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+      globalid (>= 0.6.0)
+      nokogiri (>= 1.8.5)
+    actionview (7.0.4.3)
+      activesupport (= 7.0.4.3)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activejob (7.0.4.3)
+      activesupport (= 7.0.4.3)
+      globalid (>= 0.3.6)
+    activemodel (7.0.4.3)
+      activesupport (= 7.0.4.3)
+    activerecord (7.0.4.3)
+      activemodel (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+    activestorage (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      activejob (= 7.0.4.3)
+      activerecord (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+      marcel (~> 1.0)
+      mini_mime (>= 1.1.0)
+    activesupport (7.0.4.3)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    ast (2.4.2)
+    builder (3.2.4)
+    concurrent-ruby (1.2.2)
+    crass (1.0.6)
+    date (3.3.3)
+    diff-lcs (1.5.0)
+    erubi (1.12.0)
+    globalid (1.1.0)
+      activesupport (>= 5.0)
+    i18n (1.12.0)
+      concurrent-ruby (~> 1.0)
+    json (2.6.3)
+    loofah (2.20.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    mail (2.8.1)
+      mini_mime (>= 0.1.1)
+      net-imap
+      net-pop
+      net-smtp
+    marcel (1.0.2)
+    method_source (1.0.0)
+    mini_mime (1.1.2)
+    minitest (5.18.0)
+    net-imap (0.3.4)
+      date
+      net-protocol
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.2.1)
+      timeout
+    net-smtp (0.3.3)
+      net-protocol
+    nio4r (2.5.9)
+    nokogiri (1.14.3-x86_64-linux)
+      racc (~> 1.4)
+    parallel (1.22.1)
+    parser (3.2.2.0)
+      ast (~> 2.4.1)
+    racc (1.6.2)
+    rack (2.2.6.4)
+    rack-test (2.1.0)
+      rack (>= 1.3)
+    rails (7.0.4.3)
+      actioncable (= 7.0.4.3)
+      actionmailbox (= 7.0.4.3)
+      actionmailer (= 7.0.4.3)
+      actionpack (= 7.0.4.3)
+      actiontext (= 7.0.4.3)
+      actionview (= 7.0.4.3)
+      activejob (= 7.0.4.3)
+      activemodel (= 7.0.4.3)
+      activerecord (= 7.0.4.3)
+      activestorage (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+      bundler (>= 1.15.0)
+      railties (= 7.0.4.3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.5.0)
+      loofah (~> 2.19, >= 2.19.1)
+    railties (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+      method_source
+      rake (>= 12.2)
+      thor (~> 1.0)
+      zeitwerk (~> 2.5)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.7.0)
+    rexml (3.2.5)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.1)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    rubocop (1.50.1)
+      json (~> 2.3)
+      parallel (~> 1.10)
+      parser (>= 3.2.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.28.0)
+      parser (>= 3.2.1.0)
+    ruby-progressbar (1.13.0)
+    thor (1.2.1)
+    timeout (0.3.2)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.4.2)
+    websocket-driver (0.7.5)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    zeitwerk (2.6.7)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  pgdump_scrambler!
+  rails (~> 7.0)
+  rake (~> 13.0)
+  rspec (~> 3.12)
+  rubocop
+
+BUNDLED WITH
+   2.4.12

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Shunichi Ikegami
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,79 @@
+# PgdumpScrambler
+
+Generate scrambled potgresql dump for rails application.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pgdump_scrambler', github: 'shunichi/pgdump_scrambler'
+```
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+Genarate config file.
+
+```
+bundle exec rake pgdump_scrambler:config_from_db
+```
+
+Fix column scramble functions in config/pgdump_scrambler.yml
+
+from:
+
+```
+tables:
+  users:
+    email: unspecified
+    name: unspecified
+    age: unspecified
+```
+
+to:
+
+```
+tables:
+  users:
+    email: uemail
+    name: sbytes
+    age: nop
+```
+
+Dump the scrambled database.
+
+```
+bundle exec rake pgdump_scrambler:dump
+```
+
+## scramble functions
+
+- `bytes` random bytes (each byte is one of `0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+-_`)
+- `sbytes` random bytes (each byte is one of `0123456789abcdefghijklmnopqrstuvwxyz`)
+- `digits` random digits
+- `email` random email address
+- `uemail` random unique email address
+- `inet` random ip address
+- `json` string value to random bytes, number value to random digits, keep data structure and key names
+- `nullify` NULL
+- `empty` empty string
+- `const[VALUE]` constant value
+- `nop` untouched
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/shunichi/pgdump_scrambler.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pgdump_scrambler"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/pgdump-obfuscator

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+require 'shellwords'
+
+arch = case RUBY_PLATFORM
+  when /aarch64-linux/
+    'linux-arm64'
+  when /x86_64-linux/
+    'linux-amd64'
+  when /x86_64-darwin/
+    'darwin-amd64'
+  when /arm64-darwin/
+    'darwin-arm64'
+  else
+    raise "Unsupported platform: #{RUBY_PLATFORM}"
+end
+
+cmd = File.expand_path "#{File.dirname(__FILE__)}/../libexec/pgdump-obfuscator-#{arch}"
+
+exec "#{cmd} #{Shellwords.join($*)}"

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/config/table.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+module PgdumpScrambler
+  class Config
+    class Table
+      attr_reader :name
+
+      def initialize(name, columns)
+        @name = name
+        @column_hash = columns.sort_by(&:name).map { |column| [column.name, column] }.to_h
+      end
+
+      def columns
+        @column_hash.values
+      end
+
+      def [](column_name)
+        @column_hash[column_name]
+      end
+
+      def update_with(other)
+        Table.new(name, other.column_hash.merge(@column_hash).values)
+      end
+
+      def options
+        columns.map(&:option).compact.map { |option| "-c #{name}:#{option}" }.join(' ')
+      end
+
+      def unspecifiled_columns
+        @column_hash.map(&:second).select(&:unspecifiled?).map(&:name)
+      end
+
+      protected
+
+      attr_reader :column_hash
+    end
+
+    class Column
+      SCRAMBLE_METHODS = %w[unspecified nop bytes sbytes digits email uemail inet json nullify empty].freeze
+      SCRAMBLE_CONST_REGEXP = /\Aconst\[.+\]\z/
+      NOP_METHODS = %w[unspecified nop].freeze
+      UNSPECIFIED = 'unspecified'
+      attr_reader :name
+
+      def initialize(name, scramble_method = UNSPECIFIED)
+        unless self.class.valid_scramble_method?(scramble_method)
+          raise ArgumentError, "invalid scramble_method: #{scramble_method}"
+        end
+        @name = name
+        @scramble_method = scramble_method
+      end
+
+      def scramble_method
+        @scramble_method
+      end
+
+      def unspecifiled?
+        @scramble_method == UNSPECIFIED
+      end
+
+      def option
+        unless NOP_METHODS.member?(@scramble_method)
+          m = Shellwords.escape(scramble_method)
+          "#{@name}:#{m}"
+        end
+      end
+
+      class << self
+        def valid_scramble_method?(scramble_method)
+          SCRAMBLE_CONST_REGEXP.match?(scramble_method) || SCRAMBLE_METHODS.member?(scramble_method)
+        end
+      end
+    end
+  end
+end

data/lib/pgdump_scrambler/config.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+require 'yaml'
+require 'erb'
+require 'set'
+require 'config/table'
+
+module PgdumpScrambler
+  class Config
+    IGNORED_ACTIVE_RECORD_TABLES = %w[ar_internal_metadata schema_migrations].freeze
+    IGNORED_ACTIVE_RECORD_COLUMNS = %w[id created_at updated_at].to_set.freeze
+    KEY_DUMP_PATH = 'dump_path'
+    KEY_TABLES = 'tables'
+    KEY_EXCLUDE_TABLES = 'exclude_tables'
+    KEY_PGDUMP_ARGS = 'pgdump_args'
+    KEY_S3 = 's3'
+    DEFAULT_S3_PROPERTIES = {
+      'bucket' => 'YOUR_S3_BUCKET',
+      'region' => 'YOUR_S3_REGION',
+      'prefix' => 'YOUR_S3_PATH_PREFIX',
+      'access_key_id' => "<%= ENV['AWS_ACCESS_KEY_ID'] %>",
+      'secret_key' => "<%= ENV['AWS_SECRET_KEY'] %>"
+    }
+    attr_reader :dump_path, :s3, :resolved_s3, :exclude_tables, :pgdump_args
+
+    def initialize(tables, dump_path, s3, exclude_tables, pgdump_args)
+      @table_hash = tables.sort_by(&:name).map { |table| [table.name, table] }.to_h
+      @dump_path = dump_path
+      @s3 = s3
+      @resolved_s3 = s3.map { |k, v| [k, ERB.new(v).result] }.to_h if s3
+      @exclude_tables = exclude_tables
+      @pgdump_args = pgdump_args
+    end
+
+    def table_names
+      @table_hash.keys
+    end
+
+    def table(name)
+      @table_hash[name]
+    end
+
+    def tables
+      @table_hash.values
+    end
+
+    def update_with(other)
+      new_tables = @table_hash.map do |_, table|
+        if other_table = other.table(table.name)
+          table.update_with(other_table)
+        else
+          table
+        end
+      end
+      new_tables += (other.table_names - table_names).map { |table_name| other.table(table_name) }
+      Config.new(new_tables, @dump_path, @s3, @exclude_tables, @pgdump_args)
+    end
+
+    def unspecified_columns
+      @table_hash.map do |_, table|
+        columns = table.unspecifiled_columns
+        [table.name, columns] unless columns.empty?
+      end.compact.to_h
+    end
+
+    def write(io)
+      yml = {}
+      yml[KEY_DUMP_PATH] = @dump_path
+      yml[KEY_S3] = @s3 if @s3
+      yml[KEY_EXCLUDE_TABLES] = @exclude_tables if @exclude_tables.size > 0
+      yml[KEY_TABLES] = @table_hash.map do |_, table|
+        columns = table.columns
+        unless columns.empty?
+          [
+            table.name,
+            columns.map { |column| [column.name, column.scramble_method] }.to_h,
+          ]
+        end
+      end.compact.to_h
+      YAML.dump(yml, io)
+    end
+
+    def write_file(path)
+      File.open(path, 'w') do |io|
+        write(io)
+      end
+    end
+
+    def obfuscator_options
+      tables.map(&:options).reject(&:empty?).join(' ')
+    end
+
+    class << self
+      def read(io)
+        yml = YAML.safe_load(io, permitted_classes: [], permitted_symbols: [], aliases: true)
+        if yml[KEY_TABLES]
+          tables = yml[KEY_TABLES].map do |table_name, columns|
+            Table.new(
+              table_name,
+              columns.map { |name, scramble_method| Column.new(name, scramble_method) }
+            )
+          end
+        else
+          tables = []
+        end
+        Config.new(tables, yml[KEY_DUMP_PATH], yml[KEY_S3], yml[KEY_EXCLUDE_TABLES] || [], yml[KEY_PGDUMP_ARGS])
+      end
+
+      def read_file(path)
+        open(path, 'r') do |f|
+          read(f)
+        end
+      end
+
+      if defined?(Rails)
+        def from_db
+          if defined?(Zeitwerk) && Rails.autoloaders.zeitwerk_enabled?
+            Zeitwerk::Loader.eager_load_all
+          else
+            Rails.application.eager_load!
+          end
+          klasses_by_table = ActiveRecord::Base.descendants.map { |klass| [klass.table_name, klass] }.to_h
+          table_names = ActiveRecord::Base.connection.tables.sort - IGNORED_ACTIVE_RECORD_TABLES
+          tables = table_names.map do |table_name|
+            klass = klasses_by_table[table_name]
+            if klass
+              columns = klass.columns.map(&:name).reject do |name|
+                IGNORED_ACTIVE_RECORD_COLUMNS.member?(name)
+              end.map do |name|
+                Column.new(name)
+              end
+              Table.new(table_name, columns)
+            end
+          end.compact
+          Config.new(tables, 'scrambled.dump.gz', Config::DEFAULT_S3_PROPERTIES, [], nil)
+        end
+      end
+    end
+  end
+end

data/lib/pgdump_scrambler/dumper.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+require 'open3'
+module PgdumpScrambler
+  class Dumper
+    def initialize(config, db_config = {})
+      @db_config = db_config.empty? ? load_database_yml : config
+      @config = config
+      @output_path = config.dump_path
+    end
+
+    def run
+      puts "executing pg_dump..."
+      puts full_command
+      if system(full_command)
+        puts "done!"
+      else
+        raise "pg_dump failed!"
+      end
+    end
+
+    private
+
+    def full_command
+      [pgdump_command, obfuscator_command, 'gzip -c'].compact.join(' | ') + "> #{@output_path}"
+    end
+
+    def obfuscator_command
+      if options = @config.obfuscator_options
+        command = File.expand_path('../../../bin/pgdump-obfuscator', __FILE__)
+        "#{command} #{options}"
+      end
+    end
+
+    def pgdump_command
+      command = []
+      command << "PGPASSWORD=#{Shellwords.escape(@db_config['password'])}" if @db_config['password']
+      command << 'pg_dump'
+      command << @config.pgdump_args if @config.pgdump_args
+      command << "--username=#{Shellwords.escape(@db_config['username'])}" if @db_config['username']
+      command << "--host='#{@db_config['host']}'" if @db_config['host']
+      command << "--port='#{@db_config['port']}'" if @db_config['port']
+      command << @config.exclude_tables.map { |exclude_table| "--exclude-table-data=#{exclude_table}" }.join(' ') if @config.exclude_tables.present?
+      command << @db_config['database']
+      command.join(' ')
+    end
+
+    def load_database_yml
+      if defined?(Rails)
+        db_config = open(Rails.root.join('config', 'database.yml'), 'r') do |f|
+          YAML.safe_load(f, permitted_classes: [], permitted_symbols: [], aliases: true)
+        end
+        db_config[Rails.env]
+      end
+    end
+  end
+end

data/lib/pgdump_scrambler/railtie.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+module PgdumpScrambler
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load File.expand_path('../../tasks/pgdump_scrambler_tasks.rake', __FILE__)
+    end
+  end
+end

data/lib/pgdump_scrambler/s3_request.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+require 'uri'
+require 'digest'
+require 'openssl'
+
+module PgdumpScrambler
+  class S3Request
+    def initialize(s3_path:, verb:, region:, bucket:, access_key_id:, secret_key:, time: nil)
+      @s3_path = s3_path.start_with?('/') ? s3_path : "/#{s3_path}"
+      @verb = verb
+      @time = time || Time.now.utc
+      @region = region
+      @bucket = bucket
+      @access_key_id = access_key_id
+      @secret_key = secret_key
+    end
+
+    def canonical_request
+      [
+        @verb,
+        URI.encode(@s3_path),
+        canonical_query_string,
+        "host:#{@bucket}.s3.amazonaws.com\n", # canonical headers
+        'host', # signed headers
+        'UNSIGNED-PAYLOAD'
+      ].join("\n")
+    end
+
+    def signature_key
+      date_key = hmac_sha256("AWS4#{@secret_key}", iso_date)
+      date_region_key = hmac_sha256(date_key, @region)
+      date_region_service_key = hmac_sha256(date_region_key, 's3')
+      hmac_sha256(date_region_service_key, 'aws4_request')
+    end
+
+    def signature
+      hmac_sha256_hex(signature_key, string_to_sign)
+    end
+
+    def url
+      File.join("https://#{@bucket}.s3.amazonaws.com/", "#{@s3_path}?#{canonical_query_string}&X-Amz-Signature=#{signature}")
+    end
+
+    private
+
+    def iso_time
+      @time.strftime("%Y%m%dT%H%M%SZ")
+    end
+
+    def iso_date
+      @time.strftime('%Y%m%d')
+    end
+
+    def hmac_sha256(key, message)
+      OpenSSL::HMAC.digest(OpenSSL::Digest::SHA256.new, key, message)
+    end
+
+    def hmac_sha256_hex(key, message)
+      OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, key, message)
+    end
+
+    def canonical_query_string
+      "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=#{@access_key_id}%2F#{iso_date}%2F#{@region}%2Fs3%2Faws4_request&X-Amz-Date=#{iso_time}&X-Amz-Expires=86400&X-Amz-SignedHeaders=host"
+    end
+
+    def string_to_sign
+      [
+        'AWS4-HMAC-SHA256',
+        iso_time,
+        "#{iso_date}/#{@region}/s3/aws4_request",
+        Digest::SHA256.hexdigest(canonical_request),
+      ].join("\n")
+    end
+  end
+end
+
+if $0 == __FILE__
+  # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+  require "minitest/autorun"
+  class TestS3Request < Minitest::Test
+    def setup
+      @s3_request = PgdumpScrambler::S3Request.new(verb: 'GET', s3_path: '/test.txt', region: 'us-east-1', bucket: 'examplebucket', access_key_id: 'AKIAIOSFODNN7EXAMPLE', secret_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', time: Time.utc(2013, 5, 24, 0, 0, 0))
+    end
+
+    def test_canonical_request
+      assert_equal <<~EOS.chomp, @s3_request.canonical_request
+      GET
+      /test.txt
+      X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host
+      host:examplebucket.s3.amazonaws.com
+      
+      host
+      UNSIGNED-PAYLOAD
+      EOS
+    end
+
+    def test_signature
+      assert_equal 'aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404', @s3_request.signature
+    end
+
+    def test_url
+      exected_url = 'https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404'
+      assert_equal exected_url, @s3_request.url
+    end
+  end
+end

data/lib/pgdump_scrambler/s3_uploader.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+require 'net/http'
+require 'uri'
+require_relative './s3_request'
+
+module PgdumpScrambler
+  class S3UploadError < StandardError
+    attr_reader :response
+    def initialize(response)
+      @response = response
+      super "S3 upload failed: #{response.body}"
+    end
+  end
+
+  class S3Uploader
+    def initialize(s3_path:, local_path:, region:, bucket:, access_key_id:, secret_key:)
+      raise 'missing access_key_id' if access_key_id.nil? || access_key_id.empty?
+      raise 'missing secret_key' if secret_key.nil? || secret_key.empty?
+      @s3_request = S3Request.new(s3_path: s3_path, verb: 'PUT', region: region, bucket: bucket, access_key_id: access_key_id, secret_key: secret_key)
+      @local_path = local_path
+    end
+
+    def run
+      uri = URI.parse(@s3_request.url)
+      puts "upload #{@local_path} to #{uri.host}#{uri.path}"
+      open(@local_path, 'r') do |io|
+        uri_path = uri.path
+        uri_path += "?#{uri.query}" if uri.query
+        req = Net::HTTP::Put.new(uri_path)
+        req.body_stream = io
+        req.content_length = io.size
+        req.content_type = 'application/octet-stream'
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        res = http.request(req)
+        if res.code != '200'
+          raise S3UploadError.new(res)
+        end
+      end
+    end
+  end
+end

data/lib/pgdump_scrambler/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module PgdumpScrambler
+  VERSION = "0.4.0"
+end

data/lib/pgdump_scrambler.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+require "pgdump_scrambler/version"
+require "pgdump_scrambler/config"
+require "pgdump_scrambler/dumper"
+require "pgdump_scrambler/s3_uploader"
+if defined?(Rails)
+  require 'pgdump_scrambler/railtie'
+end
+
+module PgdumpScrambler
+end

data/lib/tasks/pgdump_scrambler_tasks.rake

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+namespace :pgdump_scrambler do
+  default_config_path = ENV['SCRAMBLER_CONFIG_PATH'] || 'config/pgdump_scrambler.yml'
+
+  desc 'create config from database'
+  task config_from_db: :environment do
+    config =
+      if File.exist?(default_config_path)
+        puts "#{default_config_path} found!\nmerge existing config with config from database"
+        PgdumpScrambler::Config
+          .read_file(default_config_path)
+          .update_with(PgdumpScrambler::Config.from_db)
+      else
+        puts "craete config from database"
+        PgdumpScrambler::Config.from_db
+      end
+    config.write_file(default_config_path)
+  end
+
+  desc 'check if new columns exist'
+  task check: :environment do
+    config = PgdumpScrambler::Config
+      .read_file(default_config_path)
+      .update_with(PgdumpScrambler::Config.from_db)
+    unspecified_columns = config.unspecified_columns
+    count = unspecified_columns.sum { |_, columns| columns.size }
+    if count > 0
+      unspecified_columns.each_key do |table_name|
+        puts "#{table_name}:"
+        unspecified_columns[table_name].each do |column_name|
+          puts "  #{column_name}"
+        end
+      end
+      puts "#{count} unspecified columns found!"
+      exit 1
+    else
+      puts "No unspecified columns found."
+    end
+  end
+
+  desc 'create scrambled dump'
+  task dump: :environment do
+    config = PgdumpScrambler::Config.read_file(default_config_path)
+    PgdumpScrambler::Dumper.new(config).run
+  end
+
+  desc 'create scrambled dump'
+  task clear_dump: :environment do
+    config = PgdumpScrambler::Config.read_file(default_config_path)
+    if File.exists? config.dump_path
+      File.delete(config.dump_path)
+      puts "Dump file #{config.dump_path} has been deleted."
+    end
+  end
+
+  desc 'upload to s3'
+  task s3_upload: :environment do
+    config = PgdumpScrambler::Config.read_file(default_config_path)
+    uploader = PgdumpScrambler::S3Uploader.new(
+      s3_path: File.join(config.resolved_s3['prefix'], File::basename(config.dump_path)),
+      local_path: config.dump_path,
+      region: config.resolved_s3['region'],
+      bucket: config.resolved_s3['bucket'],
+      access_key_id: config.resolved_s3['access_key_id'],
+      secret_key: config.resolved_s3['secret_key']
+    )
+    uploader.run
+  end
+end

data/pgdump_scrambler.gemspec

@@ -0,0 +1,37 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "pgdump_scrambler/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "pgdump_scrambler"
+  spec.version       = PgdumpScrambler::VERSION
+  spec.authors       = ["Shunichi Ikegami"]
+  spec.email         = ["sike.tm@gmail.com"]
+
+  spec.summary       = %q{scramble pg_dump columns}
+  spec.description   = %q{scramble pg_dump columns.}
+  spec.homepage      = 'https://github.com/shunichi/pgdump_scrambler'
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata["allowed_push_host"] = 'https://rubygems.org'
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.12"
+  spec.add_development_dependency "rails", "~> 7.0"
+  spec.add_development_dependency "rubocop"
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: pgdump_scrambler
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- Shunichi Ikegami
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-10-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: scramble pg_dump columns.
+email:
+- sike.tm@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- ".ruby-version"
+- ".travis.yml"
+- ".vscode/tasks.json"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/pgdump-obfuscator
+- bin/setup
+- lib/config/table.rb
+- lib/pgdump_scrambler.rb
+- lib/pgdump_scrambler/config.rb
+- lib/pgdump_scrambler/dumper.rb
+- lib/pgdump_scrambler/railtie.rb
+- lib/pgdump_scrambler/s3_request.rb
+- lib/pgdump_scrambler/s3_uploader.rb
+- lib/pgdump_scrambler/version.rb
+- lib/tasks/pgdump_scrambler_tasks.rake
+- libexec/pgdump-obfuscator-darwin-amd64
+- libexec/pgdump-obfuscator-darwin-arm64
+- libexec/pgdump-obfuscator-linux-amd64
+- libexec/pgdump-obfuscator-linux-arm64
+- pgdump_scrambler.gemspec
+homepage: https://github.com/shunichi/pgdump_scrambler
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: scramble pg_dump columns
+test_files: []