database_sanitizer 0.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0ae899e9c69e083b9ad8adb9f427a384c898621d
+  data.tar.gz: 704c9bb2b7ee445b459078edf86fd6198023cd28
+SHA512:
+  metadata.gz: 0045968c62ea497b1ba352b60c5eb58c629ce2e1fcaf5480f73afabd68c8a8ba234b7a1e62a16e3563533a9e4d4e8f0fc54446af7efb09c6f0a57b6a62306d09
+  data.tar.gz: c1ebeac6a04345da7edbef0cffb4beee17276e47d6bdd8add408f1a6ec9a7e4c31beac21a197667c369eac31773857996758e3899a9a87f9891da0260c4f88be

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in database_exporter.gemspec
+gemspec
+
+group :development do
+  gem 'pry'
+  gem 'rspec'
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Marton Somogyi
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# DatabaseExporter
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'database_exporter'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install database_exporter
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/database_exporter/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/database_sanitizer

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+require 'yaml'
+require 'optparse'
+
+require 'database_sanitizer'
+
+ENV['SOURCE'] ||= 'postgres'
+ENV['TARGET'] ||= "#{ENV['SOURCE']}_sanitized"
+
+options = {dbconf: File.expand_path('config/database.yml', Dir.pwd)}
+OptionParser.new do |opts|
+  opts.on('-t', '--tables TABLES', Array, 'Restrict export scope to these tables') { |tables| options[:tables] = tables }
+  opts.on('-e', '--exclude EXCLUDE_TABLES', Array, 'Exclude these tables') { |tables| options[:exclude_tables] = tables }
+  opts.on('-r', '--transformers TRANSFORMERS', 'Ruby module containing custom transformer definitions') { |trans| options[:transformers] = trans }
+  opts.on('--conf DBCONF', 'Database config YAML') { |dbconf| options[:dbconf] = dbconf }
+  opts.on('--schema SCHEMA', 'Database schema from `rake db:schema:dump`') { |schema| options[:schema] = schema }
+end.parse!
+
+if options[:transformers]
+  begin require File.expand_path(options[:transformers], Dir.pwd)
+  rescue LoadError; puts $!
+  end
+end
+
+DBCONF = YAML::load IO.read options[:dbconf] rescue abort "Error reading #{options[:dbconf]}: #{$!}"
+
+begin
+  db = ENV['SOURCE']
+  DatabaseSanitizer::Source.establish_connection(DBCONF[db]).connection
+  db = ENV['TARGET']
+  ActiveRecord::Base.establish_connection(DBCONF[db]).connection
+rescue StandardError
+  abort "Couldn't connect to #{options[:dbconf]}[#{db}]: #{$!}"
+end
+
+DatabaseSanitizer.export DatabaseSanitizer::Source.connection, ActiveRecord::Base.connection, tables: options[:tables], exclude: options[:exclude_tables], schema: options[:schema]
+

data/database_sanitizer.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+require File.expand_path('../lib/database_sanitizer/version', __FILE__)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'database_sanitizer'
+  spec.version       = DatabaseSanitizer::VERSION
+  spec.authors       = ['Marton Somogyi']
+  spec.email         = ['msomogyi@whitepages.com']
+  spec.summary       = %q{Export your SQL database}
+  spec.description   = %q{Duplicate a databse with sanitization options using SQL comments}
+  spec.homepage      = 'https://github.com/mcbuddha/database_sanitizer'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake', '~> 10'
+  spec.add_development_dependency 'pg', '~> 0.17'
+
+  spec.add_runtime_dependency 'activerecord', '~> 3'
+  spec.add_runtime_dependency 'activerecord-comments', '~> 0'
+  spec.add_runtime_dependency 'progress', '~> 3'
+  spec.add_runtime_dependency 'iconv', '~> 1'
+end

data/lib/database_sanitizer/transformers.rb

@@ -0,0 +1,10 @@
+module DatabaseSanitizer
+  Transformers = {
+    'email' => ->(i, rec) { "email#{i.to_s.rjust(5, ?0)}@#{rec.split(?@)[1]}"},
+    'wipe' => proc { nil },
+    'zero' => proc { 0 },
+    'empty_string' => proc { '' },
+    'name' => proc { 'John Doe' },
+    'phone_number' => ->(i, rec) { rec.nil? ? rec : "#{rec[0,3]}#{i.to_s.rjust rec.length-3, ?0}" }
+  }
+end

data/lib/database_sanitizer/version.rb

@@ -0,0 +1,3 @@
+module DatabaseSanitizer
+  VERSION = '0.0.6'
+end

data/lib/database_sanitizer.rb

@@ -0,0 +1,66 @@
+require 'database_sanitizer/version'
+require 'active_record/comments'
+require 'progress'
+
+module DatabaseSanitizer
+  class Source < ActiveRecord::Base
+  end
+end
+
+require 'database_sanitizer/transformers'
+
+module DatabaseSanitizer
+  class << self
+    def extract_transformer comment; comment ? comment[/sanitize: ?(\w+)/,1] : nil; end
+
+    def read_comments conn, tables
+      tables.inject({}) do |transformers, table|
+        transformers[table.to_sym] = conn.retrieve_column_comments(table.to_sym).inject({}) do |table_transformers, column|
+          transformer_key = extract_transformer column[1]
+          unless transformer_key.nil? || Transformers.include?(transformer_key)
+            abort "Transformer '#{transformer_key}' not found (#{table}.#{column[0]})"
+          end
+          table_transformers[column[0]] = transformer_key && Transformers[transformer_key]
+          table_transformers
+        end
+        transformers
+      end
+    end
+
+    def duplicate_schema schema=nil
+      schema_src = nil
+      if schema.nil?
+        schema_sio = StringIO.new
+        puts 'Dumping schema.rb...'
+        ActiveRecord::SchemaDumper.dump(Source.connection, schema_sio)
+        puts 'Loading schema.rb...'
+        ActiveRecord::Migration.suppress_messages { eval schema_sio.string }
+      else
+        puts 'Reading schema SQL...'
+        schema_src = IO.read File.expand_path(schema, Dir.pwd)
+        ActiveRecord::Migration.suppress_messages { ActiveRecord::Base.connection.exec_query schema_src }
+      end
+    end
+
+    def export src, dest, opts={}
+      duplicate_schema opts[:schema]
+      tables = (opts[:tables] || src.tables.collect(&:to_s)) - (opts[:exclude] || [])
+      transformers = read_comments dest, tables
+
+      max_tbl_name_len = transformers.keys.map(&:length).sort.last || 0
+      tables.with_progress('Exporting').each do |table|
+        result = src.exec_query "SELECT * FROM #{table}"
+        cols = result.columns.join ','
+        dest.transaction do
+          result.rows.with_progress(table.rjust max_tbl_name_len).each_with_index do |src_row, row_i|
+            values = result.columns.each_with_index.map do |col, col_i|
+              transformer = transformers[table.to_sym][col.to_sym]
+              dest.quote transformer ? transformer.(row_i, src_row[col_i]) : src_row[col_i]
+            end
+            dest.insert_sql "INSERT INTO #{dest.quote_table_name table} (#{cols}) VALUES (#{values.join ','})"
+          end
+        end
+      end
+    end
+  end
+end

data/spec/config/database.yml

@@ -0,0 +1,18 @@
+postgres:
+  adapter: postgresql
+  database: database_exporter_test
+  host: 127.0.0.1
+  username: postgres
+  password: postgres
+
+mysql:
+  adapter: mysql2
+  database: database_exporter_test
+  user: root
+  password: password
+
+sqlite:
+  adapter: sqlite3
+  database: test/db/database_exporter_test
+  pool: 5
+  timeout: 5000

data/spec/database_exporter_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+describe DatabaseExporter do
+  describe '#extract_transformer', nodb: true do
+    context 'should return nil for no transformer' do
+      ['no tag comment', nil, '', 'sanitize no tag'].each do |comment|
+        it { expect(described_class.extract_transformer comment).to be_nil }
+      end
+    end
+
+    context 'should return transformer' do
+      [
+       'sanitize: test_tr',
+       'random sanitize: test_tr comment',
+       'some sanitize: test_tr, sanitize: other',
+       'without sanitize:test_tr space',
+       'trailing sanitize: test_tr'
+      ].each do |comment|
+        it { expect(described_class.extract_transformer comment).to eq('test_tr') }
+      end
+    end
+  end
+
+  describe '#read_comments' do
+    before do
+      ActiveRecord::Migration.suppress_messages do
+        ActiveRecord::Schema.define do
+          change_table :test do |t|
+            t.string :field3
+            t.string :field4
+          end
+        end
+      end
+      comments = {
+        field1: 'comment no tag',
+        field2: nil,
+        field3: 'comment sanitize: name',
+        field4: 'sanitize:email'
+      }.each { |col, com| ActiveRecord::Base.connection.set_column_comment :test, col, com }
+    end
+
+    context 'some defined' do
+      it 'should get transformers' do
+        transformers = described_class.read_comments(ActiveRecord::Base.connection, [:test])[:test]
+        expect(transformers[:field1]).to be_nil
+        expect(transformers[:field2]).to be_nil
+        expect(transformers[:field3]).to be_kind_of(Proc)
+        expect(transformers[:field4]).to be_kind_of(Proc)
+      end
+    end
+
+    context 'some undefined' do
+      before { ActiveRecord::Base.connection.set_column_comment :test, :field2, 'sanitize:undef' }
+      it 'should abort' do
+        expect(lambda {described_class.read_comments ActiveRecord::Base.connection, [:test]}).to raise_error(SystemExit)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,28 @@
+$: << File.expand_path('../lib', __FILE__)
+
+require 'yaml'
+require 'pry'
+
+require 'database_exporter'
+
+DBCONF = YAML::load(IO.read(File.expand_path('../config/database.yml', __FILE__)))
+ENV['DB'] ||= 'postgres'
+ActiveRecord::Base.establish_connection DBCONF[ENV['DB']]
+
+RSpec.configure do |c|
+  c.around(:each) do |ex|
+    unless ex.metadata[:nodb]
+      ActiveRecord::Migration.suppress_messages do
+        ActiveRecord::Schema.define do
+          create_table :test do |t|
+            t.string :field1
+            t.integer :field2
+          end
+        end
+      end
+    end
+    ex.run
+    ActiveRecord::Migration.suppress_messages { ActiveRecord::Schema.define { drop_table :test } } unless ex.metadata[:nodb]
+  end
+end
+

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: database_sanitizer
+version: !ruby/object:Gem::Version
+  version: 0.0.6
+platform: ruby
+authors:
+- Marton Somogyi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: activerecord-comments
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: progress
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: iconv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+description: Duplicate a databse with sanitization options using SQL comments
+email:
+- msomogyi@whitepages.com
+executables:
+- database_sanitizer
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/database_sanitizer
+- database_sanitizer.gemspec
+- lib/database_sanitizer.rb
+- lib/database_sanitizer/transformers.rb
+- lib/database_sanitizer/version.rb
+- spec/config/database.yml
+- spec/database_exporter_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/mcbuddha/database_sanitizer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Export your SQL database
+test_files:
+- spec/config/database.yml
+- spec/database_exporter_spec.rb
+- spec/spec_helper.rb