db_subsetter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8b69d7c3714c570b4a1f22b2ccff5e6e74c8b670
+  data.tar.gz: dafe34226a0da60c16d2823cc6d76e12b2f9837a
+SHA512:
+  metadata.gz: 6a7deca37ab7b6267f9b9f0fbabd7dd221278e93366623df347a46d24aa9a6980c71ec0f9f0c3a3d41b4aaf3b44032ee185092671f1c6517fd9c70e449265c25
+  data.tar.gz: d50010b3ee53ce2b2a13c95dcae3f00b08681a0f802251d85c6e6764375b4b499c9db817087a2b62c011bed578a161464b9c36abfeaf119ad772e7a2f12cda88

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.12.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in db_subsetter.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Joe Francis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,48 @@
+# db_subsetter
+
+Extract a subset of a relational database for use in development or testing.  Provides a simple API to filter rows and preserve referential integrity.  The extracted data is packed into a [SQLite](https://www.sqlite.org/) database to allow easy copying.
+
+Developing against a realistic dataset extracted from production provides a lot of advantages over starting with an empty database.  This tools was inspired by [rdbms-subsetter](https://github.com/18F/rdbms-subsetter) and [yaml_db](https://github.com/yamldb/yaml_db/) and combines some of the best attributes of both.
+
+When working against a legacy database, automatic relationship management does not always work out.  It can also be desirable to extract similar subsets of data over time to simplify testing.  We provide an API to allow you to quickly define how you want to filter each table for your subset.  We also provide tools to help calibrate your filters to extract a subset of a reasonable size.
+
+ActiveRecord is used for database access, however you *do not* need to have ActiveRecord models for all tables you wish to subset.  Any database supported by ActiveRecord should work.  In theory, you should be able to subset from database and import into another (i.e., MySQL -> Postgres), however in practice this may or may not work well depending on exactly what data types are used.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'db_subsetter'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install db_subsetter
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/lostapathy/db_subsetter.
+
+
+
+## License
+
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "db_subsetter"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/db_subsetter.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'db_subsetter/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "db_subsetter"
+  spec.version       = DbSubsetter::VERSION
+  spec.authors       = ["Joe Francis"]
+  spec.email         = ["joe@lostapathy.com"]
+
+  spec.summary       = %q{Extract a subset of a relational database for use in development or
+                          testing.  Provides a simple API to filter rows and preserve referential
+                          integrity.}
+  #spec.description   = %q{TODO: Write a longer description or delete this line.}
+  spec.homepage      = "https://github.com/lostapathy/db_subsetter"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+
+  spec.add_dependency "activerecord", "~> 4.2"
+  spec.add_dependency "sqlite3", "~> 1.3"
+end
+

data/lib/db_subsetter/exporter.rb

@@ -0,0 +1,117 @@
+require 'sqlite3'
+
+module DbSubsetter
+  class Exporter
+    attr_writer :filter
+
+    def all_tables
+      ActiveRecord::Base.connection.tables
+    end
+
+    def tables
+      filter.tables
+    end
+
+    def total_row_counts
+      tables.each.map do |table|
+        query = Arel::Table.new(table, ActiveRecord::Base).project("count(1) AS num_rows")
+        rows = ActiveRecord::Base.connection.select_one(query.to_sql)["num_rows"]
+        {table => rows}
+      end
+    end
+
+    def filtered_row_counts
+      tables.each.map do |table|
+        {table => filtered_row_count(table)}
+      end
+    end
+
+    def verify_exportability
+      tables.each do |table|
+        verify_table_exportability(table)
+      end
+    end
+
+    def export(filename)
+      verify_exportability
+
+      @output = SQLite3::Database.new(filename)
+      @output.execute("CREATE TABLE tables (name TEXT, columns TEXT)")
+      tables.each do |table|
+        export_table(table)
+      end
+    end
+
+
+
+    private
+    def max_rows
+      10000000
+    end
+
+    def insert_batch_size
+      250
+    end
+
+    def select_batch_size
+      insert_batch_size * 20
+    end
+
+    def filter
+      @filter ||= Filter.new
+      @filter.exporter = self
+      @filter
+    end
+
+    def filtered_row_count(table)
+      query = Arel::Table.new(table, ActiveRecord::Base).project( Arel.sql("count(1)") )
+      query = filter.filter(table, query)
+      ActiveRecord::Base.connection.select_one(query.to_sql).values.first
+    end
+
+    def pages(table)
+      ( filtered_row_count(table) / select_batch_size.to_f ).ceil
+    end
+
+    def order_by(table)
+      #TODO should probably allow the user to override this and manually set a sort order?
+      key = ActiveRecord::Base.connection.primary_key(table)
+      key || false
+    end
+
+    def verify_table_exportability(table)
+      raise "ERROR: Multiple pages but no primary key on: #{table}" if pages(table) > 1 && order_by(table).blank?
+      raise "ERROR: Too many rows in: #{table} (#{filtered_row_count(table)})" if( filtered_row_count(table) > max_rows )
+    end
+
+    def cleanup_types(row)
+      row.map do |field|
+        case field
+        when Date, Time then field.to_s(:db)
+        else
+          field
+        end
+      end
+    end
+
+    def export_table(table)
+      columns = ActiveRecord::Base.connection.columns(table).map{ |table| table.name }
+      @output.execute("INSERT INTO tables VALUES (?, ?)", [table, columns.to_json])
+      @output.execute("CREATE TABLE #{table.underscore} ( data TEXT )")
+      for i in 0..pages(table)
+        query = Arel::Table.new(table, ActiveRecord::Base)
+        # Need to extend this to take more than the first batch_size records
+        query = query.order(query[order_by(table)]) if order_by(table)
+
+        sql = query.skip(i * select_batch_size).take(select_batch_size).project( Arel.sql('*') ).to_sql
+
+        records = ActiveRecord::Base.connection.select_rows( sql )
+        records.each_slice(insert_batch_size) do |rows|
+          @output.execute("INSERT INTO #{table.underscore} (data) VALUES #{ Array.new(rows.size){"(?)"}.join(",")}", rows.map{|x| cleanup_types(x)}.map(&:to_json) )
+        end
+      end
+    end
+
+  end
+end
+

data/lib/db_subsetter/filter.rb

@@ -0,0 +1,24 @@
+module DbSubsetter
+  class Filter
+    attr_writer :exporter
+
+    def ignore_tables
+      []
+    end
+
+    def tables
+      @exporter.all_tables - ActiveRecord::SchemaDumper.ignore_tables - ignore_tables
+    end
+
+    def filter(table, query)
+      filter_method = "filter_#{table}"
+      if self.respond_to? filter_method
+        self.send(filter_method, query)
+      else
+        query
+      end
+    end
+
+  end
+end
+

data/lib/db_subsetter/importer.rb

@@ -0,0 +1,59 @@
+require 'sqlite3'
+
+module DbSubsetter
+  class Importer
+
+    def initialize(filename)
+      raise ArgumentError.new("invalid input file") unless File.exists?(filename)
+
+      @data = SQLite3::Database.new(filename)
+    end
+
+    def tables
+      all_tables = []
+      @data.execute("SELECT name FROM tables") do |row|
+        all_tables << row[0]
+      end
+      all_tables
+    end
+
+    def import
+     ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS=0;")
+     tables.each do |table|
+        import_table(table)
+      end
+     ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS=1;")
+    end
+
+    private
+    def import_table(table)
+      ActiveRecord::Base.connection.truncate(table)
+      @data.execute("SELECT data FROM #{table.underscore}") do |row|
+        insert_sql = "INSERT INTO #{quoted_table_name(table)} (#{quoted_column_names(table).join(",")}) VALUES (#{quoted_values(row).join(",")})"
+        ActiveRecord::Base.connection.execute(insert_sql)
+      end
+    end
+
+    def quoted_values(row)
+      out = JSON.parse(row[0])
+      out = out.map{|x| ActiveRecord::Base.connection.type_cast(x, nil) } #.first, x.last) }
+      out = out.map{|x| ActiveRecord::Base.connection.quote(x) }
+      out
+    end
+
+    def columns(table)
+      raw = @data.execute("SELECT columns FROM tables WHERE name = ?", [table]).first[0]
+      JSON.parse(raw)
+    end
+
+    def quoted_table_name(table)
+      ActiveRecord::Base.connection.quote_table_name(table)
+    end
+
+    def quoted_column_names(table)
+      columns(table).map{ |column| ActiveRecord::Base.connection.quote_column_name(column) }
+    end
+
+  end
+end
+

data/lib/db_subsetter/version.rb

@@ -0,0 +1,3 @@
+module DbSubsetter
+  VERSION = "0.1.0"
+end

data/lib/db_subsetter.rb

@@ -0,0 +1,8 @@
+require "db_subsetter/version"
+require "db_subsetter/filter"
+require "db_subsetter/exporter"
+require "db_subsetter/importer"
+
+module DbSubsetter
+  # Your code goes here...
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: db_subsetter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Joe Francis
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-07-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: 
+email:
+- joe@lostapathy.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- db_subsetter.gemspec
+- lib/db_subsetter.rb
+- lib/db_subsetter/exporter.rb
+- lib/db_subsetter/filter.rb
+- lib/db_subsetter/importer.rb
+- lib/db_subsetter/version.rb
+homepage: https://github.com/lostapathy/db_subsetter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Extract a subset of a relational database for use in development or testing.  Provides
+  a simple API to filter rows and preserve referential integrity.
+test_files: []
+has_rdoc: