rdbms_sampler 1.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d9834c6b4cba2c6c8271f9e8f744e43d1c8350b3465db252fb85b059ea1c93cd
+  data.tar.gz: feee1ba4541dc3e05eb3697272f0bdb049c5bdfbfa17344ad5b9d58675df5672
+SHA512:
+  metadata.gz: 3646c28b02866c51364d89ca2e97b847fe6d1a72bc5904e5085b5eae7267effac3ac185dca8ccda389cdc645359e106324a5a68e5bf3691ac56c9138e2e0116b
+  data.tar.gz: fa0096829e7bae368a20b934284400aa39fc7c87e1840a39a381d7e970e6032a01422a721ba2d9d0b3cc8b17d130ae57c514191dae71a7ab443167f8ff71073e

data/.gitignore

@@ -0,0 +1,8 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.idea
+*.sql
+og
+*~

data/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in rdbms_sampler.gemspec
+gemspec
+
+

data/README.md

@@ -0,0 +1,89 @@
+RDBMS Sampler
+=============
+
+Command line utility for extracting a sample (subset of all records) from a relational 
+database system (such as MySQL) while *maintaining the referential integrity* of the sample. 
+
+Description
+-----------
+
+Need e.g. 1000 rows from each of your production tables, but feel the pain of making 
+sure to include dependent rows, their dependents and so on, ad infinitum?
+
+Look no further. This tiny utility will take care that referential dependencies are
+fulfilled by recursively expanding the row sample with unfilled dependencies until
+the sample is referentially consistent.
+
+Installation
+------------
+
+    Install with `gem install rdbms_sampler`.
+    
+    Alternatively, clone the repository and install dependencies with `bundle install`. 
+    Then execute with `bundle exec rdbms_sampler ...`.
+
+Commands
+--------
+
+    help        Display global or [command] help documentation.
+    sample      Extract a sample from the given connection
+
+Options
+-------
+
+    --adapter NAME
+        ActiveRecord adapter to use
+
+    --databases NAMES
+        Comma-separated list of databases to sample
+
+    --username USER
+        Username for connection
+
+    --password PASSWORD
+        Password for connection
+
+    --encoding ENCODING
+        Encoding for connection
+
+    --host HOST
+        Host name or IP for connection
+
+    --socket PATH
+        Socket for connection
+
+    --rows NUM
+        Number of rows to sample per table
+
+    --log PATH
+        Log queries to PATH
+
+Global Options
+--------------
+
+    -h, --help
+        Display help documentation
+
+    -v, --version
+        Display version information
+
+    -t, --trace
+        Display backtrace when an error occurs
+
+Usage
+-----
+
+    rdbms_sampler --databases DB1,DB2 --username USER --password PASS --rows 100 > sample.sql
+
+
+
+CAVEATS
+-------
+
+Only single-column foreign keys are currently handled. 
+
+Additionally, due to a bug in the current implementation, if a referenced column 
+is named anything but `id`, referenced rows might get included multiple times.
+
+You will probably need to disable foreign key check *during import*, since inserts in 
+the output are not ordered with respect to referential integrity.

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler/gem_tasks'
+

data/bin/rdbms_sampler

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+require "rdbms_sampler"
+require "commander/import"
+require "logger"
+
+program :version, RdbmsSampler::VERSION
+program :description, 'Extract a sample of desired size from a database while ensuring referential integrity.'
+default_command :sample
+
+command :sample do |c|
+  c.description = 'Extract a sample from the given connection'
+  c.option '--adapter NAME', String, 'ActiveRecord adapter to use'
+  c.option '--databases NAMES', String, 'Comma-separated names of databases to sample'
+  c.option '--username USER', String, 'Username for connection'
+  c.option '--password PASSWORD', String, 'Password for connection'
+  c.option '--encoding ENCODING', String, 'Encoding for connection'
+  c.option '--socket PATH', String, 'Socket for connection'
+  c.option '--host HOST_NAME', String, 'Host name'
+  c.option '--rows NUM', Integer, 'Number of rows to sample per table'
+  c.option '--log PATH', String, 'Log queries to PATH'
+  c.when_called do |args, options|
+    options.default \
+      :adapter => 'mysql2',
+      :username => 'root',
+      :encoding => 'utf8',
+      :cast => false,
+      :rows => 1000
+    ActiveRecord::Base.logger = Logger.new(options.log) if options.log
+    warn 'Connecting...'
+    ActiveRecord::Base.establish_connection(options.__hash__).with_connection do |conn|
+      schemas = options.databases.split(/,/)
+      puts RdbmsSampler::Sample.new(conn: conn, rows_per_table: options.rows, schemas: schemas).to_sql
+    end
+  end
+end
+

data/lib/rdbms_sampler.rb

@@ -0,0 +1,5 @@
+require "rdbms_sampler/version"
+require "rdbms_sampler/sample"
+require "active_record"
+
+

data/lib/rdbms_sampler/dependency.rb

@@ -0,0 +1,35 @@
+module RdbmsSampler
+  class Dependency
+
+    attr_reader :parent_schema
+    attr_reader :parent_table
+    attr_reader :parent_key
+    attr_reader :child_schema_name
+    attr_reader :child_table_name
+    attr_reader :child_key
+    attr_reader :value
+
+    def initialize(parent_schema, parent_table, parent_key, child_schema, child_table, child_key, value)
+      @parent_schema = parent_schema
+      @parent_table = parent_table
+      @parent_key = parent_key
+      @child_schema_name = child_schema
+      @child_table_name = child_table
+      @child_key = child_key
+      @value = value
+    end
+
+    def identifier
+      "#{child_schema_name}.#{child_table_name}"
+    end
+
+    def eql? other
+      identifier == other.identifier and child_key == other.child_key and value == other.value
+    end
+
+    def to_s
+      "reference from #{parent_schema}.#{parent_table}.#{parent_key} to #{identifier}[#{child_key}=#{value}]"
+    end
+
+  end
+end

data/lib/rdbms_sampler/foreign_key.rb

@@ -0,0 +1,26 @@
+module RdbmsSampler
+
+  class ForeignKey
+    attr_reader :constraint_name
+    attr_reader :schema
+    attr_reader :table
+    attr_reader :key
+    attr_reader :referenced_schema
+    attr_reader :referenced_table
+    attr_reader :referenced_key
+
+    def initialize(constraint_name, schema, table, key,
+                   referenced_schema, referenced_table, referenced_key)
+
+      @constraint_name = constraint_name
+      @schema = schema
+      @table = table
+      @key = key
+      @referenced_schema = referenced_schema
+      @referenced_table = referenced_table
+      @referenced_key = referenced_key
+    end
+
+  end
+
+end

data/lib/rdbms_sampler/sample.rb

@@ -0,0 +1,76 @@
+require 'rdbms_sampler/table_sample'
+require 'active_support/core_ext/array'
+
+module RdbmsSampler
+
+  class Sample
+
+    def initialize(options ={})
+      @connection = options[:conn]
+      @rows_per_table = options[:rows_per_table] || 1000
+      @table_samples = {}
+      @schemas = options[:schemas]
+      @computed = false
+    end
+
+    def compute!
+      quoted_schema_names = @schemas.collect do |name|
+        @connection.quote_table_name(name)
+      end
+      warn "Discovering tables in databases: #{quoted_schema_names.to_sentence}..."
+      tables_without_views.each do |schema_name, table_name|
+        table_sample = TableSample.new(@connection, schema_name, table_name, @rows_per_table)
+        @table_samples[table_sample.identifier] = table_sample
+      end
+      return warn 'No tables found!' unless @table_samples.count > 0
+      warn "Sampling #{@table_samples.count} tables..."
+      @table_samples.values.map &:sample!
+      warn 'Ensuring referential integrity...'
+      begin
+        new_dependencies = 0
+        @table_samples.values.each do |table_sample|
+          newly_added = table_sample.ensure_referential_integrity(self)
+          if newly_added > 0
+            new_dependencies += newly_added
+            warn "  Expanded sample with #{newly_added} new rows referenced from table #{table_sample.quoted_name}"
+          end
+        end
+        warn " Discovered #{new_dependencies} new dependencies" if new_dependencies > 0
+      end while new_dependencies > 0
+      warn 'Referential integrity obtained'
+
+      warn 'Final sample contains:'
+      @table_samples.values.each do |table_sample|
+        warn "  #{table_sample.size} row(s) from `#{table_sample.identifier}`"
+      end
+      @computed = true
+    end
+
+    # @param [Dependency]
+    # @return [TableSample]
+    def table_sample_for_dependency(dependency)
+      raise "Table sample for [#{dependency.identifier}] not found" unless @table_samples.include? dependency.identifier
+      @table_samples[dependency.identifier]
+    end
+
+    def to_sql
+      compute! unless @computed
+      @table_samples.values.collect(&:to_sql) * "\n"
+    end
+
+    private
+
+    def tables_without_views
+      quoted_schema_names = @schemas.collect { |name|
+        @connection.quote(name)
+      }.join(', ')
+      @connection.execute <<SQL
+        SELECT TABLE_SCHEMA, TABLE_NAME
+        FROM INFORMATION_SCHEMA.TABLES
+        WHERE TABLE_TYPE = 'BASE TABLE'
+          AND TABLE_SCHEMA IN (#{quoted_schema_names})
+SQL
+    end
+  end
+
+end

data/lib/rdbms_sampler/table_sample.rb

@@ -0,0 +1,169 @@
+require 'pry'
+require 'rdbms_sampler/dependency'
+require 'rdbms_sampler/foreign_key'
+
+module RdbmsSampler
+
+  class TableSample
+
+    attr_reader :pending_dependencies
+
+    def initialize(connection, schema_name, table_name, size = 1000)
+      @schema = schema_name
+      @table = table_name
+      @connection = connection
+      @size = size
+      @pending_dependencies = Set.new
+      @sample = Set.new
+      @sampled = false
+      @sampled_ids = Set.new
+    end
+
+    def sample!
+      fetch(@size) unless @sampled
+      @sample
+    end
+
+    def size
+      @sampled ? @sample.size : @size
+    end
+
+    def identifier
+      "#{@schema}.#{@table}"
+    end
+
+    # Add the given dependency to the sample
+    # @param [Dependency] dependency
+    def fulfil(dependency)
+      return 0 if fulfilled?(dependency)
+      quoted_column = @connection.quote_column_name dependency.child_key
+      quoted_value = @connection.quote dependency.value
+      sql = "SELECT * FROM #{quoted_name} WHERE #{quoted_column} = #{quoted_value}"
+      row = @connection.select_one(sql)
+      raise "Could not fulfil #{dependency} using query [#{sql}]" if row.nil?
+      add row
+    end
+
+    # @param [Dependency] dependency
+    def fulfilled?(dependency)
+      # FIXME: Only handles `id` column
+      return false if dependency.child_key != 'id'
+
+      @sampled_ids.include?(dependency.value)
+    end
+
+    # Add a row to the table sample.
+    # Returns number of new dependencies introduced.
+    def add(row)
+      return 0 unless @sample.add? row
+      @sampled_ids.add row['id'] if row['id']
+      dependencies_for(row).collect { |dep|
+        1 if @pending_dependencies.add?(dep)
+      }.compact.sum
+    end
+
+    # @param [Sample] sample
+    def ensure_referential_integrity(sample)
+      dependencies_in_progress = @pending_dependencies
+      @pending_dependencies = Set.new
+      dependencies_in_progress.map { |dependency|
+        dependency_sample = sample.table_sample_for_dependency(dependency)
+        dependency_sample.fulfil(dependency)
+      }.compact.sum
+    end
+
+    def to_sql
+      ret = "\n-- Sample from #{quoted_name} (#{@sample.count} rows)\n"
+      unless @sample.empty?
+        quoted_cols = @sample.first.keys.collect { |col| @connection.quote_column_name col }
+        # INSERT in batches to reduce the likelihood of hitting `max_allowed_packet`
+        @sample.each_slice(250) do |rows|
+          values = rows.collect { |row|
+            row.values.map { |val|
+              @connection.quote(val)
+            } * ','
+          } * "),\n  ("
+          ret << "INSERT INTO #{quoted_name} \n  (#{quoted_cols * ','}) \nVALUES \n  (#{values});\n"
+        end
+      end
+      ret
+    end
+
+    def quoted_name
+      @connection.quote_table_name(@schema)+'.'+@connection.quote_table_name(@table)
+    end
+
+    protected
+
+    def fetch(count = 1000)
+      sql = "SELECT * FROM #{quoted_name}"
+      unless (pks = self.primary_keys).count == 0
+        order_by = @connection.quote_column_name(pks.first)
+        sql += " ORDER BY #{order_by} DESC"
+      end
+      sql += " LIMIT #{count}"
+      warn "  Sampling #{count} rows from #{quoted_name}..."
+      @connection.select_all(sql).each { |row| add(row) }
+      @sampled = true
+    end
+
+    # @param [ForeignKey] fk
+    # @param [Array] row
+    def dependency_for(fk, row)
+      unless (value = row[fk.key]).nil?
+        Dependency.new(fk.schema, fk.table, fk.key, fk.referenced_schema, fk.referenced_table, fk.referenced_key, value)
+      end
+    end
+
+    # @param [Array] row
+    def dependencies_for(row)
+      foreign_keys.collect { |fk| dependency_for(fk, row) }.compact
+    end
+
+    def foreign_keys
+      @fks ||= discover_foreign_keys
+    end
+
+    def discover_foreign_keys
+      quoted_schema = @connection.quote @schema
+      quoted_table = @connection.quote @table
+
+      sql = <<SQL
+      SELECT
+        fk.constraint_name,
+        fk.table_schema,
+        fk.table_name,
+        fk.column_name,
+        fk.referenced_table_schema,
+        fk.referenced_table_name,
+        fk.referenced_column_name
+      FROM information_schema.key_column_usage fk
+      WHERE fk.referenced_column_name IS NOT NULL
+      AND fk.table_schema = #{quoted_schema}
+      AND fk.table_name = #{quoted_table}
+SQL
+
+      @connection.execute(sql).map do |row|
+        ForeignKey.new(*row)
+      end
+    end
+
+    def primary_keys
+      quoted_schema = @connection.quote @schema
+      quoted_table = @connection.quote @table
+
+      sql = <<SQL
+      SELECT column_name
+      FROM information_schema.key_column_usage
+      WHERE constraint_name = 'PRIMARY'
+      AND table_schema = #{quoted_schema}
+      AND table_name = #{quoted_table}
+SQL
+
+      @connection.execute(sql).map do |row|
+        row.first
+      end
+    end
+
+  end
+end

data/lib/rdbms_sampler/version.rb

@@ -0,0 +1,3 @@
+module RdbmsSampler
+  VERSION = "1.1.1"
+end

data/rdbms_sampler.gemspec

@@ -0,0 +1,29 @@
+$:.push File.expand_path("../lib", __FILE__)
+require "rdbms_sampler/version"
+
+Gem::Specification.new do |s|
+  s.name        = "rdbms_sampler"
+  s.version     = RdbmsSampler::VERSION
+  s.licenses    = ['MIT']
+  s.authors     = ["Christian Rishoj"]
+  s.email       = ["christian@rishoj.net"]
+  s.homepage    = "https://github.com/crishoj/rdbms_sampler"
+  s.summary     = %q{Extract a sample of records from a database while maintaining referential integrity.}
+  s.description = %q{Ever found yourself wanting a modest amount of fresh rows from a production database for development purposes, but
+put back by the need to maintain referential integrity in the extracted data sample? This data sampler utility will
+take care that referential dependencies are fulfilled by recursively fetching any rows referred to by the sample.}
+
+  s.rubyforge_project = "rdbms_sampler"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "schema_plus_foreign_keys", '~> 0.1', '>= 0.1.7'
+  s.add_dependency "activerecord", '~> 5.0'
+  s.add_dependency "commander", '~> 4.4'
+  s.add_dependency "mysql2", '~> 0.4', '>= 0.4.4'
+
+  s.add_development_dependency "pry", '~> 0.9'
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: rdbms_sampler
+version: !ruby/object:Gem::Version
+  version: 1.1.1
+platform: ruby
+authors:
+- Christian Rishoj
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-02-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: schema_plus_foreign_keys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.7
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: commander
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.4'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: |-
+  Ever found yourself wanting a modest amount of fresh rows from a production database for development purposes, but
+  put back by the need to maintain referential integrity in the extracted data sample? This data sampler utility will
+  take care that referential dependencies are fulfilled by recursively fetching any rows referred to by the sample.
+email:
+- christian@rishoj.net
+executables:
+- rdbms_sampler
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/rdbms_sampler
+- lib/rdbms_sampler.rb
+- lib/rdbms_sampler/dependency.rb
+- lib/rdbms_sampler/foreign_key.rb
+- lib/rdbms_sampler/sample.rb
+- lib/rdbms_sampler/table_sample.rb
+- lib/rdbms_sampler/version.rb
+- rdbms_sampler.gemspec
+homepage: https://github.com/crishoj/rdbms_sampler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: rdbms_sampler
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Extract a sample of records from a database while maintaining referential
+  integrity.
+test_files: []