data_sampler 0.0.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.idea

data/Gemfile

@@ -0,0 +1,9 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in data_sampler.gemspec
+gemspec
+
+# These are here to coerce RubyMine into recognising these dependencies
+gem "activerecord"
+gem "schema_plus"
+

data/README

@@ -0,0 +1,47 @@
+
+Ever found yourself wanting a modest amount of fresh rows from a production database for development purposes, but
+put back by the need to maintain referential integrity in the extracted data sample? This data sampler utility will
+take care that referential dependencies are fulfilled by recursively fetching any rows referred to by the sample.
+
+  COMMANDS:
+
+    help                 Display global or [command] help documentation.
+    sample               Extract a sample from the given connection
+
+  OPTIONS:
+
+    --adapter NAME
+        ActiveRecord adapter to use
+
+    --database NAME
+        Name of database to sample
+
+    --username USER
+        Username for connection
+
+    --password PASSWORD
+        Password for connection
+
+    --encoding ENCODING
+        Encoding for connection
+
+    --socket PATH
+        Socket for connection
+
+    --rows NUM
+        Number of rows to sample per table
+
+    --log PATH
+        Log queries to PATH
+
+  GLOBAL OPTIONS:
+
+    -h, --help
+        Display help documentation
+
+    -v, --version
+        Display version information
+
+    -t, --trace
+        Display backtrace when an error occurs
+

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler/gem_tasks'
+

data/bin/data_sampler

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+require "data_sampler"
+require "commander/import"
+require "logger"
+
+program :version, DataSampler::VERSION
+program :description, 'Extract a sample of desired size from a database while ensuring referential integrity.'
+default_command :sample
+
+command :sample do |c|
+  c.description = 'Extract a sample from the given connection'
+  c.option '--adapter NAME', String, 'ActiveRecord adapter to use'
+  c.option '--database NAME', String, 'Name of database to sample'
+  c.option '--username USER', String, 'Username for connection'
+  c.option '--password PASSWORD', String, 'Password for connection'
+  c.option '--encoding ENCODING', String, 'Encoding for connection'
+  c.option '--socket PATH', String, 'Socket for connection'
+  c.option '--rows NUM', Integer, 'Number of rows to sample per table'
+  c.option '--log PATH', String, 'Log queries to PATH'
+  c.when_called do |args, options|
+    options.default \
+      :adapter  => 'mysql',
+      :database => 'test',
+      :username => 'root',
+      :encoding => 'utf8',
+      :socket   => '/opt/local/var/run/mysql5/mysqld.sock',
+      :rows     => 1000
+    ActiveRecord::Base.logger = Logger.new(options.log) if options.log
+    ActiveRecord::Base.establish_connection(options.__hash__).with_connection do |conn|
+      puts DataSampler::Sample.new(conn, options.rows).to_sql
+    end
+  end
+end
+

data/data_sampler.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "data_sampler/version"
+
+Gem::Specification.new do |s|
+  s.name        = "data_sampler"
+  s.version     = DataSampler::VERSION
+  s.authors     = ["Christian Rishoj"]
+  s.email       = ["christian@rishoj.net"]
+  s.homepage    = "https://github.com/crishoj/data_sampler"
+  s.summary     = %q{Extract a sample of records from a database while maintaining referential integrity.}
+  s.description = %q{Ever found yourself wanting a modest amount of fresh rows from a production database for development purposes, but
+put back by the need to maintain referential integrity in the extracted data sample? This data sampler utility will
+take care that referential dependencies are fulfilled by recursively fetching any rows referred to by the sample.}
+
+  s.rubyforge_project = "data_sampler"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "schema_plus"
+  s.add_dependency "activerecord"
+end

data/lib/data_sampler/dependency.rb

@@ -0,0 +1,21 @@
+module DataSampler
+  class Dependency
+
+    attr_reader :table_name
+    attr_reader :keys
+
+    def initialize(table_name, keys)
+      @table_name = table_name
+      @keys = keys
+    end
+
+    def eql? other
+      table_name == other.table_name and keys == other.keys
+    end
+
+    def to_s
+      "#{keys} in table #{table_name}"
+    end
+
+  end
+end

data/lib/data_sampler/sample.rb

@@ -0,0 +1,41 @@
+require "data_sampler/table_sample"
+
+module DataSampler
+
+  class Sample
+
+    def initialize(connection, rows_per_table = 1000)
+      @connection = connection
+      @rows_per_table = rows_per_table
+      @table_samples = {}
+      @computed = false
+    end
+
+    def compute!
+      @connection.tables.each do |table_name|
+        # Workaround for inconsistent casing in table definitions (http://bugs.mysql.com/bug.php?id=60773)
+        table_name.downcase!
+        @table_samples[table_name] = TableSample.new(@connection, table_name, @rows_per_table)
+      end
+      warn "Sampling #{@table_samples.count} tables..."
+      @table_samples.values.map &:sample!
+      warn "Ensuring referential integrity..."
+      begin
+        new_dependencies = 0
+        @table_samples.values.each do |table_sample|
+          new_dependencies += 1 if table_sample.ensure_referential_integrity(@table_samples)
+        end
+        warn " - discovered #{new_dependencies} new dependencies" if new_dependencies > 0
+      end while new_dependencies > 0
+      warn " - referential integrity obtained"
+      @computed = true
+    end
+
+    def to_sql
+      compute! unless @computed
+      @table_samples.values.collect(&:to_sql) * "\n"
+    end
+
+  end
+
+end

data/lib/data_sampler/table_sample.rb

@@ -0,0 +1,117 @@
+require "data_sampler/dependency"
+
+module DataSampler
+
+  class TableSample
+
+    attr_reader :table_name
+    attr_reader :pending_dependencies
+
+    def initialize(connection, table_name, size = 1000)
+      @table_name = table_name
+      @connection = connection
+      @size = size
+      @pending_dependencies = Set.new
+      @sample = Set.new
+      @sampled = false
+      @sampled_ids = Set.new
+    end
+
+    def sample!
+      fetch_sample(@size) unless @sampled
+      @sample
+    end
+
+    def fulfil(dependency)
+      return if fulfilled?(dependency)
+      where = dependency.keys.collect { |col, val| "#{@connection.quote_column_name col} = #{@connection.quote val}" } * ' AND '
+      sql = "SELECT * FROM #{@connection.quote_table_name @table_name} WHERE " + where
+      add @connection.select_one(sql)
+    end
+
+    def fulfilled?(dependency)
+      # FIXME: Only checks id column
+      if dependency.keys.values.size == 1
+        dependency.keys.each_pair do |key, val|
+          if key == 'id'
+            return true if @sampled_ids.include?(val)
+          end
+        end
+      end
+      false
+    end
+
+    def add(row)
+      return false unless @sample.add? row
+      @sampled_ids.add row['id'] if row['id']
+      any_new = false
+      dependencies_for(row).each do |dep|
+        any_new = true if @pending_dependencies.add?(dep)
+      end
+      any_new
+    rescue ActiveRecord::StatementInvalid => e
+      # Don't choke on unknown table engines, such as Sphinx
+    end
+
+    def ensure_referential_integrity(table_samples)
+      any_new = false
+      deps_in_progress = @pending_dependencies
+      @pending_dependencies = Set.new
+      deps_in_progress.each do |dependency|
+        any_new = true if table_samples[dependency.table_name].fulfil(dependency)
+      end
+      any_new
+    end
+
+    def to_sql
+      ret = ["-- #{@table_name}: #{@sample.count} rows"]
+      unless @sample.empty?
+        quoted_cols = @sample.first.keys.collect { |col| @connection.quote_column_name col }
+        sql = "INSERT INTO #{@connection.quote_table_name @table_name} (#{quoted_cols * ','})"
+        @sample.each do |row|
+          quoted_vals = row.values.collect { |val| @connection.quote val }
+          ret << sql + " VALUES (#{quoted_vals * ','})"
+        end
+      end
+      ret * "\n"
+    end
+
+    protected
+
+    def fetch_sample(count)
+      sql = "SELECT * FROM #{@connection.quote_table_name @table_name}"
+      pk = @connection.primary_key(@table_name)
+      sql += " ORDER BY #{@connection.quote_column_name pk} DESC" unless pk.nil?
+      sql += " LIMIT #{count}"
+      @connection.select_all(sql).each { |row| add(row) }
+    rescue ActiveRecord::StatementInvalid => e
+      # Don't choke on unknown table engines, such as Sphinx
+      []
+    end
+
+    def samplable?
+      # We shouldn't be sampling views
+      @connection.views.grep(@table_name).empty?
+    end
+
+    def dependency_for(fk, row)
+      ref = {}
+      cols = fk.column_names.dup
+      raise "No column names in foreign key #{fk.inspect}" if cols.empty?
+      fk.references_column_names.each do |ref_col|
+        col = cols.shift
+        ref[ref_col] = row[col] unless row[col].nil?
+      end
+      Dependency.new(fk.references_table_name, ref) unless ref.empty?
+    end
+
+    def dependencies_for(row)
+      foreign_keys.collect { |fk| dependency_for(fk, row) }.compact
+    end
+
+    def foreign_keys
+      @fks ||= @connection.foreign_keys(@table_name)
+    end
+
+  end
+end

data/lib/data_sampler/version.rb

@@ -0,0 +1,3 @@
+module DataSampler
+  VERSION = "0.0.1"
+end

data/lib/data_sampler.rb

@@ -0,0 +1,5 @@
+require "data_sampler/version"
+require "data_sampler/sample"
+require "active_record"
+require 'schema_plus'
+

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: data_sampler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Christian Rishoj
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-08-03 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: schema_plus
+  requirement: &70132762292640 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70132762292640
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: &70132762292220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70132762292220
+description: ! 'Ever found yourself wanting a modest amount of fresh rows from a production
+  database for development purposes, but
+
+  put back by the need to maintain referential integrity in the extracted data sample?
+  This data sampler utility will
+
+  take care that referential dependencies are fulfilled by recursively fetching any
+  rows referred to by the sample.'
+email:
+- christian@rishoj.net
+executables:
+- data_sampler
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README
+- Rakefile
+- bin/data_sampler
+- data_sampler.gemspec
+- lib/data_sampler.rb
+- lib/data_sampler/dependency.rb
+- lib/data_sampler/sample.rb
+- lib/data_sampler/table_sample.rb
+- lib/data_sampler/version.rb
+homepage: https://github.com/crishoj/data_sampler
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: data_sampler
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Extract a sample of records from a database while maintaining referential
+  integrity.
+test_files: []