distyll 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b258d9e95565fd009d54bc4a70893f7ec7644b2e
+  data.tar.gz: 06714e3d3c272d347c331e27bf9bfbf6d78f1df4
+SHA512:
+  metadata.gz: aa597d98f6dc4ae3848ae092efc3a2e2f3f511be6a05468f89527e74004f53fccd45ceb8b8b25ea053b846b4c7f1fa5149576e20f54aed61bdc3817aaf4d9a28
+  data.tar.gz: 92a4ef0ea98c1ccdbb3b38564e8438aa0b5502e6c3ecdad89046dc3eb038bb4a33a7b1947d388a61d7b10327bc555b0e4312eec2fdaabdc69e7435102f2724bb

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,14 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "rdoc", "~> 3.12"
+  gem "bundler", "~> 1.0"
+  gem "jeweler", "~> 2.0.1"
+  gem "simplecov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,81 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (4.1.1)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    addressable (2.3.6)
+    builder (3.2.2)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    docile (1.1.3)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    git (1.2.6)
+    github_api (0.11.3)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.1)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 1.2)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.0)
+      oauth2
+    hashie (2.1.1)
+    highline (1.6.21)
+    i18n (0.6.9)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+    json (1.8.1)
+    jwt (0.1.13)
+      multi_json (>= 1.5)
+    mini_portile (0.6.0)
+    minitest (5.3.4)
+    multi_json (1.10.0)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.2.1)
+      mini_portile (= 0.6.0)
+    oauth2 (0.9.3)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 0.1.8)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.5.2)
+    rake (10.3.2)
+    rdoc (3.12.2)
+      json (~> 1.4)
+    shoulda (3.5.0)
+      shoulda-context (~> 1.0, >= 1.0.1)
+      shoulda-matchers (>= 1.4.1, < 3.0)
+    shoulda-context (1.2.1)
+    shoulda-matchers (2.6.1)
+      activesupport (>= 3.0.0)
+    simplecov (0.8.2)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    thread_safe (0.3.3)
+    tzinfo (1.1.0)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  jeweler (~> 2.0.1)
+  rdoc (~> 3.12)
+  shoulda
+  simplecov

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2014 Mason F. Matthews
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,55 @@
+= distyll
+
+Suppose that you're writing code for a project that's been in production for a long time.  All said and done,
+the production database size is on the order of 100 GB.  When you've finished writing your new feature, you
+test it on your seeds.  All is well.  However, how do you know that it will work on production data?  How do
+you know that your seeds accurately reflect the variance (and oddities) present in the production data set?
+
+Distyll attempts to solve this by creating a "recent" subset of the production data set.  This could be done
+naively by taking all of the records across the whole database with a created_at above a certain time.
+However, a record created today may have an associated record (via a foreign key) which was created five
+years ago.  If you slice the entire database by created_at timestamps, you'll have foreign keys which point
+nowhere.  Not very helpful for ensuring that your new feature works on production data.
+
+Distyll's solution is to start from a set of "core" ActiveRecord models supplied at initialization time
+(plus a date threshold for these models), and only pull those that have been created since the date
+threshold.  It then traverses all belongs_to relationships from those core models and pulls in all of those
+related records.
+
+Consequently, you end up with a data set that is representative of production, is internally consistent,
+and is smaller.
+
+== Using distyll in your project
+
+1. Add <code>gem 'distyll'</code> to your gemfile
+1. Run <code>bundle install</code>
+1. Add a <code>distyll:</code> database to your database.yml
+1. Run <code>rake db:create RAILS_ENV=distyll</code>
+1. Run <code>rake db:schema:load RAILS_ENV=distyll</code>
+1. Run <code>rails console</code>
+1. Call <code>Distyll.new(model_names, created_since)</code>, passing it an array of strings of the core models and a date after which core records will be copied.
+
+If you need to clear out the distyll database and try again with different parameters, just go back to the <code>schema:load</code> step and continue from there.
+
+== Contributing to distyll
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Next Steps for distyll
+
+* Distyll only traverses belongs_to associations for now.  Need to consider other association types.
+* Is likely to cause problems with single table inheritance.  Could probably refer to table names rather than model names when traversing relationships... but this would still be an issue for the base models.
+* Currently performs "IN" query.  In Oracle, this is limited to 1000 values, so I would need to chunk them for that DBMS.
+* Tests.  I know.  I just don't yet have my head around how to test something that's SO model- and database-centric, when those models and databases aren't present in the gem.  Any advice would be appreciated.
+
+== Copyright
+
+Copyright (c) 2014 Mason F. Matthews. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,51 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = "distyll"
+  gem.homepage = "http://github.com/masonfmatthews/distyll"
+  gem.license = "MIT"
+  gem.summary = %Q{A gem for distilling a massive database into a recent (and smaller) data set}
+  gem.description = %Q{Have you ever had a 100GB production database and been unable to test on an internally consistent subset of the data?  Distyll is your answer.}
+  gem.email = "mason.f.matthews@gmail.com"
+  gem.authors = ["Mason F. Matthews"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+desc "Code coverage detail"
+task :simplecov do
+  ENV['COVERAGE'] = "true"
+  Rake::Task['test'].execute
+end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "distyll #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.0

data/distyll.gemspec

@@ -0,0 +1,62 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "distyll"
+  s.version = "0.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Mason F. Matthews"]
+  s.date = "2014-06-03"
+  s.description = "Have you ever had a 100GB production database and been unable to test on an internally consistent subset of the data?  Distyll is your answer."
+  s.email = "mason.f.matthews@gmail.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "distyll.gemspec",
+    "lib/distyll.rb",
+    "test/helper.rb",
+    "test/test_distyll.rb"
+  ]
+  s.homepage = "http://github.com/masonfmatthews/distyll"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "2.0.14"
+  s.summary = "A gem for distilling a massive database into a recent (and smaller) data set"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 2.0.1"])
+      s.add_development_dependency(%q<simplecov>, [">= 0"])
+    else
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+      s.add_dependency(%q<simplecov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<rdoc>, ["~> 3.12"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
+    s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+    s.add_dependency(%q<simplecov>, [">= 0"])
+  end
+end
+

data/lib/distyll.rb

@@ -0,0 +1,120 @@
+class Distyll
+
+  attr_reader :base_models, :model_profiles, :created_since
+
+  def initialize(bms, cs)
+    @created_since = cs.to_date
+    @base_models = bms.map &:constantize
+    set_model_profiles
+  end
+
+  def run
+    base_models.each do |model|
+      @model_profiles[model].load_ids_by_timestamp(@created_since)
+    end
+
+    prior_count = -1
+    while prior_count != current_count
+      prior_count = current_count
+      @model_profiles.each_value &:demote_new_ids
+
+      @model_profiles.each_value do |profile|
+        profile.associations.each do |a|
+          @model_profiles[a.klass].load_ids(profile.get_new_associated_ids(a))
+        end
+      end
+    end
+
+    @model_profiles.each_value do |profile|
+      profile.copy_records
+    end
+  end
+
+
+  private
+
+  def set_model_profiles
+    @model_profiles = Hash.new
+    base_models.each do |bm|
+      @model_profiles = potentially_add_profiles(bm, @model_profiles)
+    end
+  end
+
+  def potentially_add_profiles(model, profiles)
+    return profiles if profiles.include? model
+    profiles[model] = DistyllModelProfile.new(model)
+    profiles[model].associations.each do |a|
+      profiles = potentially_add_profiles(a.klass, profiles)
+    end
+    profiles
+  end
+
+  def current_count
+    model_profiles.each_value.sum &:get_id_count
+  end
+
+end
+
+
+
+class DistyllModelProfile
+  attr_reader :model, :record_count, :associations, :all_ids, :last_ids, :new_ids
+
+  def initialize(m)
+    @model = m
+    @record_count = m.count
+    @all_ids = Array.new
+    @last_ids = Array.new
+    @new_ids = Array.new
+    set_associations
+  end
+
+  def demote_new_ids
+    @last_ids = @new_ids
+    @new_ids = Array.new
+  end
+
+  def load_ids_by_timestamp(timestamp)
+    ids = model.where("created_at >= ?", timestamp).select(:id).map &:id
+    @new_ids += ids
+    @all_ids += ids
+  end
+
+  def load_ids(ids)
+    @new_ids += ids
+    @all_ids += ids
+  end
+
+  def get_id_count
+    @all_ids = @all_ids.uniq || []
+    @all_ids.count
+  end
+
+  def get_new_associated_ids(a)
+    model.where(id: last_ids).select(a.foreign_key).map { |r| r.send(a.foreign_key) }
+  end
+
+  def copy_records
+    return nil if all_ids.blank?
+
+    records = model.where(id: all_ids).load
+
+    model.establish_connection("distyll")
+    records.each { |record| model.new(record.attributes).save!(validate: false) }
+    model.establish_connection(Rails.env)
+
+    records
+  end
+
+  private
+
+  def set_associations
+    @associations = Array.new
+    model.reflect_on_all_associations.each do |association|
+      if association.belongs_to? && association.through_reflection.nil?
+        @associations << association
+      end
+    end
+  end
+
+end

data/test/helper.rb

@@ -0,0 +1,34 @@
+require 'simplecov'
+
+module SimpleCov::Configuration
+  def clean_filters
+    @filters = []
+  end
+end
+
+SimpleCov.configure do
+  clean_filters
+  load_adapter 'test_frameworks'
+end
+
+ENV["COVERAGE"] && SimpleCov.start do
+  add_filter "/.rvm/"
+end
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'distyll'
+
+class Test::Unit::TestCase
+end

data/test/test_distyll.rb

@@ -0,0 +1,5 @@
+require 'helper'
+
+class TestDistyll < Test::Unit::TestCase
+
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: distyll
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Mason F. Matthews
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Have you ever had a 100GB production database and been unable to test
+  on an internally consistent subset of the data?  Distyll is your answer.
+email: mason.f.matthews@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- .document
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- distyll.gemspec
+- lib/distyll.rb
+- test/helper.rb
+- test/test_distyll.rb
+homepage: http://github.com/masonfmatthews/distyll
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: A gem for distilling a massive database into a recent (and smaller) data
+  set
+test_files: []