git_friendly_dumper 0.0.1

data/features/step_definitions/app_steps.rb

@@ -0,0 +1,25 @@
+Given /I am in an empty app/ do
+  steps %q{
+    Given a directory named "app"
+    Given I cd to "app"
+  }
+  in_current_dir { `ln -s ../../../lib lib` }
+end
+
+Given /a Rakefile exists which has an environment task and loads git_friendly_dumper tasks/ do
+  steps %q{
+    Given a file named "Rakefile" with:
+    """
+    $LOAD_PATH.unshift("lib")
+    require 'rake'
+    require 'active_record'
+    
+    load "lib/tasks/git_friendly_dumper_tasks.rake"
+    
+    task :environment do
+      require 'active_record'
+      ActiveRecord::Base.establish_connection(:adapter  => "sqlite3", :database => "test.sqlite3")
+    end
+    """
+  }
+end

data/features/step_definitions/database_steps.rb

@@ -0,0 +1,172 @@
+Given /^there is a connection$/ do
+  ActiveRecord::Base.connection.should_not be_nil
+end
+
+Given /^an empty database$/ do
+  ActiveRecord::Base.establish_connection(:adapter  => "sqlite3", :database => "#{current_dir}/test.sqlite3")
+  db_table_names.each do |table|
+    ActiveRecord::Base.connection.drop_table table
+  end
+end
+
+When /^I refresh the database tables cache$/ do
+  # prevents exception SQLite3::SchemaChangedException: no such table: users: SELECT * FROM "users"  (ActiveRecord::StatementInvalid)
+  # TODO: is there a public API for this?
+  ActiveRecord::Base.connection.tables 
+end
+
+When(/^I execute the schema "([^"]*)"$/) do |schema_path|
+  schema_definition = File.read(File.join(current_dir, schema_path))
+  ActiveRecord::Migration.suppress_messages do
+    ActiveRecord::Schema.define do
+      eval schema_definition
+    end
+  end
+end
+
+Then(/^a "([^"]*)" table should exist with structure:$/) do |table_name, table|
+  # table is a Cucumber::Ast::Table
+  # TODO: make this db independent, currenlty for sqlite3 only
+  table.diff! ActiveRecord::Base.connection.send :table_structure, table_name
+end
+
+Then /^list the table names$/ do
+  announce db_table_names.to_sentence
+end
+
+Then(/^the database should have tables:$/) do |table|
+  table.diff! db_table_names.map {|c| [c]}, :surplus_row => true, :surplus_col => true
+end
+
+Then(/^the database should not have table "([^"]*)"$/) do |table_name|
+  db_table_names.should_not include(table_name)
+end
+
+Then /^show me the tables$/ do
+  db_table_names.each do |table_name|
+    pp table_name
+    pp(table_contents(table_name))
+  end
+end
+
+Then /^show me the "([^"]*)" table$/ do |table_name|
+  announce table_contents(table_name).to_yaml
+end
+
+Given /^the database has a "([^"]*)" table( \(with timestamps\))?:$/ do |table_name, timestamps, table|
+  create_table(table_name, timestamps)
+  
+  columns = []
+  table.headers.each do |column_def|
+    raise "column_def should look like 'column_name (type)'" unless match = column_def.match(/(\w+) \((\w+)\)/)
+    add_column_to_table(table_name, match[1], match[2])
+    columns << match[1]
+  end
+  
+  table.rows.each do |row|
+    attrs = {}
+    columns.each_with_index do |column_name, index|
+      attrs[column_name] = row[index]
+    end
+    insert_record_into_table(table_name, attrs)
+  end
+end
+
+Then /^the "([^"]*)" table should match exactly:$/ do |table_name, table|
+  table.diff! table_to_strings(table_contents(table_name)), :surplus_col => true
+end
+
+Then /^the "([^"]*)" table should match exactly \(ignoring (ids)?(?: and )?(timestamps)?\):$/ do |table_name, ids, timestamps, table|
+  table.diff! table_to_strings(table_contents(table_name, :ids => !ids, :timestamps => !timestamps)), :surplus_col => true
+end
+
+When /^I destroy record (\d+) from the "([^"]*)" table$/ do |id, table_name|
+  class_for_table(table_name).destroy(id)
+end
+
+Then /^the data in the dumped "([^"]*)" yaml files should match the database contents$/ do |table_name|
+  records = class_for_table(table_name).all
+  fixtures = fixtures_for_table(table_name)
+  records.count.should == fixtures.length
+  records.each {|record| match_fixture_file_against_record(record)}
+end
+
+module FixtureHelpers
+  def fixture_path_for(record)
+    fixture_id_path = ("%08d" % record.id).scan(/..../).join('/')
+    File.join current_dir, "db/dump", record.class.table_name, "#{fixture_id_path}.yml"
+  end
+
+  def replace_record_with_fixture!(record)
+    require 'active_record/fixtures'
+    fixture_path =fixture_path_for(record)
+    fixture = ActiveRecord::Fixture.new(YAML.load(File.read(fixture_path)), record.class)
+    record.destroy
+    ActiveRecord::Base.connection.insert_fixture fixture, record.class.table_name
+    record.class.find(record.id)
+  end
+  
+  def match_fixture_file_against_record(record)
+    record.attributes.dup.should == replace_record_with_fixture!(record).attributes
+    announce "#{table_name.singularize} #{record.id} data matches its fixture data #{fixture_path_for(record)}" if @announce
+  end
+  
+  def fixtures_for_table(table_name)
+    fixtures = Dir.glob File.join(current_dir, "db/dump", table_name, '**', '*.yml')
+    announce "Fixtures for #{table_name}:\n#{fixtures.join("\n")}\n" if @announce
+    fixtures
+  end
+end
+World(FixtureHelpers)
+
+module DatabaseHelpers
+  def create_table(name, timestamps = false)
+    ActiveRecord::Base.connection.create_table name do |t|
+      t.timestamps if timestamps
+    end
+  end
+
+  def add_column_to_table(table_name, column_name, type)
+    ActiveRecord::Base.connection.change_table table_name do |t|
+      t.send type, column_name
+    end
+  end
+
+  def class_for_table(table_name)
+    @class_for_table ||= {}
+    @class_for_table[table_name] ||= begin
+      Class.new(ActiveRecord::Base).tap {|klass| klass.table_name = table_name }
+    end
+  end
+
+  def insert_record_into_table(table_name, attrs)
+    class_for_table(table_name).create! attrs
+  end
+
+  #returns ['table', 'names'] with sqlite adapter
+  def db_table_names
+    ActiveRecord::Base.connection.tables
+  end
+
+  # table_contents 'users' # gives back everything
+  # table_contents 'users', :timestamps => false # without timestamps
+  # table_contents 'users', :ids => false # without ids
+  def table_contents(table_name, opts={:timestamps => true, :ids => true})
+    contents = class_for_table(table_name).all.map(&:attributes)
+    contents.tap do |contents|
+      contents.map{|c| c.delete('id')} unless opts[:ids]
+      contents.map{|c| c.delete('updated_at'); c.delete('created_at')} unless opts[:timestamps]
+    end
+  end
+  
+  #because cucumber table#diff! expects types to match and step transforms are silly
+  def table_to_strings(table)
+    table.each do |row|
+      row.each_pair do |key, value| 
+        row[key] = value.is_a?(Time) ? value.utc.to_s(:db) : value.to_s
+      end
+    end
+  end
+end
+
+World(DatabaseHelpers)

data/features/support/announce.rb

@@ -0,0 +1,3 @@
+Before('@announce') do
+  @announce = true
+end

data/features/support/env.rb

@@ -0,0 +1,2 @@
+require 'rspec/expectations'
+require 'aruba/cucumber'

data/features/support/logger.rb

@@ -0,0 +1,7 @@
+require 'logger'
+require 'active_record'
+
+log_filename = File.join(File.dirname(__FILE__), '..', '..', 'tmp', 'test.log')
+`mkdir -p #{File.dirname(log_filename)}`
+log_file = File.open(log_filename, 'w')
+ActiveRecord::Base.logger = Logger.new(log_file)

data/lib/git_friendly_dumper.rb

@@ -0,0 +1,279 @@
+require 'fileutils'
+require 'active_support/all'
+require 'active_record/fixtures'
+
+begin; require 'progressbar'; rescue MissingSourceFile; end
+
+# Database independent and git friendly replacement for mysqldump for rails projects
+class GitFriendlyDumper
+  include FileUtils
+  
+  attr_accessor :root, :path, :connection, :tables, :force, :include_schema, :show_progress, :clobber_fixtures, :limit, :raise_error, :fixtures
+  alias_method :include_schema?, :include_schema
+  alias_method :clobber_fixtures?, :clobber_fixtures
+  alias_method :show_progress?, :show_progress
+  alias_method :force?, :force
+  alias_method :raise_error?, :raise_error
+  
+  class << self
+    def dump(options = {})
+      new(options).dump
+    end
+  
+    def load(options = {})
+      new(options).load
+    end
+  end
+  
+  def initialize(options = {})
+    options.assert_valid_keys(:root, :path, :connection, :connection_name, :tables, :force, :include_schema, :show_progress, :clobber_fixtures, :limit, :raise_error, :fixtures, :fixtures_file)
+
+    self.root = options[:root] || (defined?(Rails) && Rails.root) || pwd
+    
+    if options[:fixtures_file]
+      raise ArgumentError, "GitFriendlyDumper cannot specify both :fixtures and :fixtures_file" if options[:fixtures].present?
+      options[:fixtures] = File.read(options[:fixtures_file]).split("\n").map(&:squish).reject(&:blank?)
+    end
+    
+    if options[:fixtures] && (options[:include_schema] || options[:clobber_fixtures])
+      raise ArgumentError, "GitFriendlyDumper if :fixtures option given, neither :include_schema nor :clobber_fixtures can be given"
+    end
+    
+    if options[:show_progress] && !defined?(ProgressBar)
+      raise RuntimeError, "GitFriendlyDumper requires the progressbar gem for progress option.\n  sudo gem install progressbar"
+    end
+    
+    self.path             = File.expand_path(options[:path] || 'db/dump')
+    self.tables           = options[:tables]
+    self.fixtures         = options[:fixtures]
+    self.limit            = options.key?(:limit) ? options[:limit].to_i : 2500
+    self.raise_error      = options.key?(:raise_error) ? options[:raise_error] : true
+    self.force            = options.key?(:force) ? options[:force] : false
+    self.include_schema   = options.key?(:include_schema) ? options[:include_schema] : false
+    self.show_progress    = options.key?(:show_progress) ? options[:show_progress] : false
+    self.clobber_fixtures = options.key?(:clobber_fixtures) ? options[:clobber_fixtures] : (options[:tables].blank? ? true : false)
+    self.connection       = options[:connection] || begin
+      if options[:connection_name]
+        ActiveRecord::Base.establish_connection(options[:connection_name])
+      end
+      ActiveRecord::Base.connection
+    end
+  end
+  
+  def dump
+    if fixtures
+      raise ArgumentError, "Cannot dump when :fixtures option is given"
+    end
+    self.tables ||= db_tables
+    tables.delete('schema_migrations') unless include_schema?
+    if force? || (tables & fixtures_tables).empty? || confirm?(:dump)
+      puts "Dumping data#{' and structure' if include_schema?} from #{current_database_name} to #{path.sub("#{root}/",'')}\n"
+      clobber_all_fixtures if clobber_fixtures?
+      connection.transaction do
+        tables.each {|table| dump_table(table) }
+      end
+    end
+  end
+  
+  def load
+    fixtures ? load_fixtures : load_tables
+  end
+
+private
+  def current_database_name
+    @current_database_name ||= (connection.respond_to?(:current_database) && connection.current_database) || 'database'
+  end
+  
+  def confirm?(type)
+    dump_path = path.sub("#{root}/", '')
+    if clobber_fixtures? && type == :dump
+      puts "\nWARNING: all fixtures in #{dump_path}"
+    else
+      puts "\nWARNING: the following #{type == :dump ? 'fixtures' : 'tables'} in #{type == :dump ? dump_path : current_database_name}:"
+      puts "  " + tables.join("\n  ")
+    end
+    if fixtures
+      puts "will have records replaced by the specified #{fixtures.length} fixtures (deleting if fixture file is missing)"
+    else
+      puts "will be replaced with #{type == :dump ? 'records' : 'fixtures'}#{' and table schemas' if include_schema?} from #{type == :dump ? current_database_name : dump_path}."
+    end
+    puts "Do you wish to proceed? (type 'yes' to proceed)"
+    proceed = ($stdin.gets.downcase.strip == 'yes')
+    puts "#{type.to_s.capitalize} cancelled at user's request." unless proceed
+    proceed
+  end
+
+  def fixtures_tables
+    @fixture_tables ||= Dir[File.join(path, '*')].select{|f| File.directory?(f)}.map{|f| File.basename(f)}
+  end
+  
+  def db_tables
+    @db_tables ||= connection.tables
+  end
+  
+  def load_tables
+    self.tables ||= fixtures_tables
+    tables.delete('schema_migrations') unless include_schema?
+    if force? || (tables & db_tables).empty? || confirm?(:load)
+      puts "Loading data#{' and structure' if include_schema?} into #{current_database_name} from #{path.sub("#{root}/",'')}\n"
+      connection.transaction do
+        tables.each {|table| load_table(table) }
+      end
+    end
+  end
+  
+  def load_fixtures
+    fixtures_tables = []
+    fixtures.map! do |fixture|
+      raise ArgumentError, "Fixture filename error: #{fixture} should be a relative filename e.g. users/0000/0001.yml" unless fixture =~ /^\w+\/\d+\/\d+\.yml$/
+      table = fixture.split('/').first
+      if (!tables || tables.include?(table))
+        unless fixtures_tables.include?(table)
+          begin
+            "::#{table.classify}".constantize
+          rescue NameError
+            eval "class ::#{table.classify} < ActiveRecord::Base; end"
+          end
+          fixtures_tables << table
+        end
+        fixture
+      end
+    end
+    fixtures.compact!
+  
+    self.tables = fixtures_tables
+  
+    if force? || (tables & db_tables).empty? || confirm?(:load)
+      puts "Loading fixtures into #{current_database_name} from #{path.sub("#{root}/",'')}\n"
+      show_progress? && (progress_bar = ProgressBar.new("fixtures", fixtures.length))
+      connection.transaction do
+        fixtures.each do |fixture|
+          match_data = fixture.match(/(\w+)\/(.+)\.yml/)
+          table, id, file = match_data[1], match_data[2].sub('/','').to_i, File.join(path, fixture)
+          
+          raise "Couldn't determine id from #{fixture} (id was #{id})" if id < 1
+          connection.delete("DELETE FROM #{table} WHERE id=#{id};")
+          load_fixture(table.classify.constantize, table, file) if File.exist?(file)
+          show_progress? && progress_bar.inc
+        end
+      end
+      show_progress && progress_bar.finish
+    end
+  end
+  
+  def dump_table(table)
+    clobber_fixtures_for_table(table)
+    count = connection.select_value("SELECT COUNT(*) FROM %s" % table).to_i
+    show_progress? && (progress_bar = ProgressBar.new(table, count))
+    
+    offset = 0
+    while (records = select_records(table, offset)).any?
+      dump_records(table, records, show_progress? && progress_bar)
+      offset += limit
+    end
+    
+    show_progress? && progress_bar.finish
+    dump_table_schema(table) if include_schema?
+  rescue ActiveRecord::ActiveRecordError => e
+    puts "dumping #{table} failed: #{e.message}"
+    puts "Partial dump files have been left behind and you should clean up before continuing (e.g. git status, git checkout, git clean)."
+    raise e if raise_error?
+  end
+  
+  def select_records(table, offset)
+    connection.select_all("SELECT * FROM %s LIMIT #{limit} OFFSET #{offset}" % table)
+  end
+  
+  def dump_records(table, records, progress_bar)
+    records.each_with_index do |record, index|
+      id = record['id'] ? record['id'].to_i : index + 1
+      fixture_file = File.join(path, table, *id_path(id)) + ".yml"
+      `mkdir -p #{File.dirname(fixture_file)}`
+      File.open(fixture_file, "w") do |record_file|
+        record_file.write record.to_yaml
+      end
+      show_progress? && progress_bar.inc
+    end
+  end
+  
+  def load_table(table)
+    # create a placeholder AR class for the table without loading anything from the app.
+    klass = eval "class #{table.classify} < ActiveRecord::Base; end"
+    include_schema? ? load_table_schema(table) : clobber_records(table)
+    files = Dir[File.join(path, table, '**', '*.yml')]
+    show_progress? && (progress_bar = ProgressBar.new(table, files.length))
+    files.each do |file|
+      load_fixture(klass, table, file)
+      show_progress? && progress_bar.inc
+    end
+    show_progress? && progress_bar.finish
+  rescue ActiveRecord::ActiveRecordError => e
+    puts "loading #{table} failed - check log for details"
+    raise e if raise_error?
+  end
+  
+  def load_fixture(klass, table, file)
+    fixture = ActiveRecord::Fixture.new(YAML.load(File.read(file)), klass)
+    begin
+      connection.insert_fixture fixture, table
+    rescue ActiveRecord::ActiveRecordError => e
+      puts "loading fixture #{file} failed - check log for details"
+      raise e if raise_error?
+    end
+  end
+  
+  def dump_table_schema(table)
+    File.open(File.join(path, table, 'schema.rb'), "w") do |schema_file|
+      if table == 'schema_migrations'
+        schema_file.write schema_migrations_schema
+      else
+        schema_dumper.send :table, table, schema_file
+      end
+    end
+  end
+
+  def schema_migrations_schema
+    <<-end_eval
+  create_table "schema_migrations", :force => true, :id => false do |t|
+    t.string "version", :null => false
+  end
+  add_index :schema_migrations, :version, :unique => true, :name => 'unique_schema_migrations'
+    end_eval
+  end
+  
+  def schema_dumper
+    @schema_dumper ||= ActiveRecord::SchemaDumper.send :new, @connection
+  end
+  
+  def load_table_schema(table)
+    schema_definition = File.read(File.join(path, table, 'schema.rb'))
+    ActiveRecord::Migration.suppress_messages do 
+      ActiveRecord::Schema.define do
+        eval schema_definition
+      end
+    end
+  end
+
+  def clobber_fixtures_for_table(table)
+    `rm -rf #{File.join(path, table)}`
+    `mkdir -p #{File.join(path, table)}`
+  end
+  
+  def clobber_all_fixtures
+    fixtures_tables.each {|table| clobber_fixtures_for_table(table)}
+  end
+  
+  def clobber_records(table)
+    connection.delete "DELETE FROM #{table}"
+  end
+  
+  # Partitions the given id into an array of path components.
+  #
+  # For example, given an id of 1
+  # <tt>["0000", "0001"]</tt>
+  #
+  # Currently only integer ids are supported
+  def id_path(id)
+    ("%08d" % id).scan(/..../)
+  end
+end