ydd 0.0.1

data/.rvmrc

@@ -0,0 +1,6 @@
+
+if [[ -d "$rvm_path/environments" && -s "$rvm_path/environments/ree-1.8.7-2010.02@ydd" ]] ; then
+  \. "$rvm_path/environments/ree-1.8.7-2010.02@ydd"
+else
+  rvm --create use  "ree-1.8.7-2010.02@ydd"
+fi

data/README.markdown

@@ -0,0 +1,44 @@
+# ydd - Yaml DB Dumber
+
+A fork of the awesome [yaml_db](http://github.com/ludicast/yaml_db).
+
+## Installation
+
+This plugin can now be installed as a gem via 
+
+    gem install ydd
+
+Note that when doing this, the rake tasks won't automatically be available to your app.  You'll have to explicitly include them via the approaches followed: [here](http://ggr.com/how-to-include-a-gems-rake-tasks-in-your-rails-app.html), [here](https://rails.lighthouseapp.com/projects/8994/tickets/510-rake-tasks-not-included-for-gems), [and here](https://rails.lighthouseapp.com/projects/8994/tickets/59)
+
+## Usage
+
+    rake db:data:dump   ->   Dump contents of Rails database to db/data.yml
+    rake db:data:load   ->   Load contents of db/data.yml into the database
+
+Further, there are tasks db:dump and db:load which do the entire database (the equivalent of running db:schema:dump followed by db:data:load).  Also, there are other tasks recently added that allow the export of the database contents to/from multiple files (each one named after the table being dumped or loaded).
+
+    rake db:data:dump_dir   ->   Dump contents of database to curr_dir_name/tablename.extension (defaults to yaml)
+    rake db:data:load_dir   ->   Load contents of db/data_dir into database
+
+In addition, we have plugins whereby you can export your database to/from various formats.  We only deal with yaml and csv right now, but you can easily write tools for your own formats (such as Excel or XML).  To use another format, just load setting the "class"  parameter to the class you are using.  This defaults to "YamlDb::Helper" which is a refactoring of the old yaml_db code.  We'll shorten this to use class nicknames in a little bit.
+
+## Examples
+
+One common use would be to switch your data from one database backend to another.  For example, let's say you wanted to switch from SQLite to MySQL.  You might execute the following steps:
+
+1. rake db:dump
+
+2. Edit config/database.yml and change your adapter to mysql, set up database params
+
+3. mysqladmin create [database name]
+
+4. rake db:load
+
+## Credits
+
+Created by Orion Henry and Adam Wiggins.  Major updates by Ricardo Chimal, Jr.
+
+Patches contributed by Michael Irwin, Tom Locke, and Tim Galeckas.
+
+Send questions, feedback, or patches to the Heroku mailing list: http://groups.google.com/group/heroku
+

data/Rakefile

@@ -0,0 +1,23 @@
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "ydd"
+    gem.summary = %Q{ydd dumps / loads rails app databases for backup purposes.}
+    gem.description = %Q{
+YDD is a tool (a fork of yaml_db really) to make it generally easy
+to use it for things like copying databases from production to staging
+and the like.
+}
+    gem.email = "sutto@sutto.net"
+    gem.homepage = "http://github.com/YouthTree/ydd"
+    gem.authors = ["Adam Wiggins","Orion Henry", "Darcy Laycock"]
+    gem.add_dependency 'thor', '~> 0.14'
+    gem.add_dependency 'activesupport', '~> 3.0.0'
+    gem.add_dependency 'activerecord', '~> 3.0.0'
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/bin/ydd

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path("../lib", File.dirname(__FILE__))
+require 'ydd'
+YDD::Application.start

data/lib/ydd.rb

@@ -0,0 +1,68 @@
+require 'yaml'
+require 'active_support'
+require 'active_record'
+require 'fileutils'
+
+module YDD
+  class Error < StandardError; end
+  
+  autoload :SchemaManager,       'ydd/schema_manager'
+  autoload :DataManager,         'ydd/data_manager'
+  autoload :YamlDB,              'ydd/yaml_db'
+  autoload :SerializationHelper, 'ydd/serialization_helper'
+  autoload :Application,         'ydd/application'
+
+  def self.env=(value)
+    if value.blank?
+      @@env = nil
+    else
+      @@env = value
+    end
+  end
+  
+  def self.env
+    @@env ||= (ENV['YDD_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development')
+  end
+  
+  def self.connection
+    ActiveRecord::Base.connection
+  end
+  
+  def self.configuration_from(file)
+    require 'erb'
+    parsed = ERB.new(File.read(file)).result
+    YAML.load(parsed)[env]
+  end
+  
+  def self.connect_from(path)
+    raise Error, "Invalid database config at #{path}" if !File.exists?(path)
+    connect configuration_from(path)
+  end
+  
+  def self.connect(specification)
+    ActiveRecord::Base.establish_connection(specification)
+  end
+  
+  def self.dump(directory)
+    FileUtils.mkdir_p directory
+    SchemaManager.dump File.join(directory, "schema.rb")
+    DataManager.dump   File.join(directory, "data.yml")
+  end
+  
+  def self.load(directory)
+    if !File.directory?(directory)
+      raise Error, "Please provide a valid directory - #{directory} doesn't exist."
+    end
+    check_files! directory, "schema.rb", "data.yml"
+    SchemaManager.load File.join(directory, "schema.rb")
+    DataManager.load   File.join(directory, "data.yml")
+  end
+  
+  def self.check_files!(dir, *files)
+    files.each do |file|
+      path = File.join(dir, file)
+      raise Error, "#{file} doesn't exist in #{dir}" if !File.readable?(path)
+    end
+  end
+  
+end

data/lib/ydd/application.rb

@@ -0,0 +1,61 @@
+require 'thor'
+
+module YDD
+  class Application < Thor
+    include Thor::Actions
+    
+    method_options :env => :string, :force => :boolean
+    
+    desc "dump DUMP_DIR [APP_DIR='.'] [OPTIONS]", "Dumps the database contents and schema."
+    def dump(dump_dir, app_dir = ".")
+      setup! dump_dir, app_dir, :dump => true
+      say "Starting dump..."
+      YDD.dump @dump_dir
+    end
+    
+    desc "load DUMP_DIR [APP_DIR='.'] [OPTIONS]", "Loads the database contents and schema."
+    def load(dump_dir, app_dir = ".")
+      setup! dump_dir, app_dir
+      say "Starting load..."
+      YDD.load @dump_dir
+    end
+    
+    protected
+    
+    def setup_directories!(dump_dir, app_dir, extra = {})
+      @dump_dir  = File.expand_path(dump_dir)
+      @app_dir   = File.expand_path(app_dir)
+      @db_config = File.join(@app_dir, "config", "database.yml")
+      if !File.directory?(@app_dir)
+        die "The given application directory, #{@application}, does not exist."
+      elsif !File.exist?(@db_config)
+        die "#{@db_config} does not exist."
+      elsif YDD.configuration_from(@db_config).blank?
+        die "Unable to find database configuration for #{YDD.env}."
+      elsif extra[:dump] && !Dir[File.join(@dump_dir, "**", "*")].empty? && !options.force?
+        die "The given dump dir, #{@dump_dir}, is not empty."
+      end
+    end
+    
+    def setup!(dump_dir, app_dir, extra = {})
+      validate_environment!
+      setup_directories!    dump_dir, app_dir, extra
+      connect_to_database!
+    end
+    
+    def connect_to_database!
+      YDD.connect_from @db_config
+    end
+    
+    def validate_environment!
+      YDD.env = options.env if options.env.present?
+    end
+    
+    def die(error, code = 1)
+      say error
+      exit code
+    end
+    
+  end
+
+end

data/lib/ydd/data_manager.rb

@@ -0,0 +1,15 @@
+module YDD
+  class DataManager
+    
+    def self.dump(path)
+      SerializationHelper::Base.new(YamlDB::Helper).dump path
+    end
+    
+    def self.load(path)
+      if File.exists?(path)
+        SerializationHelper::Base.new(YamlDB::Helper).load path
+      end
+    end
+    
+  end
+end

data/lib/ydd/schema_manager.rb

@@ -0,0 +1,15 @@
+module YDD
+  class SchemaManager
+    
+    def self.dump(path)
+      File.open(path, "w") do |file|
+        ActiveRecord::SchemaDumper.dump(YDD.connection, file)
+      end
+    end
+    
+    def self.load(path)
+      Kernel.load path if File.exists?(path)
+    end
+    
+  end
+end

data/lib/ydd/serialization_helper.rb

@@ -0,0 +1,177 @@
+module YDD
+  module SerializationHelper
+
+    class Base
+      attr_reader :extension
+
+      def initialize(helper)
+        @dumper    = helper.dumper
+        @loader    = helper.loader
+        @extension = helper.extension
+      end
+
+      def dump(filename)
+        disable_logger
+        @dumper.dump(File.new(filename, "w"))
+        reenable_logger
+      end
+
+      def load(filename, truncate = true)
+        disable_logger
+        @loader.load(File.new(filename, "r"), truncate)
+        reenable_logger
+      end
+
+      def disable_logger
+        @@old_logger = ActiveRecord::Base.logger
+        ActiveRecord::Base.logger = nil
+      end
+
+      def reenable_logger
+        ActiveRecord::Base.logger = @@old_logger
+      end
+    end
+  
+    class Load
+      def self.load(io, truncate = true)
+        YDD.connection.transaction do
+          load_documents(io, truncate)
+        end
+      end
+
+      def self.truncate_table(table)
+        begin
+          YDD.connection.execute("TRUNCATE #{SerializationHelper::Utils.quote_table(table)}")
+        rescue Exception
+          YDD.connection.execute("DELETE FROM #{SerializationHelper::Utils.quote_table(table)}")
+        end
+      end
+
+      def self.load_table(table, data, truncate = true)
+        column_names = data['columns']
+        if truncate
+          truncate_table(table)
+        end
+        load_records(table, column_names, data['records'])
+        reset_pk_sequence!(table)
+      end
+
+      def self.load_records(table, column_names, records)
+        if column_names.nil?
+          return
+        end
+        columns = column_names.map{|cn| YDD.connection.columns(table).detect{|c| c.name == cn}}
+        quoted_column_names = column_names.map { |column| YDD.connection.quote_column_name(column) }.join(',')
+        quoted_table_name = SerializationHelper::Utils.quote_table(table)
+        records.each do |record|
+          quoted_values = record.zip(columns).map{|c| YDD.connection.quote(c.first, c.last)}.join(',')
+          YDD.connection.execute("INSERT INTO #{quoted_table_name} (#{quoted_column_names}) VALUES (#{quoted_values})")
+        end
+      end
+
+      def self.reset_pk_sequence!(table_name)
+        if YDD.connection.respond_to?(:reset_pk_sequence!)
+          YDD.connection.reset_pk_sequence!(table_name)
+        end
+      end    
+
+      
+    end
+
+    module Utils
+
+      def self.unhash(hash, keys)
+        keys.map { |key| hash[key] }
+      end
+
+      def self.unhash_records(records, keys)
+        records.each_with_index do |record, index|
+          records[index] = unhash(record, keys)
+        end
+
+        records
+      end
+
+      def self.convert_booleans(records, columns)
+        records.each do |record|
+          columns.each do |column|
+            next if is_boolean(record[column])
+            record[column] = (record[column] == 't' or record[column] == '1')
+          end
+        end
+        records
+      end
+
+      def self.boolean_columns(table)
+        columns = YDD.connection.columns(table).reject { |c| silence_warnings { c.type != :boolean } }
+        columns.map { |c| c.name }
+      end
+
+      def self.is_boolean(value)
+        value.kind_of?(TrueClass) or value.kind_of?(FalseClass)
+      end
+
+      def self.quote_table(table)
+        YDD.connection.quote_table_name(table)
+      end
+
+    end
+
+    class Dump
+      def self.before_table(io, table)
+
+      end
+
+      def self.dump(io)
+        tables.each do |table|
+          before_table(io, table)
+          dump_table(io, table)
+          after_table(io, table)
+        end
+      end
+
+      def self.after_table(io, table)
+
+      end
+
+      def self.tables
+        YDD.connection.tables.reject { |table| ['schema_info', 'schema_migrations'].include?(table) }
+      end
+
+      def self.dump_table(io, table)
+        return if table_record_count(table).zero?
+
+        dump_table_columns(io, table)
+        dump_table_records(io, table)
+      end
+
+      def self.table_column_names(table)
+        YDD.connection.columns(table).map { |c| c.name }
+      end
+
+
+      def self.each_table_page(table, records_per_page=1000)
+        total_count = table_record_count(table)
+        pages = (total_count.to_f / records_per_page).ceil - 1
+        id = table_column_names(table).first
+        boolean_columns = SerializationHelper::Utils.boolean_columns(table)
+        quoted_table_name = SerializationHelper::Utils.quote_table(table)
+
+        (0..pages).to_a.each do |page|
+          sql = YDD.connection.add_limit_offset!("SELECT * FROM #{quoted_table_name} ORDER BY #{id}",
+                                                                :limit => records_per_page, :offset => records_per_page * page
+          )
+          records = YDD.connection.select_all(sql)
+          records = SerializationHelper::Utils.convert_booleans(records, boolean_columns)
+          yield records
+        end
+      end
+
+      def self.table_record_count(table)
+        YDD.connection.select_one("SELECT COUNT(*) FROM #{SerializationHelper::Utils.quote_table(table)}").values.first.to_i
+      end
+
+    end
+
+  end
+end

data/lib/ydd/yaml_db.rb

@@ -0,0 +1,64 @@
+module YDD
+  module YamlDB
+    module Helper
+      def self.loader
+        YamlDB::Load 
+      end
+
+      def self.dumper
+        YamlDB::Dump
+      end
+
+      def self.extension
+        "yml"
+      end
+    end
+
+
+    module Utils
+      def self.chunk_records(records)
+        yaml = [ records ].to_yaml
+        yaml.sub!("--- \n", "")
+        yaml.sub!('- - -', '  - -')
+        yaml
+      end
+
+    end
+
+    class Dump < SerializationHelper::Dump
+ 
+      def self.dump_table_columns(io, table)
+        io.write("\n")
+        io.write({ table => { 'columns' => table_column_names(table) } }.to_yaml)
+      end
+
+      def self.dump_table_records(io, table)
+        table_record_header(io)
+
+        column_names = table_column_names(table)
+
+        each_table_page(table) do |records|
+          rows = SerializationHelper::Utils.unhash_records(records, column_names)
+          io.write(YamlDB::Utils.chunk_records(records))
+        end
+      end
+
+      def self.table_record_header(io)
+        io.write("  records: \n")
+      end
+
+    end
+
+    class Load < SerializationHelper::Load
+      def self.load_documents(io, truncate = true) 
+          YAML.load_documents(io) do |ydoc|
+            ydoc.keys.each do |table_name|
+              next if ydoc[table_name].nil?
+              load_table(table_name, ydoc[table_name], truncate)
+            end
+          end
+      end
+    end
+
+  end
+end

data/tasks/yaml_db_tasks.rake

@@ -0,0 +1,45 @@
+namespace :db do
+	desc "Dump schema and data to db/schema.rb and db/data.yml"
+	task(:dump => [ "db:schema:dump", "db:data:dump" ])
+
+	desc "Load schema and data from db/schema.rb and db/data.yml"
+	task(:load => [ "db:schema:load", "db:data:load" ])
+
+	namespace :data do
+		def db_dump_data_file (extension = "yml")
+			"#{dump_dir}/data.#{extension}"
+        end
+            
+        def dump_dir(dir = "")
+          "#{RAILS_ROOT}/db#{dir}"
+        end
+
+		desc "Dump contents of database to db/data.extension (defaults to yaml)"
+		task :dump => :environment do
+            format_class = ENV['class'] || "YamlDb::Helper"
+            helper = format_class.constantize
+			SerializationHelper::Base.new(helper).dump db_dump_data_file helper.extension
+		end
+
+		desc "Dump contents of database to curr_dir_name/tablename.extension (defaults to yaml)"
+		task :dump_dir => :environment do
+            format_class = ENV['class'] || "YamlDb::Helper"
+            dir = ENV['dir'] || "#{Time.now.to_s.gsub(/ /, '_')}"
+            SerializationHelper::Base.new(format_class.constantize).dump_to_dir dump_dir("/#{dir}")
+		end
+
+		desc "Load contents of db/data.extension (defaults to yaml) into database"
+		task :load => :environment do
+            format_class = ENV['class'] || "YamlDb::Helper"
+            helper = format_class.constantize
+			SerializationHelper::Base.new(helper).load (db_dump_data_file helper.extension)
+		end
+
+		desc "Load contents of db/data_dir into database"
+		task :load_dir  => :environment do
+          dir = ENV['dir'] || "base"
+          format_class = ENV['class'] || "YamlDb::Helper"
+	      SerializationHelper::Base.new(format_class.constantize).load_from_dir dump_dir("/#{dir}")
+		end
+	end
+end

data/ydd.gemspec

@@ -0,0 +1,64 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{ydd}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Adam Wiggins", "Orion Henry", "Darcy Laycock"]
+  s.date = %q{2010-09-18}
+  s.default_executable = %q{ydd}
+  s.description = %q{
+YDD is a tool (a fork of yaml_db really) to make it generally easy
+to use it for things like copying databases from production to staging
+and the like.
+}
+  s.email = %q{sutto@sutto.net}
+  s.executables = ["ydd"]
+  s.extra_rdoc_files = [
+    "README.markdown"
+  ]
+  s.files = [
+    ".rvmrc",
+     "README.markdown",
+     "Rakefile",
+     "VERSION",
+     "bin/ydd",
+     "lib/ydd.rb",
+     "lib/ydd/application.rb",
+     "lib/ydd/data_manager.rb",
+     "lib/ydd/schema_manager.rb",
+     "lib/ydd/serialization_helper.rb",
+     "lib/ydd/yaml_db.rb",
+     "tasks/yaml_db_tasks.rake",
+     "ydd.gemspec"
+  ]
+  s.homepage = %q{http://github.com/YouthTree/ydd}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{ydd dumps / loads rails app databases for backup purposes.}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<thor>, ["~> 0.14"])
+      s.add_runtime_dependency(%q<activesupport>, ["~> 3.0.0"])
+      s.add_runtime_dependency(%q<activerecord>, ["~> 3.0.0"])
+    else
+      s.add_dependency(%q<thor>, ["~> 0.14"])
+      s.add_dependency(%q<activesupport>, ["~> 3.0.0"])
+      s.add_dependency(%q<activerecord>, ["~> 3.0.0"])
+    end
+  else
+    s.add_dependency(%q<thor>, ["~> 0.14"])
+    s.add_dependency(%q<activesupport>, ["~> 3.0.0"])
+    s.add_dependency(%q<activerecord>, ["~> 3.0.0"])
+  end
+end
+

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification 
+name: ydd
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Adam Wiggins
+- Orion Henry
+- Darcy Laycock
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-09-18 00:00:00 +08:00
+default_executable: ydd
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: thor
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 0
+        - 14
+        version: "0.14"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 3
+        - 0
+        - 0
+        version: 3.0.0
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 3
+        - 0
+        - 0
+        version: 3.0.0
+  type: :runtime
+  version_requirements: *id003
+description: |
+  
+  YDD is a tool (a fork of yaml_db really) to make it generally easy
+  to use it for things like copying databases from production to staging
+  and the like.
+
+email: sutto@sutto.net
+executables: 
+- ydd
+extensions: []
+
+extra_rdoc_files: 
+- README.markdown
+files: 
+- .rvmrc
+- README.markdown
+- Rakefile
+- VERSION
+- bin/ydd
+- lib/ydd.rb
+- lib/ydd/application.rb
+- lib/ydd/data_manager.rb
+- lib/ydd/schema_manager.rb
+- lib/ydd/serialization_helper.rb
+- lib/ydd/yaml_db.rb
+- tasks/yaml_db_tasks.rake
+- ydd.gemspec
+has_rdoc: true
+homepage: http://github.com/YouthTree/ydd
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: ydd dumps / loads rails app databases for backup purposes.
+test_files: []
+