pt-osc 0.0.1

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pt-osc.gemspec
+gemspec

data/README.md

@@ -0,0 +1,35 @@
+## `pt-online-schema-change` migrations
+
+Runs regular Rails/ActiveRecord migrations via the [Percona Toolkit pt-online-schema-change tool](http://www.percona.com/doc/percona-toolkit/2.1/pt-online-schema-change.html).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'pt-osc'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pt-osc
+
+## Usage
+
+Set your database adapter to be `pt_osc` in your application's database.yml.
+Specify `pt-online-schema-change` flags in a `percona` hash in the config.
+e.g.
+```yaml
+environment:
+  host: localhost
+  username: root
+  database: rails
+  percona:
+    defaults-file: /etc/mysql/percona-user.cnf
+    recursion-method: "'dsn=D=percona,t=slaves'"
+```
+
+Additional options for the `percona` hash include:
+  - `run_mode`: Specify `'execute'` to actually run `pt-online-schema-change` when the migration runs. Specify `'print'` to output the commands to run to STDOUT instead. Default is `'print'`.

data/Rakefile

@@ -0,0 +1,20 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs.push 'test'
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+desc 'Run tests'
+task :default => :test
+
+load 'active_record/railties/databases.rake'
+
+namespace :db do
+  task :test_create do
+    test_spec = YAML.load_file('./test/config/database.yml')['test']
+    test_spec['adapter'] = 'mysql2'
+    create_database(test_spec)
+  end
+end

data/lib/active_record/connection_adapters/pt_osc_adapter.rb

@@ -0,0 +1,134 @@
+require 'active_record/connection_adapters/mysql2_adapter'
+
+module ActiveRecord
+  class Base
+    # Establishes a connection to the database that's used by all Active Record objects.
+    def self.pt_osc_connection(config)
+      config[:username] = 'root' if config[:username].nil?
+
+      if Mysql2::Client.const_defined? :FOUND_ROWS
+        config[:flags] = Mysql2::Client::FOUND_ROWS
+      end
+
+      client = Mysql2::Client.new(config.symbolize_keys)
+      options = [config[:host], config[:username], config[:password], config[:database], config[:port], config[:socket], 0]
+      ConnectionAdapters::PtOscAdapter.new(client, logger, options, config)
+    end
+  end
+
+  module ConnectionAdapters
+    class PtOscAdapter < Mysql2Adapter
+      ADAPTER_NAME = 'pt-osc'
+
+      # Renames a table.
+      #
+      # Example:
+      #   rename_table('octopuses', 'octopi')
+      def rename_table(table_name, new_name)
+        add_command(table_name, "RENAME TO #{quote_table_name(new_name)}")
+      end
+
+      def add_column(table_name, column_name, type, options = {})
+        add_command(table_name, add_column_sql(table_name, column_name, type, options))
+      end
+
+      def change_column(table_name, column_name, type, options = {}) #:nodoc:
+        add_command(table_name, change_column_sql(table_name, column_name, type, options))
+      end
+
+      def rename_column(table_name, column_name, new_column_name) #:nodoc:
+        add_command(table_name, rename_column_sql(table_name, column_name, new_column_name))
+      end
+
+      # Removes the column(s) from the table definition.
+      # ===== Examples
+      #  remove_column(:suppliers, :qualification)
+      #  remove_columns(:suppliers, :qualification, :experience)
+      def remove_column(table_name, *column_names)
+        if column_names.flatten!
+          message = 'Passing array to remove_columns is deprecated, please use ' +
+            'multiple arguments, like: `remove_columns(:posts, :foo, :bar)`'
+          ActiveSupport::Deprecation.warn message, caller
+        end
+
+        columns_for_remove(table_name, *column_names).each do |column_name|
+          add_command(table_name, "DROP COLUMN #{column_name}")
+        end
+      end
+
+      # Adds a new index to the table. +column_name+ can be a single Symbol, or
+      # an Array of Symbols.
+      #
+      # The index will be named after the table and the column name(s), unless
+      # you pass <tt>:name</tt> as an option.
+      #
+      # ===== Examples
+      #
+      # ====== Creating a simple index
+      #  add_index(:suppliers, :name)
+      # generates
+      #  CREATE INDEX suppliers_name_index ON suppliers(name)
+      #
+      # ====== Creating a unique index
+      #  add_index(:accounts, [:branch_id, :party_id], :unique => true)
+      # generates
+      #  CREATE UNIQUE INDEX accounts_branch_id_party_id_index ON accounts(branch_id, party_id)
+      #
+      # ====== Creating a named index
+      #  add_index(:accounts, [:branch_id, :party_id], :unique => true, :name => 'by_branch_party')
+      # generates
+      #  CREATE UNIQUE INDEX by_branch_party ON accounts(branch_id, party_id)
+      #
+      # ====== Creating an index with specific key length
+      #  add_index(:accounts, :name, :name => 'by_name', :length => 10)
+      # generates
+      #  CREATE INDEX by_name ON accounts(name(10))
+      #
+      #  add_index(:accounts, [:name, :surname], :name => 'by_name_surname', :length => {:name => 10, :surname => 15})
+      # generates
+      #  CREATE INDEX by_name_surname ON accounts(name(10), surname(15))
+      #
+      # ====== Creating an index with a sort order (desc or asc, asc is the default)
+      #  add_index(:accounts, [:branch_id, :party_id, :surname], :order => {:branch_id => :desc, :part_id => :asc})
+      # generates
+      #  CREATE INDEX by_branch_desc_party ON accounts(branch_id DESC, party_id ASC, surname)
+      #
+      # Note: mysql doesn't yet support index order (it accepts the syntax but ignores it)
+      #
+      def add_index(table_name, column_name, options = {})
+        index_name, index_type, index_columns = add_index_options(table_name, column_name, options)
+        add_command(table_name, "ADD #{index_type} INDEX #{quote_column_name(index_name)} (#{index_columns})")
+      end
+
+      def remove_index!(table_name, index_name) #:nodoc:
+        add_command(table_name, "DROP INDEX #{quote_column_name(index_name)}")
+      end
+
+      def clear_commands
+        @osc_commands = {}
+      end
+
+      def get_commands_string(table_name)
+        @osc_commands ||= {}
+        @osc_commands[table_name] ||= []
+        @osc_commands[table_name].join(',')
+      end
+
+      def get_commanded_tables
+        @osc_commands.keys
+      end
+
+      protected
+      def add_command(table_name, command)
+        @osc_commands ||= {}
+        @osc_commands[table_name] ||= []
+        @osc_commands[table_name] << command
+      end
+
+      def get_commands(table_name)
+        @osc_commands ||= {}
+        @osc_commands[table_name]
+      end
+    end
+  end
+end

data/lib/active_record/pt_osc_migration.rb

@@ -0,0 +1,150 @@
+module ActiveRecord
+  class PtOscMigration < Migration
+    # @TODO whitelist all valid pt-osc flags
+    DEFAULT_FLAGS = {
+      'defaults-file' => nil,
+      'recursion-method' => nil,
+      'execute' => false,
+    }.freeze
+
+    def migrate(direction)
+      return unless respond_to?(direction)
+
+      run_mode = percona_config[:run_mode] || 'print'
+      raise ArgumentError.new('Invalid run_mode specified in database config') unless run_mode.in? %w(print execute)
+
+      case direction
+        when :up   then announce 'migrating'
+        when :down then announce 'reverting'
+      end
+
+      time   = nil
+      ActiveRecord::Base.connection_pool.with_connection do |conn|
+        @connection = conn
+        if respond_to?(:change)
+          if direction == :down
+            recorder = CommandRecorder.new(@connection)
+            suppress_messages do
+              @connection = recorder
+              change
+            end
+            @connection = conn
+            time = Benchmark.measure {
+              self.revert {
+                recorder.inverse.each do |cmd, args|
+                  send(cmd, *args)
+                end
+              }
+            }
+          else
+            time = Benchmark.measure { change }
+          end
+        else
+          time = Benchmark.measure { send(direction) }
+        end
+
+        case run_mode
+          when 'execute' then time += Benchmark.measure { execute_pt_osc }
+          when 'print' then print_pt_osc
+        end
+
+        @connection = nil
+      end
+
+      case direction
+        when :up   then announce 'migrated (%.4fs)' % time.real; write
+        when :down then announce 'reverted (%.4fs)' % time.real; write
+      end
+    end
+
+    protected
+    def execute_pt_osc
+      return unless @connection.is_a? ActiveRecord::ConnectionAdapters::PtOscAdapter
+
+      @connection.get_commanded_tables.each do |table_name|
+        execute_sql = @connection.get_commands_string(table_name)
+
+        Rails.logger.tagged('pt-osc') do |logger|
+
+          database_name = database_config[:database]
+
+          logger.info "Running on #{database_name}|#{table_name}: #{execute_sql}"
+          announce 'running pt-online-schema-change'
+
+          [true, false].each do |dry_run|
+            command = percona_command(execute_sql, database_name, table_name, execute: !dry_run)
+            logger.info "Command is #{command}"
+            success = Kernel.system command
+            if success
+              logger.info "Successfully #{dry_run ? 'dry ran' : 'executed'} on #{database_name}|#{table_name}: #{execute_sql}"
+            else
+              failure_message = "Unable to #{dry_run ? 'dry run' : 'execute'} query on #{database_name}|#{table_name}: #{execute_sql}"
+              logger.error failure_message
+              raise RuntimeError.new(failure_message)
+            end
+          end
+        end
+      end
+
+      @connection.clear_commands
+    end
+
+    def print_pt_osc
+      return unless @connection.is_a? ActiveRecord::ConnectionAdapters::PtOscAdapter
+
+      database_name = database_config[:database]
+
+      @connection.get_commanded_tables.each do |table_name|
+        execute_sql = @connection.get_commands_string(table_name)
+
+        announce 'Run the following commands:'
+
+        [true, false].each do |dry_run|
+          write percona_command(execute_sql, database_name, table_name, execute: !dry_run)
+        end
+
+      end
+
+      @connection.clear_commands
+    end
+
+    def percona_command(execute_sql, database_name, table_name, options = {})
+      command = "pt-online-schema-change --alter '#{execute_sql}' D=#{database_name},t=#{table_name}"
+
+      # Whitelist
+      options = HashWithIndifferentAccess.new(options)
+      options = options.slice(*DEFAULT_FLAGS.keys)
+
+      # Merge config
+      config = percona_config
+      if config
+        config.slice(*DEFAULT_FLAGS.keys).each do |key, value|
+          options[key] ||= value
+        end
+      end
+
+      # Set defaults
+      DEFAULT_FLAGS.each do |key, value|
+        options[key] ||= value unless value.nil?
+      end
+
+      # Determine run mode
+      command += options.delete(:execute) ? ' --execute' : ' --dry-run'
+
+      options.each do |key, value|
+        command += " --#{key} #{value}"
+      end
+
+      command
+    end
+
+    def database_config
+      # @TODO better way to config?
+      @connection.instance_variable_get(:@config) || ActiveRecord::Base.connection_config
+    end
+
+    def percona_config
+      database_config[:percona]
+    end
+  end
+end

data/lib/pt-osc.rb

@@ -0,0 +1,3 @@
+require 'pt-osc/version'
+require 'active_record/connection_adapters/pt_osc_adapter'
+require 'active_record/pt_osc_migration'

data/lib/pt-osc/version.rb

@@ -0,0 +1,5 @@
+module Pt
+  module Osc
+    VERSION = '0.0.1'
+  end
+end

data/pt-osc.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pt-osc/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'pt-osc'
+  spec.version       = Pt::Osc::VERSION
+  spec.authors       = ['Steve Rice']
+  spec.email         = ['steve@pagerduty.com']
+  spec.summary       = 'Rails migrations via pt-online-schema-change'
+  spec.description   = 'Runs regular Rails/ActiveRecord migrations via the Percona Toolkit pt-online-schema-change tool.'
+  spec.homepage      = 'https://github.com/steverice/pt-osc'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'shoulda'
+  spec.add_development_dependency 'faker'
+  spec.add_development_dependency 'mocha'
+
+  # For testing using dummy Rails app
+  spec.add_development_dependency 'rails', '~> 3.2'
+  spec.add_development_dependency 'sqlite3'
+
+  spec.add_runtime_dependency 'activerecord', '~> 3.2'
+  spec.add_runtime_dependency 'mysql2'
+end

data/test/config/database.yml

@@ -0,0 +1,6 @@
+test:
+  pool: 5
+  host: localhost
+  username: root
+  database: pt_osc_test
+  reconnect: true

data/test/dummy/.gitignore

@@ -0,0 +1 @@
+log/*

data/test/dummy/README.rdoc

@@ -0,0 +1,261 @@
+== Welcome to Rails
+
+Rails is a web-application framework that includes everything needed to create
+database-backed web applications according to the Model-View-Control pattern.
+
+This pattern splits the view (also called the presentation) into "dumb"
+templates that are primarily responsible for inserting pre-built data in between
+HTML tags. The model contains the "smart" domain objects (such as Account,
+Product, Person, Post) that holds all the business logic and knows how to
+persist themselves to a database. The controller handles the incoming requests
+(such as Save New Account, Update Product, Show Post) by manipulating the model
+and directing data to the view.
+
+In Rails, the model is handled by what's called an object-relational mapping
+layer entitled Active Record. This layer allows you to present the data from
+database rows as objects and embellish these data objects with business logic
+methods. You can read more about Active Record in
+link:files/vendor/rails/activerecord/README.html.
+
+The controller and view are handled by the Action Pack, which handles both
+layers by its two parts: Action View and Action Controller. These two layers
+are bundled in a single package due to their heavy interdependence. This is
+unlike the relationship between the Active Record and Action Pack that is much
+more separate. Each of these packages can be used independently outside of
+Rails. You can read more about Action Pack in
+link:files/vendor/rails/actionpack/README.html.
+
+
+== Getting Started
+
+1. At the command prompt, create a new Rails application:
+       <tt>rails new myapp</tt> (where <tt>myapp</tt> is the application name)
+
+2. Change directory to <tt>myapp</tt> and start the web server:
+       <tt>cd myapp; rails server</tt> (run with --help for options)
+
+3. Go to http://localhost:3000/ and you'll see:
+       "Welcome aboard: You're riding Ruby on Rails!"
+
+4. Follow the guidelines to start developing your application. You can find
+the following resources handy:
+
+* The Getting Started Guide: http://guides.rubyonrails.org/getting_started.html
+* Ruby on Rails Tutorial Book: http://www.railstutorial.org/
+
+
+== Debugging Rails
+
+Sometimes your application goes wrong. Fortunately there are a lot of tools that
+will help you debug it and get it back on the rails.
+
+First area to check is the application log files. Have "tail -f" commands
+running on the server.log and development.log. Rails will automatically display
+debugging and runtime information to these files. Debugging info will also be
+shown in the browser on requests from 127.0.0.1.
+
+You can also log your own messages directly into the log file from your code
+using the Ruby logger class from inside your controllers. Example:
+
+  class WeblogController < ActionController::Base
+    def destroy
+      @weblog = Weblog.find(params[:id])
+      @weblog.destroy
+      logger.info("#{Time.now} Destroyed Weblog ID ##{@weblog.id}!")
+    end
+  end
+
+The result will be a message in your log file along the lines of:
+
+  Mon Oct 08 14:22:29 +1000 2007 Destroyed Weblog ID #1!
+
+More information on how to use the logger is at http://www.ruby-doc.org/core/
+
+Also, Ruby documentation can be found at http://www.ruby-lang.org/. There are
+several books available online as well:
+
+* Programming Ruby: http://www.ruby-doc.org/docs/ProgrammingRuby/ (Pickaxe)
+* Learn to Program: http://pine.fm/LearnToProgram/ (a beginners guide)
+
+These two books will bring you up to speed on the Ruby language and also on
+programming in general.
+
+
+== Debugger
+
+Debugger support is available through the debugger command when you start your
+Mongrel or WEBrick server with --debugger. This means that you can break out of
+execution at any point in the code, investigate and change the model, and then,
+resume execution! You need to install ruby-debug to run the server in debugging
+mode. With gems, use <tt>sudo gem install ruby-debug</tt>. Example:
+
+  class WeblogController < ActionController::Base
+    def index
+      @posts = Post.all
+      debugger
+    end
+  end
+
+So the controller will accept the action, run the first line, then present you
+with a IRB prompt in the server window. Here you can do things like:
+
+  >> @posts.inspect
+  => "[#<Post:0x14a6be8
+          @attributes={"title"=>nil, "body"=>nil, "id"=>"1"}>,
+       #<Post:0x14a6620
+          @attributes={"title"=>"Rails", "body"=>"Only ten..", "id"=>"2"}>]"
+  >> @posts.first.title = "hello from a debugger"
+  => "hello from a debugger"
+
+...and even better, you can examine how your runtime objects actually work:
+
+  >> f = @posts.first
+  => #<Post:0x13630c4 @attributes={"title"=>nil, "body"=>nil, "id"=>"1"}>
+  >> f.
+  Display all 152 possibilities? (y or n)
+
+Finally, when you're ready to resume execution, you can enter "cont".
+
+
+== Console
+
+The console is a Ruby shell, which allows you to interact with your
+application's domain model. Here you'll have all parts of the application
+configured, just like it is when the application is running. You can inspect
+domain models, change values, and save to the database. Starting the script
+without arguments will launch it in the development environment.
+
+To start the console, run <tt>rails console</tt> from the application
+directory.
+
+Options:
+
+* Passing the <tt>-s, --sandbox</tt> argument will rollback any modifications
+  made to the database.
+* Passing an environment name as an argument will load the corresponding
+  environment. Example: <tt>rails console production</tt>.
+
+To reload your controllers and models after launching the console run
+<tt>reload!</tt>
+
+More information about irb can be found at:
+link:http://www.rubycentral.org/pickaxe/irb.html
+
+
+== dbconsole
+
+You can go to the command line of your database directly through <tt>rails
+dbconsole</tt>. You would be connected to the database with the credentials
+defined in database.yml. Starting the script without arguments will connect you
+to the development database. Passing an argument will connect you to a different
+database, like <tt>rails dbconsole production</tt>. Currently works for MySQL,
+PostgreSQL and SQLite 3.
+
+== Description of Contents
+
+The default directory structure of a generated Ruby on Rails application:
+
+  |-- app
+  |   |-- assets
+  |   |   |-- images
+  |   |   |-- javascripts
+  |   |   `-- stylesheets
+  |   |-- controllers
+  |   |-- helpers
+  |   |-- mailers
+  |   |-- models
+  |   `-- views
+  |       `-- layouts
+  |-- config
+  |   |-- environments
+  |   |-- initializers
+  |   `-- locales
+  |-- db
+  |-- doc
+  |-- lib
+  |   |-- assets
+  |   `-- tasks
+  |-- log
+  |-- public
+  |-- script
+  |-- test
+  |   |-- fixtures
+  |   |-- functional
+  |   |-- integration
+  |   |-- performance
+  |   `-- unit
+  |-- tmp
+  |   `-- cache
+  |       `-- assets
+  `-- vendor
+      |-- assets
+      |   |-- javascripts
+      |   `-- stylesheets
+      `-- plugins
+
+app
+  Holds all the code that's specific to this particular application.
+
+app/assets
+  Contains subdirectories for images, stylesheets, and JavaScript files.
+
+app/controllers
+  Holds controllers that should be named like weblogs_controller.rb for
+  automated URL mapping. All controllers should descend from
+  ApplicationController which itself descends from ActionController::Base.
+
+app/models
+  Holds models that should be named like post.rb. Models descend from
+  ActiveRecord::Base by default.
+
+app/views
+  Holds the template files for the view that should be named like
+  weblogs/index.html.erb for the WeblogsController#index action. All views use
+  eRuby syntax by default.
+
+app/views/layouts
+  Holds the template files for layouts to be used with views. This models the
+  common header/footer method of wrapping views. In your views, define a layout
+  using the <tt>layout :default</tt> and create a file named default.html.erb.
+  Inside default.html.erb, call <% yield %> to render the view using this
+  layout.
+
+app/helpers
+  Holds view helpers that should be named like weblogs_helper.rb. These are
+  generated for you automatically when using generators for controllers.
+  Helpers can be used to wrap functionality for your views into methods.
+
+config
+  Configuration files for the Rails environment, the routing map, the database,
+  and other dependencies.
+
+db
+  Contains the database schema in schema.rb. db/migrate contains all the
+  sequence of Migrations for your schema.
+
+doc
+  This directory is where your application documentation will be stored when
+  generated using <tt>rake doc:app</tt>
+
+lib
+  Application specific libraries. Basically, any kind of custom code that
+  doesn't belong under controllers, models, or helpers. This directory is in
+  the load path.
+
+public
+  The directory available for the web server. Also contains the dispatchers and the
+  default HTML files. This should be set as the DOCUMENT_ROOT of your web
+  server.
+
+script
+  Helper scripts for automation and generation.
+
+test
+  Unit and functional tests along with fixtures. When using the rails generate
+  command, template test files will be generated for you and placed in this
+  directory.
+
+vendor
+  External libraries that the application depends on. Also includes the plugins
+  subdirectory. If the app has frozen rails, those gems also go here, under
+  vendor/rails/. This directory is in the load path.