ey_snaplock 0.0.8

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Engine Yard Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,21 @@
+EY Snaplock
+===========
+
+almost known as ey-snapshot-really
+
+lock the filesystem and almost really run the snapshot.
+
+Postgres testing notes,
+
+wal_level = hot_standby
+archive_mode = on
+archive_command = '/usr/bin/true'
+
+![EY Snaplock](http://img825.imageshack.us/img825/4323/snaplock.jpg "EY Snaplock")
+
+Release process:
+  edit version file
+  gem build ey_snaplock.gemspec
+  ey-gem upload ey_snaplock-0.0.5.gem --server public
+  edit version file again
+  commit

data/bin/ey-snaplock

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'ey_snaplock'
+
+EY::Snaplock.call(*ARGV)

data/lib/ey_snaplock/database/mysql.rb

@@ -0,0 +1,66 @@
+module EY
+  class Snaplock
+    class Database
+      class MySQL
+        def initialize(uri)
+          @mysql = mysql_command(uri)
+        end
+
+        def with_lock(timeout)
+          aquire_lock_within_timeout(timeout)
+          write_master_status(ENV["MASTER_STATUS_FILE"] || "/db/mysql/.snapshot_backup_master_status.txt")
+          yield
+        rescue Timeout::Error
+          lock_timeout(timeout)
+        ensure
+          release_lock
+        end
+
+        private
+
+        def write_master_status(file)
+          master_status_cmd = "#{@mysql} -e'SHOW MASTER STATUS\\G' > #{file}"
+          system(master_status_cmd)
+        end
+
+        def aquire_lock_within_timeout(timeout)
+          pipe = IO.popen(@mysql, 'w')
+          @read_lock_pid = pipe.pid
+          pipe.puts('flush tables with read lock;')
+
+          SystemTimer.timeout_after(timeout) do
+            until locked?
+              sleep 1
+            end
+          end
+        end
+
+        def locked?
+          waiting_read_lock_thread_id == ''
+        end
+
+        def waiting_read_lock_thread_id
+          %x<#{@mysql} -N -e 'show full processlist;' | grep 'flush tables with read lock' | awk '{print $1}'>
+        end
+
+        def lock_timeout(timeout)
+          thread_id = waiting_read_lock_thread_id
+          system("#{@mysql} -e'kill #{thread_id};'") unless thread_id == ''
+          abort "Read lock not acquired after #{timeout} second timeout.  Killed request and aborting backup."
+        end
+
+        def release_lock
+          Process.kill('TERM', @read_lock_pid) # unlock tables
+        end
+
+        def mysql_command(uri)
+          command = "mysql"
+          command << " -u" << (uri.user || 'root')
+          command << " -p" <<  uri.password        if uri.password
+          command << " -h" <<  uri.host            if uri.host
+          command
+        end
+      end
+    end
+  end
+end

data/lib/ey_snaplock/database/postgresql9.rb

@@ -0,0 +1,46 @@
+module EY
+  class Snaplock
+    class Database
+      class Postgresql
+        # Creates an object called @postgresql for usage with IO.popen.
+        def initialize(uri)
+          @postgresql = postgresql_command(uri)
+        end
+
+        # Default function being called (e.g. database.with_lock)
+        def with_lock(timeout) #timeout is ignored
+          aquire_lock
+          yield
+        ensure
+          release_lock
+        end
+
+        # http://wiki.postgresql.org/wiki/Hot_Standby
+        # We don't need to acquire a read lock for the snapshot.  However we do need to create an backup of the base database and force a checkpoint to ensure the data is written so when the slave comes up it has enough data to start.
+        def aquire_lock
+          pipe = IO.popen(@postgresql, 'w')
+          @read_lock_pid = pipe.pid
+          pipe.puts("select pg_start_backup('backup',true);")
+          sleep 1
+          pipe.close
+        end
+
+        # The backup should have been started to finish this and actually force the checkpoint we need to stop the backup.  This completes the only action that PostgreSQL should require for a hot_standby.
+        def release_lock
+          pipe2 = IO.popen(@postgresql, 'w')
+          @read_lock_pid2 = pipe2.pid
+          pipe2.puts("select pg_stop_backup();")
+          sleep 1
+          pipe2.close
+        end
+
+        # Construct to help prepare the command.
+        def postgresql_command(uri)
+          command = "psql"
+          command << " -U" << (uri.user || 'postgres')
+          command
+        end
+      end
+    end
+  end
+end

data/lib/ey_snaplock/database.rb

@@ -0,0 +1,33 @@
+require 'addressable/uri'
+require 'ey_snaplock/database/mysql'
+require 'ey_snaplock/database/postgresql9'
+module EY
+  class Snaplock
+    class Database
+      def self.for(database_uri)
+        uri = parse_uri(database_uri)
+        # Determine what kind of database we need to snaplock, current two supported schemes are:
+        # mysql://root:password@localhost/
+        # postgres9:://postgres:localhost/
+        case uri.scheme
+        when 'mysql'
+          MySQL.new(uri)
+        when "postgres9"
+          Postgresql.new(uri)
+          # Unknown url schema provided, print the error message and exit with an exit code of 1
+        else
+          abort "Don't know how to lock database with URI: #{database_uri.inspect}"
+        end
+      end
+
+      def self.parse_uri(uri)
+        begin
+          Addressable::URI.parse(uri)
+        rescue TypeError, Addressable::URI::InvalidURIError
+          $stderr.puts "Error parsing database URI: #{uri.inspect}"
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/ey_snaplock/version.rb

@@ -0,0 +1,5 @@
+module EY
+  class Snaplock
+    VERSION = '0.0.8'
+  end
+end

data/lib/ey_snaplock.rb

@@ -0,0 +1,84 @@
+require 'net/http'
+require 'net/https'
+require 'ey_snaplock/database'
+require 'system_timer'
+
+module EY
+  class Snaplock
+    REQUEST_TIMEOUT = 15
+    PER_DATABASE_TIMEOUT = 5
+
+    def self.call(*argv)
+      new(*argv).call
+      exit 0
+    end
+
+    def initialize(callback_uri = nil, *database_uris)
+      if callback_uri.nil?
+        $stderr.puts "Usage: ey-snaplock CALLBACK_URI [DATABASE_URI] [DATABASE_URI] ..."
+        abort "Error: CALLBACK_URI is required."
+      end
+
+      callback = request(callback_uri)
+      callback = timeout(&callback)
+      callback = sync_filesystem(&callback)
+
+      database_uris.each do |database_uri| # it'd be nice to parallelize these database locks when we have more than one
+        callback = database_lock(database_uri, PER_DATABASE_TIMEOUT, &callback)
+      end
+      @callback = callback
+    end
+
+    def call
+      success, code, body = @callback.call
+      if success
+        exit
+      else
+        abort "Error: Received unsuccessful response #{code} from callback:\n#{body}"
+      end
+    end
+
+    def request(uri)
+      uri = URI.parse(uri)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true if uri.scheme == "https"
+
+      lambda do
+        response = http.request_post(post_path_for_uri(uri), "", "Content-Type" => "application/x-www-form-urlencoded")
+        [(200...300).include?(response.code.to_i), response.code.to_i, response.body]
+      end
+    end
+
+    def timeout
+      lambda do
+        begin
+          SystemTimer.timeout_after(REQUEST_TIMEOUT) { yield }
+        rescue Timeout::Error
+          $stderr.puts "Timeout Exceeded: Callback request took longer than #{REQUEST_TIMEOUT} seconds."
+          raise
+        end
+      end
+    end
+
+    def sync_filesystem(&block)
+      lambda do
+        `sync && sync && sync`
+        yield
+      end
+    end
+
+    def database_lock(database_uri, timeout, &block)
+      database = Database.for(database_uri)
+      lambda do
+        database.with_lock(timeout, &block)
+      end
+    end
+
+    private
+
+    def post_path_for_uri(uri)
+      [uri.path, uri.query].reject{ |str| str.nil?}.join("?")
+    end
+
+  end
+end

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification 
+name: ey_snaplock
+version: !ruby/object:Gem::Version 
+  hash: 15
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 8
+  version: 0.0.8
+platform: ruby
+authors: 
+- Engine Yard
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-11-29 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: addressable
+  type: :runtime
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 2
+        - 2
+        - 2
+        version: 2.2.2
+  prerelease: false
+  requirement: *id001
+- !ruby/object:Gem::Dependency 
+  name: SystemTimer
+  type: :runtime
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 1
+        - 2
+        - 1
+        version: 1.2.1
+  prerelease: false
+  requirement: *id002
+- !ruby/object:Gem::Dependency 
+  name: rake
+  type: :development
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 49
+        segments: 
+        - 0
+        - 8
+        - 7
+        version: 0.8.7
+  prerelease: false
+  requirement: *id003
+- !ruby/object:Gem::Dependency 
+  name: realweb
+  type: :development
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 21
+        segments: 
+        - 0
+        - 2
+        - 1
+        version: 0.2.1
+  prerelease: false
+  requirement: *id004
+- !ruby/object:Gem::Dependency 
+  name: cucumber
+  type: :development
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 55
+        segments: 
+        - 0
+        - 10
+        - 0
+        version: 0.10.0
+  prerelease: false
+  requirement: *id005
+- !ruby/object:Gem::Dependency 
+  name: aruba
+  type: :development
+  version_requirements: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 0
+        - 3
+        version: "0.3"
+  prerelease: false
+  requirement: *id006
+- !ruby/object:Gem::Dependency 
+  name: sinatra
+  type: :development
+  version_requirements: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  prerelease: false
+  requirement: *id007
+description: Server side components for Engine Yard's snapshotting process
+email: 
+executables: 
+- ey-snaplock
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/ey-snaplock
+- lib/ey_snaplock/database/mysql.rb
+- lib/ey_snaplock/database/postgresql9.rb
+- lib/ey_snaplock/database.rb
+- lib/ey_snaplock/version.rb
+- lib/ey_snaplock.rb
+- LICENSE
+- README.md
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Server side components for Engine Yard's snapshotting process
+test_files: []
+