ey_snaplock 0.1.0 → 0.1.1.pre

data/lib/ey_snaplock/database/mysql.rb

@@ -20,6 +20,22 @@ module EY
           release_lock
         end
 
+        def has_long_running_queries?(cutoff_in_seconds)
+          %x<#{@mysql} -B -N -e 'show processlist;'>.split("\n").each do |process_entry|
+            id, user, host, db, command, time, state, info = process_entry.split("\t")
+
+            next if %w{root system}.include? user
+            next if ['Sleep', 'Binlog Dump'].include? command
+
+            return true if time.to_i > cutoff_in_seconds
+          end
+          false
+        end
+
+        def dump_process_list(logfile)
+          system("#{@mysql} -N -e 'show processlist;' >> #{logfile}")
+        end
+
         private
 
         def write_master_status(file)

data/lib/ey_snaplock/version.rb

@@ -1,5 +1,5 @@
 module EY
   class Snaplock
-    VERSION = '0.1.0'
+    VERSION = '0.1.1.pre'
   end
 end

data/lib/ey_snaplock.rb

@@ -7,6 +7,7 @@ module EY
   class Snaplock
     REQUEST_TIMEOUT = 15
     PER_DATABASE_TIMEOUT = 5
+    LONG_RUNNING_QUERY_RETRIES = 3
 
     def self.call(*argv)
       new(*argv).call
@@ -70,6 +71,25 @@ module EY
     def database_lock(database_uri, timeout, &block)
       database = Database.for(database_uri)
       with_locked_db = block
+
+      if database.respond_to?(:has_long_running_queries?)
+        log_file = '/var/log/ey-snaplock.' + Time.now.strftime("%Y-%m-%dT%H-%M-%S") + '.log'
+        remaining_retries = LONG_RUNNING_QUERY_RETRIES
+
+        while database.has_long_running_queries?(PER_DATABASE_TIMEOUT) && remaining_retries >= 0
+          if remaining_retries == 0
+            database.dump_process_list(log_file)
+            clean_up_old_log_files
+            $stderr.puts "Aborting Snaplock due to long running queries.  Process list dumped to #{log_file}."
+            exit 1
+          else
+            remaining_retries -= 1
+            sleep PER_DATABASE_TIMEOUT
+          end
+        end
+
+      end
+
       lambda do
         lock_filename = database.lock_filename
         file = File.open(lock_filename, File::RDWR|File::EXCL|File::CREAT) rescue nil
@@ -90,6 +110,11 @@ module EY
       end
     end
 
+    def clean_up_old_log_files(logs_to_keep = 10)
+      ordinal = logs_to_keep.succ.to_s
+      system("ls -r /var/log/ey-snaplock.*.log | tail -n +#{ordinal} | xargs -I@ rm @")
+    end
+
     private
 
     def post_path_for_uri(uri)

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: ey_snaplock
 version: !ruby/object:Gem::Version
-  version: 0.1.0
-  prerelease: 
+  version: 0.1.1.pre
+  prerelease: 6
 platform: ruby
 authors:
 - Engine Yard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-17 00:00:00.000000000 Z
+date: 2012-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -138,12 +138,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Server side components for Engine Yard's snapshotting process