daemon_runner 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d36f55746d2502664fad4104b312003f1b473b95
-  data.tar.gz: 6729f091a55d2ee3ebee5d0e72788589b88367c0
+  metadata.gz: 53c26321dab0ca599634cdf709c5fbed7f0d7567
+  data.tar.gz: 054a65b2f31eac817e24fa25d9ad068e516efa1a
 SHA512:
-  metadata.gz: 6eaa76309f8b31a68ceb7ce508e7047c050fee14215f7330c28bf551f88c190bbcd37f3106aea7f595178441ab8560a5dcbffe852bf8a62fa0a75b515826a0d3
-  data.tar.gz: af1b9a90f5b429831c47b6df4091e175c493137d01b147bc9e43e1fce25e136167b0e64dade105b0884342e43ee25033ed8f8ade57fcbe22e27e4cb315a7a3b1
+  metadata.gz: cca748f13e715c7170f77f6ccf83e91c0121e6352bb029f0bbf910d57f4d6d2a2f8dfbf7170d3d381f8aff3bc52ebbd7d685acd6036fe85ff94c781d08d7ffd1
+  data.tar.gz: 6b13b67488eb4704057eed3d715d456c80876aadad2fc2cb5dd60694da6f88d85eeefbbcedf5dca87f04b631f9c94ff19e0c2c1e41bb7bb180c37b5bae4f6795

data/.gitignore

@@ -7,3 +7,6 @@
 /pkg/
 /spec/reports/
 /tmp/
+VERSION
+*.gem
+/vendor

data/Gemfile

@@ -3,6 +3,8 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in daemon_runner.gemspec
 gemspec
 
+gem 'diplomat', github: 'WeAreFarmGeek/diplomat', ref: 'e64b0dec3b3616ded40bb64139a388dfc1a68232'
+
 group :development do
   gem 'bundler', '~> 1.7'
   gem 'thor-scmversion', '= 1.7.0'

data/README.md

@@ -148,6 +148,29 @@ def schedule
 end
 ```
 
+### Retries
+Simple interface to retry requests that are known to fails sometimes.  To add a retry wrap the code like this:
+
+```
+DaemonRunner::RetryErrors.retry do
+  my_not_so_good_network_service_that_fails_sometimes
+end
+```
+
+* `options` - Options hash to pass to `retry` (**optional**)
+    * :retries - Number of times to retry an exception (**optional**, _default_: 3)
+    * :exceptions - Array of exceptions to catch and retry (**optional**, _default_: `[Faraday::ClientError]`)
+
+### Locking
+Locking can be done either via an exclusive lock or a semaphore lock.  The major difference is that with a semaphore lock you can define how many nodes can obtain the lock.
+
+#### Exclusive Lock
+**TBD**
+
+#### Semaphore Lock
+For an example of how to implement semaphore locking take a look at the [example](/examples/example_semaphore.rb)
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/daemon_runner.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "mixlib-shellout", "~> 2.2"
   spec.add_dependency "diplomat", "~> 1.0"
   spec.add_dependency "rufus-scheduler", "~> 3.2"
+  spec.add_dependency "retryable", "~> 2.0"
 
   spec.add_development_dependency "bundler", "~> 1.12"
   spec.add_development_dependency "rake", "~> 10.0"

data/examples/example1.rb

@@ -37,12 +37,17 @@ class MyService
         end
 
         def run!(args)
+          sleep 10
           name = args[0]
           reason = args[1]
           puts name
           puts reason
           name
         end
+
+        def task_id
+          'long-running-task'
+        end
       end
     end
   end
@@ -51,10 +56,13 @@ end
 class MyService
   class Tasks
     class Quiz
-      def schedule
-        [:interval, '30s']
+      def initialize
+        @task_id = 'long-running-task'
+        @schedule = [:interval, '30s']
       end
+
       def foo!(args)
+        sleep 10
         puts 'Firing error'
         sargs
       end

data/examples/example_semaphore.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/daemon_runner'
+require 'dev/consul'
+
+@service = 'myservice'
+@lock_count = 3
+@locked = false
+
+@semaphore = DaemonRunner::Semaphore.start(@service)
+DaemonRunner::Semaphore.lock(@lock_count)

data/examples/example_semaphore_release.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/daemon_runner'
+require 'dev/consul'
+
+@service = 'myreleaseservice'
+@lock_count = 3
+@locked = false
+@lock_time = 10
+
+DaemonRunner::Semaphore.lock(@service, @lock_count) do
+  @lock_time.downto(0).each do |i|
+   puts "Releasing lock in #{i} seconds"
+   sleep 1
+  end
+end

data/examples/run_consul.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'dev/consul'
+
+::Dev::Consul.run
+::Dev::Consul.wait
+::Dev::Consul.block

data/lib/daemon_runner.rb

@@ -1,5 +1,7 @@
 require_relative 'daemon_runner/version'
 require_relative 'daemon_runner/logger'
+require_relative 'daemon_runner/retry_errors'
 require_relative 'daemon_runner/client'
 require_relative 'daemon_runner/shell_out'
 require_relative 'daemon_runner/session'
+require_relative 'daemon_runner/semaphore'

data/lib/daemon_runner/client.rb

@@ -17,15 +17,22 @@ module DaemonRunner
       # @param [RuntimeError] error the body of the error
       def scheduler.on_error(job, error)
         error_sleep_time = job[:error_sleep_time]
+        on_error_release_lock = job[:on_error_release_lock]
         logger = job[:logger]
         task_id = job[:task_id]
 
+        mutex = @mutexes[task_id]
+
         logger.error "#{task_id}: #{error}"
+        logger.debug "#{task_id}: #{error.backtrace.join("\n")}"
         logger.debug "#{task_id}: Suspending #{task_id} for #{error_sleep_time} seconds"
-        job.pause
+
+        # Unlock the job mutex if the job owns it and on_error_release_lock is true
+        mutex.unlock if on_error_release_lock && mutex.owned?
+
         sleep error_sleep_time
+
         logger.debug "#{task_id}: Resuming #{task_id}"
-        job.resume
       end
     end
 
@@ -58,6 +65,16 @@ module DaemonRunner
       [:interval, loop_sleep_time]
     end
 
+    # @return [Boolean] Whether to release a mutex lock if an error occurs in a task.
+    def on_error_release_lock
+      return @on_error_release_lock unless @on_error_release_lock.nil?
+      @on_error_release_lock = if options[:on_error_release_lock].nil?
+                        true
+                      else
+                        options[:on_error_release_lock]
+                      end
+    end
+
     # @return [Fixnum] Number of seconds to sleep between loop interactions.
     def loop_sleep_time
       return @loop_sleep_time unless @loop_sleep_time.nil?
@@ -123,11 +140,14 @@ module DaemonRunner
 
       out[:method] = task[1]
 
-      out[:task_id] = if out[:instance].respond_to?(:task_id)
-        out[:instance].send(:task_id).to_s
+      if out[:instance].instance_variable_defined?(:@task_id)
+        out[:task_id] = out[:instance].instance_variable_get(:@task_id)
+      elsif out[:instance].respond_to?(:task_id)
+        out[:task_id] = out[:instance].send(:task_id).to_s
       else
-        "#{out[:class_name]}.#{out[:method]}"
+        out[:task_id] = "#{out[:class_name]}.#{out[:method]}"
       end
+
       raise ArgumentError, 'Invalid task id' if out[:task_id].nil? || out[:task_id].empty?
 
       out[:args] = task[2..-1].flatten
@@ -140,10 +160,12 @@ module DaemonRunner
     def parse_schedule(instance)
       valid_types = [:in, :at, :every, :interval, :cron]
       out = {}
-      task_schedule = if instance.respond_to?(:schedule)
-        instance.send(:schedule)
+      if instance.instance_variable_defined?(:@schedule)
+        task_schedule = instance.instance_variable_get(:@schedule)
+      elsif instance.respond_to?(:schedule)
+        task_schedule = instance.send(:schedule)
       else
-        schedule
+        task_schedule = schedule
       end
 
       raise ArgumentError, 'Malformed schedule definition, should be [TYPE, DURATION]' if task_schedule.length < 2
@@ -151,6 +173,7 @@ module DaemonRunner
 
       out[:type] = task_schedule[0].to_sym
       out[:schedule] = task_schedule[1]
+      out[:extra_opts] = task_schedule[2] if task_schedule.length > 2
       out
     end
 
@@ -171,12 +194,16 @@ module DaemonRunner
       schedule_log_line += " with schedule: #{schedule[:schedule]}"
       logger.debug schedule_log_line
 
-      scheduler.send(schedule[:type], schedule[:schedule], :overlap => false, :job => true) do |job|
+      opts = { :overlap => false, :job => true, :mutex => task_id }
+      opts.merge!(schedule[:extra_opts]) if schedule.key?(:extra_opts)
+
+      scheduler.send(schedule[:type], schedule[:schedule], opts) do |job|
         log_line = "#{task_id}: Running #{class_name}.#{method}"
         log_line += "(#{args})" unless args.empty?
         logger.debug log_line
 
         job[:error_sleep_time] = error_sleep_time
+        job[:on_error_release_lock] = on_error_release_lock
         job[:logger] = logger
         job[:task_id] = task_id
 

data/lib/daemon_runner/retry_errors.rb

@@ -0,0 +1,24 @@
+require 'retryable'
+
+module DaemonRunner
+  #
+  # Retry Errors
+  #
+  class RetryErrors
+    extend Logger
+
+    class << self
+      def retry(retries: 3, exceptions: [Faraday::ClientError], &block)
+        properties = {
+          on: exceptions,
+          sleep: lambda { |c| 2**c * 0.3 },
+          tries: retries
+        }
+        Retryable.retryable(properties) do |retries, exception|
+          logger.warn "try #{retries} failed with exception: #{exception}" if retries > 0
+          block.call
+        end
+      end
+    end
+  end
+end

data/lib/daemon_runner/semaphore.rb

@@ -0,0 +1,284 @@
+module DaemonRunner
+  #
+  # Manage semaphore locks with Consul
+  #
+  class Semaphore
+    include Logger
+
+    class << self
+      include Logger
+
+      # Acquire a lock with the current session
+      #
+      # @param limit [Integer] The number of nodes that can request the lock
+      # @see #initialize for extra `options`
+      # @return [DaemonRunner::Semaphore] instance of the semaphore class
+      #
+      def lock(name, limit = 3, **options)
+        options.merge!(name: name)
+        semaphore = Semaphore.new(options)
+        semaphore.lock
+        if block_given?
+          lock_thr = semaphore.renew
+          yield
+        end
+        semaphore
+      rescue Exception => e
+        logger.error e
+        logger.debug e.backtrace.join("\n")
+        raise
+      ensure
+        lock_thr.kill
+        semaphore.release
+      end
+    end
+
+    # The Consul session
+    attr_reader :session
+
+    # The current state of the semaphore
+    attr_reader :state
+
+    # The current semaphore members
+    attr_reader :members
+
+    # The current lock modify index
+    attr_reader :lock_modify_index
+
+    # The lock content
+    attr_reader :lock_content
+
+    # The Consul key prefix
+    attr_reader :prefix
+
+    # The number of nodes that can obtain a semaphore lock
+    attr_reader :limit
+
+    # @param name [String] The name of the session, it is also used in the `prefix`
+    # @param prefix [String|NilClass] The Consul Kv prefix
+    # @param lock [String|NilClass] The path to the lock file
+    def initialize(name:, prefix: nil, lock: nil, limit: 3)
+      create_session(name)
+      @prefix = prefix.nil? ? "service/#{name}/lock/" : prefix
+      @prefix += '/' unless @prefix.end_with?('/')
+      @lock = lock.nil? ? "#{@prefix}.lock" : lock
+      @lock_modify_index = nil
+      @lock_content = nil
+      @limit = set_limit(limit)
+      @reset = false
+    end
+
+    # Obtain a lock with the current session
+    #
+    # @return [Boolean] `true` if the lock was obtained
+    #
+    def lock
+      contender_key
+      semaphore_state
+      try_lock
+    end
+
+    # Renew lock watching for changes
+    # @return [Thread] Thread running a blocking call maintaining the lock state
+    #
+    def renew
+      thr = Thread.new do
+        loop do
+          if renew?
+            semaphore_state
+            try_lock
+          end
+        end
+      end
+      thr
+    end
+
+    # Release a lock with the current session
+    #
+    # @return [Boolean] `true` if the lock was released
+    #
+    def release
+      semaphore_state
+      try_release
+    end
+
+
+    def create_session(name)
+      ::DaemonRunner::RetryErrors.retry(exceptions: [DaemonRunner::Session::CreateSessionError]) do
+        @session = Session.start(name, behavior: 'delete')
+      end
+    end
+
+    def set_limit(new_limit)
+      if lock_exists?
+        if new_limit.to_i != @limit.to_i
+          logger.warn 'Limit in lockfile and @limit do not match using limit from lockfile'
+        end
+        @limit = lock_content['Limit']
+      else
+        @limit = new_limit
+      end
+    end
+
+    # Create a contender key
+    def contender_key(value = 'none')
+      if value.nil? || value.empty?
+        raise ArgumentError, 'Value cannot be empty or nil'
+      end
+      key = "#{prefix}/#{session.id}"
+      ::DaemonRunner::RetryErrors.retry do
+        @contender_key = Diplomat::Lock.acquire(key, session.id, value)
+      end
+      @contender_key
+    end
+
+    # Get the current semaphore state by fetching all
+    # conterder keys and the lock key
+    def semaphore_state
+      options = { decode_values: true, recurse: true }
+      @state = Diplomat::Kv.get(prefix, options, :return)
+      decode_semaphore_state unless state.empty?
+      state
+    end
+
+    def try_lock
+      prune_members
+      do_update = add_self_to_holders
+      @reset = false
+      if do_update
+        format_holders
+        @locked = write_lock
+      end
+      log_lock_state
+    end
+
+    def try_release
+      do_update = remove_self_from_holders
+      if do_update
+        format_holders
+        @locked = !write_lock
+      end
+      DaemonRunner::Session.release(prefix)
+      session.destroy!
+      log_release_state
+    end
+
+    # Write a new lock file if the number of contenders is less than `limit`
+    # @return [Boolean] `true` if the lock was written succesfully
+    def write_lock
+      index = lock_modify_index.nil? ? 0 : lock_modify_index
+      value = generate_lockfile
+      return true if value == true
+      Diplomat::Kv.put(@lock, value, cas: index)
+    end
+
+    # Start a blocking query on the prefix, if there are changes
+    # we need to try to obtain the lock again.
+    #
+    # @return [Boolean] `true` if there are changes,
+    #  `false` if the request has timed out
+    def renew?
+      logger.debug("Watching Consul #{prefix} for changes")
+      options = { recurse: true }
+      changes = Diplomat::Kv.get(prefix, options, :wait, :wait)
+      logger.info("Changes on #{prefix} detected") if changes
+      changes
+    rescue StandardError => e
+      logger.error(e)
+    end
+
+    private
+
+    # Decode raw response from Consul
+    # Set `@lock_modify_index`, `@lock_content`, and `@members`
+    # @returns [Array] List of members
+    def decode_semaphore_state
+      lock_key = state.find { |k| k['Key'] == @lock }
+      member_keys = state.delete_if { |k| k['Key'] == @lock }
+      member_keys.map! { |k| k['Key'] }
+
+      unless lock_key.nil?
+        @lock_modify_index = lock_key['ModifyIndex']
+        @lock_content = JSON.parse(lock_key['Value'])
+      end
+      @members = member_keys.map { |k| k.split('/')[-1] }
+    end
+
+    # Returns current state of lockfile
+    def lock_exists?
+      (!lock_modify_index.nil? && !lock_content.nil?) && !@reset
+    end
+
+    # Get the active members from the lock file, removing any _dead_ members.
+    # This is accomplished by using the contenders keys(`@members`) to get the
+    # list of all alive members.  So we can easily remove any nodes that don't
+    # appear in that list.
+    def prune_members
+      @holders = if lock_exists?
+        holders = lock_content['Holders']
+        return @holders = [] if holders.nil?
+        holders = holders.keys
+        holders & members
+      else
+        []
+      end
+    end
+
+    # Add our session.id to the holders list if holders is less than limit
+    def add_self_to_holders
+      @holders.uniq!
+      @reset = true if @holders.length == 0
+      return true if @holders.include? session.id
+      if @holders.length < limit
+        @holders << session.id
+      end
+    end
+
+    # Remove our session.id from the holders list
+    def remove_self_from_holders
+      return unless lock_exists?
+      @holders = lock_content['Holders']
+      @holders = @holders.keys
+      @holders.delete(session.id)
+      @holders
+    end
+
+    # Format the list of holders for the lock file
+    def format_holders
+      @holders.uniq!
+      @holders.sort!
+      holders = {}
+      logger.debug "Holders are: #{@holders.join(',')}"
+      @holders.map { |m| holders[m] = true }
+      @holders = holders
+    end
+
+    # Generate JSON formatted lockfile content, only if the content has changed
+    def generate_lockfile
+      if lock_exists? && lock_content['Holders'] == @holders
+        logger.info 'Holders are unchanged, not updating'
+        return true
+      end
+      lockfile_format = {
+        'Limit' => limit,
+        'Holders' => @holders
+      }
+      JSON.generate(lockfile_format)
+    end
+
+    def log_lock_state
+      log_lock(locked: @locked)
+    end
+
+    def log_release_state
+      log_lock(locked: !@locked, end_string: 'released')
+    end
+
+    def log_lock(locked: true, end_string: 'obtained')
+      msg = 'Lock %{text} %{end_string}'
+      text = locked == true ? 'succesfully' : 'could not be'
+      msg = msg % { text: text, end_string: end_string }
+      logger.info msg
+    end
+  end
+end

data/lib/daemon_runner/session.rb

@@ -5,16 +5,21 @@ module DaemonRunner
   # Manage distributed locks with Consul
   #
   class Session
+    class SessionError < RuntimeError; end
+    class CreateSessionError < SessionError; end
+
     include Logger
 
     class << self
       attr_reader :session
 
       def start(name, **options)
-        @session ||= Session.new(name, options).renew!
+        @session = Session.new(name, options).renew!
+        raise CreateSessionError, 'Failed to create session' if @session == false
+        @session.verify_session
+        @session
       end
 
-
       # Acquire a lock with the current session, or initialize a new session
       #
       # @param path [String]  A path in the Consul key-value space to lock
@@ -22,7 +27,7 @@ module DaemonRunner
       # @return [Boolean] `true` if the lock was acquired
       #
       def lock(path)
-        Diplomat::Lock.acquire(path, session.id)
+        Diplomat::Lock.wait_to_acquire(path, session.id)
       end
 
       # Release a lock held by the current session
@@ -107,6 +112,19 @@ module DaemonRunner
       Diplomat::Session.destroy(id)
     end
 
+    # Verify wheather the session exists after a period of time
+    def verify_session(wait_time = 2)
+      logger.info(" - Wait until Consul session #{id} exists")
+      wait_time.times do
+        exists = session_exist?
+        raise CreateSessionError, 'Error creating session' unless exists
+        sleep 1
+      end
+      logger.info(" - Found Consul session #{id}")
+    rescue CreateSessionError
+      init
+    end
+
     private
 
     # Initialize a session and store it's ID
@@ -118,8 +136,13 @@ module DaemonRunner
         :LockDelay => "#{delay}s",
         :Behavior => behavior
       )
-
       logger.info(" - Initialized a Consul session #{id}")
     end
+
+    # Does the session exist
+    def session_exist?
+      sessions = Diplomat::Session.list
+      sessions.any? { |s| s['ID'] == id }
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daemon_runner
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Andrew Thompson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-25 00:00:00.000000000 Z
+date: 2016-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logging
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: retryable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -156,9 +170,14 @@ files:
 - bin/setup
 - daemon_runner.gemspec
 - examples/example1.rb
+- examples/example_semaphore.rb
+- examples/example_semaphore_release.rb
+- examples/run_consul.rb
 - lib/daemon_runner.rb
 - lib/daemon_runner/client.rb
 - lib/daemon_runner/logger.rb
+- lib/daemon_runner/retry_errors.rb
+- lib/daemon_runner/semaphore.rb
 - lib/daemon_runner/session.rb
 - lib/daemon_runner/shell_out.rb
 - lib/daemon_runner/version.rb