blender-serf 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cfaa34818e2ebbe4ed6327b625c5927d49291901
-  data.tar.gz: 167c55fac333e5205c4f1599aad75e68c50c2575
+  metadata.gz: 6a360de1c7367a78e1ae4cb4a5a1bb2095f11e3d
+  data.tar.gz: 61df6de3ba420247a8e0169005212c1f74baef39
 SHA512:
-  metadata.gz: 2cb10ede968b4329a18917e32e9ece5f974f4f1d3a7db8cdcef971641d6d2224014ad37bfe234670dd05cddd69b8294a8c8db597671c3cd48c9c62fc641f6ac7
-  data.tar.gz: ecab1c61c7e6c29c9906239558a8f81efcbb06360e6e4d88503b564d44b9eaf8fd07a26f3056659d6ce32292eb7e93df97a7ff6c179664f9f46d9e1c77853b5a
+  metadata.gz: 43bef41ba273f03c8ebe522e8de7820b215a2fd2606f587cb8916544b2d4d30819e55f8f552d4fbfd1b0a4655aa4ea7183e057aaee58c1503d3769abba33ccac
+  data.tar.gz: 4870e0f7baf6977b6ec519d3a997317b9df659cb723eb457be3f1311b4fa68a261d51841b50fb9338185f7cf4a527d2ee2b7f680ccc91cdac246c51b65a2df39

data/Gemfile

@@ -1,3 +1,5 @@
 source 'https://rubygems.org'
 
 gemspec
+
+gem 'pd-blender', github: 'PagerDuty/blender'

data/README.md

@@ -0,0 +1,135 @@
+# Blender-Serf
+
+Provides [Serf](https://serfdom.io/) based host discovery and job dispatch
+mechanism for [Blender](https://github.com/PagerDuty/blender)
+
+## Installation
+```sh
+  gem install blender-serf
+```
+
+## Usage
+
+### Config
+Blender-serf uses [Serf's RPC](https://serfdom.io/docs/agent/rpc.html)
+interface for communication between serf and ruby. Following serf RPC connection
+specific configuration options are allowed
+  - host (defaults to localhost)
+  - port (defaults to 7373)
+  - authkey (rpc authenticatioin key, default is nil)
+
+Example
+```ruby
+config(:serf, host: '127.0.0.1', port: 7373, authkey: 'foobar')
+```
+### Using serf for host discovery
+Serf provides cluster membership list via `members` and `members_filtered` RPC
+calls, among them Blender-Serf uses `members_filtered` RPC command for host discovery.
+Following is an example of obtaining all live members of a cluster
+```ruby
+  require 'blender/serf'
+  members(search(:serf, status: 'alive'))
+  ruby_task 'test' do
+    execute do |host|
+      Blender::Log.info(host)
+    end
+  end
+```
+Member list can be obtained based on serf tags or name.
+
+Filter by tag expect tags to be supplied as hash (key value pair), which
+allows grouping hosts by multiple tags
+```ruby
+  members(search(:serf, tags: {'role'=>'db'}))
+  ruby_task 'test' do
+    execute do |host|
+      Blender::Log.info(host)
+    end
+  end
+```
+
+Filter by name, the specification string can be a pattern as well
+```ruby
+  members(search(:serf, name: 'foo-baar-*'))
+  ruby_task 'test' do
+    execute do |host|
+      Blender::Log.info(host)
+    end
+  end
+```
+
+Combinations of all three filtering mechanism is valid as well
+```ruby
+  members(search(:serf, name: 'foo-baar-*', status: 'alive', tags:{'datacenter' => 'eu-east-2a'}))
+  ruby_task 'test'
+    execute do |host|
+      Blender::Log.info(host)
+    end
+  end
+```
+
+### Using serf for job dispatch
+
+Serf agents allow job execution via [event handlers](https://serfdom.io/docs/agent/event-handlers.html),
+where the execution details is captured by the handler, while trigger mechanism is controlled by
+serf event. _Blender-Serf_ uses [serf query](https://serfdom.io/docs/commands/query.html) event type
+to dispatch event and check for successfull response. The handler itself, and associated serf' configyration
+needs to be setup externally (bake it in your image, or use a configuration managemenet system like
+chef, puppet, ansible or salt for this).
+
+Following example will trigger serf query event `test` against 3 nodes, one by one
+```ruby
+  extend Blender::SerfDSL
+  members(['node_1', 'node_2', 'node_3'])
+  serf_task 'test'
+```
+A more elaborate example
+```ruby
+  extend Blender::SerfDSL
+  members(['node_1', 'node_2', 'node_3'])
+  serf_task 'start_nginx' do
+    query 'nginx'
+    payload 'start'
+    timeout 3
+  end
+```
+
+Which might be accmoplished by the following handler script (needs to be present aprior)
+```ruby
+require 'serfx/utils/handler'
+
+include Serfx::Utils::Handler
+
+
+on :query, 'nginx' do |event|
+  case event.payload
+  when 'start'
+    %x{/etc/init.d/nginx start}
+  when 'stop'
+  # ...
+  when 'check'
+  # ...
+  end
+end
+
+run
+
+```
+## Supported ruby versions
+
+Blender-serf currently support the following MRI versions:
+
+* *Ruby 1.9.3*
+* *Ruby 2.1*
+
+## License
+
+[Apache 2](http://www.apache.org/licenses/LICENSE-2.0)
+
+## Contributing
+
+1. Fork it ( https://github.com/PagerDuty/blender-serf/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/blender-serf.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'blender-serf'
-  spec.version       = '0.0.1'
+  spec.version       = '0.1.0'
   spec.authors       = ['Ranjib Dey']
   spec.email         = ['ranjib@pagerduty.com']
   spec.summary       = %q{Serf backend for blender}
@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'pd-blender'
   spec.add_dependency 'serfx'
+  spec.add_dependency 'retriable'
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'

data/lib/blender/discoveries/serf.rb

@@ -21,20 +21,38 @@ require 'blender/log'
 module Blender
   module Discovery
     class Serf
+      Property = Struct.new(:key, :hook)
+      attr_reader :attribute
+
       def initialize(opts = {})
-        @config = opts
+        @config = {}
+        @config[:host] = opts[:host] if opts.key?(:host)
+        @config[:port] = opts[:port] if opts.key?(:port)
+        @config[:authkey] = opts[:authkey] if opts.key?(:authkey)
+        @attribute = opts[:attribute] || 'name'
       end
+
       def search(opts = {})
+        opts = {} if opts.nil?
         tags = opts[:tags] || {}
         status = opts[:status] || 'alive'
         name = opts[:name]
         hosts = []
+        Blender::Log.debug("Using Serf agents '#{attribute}' as discovery return values")
+        case attribute
+        when 'name'
+          property = Property.new('Name', lambda{|data| data})
+        when 'ipaddress'
+          property = Property.new('Addr', lambda{|data| data.unpack('CCCC').join('.')})
+        else
+          raise ArgumentError, "Attribute can be either 'name' or 'ipaddress', you have provided '#{attribute}'"
+        end
         Blender::Log.debug("Serf memeber list call with arguments: Tags=#{tags}")
         Blender::Log.debug("Serf memeber list call with arguments: status=#{status}")
         Blender::Log.debug("Serf memeber list call with arguments: Name=#{name}")
         Serfx.connect(@config) do |conn|
           conn.members_filtered(tags, status, name).body['Members'].map do |m|
-            hosts << m['Name']
+            hosts << property.hook.call(m[property.key])
           end
         end
         hosts

data/lib/blender/drivers/serf.rb

@@ -56,7 +56,7 @@ module Blender
           if command.process
             command.process.call(responses)
           end
-          ExecOutput.new(exit_status(responses, nodes), responses.inspect, '')
+          exit_status(responses, nodes)
         rescue StandardError => e
           ExecOutput.new( -1, '', e.message)
         end
@@ -65,9 +65,13 @@ module Blender
       def exit_status(responses, nodes)
         case filter_by
         when :host
-          responses.size == nodes.size ? 0 : -1
+          if responses.size == nodes.size
+            ExecOutput.new(0, responses.inspect, '')
+          else
+            ExecOutput.new(-1, '', "Insufficient number of responses. Expected:#{nodes.size}, Got:#{responses.size}")
+          end
         when :tag, :none
-          0
+          ExecOutput.new(0, responses.inspect, '')
         else
           raise ArgumentError, "Unknown filter_by option: #{filter_by}"
         end

data/lib/blender/drivers/serf_async.rb

@@ -20,61 +20,39 @@ require 'blender/log'
 require 'blender/drivers/serf'
 require 'json'
 require 'blender/tasks/serf'
+require 'retriable'
 
 module Blender
   module Driver
-    class SerfAsync < Serf
-
-      def dup_command(cmd, payload)
-        Blender::Task::Serf::SerfQuery.new(
-          cmd.query,
-          payload,
-          cmd.timeout,
-          cmd.noack
-        )
-      end
-
-      def start!(cmd, host)
-        resps = serf_query(dup_command(cmd, 'start'), host)
-        status = extract_status!(resps.first)
-        unless status == 'success'
-          raise RuntimeError, "Failed to start async serf job. Status = #{status}"
-        end
-      end
-
-      def finished?(cmd, host)
-        Blender::Log.debug("Checking for status")
-        resps = serf_query(dup_command(cmd, 'check'), host)
-        Blender::Log.debug("Responses: #{resps.inspect}")
-        Blender::Log.debug("Status:#{extract_status!(resps.first)}")
-        extract_status!(resps.first) == 'finished'
+    class SerfAsync < Blender::Driver::Base
+      def initialize(config = {})
+        super
+        @driver = Blender::Driver::Serf.new(config)
       end
 
-      def reap!(cmd, host)
-        resps = serf_query(dup_command(cmd, 'reap'), host)
-        extract_status!(resps.first) == 'success'
-      end
-
-      def run_command(command, host)
-        begin
-          start!(command, host)
-          until finished?(command, host)
-            sleep 10
+      def execute(tasks, hosts)
+        Log.debug("Serf Async query on #{@driver.filter_by}s [#{hosts.inspect}]")
+        tasks.each do |task|
+          retry_options = task.metadata[:retry_options]
+          hosts.each_slice(@driver.concurrency) do |nodes|
+            Blender::Log.debug("Start query: #{task.start_query.inspect}")
+            start = @driver.run_command(task.start_query.command, nodes)
+            if start.exitstatus != 0 and !task.metadata[:ignore_failure]
+              raise ExecutionFailed, start.stderr
+            end
+            Blender::Log.debug("Using check retry options:#{retry_options}")
+            Retriable.retriable(retry_options) do
+              cmd = @driver.run_command(task.check_query.command, nodes)
+              if cmd.exitstatus != 0 and !task.metadata[:ignore_failure]
+                raise ExecutionFailed, cmd.stderr
+              end
+            end
+            cmd = @driver.run_command(task.stop_query.command, nodes)
+            if cmd.exitstatus != 0 and !task.metadata[:ignore_failure]
+              raise ExecutionFailed, cmd.stderr
+            end
           end
-          reap!(command, host)
-          ExecOutput.new(0, '', '')
-        rescue StandardError => e
-          ExecOutput.new( -1, '', e.message + e.backtrace.join("\n"))
-        end
-      end
-
-      def extract_status!(res)
-        payload = JSON.parse(res['Payload'])
-        unless payload['code'].zero?
-          raise RuntimeError, "non zero query response code: #{res}"
         end
-        Blender::Log.debug("Payload: #{payload['result'].inspect}")
-        payload['result']['status']
       end
     end
   end

data/lib/blender/serf.rb

@@ -1,4 +1,6 @@
 require 'blender/discoveries/serf'
 require 'blender/drivers/serf'
+require 'blender/drivers/serf_async'
 require 'blender/tasks/serf'
+require 'blender/tasks/serf_async'
 require 'blender/serf_dsl'

data/lib/blender/serf_dsl.rb

@@ -1,3 +1,20 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2015 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 module Blender
   module SerfDSL
     def serf_task(name, &block)
@@ -5,5 +22,10 @@ module Blender
       task.instance_eval(&block) if block_given?
       append_task(:serf, task)
     end
+    def serf_async_task(name, &block)
+      task = build_task(name, :serf_async)
+      task.instance_eval(&block) if block_given?
+      append_task(:serf_async, task)
+    end
   end
 end

data/lib/blender/tasks/serf.rb

@@ -19,17 +19,12 @@ require 'blender/tasks/base'
 module Blender
   module Task
     class Serf < Blender::Task::Base
-
       SerfQuery = Struct.new(:query, :payload, :timeout, :noack, :process)
 
       def initialize(name, metadata = {})
         super
         @command = SerfQuery.new
-        @command.query = name
-      end
-
-      def execute(&block)
-        @command.instance_eval(&block)
+        @command.query= name
       end
 
       def query(q)
@@ -48,8 +43,12 @@ module Blender
         @command.noack = bool
       end
 
-      def process(callback)
-        @command.process = callback
+      def process(&block)
+        @command.process = block if block
+      end
+
+      def execute(&block)
+        @command.instance_eval(&block)
       end
 
       def command

data/lib/blender/tasks/serf_async.rb

@@ -0,0 +1,81 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2015 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'blender/tasks/serf'
+
+module Blender
+  module Task
+    class SerfAsync < Blender::Task::Base
+      attr_reader :start_query, :stop_query, :check_query
+
+      def initialize(name, options = {})
+        super
+        @start_query = Blender::Task::Serf.new(name)
+        @start_query.payload(metadata[:start])
+        @start_query.timeout(metadata[:timeout])
+        @start_query.no_ack(metadata[:no_ack])
+        @start_query.process(&metadata[:process])
+
+        @stop_query = Blender::Task::Serf.new(name)
+        @stop_query.payload(metadata[:stop])
+        @stop_query.timeout(metadata[:timeout])
+        @stop_query.no_ack(metadata[:no_ack])
+        @stop_query.process(&metadata[:process])
+
+        @check_query = Blender::Task::Serf.new(name)
+        @check_query.payload(metadata[:check])
+        @check_query.timeout(metadata[:timeout])
+        @check_query.no_ack(metadata[:no_ack])
+        @check_query.process(&metadata[:process])
+      end
+
+      def start(&block)
+        @start_query.instance_eval(&block)
+      end
+
+      def stop(&block)
+        @stop_query.instance_eval(&block)
+      end
+
+      def check(&block)
+        @check_query.instance_eval(&block)
+      end
+
+      def execute
+      end
+
+      def command
+      end
+
+      def retry_options(options = {})
+        @metadata[:retry_options].merge!(options)
+      end
+
+      def default_metadata
+        {
+          ignore_failure: false,
+          start: 'start',
+          stop: 'stop',
+          check: 'check',
+          timeout: 5,
+          no_ack: false,
+          process: nil,
+          retry_options: { max_elapsed_time: 180 }
+        }
+      end
+    end
+  end
+end

data/spec/blender/drivers/serf_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'blender/handlers/base'
+require 'blender/cli'
 
 describe Blender::Driver::Serf do
   let(:driver) do
@@ -46,4 +47,26 @@ describe Blender::Driver::Serf do
     expect(Serfx).to receive(:connect).with(conn_opts).and_yield(conn).exactly(3).times
     tag_driver.execute(tasks, hosts)
   end
+  context '#CLI' do
+    it '#sync' do
+      conn = double('serf connection')
+      ev = double('serf event')
+      allow(conn).to receive(:query).with('test', 'start', Timeout: 7000000000, FilterNodes: ['node1']).and_yield(ev)
+      allow(conn).to receive(:query).with('test', 'start', Timeout: 7000000000, FilterNodes: ['node2']).and_yield(ev)
+      allow(conn).to receive(:query).with('test', 'start', Timeout: 7000000000, FilterNodes: ['node3']).and_yield(ev)
+      allow(Serfx).to receive(:connect).with(authkey: 'foobar').and_yield(conn)
+      Blender::CLI.start(%w{-f spec/data/example.rb})
+    end
+    it '#async' do
+      conn = double('serf connection')
+      ev = double('serf event')
+      allow(ev).to receive(:body).and_return({'success' => true})
+      %w{start stop check}.each do |payload|
+        allow(conn).to receive(:query).with('chef', payload , Timeout: 5000000000, FilterNodes: ['node1']).and_yield(ev)
+      end
+      allow(Serfx).to receive(:connect).with(authkey: 'asyncexample').and_yield(conn)
+      expect(Retriable).to receive(:retriable).with(max_elapsed_time: 300).and_yield
+      Blender::CLI.start(%w{-f spec/data/async_example.rb})
+    end
+  end
 end

data/spec/data/async_example.rb

@@ -0,0 +1,8 @@
+require 'blender/serf'
+extend SerfDSL
+config(:serf_async, authkey: 'asyncexample')
+
+serf_async_task 'chef' do
+  members ['node1']
+  retry_options(max_elapsed_time: 300)
+end

data/spec/data/example.rb

@@ -0,0 +1,10 @@
+require 'blender/serf'
+extend SerfDSL
+config(:serf, authkey: 'foobar')
+
+serf_task 'example serf task' do
+  query 'test'
+  payload 'start'
+  timeout 7
+  members ['node1', 'node2', 'node3']
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blender-serf
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Ranjib Dey
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-10 00:00:00.000000000 Z
+date: 2015-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pd-blender
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: retriable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -146,6 +160,7 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
+- README.md
 - Rakefile
 - blender-serf.gemspec
 - lib/blender/discoveries/serf.rb
@@ -154,8 +169,11 @@ files:
 - lib/blender/serf.rb
 - lib/blender/serf_dsl.rb
 - lib/blender/tasks/serf.rb
+- lib/blender/tasks/serf_async.rb
 - spec/blender/discoveries/serf_spec.rb
 - spec/blender/drivers/serf_spec.rb
+- spec/data/async_example.rb
+- spec/data/example.rb
 - spec/spec_helper.rb
 homepage: http://github.com/PagerDuty/blender-serf
 licenses:
@@ -184,5 +202,7 @@ summary: Serf backend for blender
 test_files:
 - spec/blender/discoveries/serf_spec.rb
 - spec/blender/drivers/serf_spec.rb
+- spec/data/async_example.rb
+- spec/data/example.rb
 - spec/spec_helper.rb
 has_rdoc: