blender-serf 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cfaa34818e2ebbe4ed6327b625c5927d49291901
+  data.tar.gz: 167c55fac333e5205c4f1599aad75e68c50c2575
+SHA512:
+  metadata.gz: 2cb10ede968b4329a18917e32e9ece5f974f4f1d3a7db8cdcef971641d6d2224014ad37bfe234670dd05cddd69b8294a8c8db597671c3cd48c9c62fc641f6ac7
+  data.tar.gz: ecab1c61c7e6c29c9906239558a8f81efcbb06360e6e4d88503b564d44b9eaf8fd07a26f3056659d6ce32292eb7e93df97a7ff6c179664f9f46d9e1c77853b5a

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,10 @@
+before_install:
+  - bundle install --path .bundle
+rvm:
+  - 1.9.3
+  - 2.1.0
+  - 2.1.2
+branches:
+  only:
+    - master
+script: "bundle exec rake spec"

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Rakefile

@@ -0,0 +1,17 @@
+require 'bundler/gem_tasks'
+require 'rubocop/rake_task'
+require 'rspec/core/rake_task'
+require 'yard'
+
+YARD::Rake::YardocTask.new do |t|
+  t.files   = ['lib/**/*.rb']
+end
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = %w{spec/**/*_spec.rb}
+end
+
+RuboCop::RakeTask.new(:rubocop) do |t|
+  t.patterns = %w{Rakefile Gemfile lib/**/*.rb}
+  t.fail_on_error = true
+end

data/blender-serf.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'blender-serf'
+  spec.version       = '0.0.1'
+  spec.authors       = ['Ranjib Dey']
+  spec.email         = ['ranjib@pagerduty.com']
+  spec.summary       = %q{Serf backend for blender}
+  spec.description   = %q{Discover and execute tasks using serf}
+  spec.homepage      = 'http://github.com/PagerDuty/blender-serf'
+  spec.license       = 'Apache 2'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'pd-blender'
+  spec.add_dependency 'serfx'
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'yard'
+  spec.add_development_dependency 'pry'
+end

data/lib/blender/discoveries/serf.rb

@@ -0,0 +1,44 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'serfx'
+require 'blender/log'
+
+module Blender
+  module Discovery
+    class Serf
+      def initialize(opts = {})
+        @config = opts
+      end
+      def search(opts = {})
+        tags = opts[:tags] || {}
+        status = opts[:status] || 'alive'
+        name = opts[:name]
+        hosts = []
+        Blender::Log.debug("Serf memeber list call with arguments: Tags=#{tags}")
+        Blender::Log.debug("Serf memeber list call with arguments: status=#{status}")
+        Blender::Log.debug("Serf memeber list call with arguments: Name=#{name}")
+        Serfx.connect(@config) do |conn|
+          conn.members_filtered(tags, status, name).body['Members'].map do |m|
+            hosts << m['Name']
+          end
+        end
+        hosts
+      end
+    end
+  end
+end

data/lib/blender/drivers/serf.rb

@@ -0,0 +1,105 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'serfx'
+require 'blender/exceptions'
+require 'blender/log'
+require 'blender/drivers/base'
+
+module Blender
+  module Driver
+    class Serf < Base
+
+      attr_reader :filter_by, :concurrency, :filter_tag
+
+      def initialize(config = {})
+        cfg = config.dup
+        @filter_by =  cfg.delete(:filter_by) || :host
+        @concurrency = cfg.delete(:concurrency) || 1
+        if @filter_by == :tag
+          @filter_tag =  cfg.delete(:filter_tag)
+          raise ArgumentError, 'Must specify filter_tag when filter_by is set to :tag' unless @filter_tag
+        end
+        super(cfg)
+      end
+
+      def serf_query(command, host)
+        responses = []
+        Log.debug("Invoking serf query '#{command.query}' with payload '#{command.payload}' against #{@current_host}")
+        Log.debug("Serf RPC address #{config[:host]}:#{config[:port]}")
+        Serfx.connect(config) do |conn|
+          conn.query(*query_opts(command, host)) do |event|
+            responses <<  event
+            stdout.puts event.inspect
+          end
+        end
+        responses
+      end
+
+      def run_command(command, nodes)
+        begin
+          responses = serf_query(command, nodes)
+          if command.process
+            command.process.call(responses)
+          end
+          ExecOutput.new(exit_status(responses, nodes), responses.inspect, '')
+        rescue StandardError => e
+          ExecOutput.new( -1, '', e.message)
+        end
+      end
+
+      def exit_status(responses, nodes)
+        case filter_by
+        when :host
+          responses.size == nodes.size ? 0 : -1
+        when :tag, :none
+          0
+        else
+          raise ArgumentError, "Unknown filter_by option: #{filter_by}"
+        end
+      end
+
+      def query_opts(command, nodes)
+        opts = { Timeout: (command.timeout || 15)*1e9.to_i}
+        case filter_by
+        when :host
+          opts.merge!(FilterNodes: nodes)
+        when :tag
+          raise 'filter by :tag only supports single tag' unless nodes.size == 1
+          opts.merge!(FilterTags: {filter_tag => nodes.first})
+        when :none
+          raise 'filter by :none only supported with localhost' unless nodes == ['localhost']
+        else
+          raise ArgumentError, "Unknown filter_by option: #{filter_by}"
+        end
+        [ command.query, command.payload, opts]
+      end
+
+      def execute(tasks, hosts)
+        Log.debug("Serf query on #{filter_by}s [#{hosts.inspect}]")
+        tasks.each do |task|
+          hosts.each_slice(concurrency) do |nodes|
+            cmd = run_command(task.command, nodes)
+            if cmd.exitstatus != 0 and !task.metadata[:ignore_failure]
+              raise ExecutionFailed, cmd.stderr
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/blender/drivers/serf_async.rb

@@ -0,0 +1,81 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'blender/exceptions'
+require 'blender/log'
+require 'blender/drivers/serf'
+require 'json'
+require 'blender/tasks/serf'
+
+module Blender
+  module Driver
+    class SerfAsync < Serf
+
+      def dup_command(cmd, payload)
+        Blender::Task::Serf::SerfQuery.new(
+          cmd.query,
+          payload,
+          cmd.timeout,
+          cmd.noack
+        )
+      end
+
+      def start!(cmd, host)
+        resps = serf_query(dup_command(cmd, 'start'), host)
+        status = extract_status!(resps.first)
+        unless status == 'success'
+          raise RuntimeError, "Failed to start async serf job. Status = #{status}"
+        end
+      end
+
+      def finished?(cmd, host)
+        Blender::Log.debug("Checking for status")
+        resps = serf_query(dup_command(cmd, 'check'), host)
+        Blender::Log.debug("Responses: #{resps.inspect}")
+        Blender::Log.debug("Status:#{extract_status!(resps.first)}")
+        extract_status!(resps.first) == 'finished'
+      end
+
+      def reap!(cmd, host)
+        resps = serf_query(dup_command(cmd, 'reap'), host)
+        extract_status!(resps.first) == 'success'
+      end
+
+      def run_command(command, host)
+        begin
+          start!(command, host)
+          until finished?(command, host)
+            sleep 10
+          end
+          reap!(command, host)
+          ExecOutput.new(0, '', '')
+        rescue StandardError => e
+          ExecOutput.new( -1, '', e.message + e.backtrace.join("\n"))
+        end
+      end
+
+      def extract_status!(res)
+        payload = JSON.parse(res['Payload'])
+        unless payload['code'].zero?
+          raise RuntimeError, "non zero query response code: #{res}"
+        end
+        Blender::Log.debug("Payload: #{payload['result'].inspect}")
+        payload['result']['status']
+      end
+    end
+  end
+end

data/lib/blender/serf.rb

@@ -0,0 +1,4 @@
+require 'blender/discoveries/serf'
+require 'blender/drivers/serf'
+require 'blender/tasks/serf'
+require 'blender/serf_dsl'

data/lib/blender/serf_dsl.rb

@@ -0,0 +1,9 @@
+module Blender
+  module SerfDSL
+    def serf_task(name, &block)
+      task = build_task(name, :serf)
+      task.instance_eval(&block) if block_given?
+      append_task(:serf, task)
+    end
+  end
+end

data/lib/blender/tasks/serf.rb

@@ -0,0 +1,60 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'blender/tasks/base'
+
+module Blender
+  module Task
+    class Serf < Blender::Task::Base
+
+      SerfQuery = Struct.new(:query, :payload, :timeout, :noack, :process)
+
+      def initialize(name, metadata = {})
+        super
+        @command = SerfQuery.new
+        @command.query = name
+      end
+
+      def execute(&block)
+        @command.instance_eval(&block)
+      end
+
+      def query(q)
+        @command.query = q
+      end
+
+      def timeout(t)
+        @command.timeout = t
+      end
+
+      def payload(pl)
+        @command.payload = pl
+      end
+
+      def no_ack(bool)
+        @command.noack = bool
+      end
+
+      def process(callback)
+        @command.process = callback
+      end
+
+      def command
+        @command
+      end
+    end
+  end
+end

data/spec/blender/discoveries/serf_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+require 'blender'
+require 'blender/discoveries/serf'
+require 'blender/serf_dsl'
+
+describe Blender::Discovery::Serf do
+  let(:scheduler) do
+    Blender::Scheduler.new('test').extend Blender::SerfDSL
+  end
+  let(:discovery){described_class.new(host: '1.2.3.4')}
+
+  it '#search' do
+    conn = double('connection')
+    response = double('response', body: {'Members'=> [{'Name'=>'a'}]})
+    expect(conn).to receive(:members_filtered).with({}, "alive", "foo").and_return(response)
+    expect(Serfx).to receive(:connect).with(host: '1.2.3.4').and_yield(conn)
+    expect(discovery.search(name: 'foo')).to eq(['a'])
+  end
+    describe'#serf_task' do
+      subject(:task){scheduler.tasks.first}
+      before do
+        scheduler.serf_task('test') do
+          members ['b']
+          query 'foo'
+          payload 'bar'
+          no_ack true
+        end
+      end
+      it 'should have correct host list' do
+        expect(task.hosts).to eq(['b'])
+      end
+      it 'should use the serf task subclass' do
+        expect(task).to be_kind_of(Blender::Task::Serf)
+      end
+      it 'should use the serquery inner class for command' do
+        expect(task.command).to be_kind_of(Blender::Task::Serf::SerfQuery)
+      end
+      it 'should use serf driver subclass' do
+        expect(task.driver).to be_kind_of(Blender::Driver::Serf)
+      end
+      it 'should allow setting up serf query and payload' do
+        expect(task.command.query).to eq('foo')
+        expect(task.command.payload).to eq('bar')
+      end
+    end
+end

data/spec/blender/drivers/serf_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+require 'blender/handlers/base'
+
+describe Blender::Driver::Serf do
+  let(:driver) do
+    described_class.new(
+      events: Blender::Handlers::Base.new,
+      host: 'foo',
+      port: 123,
+      authkey: 'xyz'
+    )
+  end
+  let(:tag_driver) do
+    described_class.new(
+      events: Blender::Handlers::Base.new,
+      host: 'foo',
+      port: 123,
+      authkey: 'xyz',
+      filter_by: :tag,
+      filter_tag: 'fake-tag'
+    )
+  end
+  let(:hosts) {['h1']}
+  let(:tasks){ Array.new(3){|n| create_serf_task("t#{n}")}}
+  it '#execute serf query' do
+    conn = double('connection')
+    conn_opts = { host: 'foo', port: 123, authkey: 'xyz'}
+    query_opts = [
+      'test-query',
+      'test-payload',
+      {:FilterNodes=>["h1"], :Timeout=>15000000000}
+    ]
+    expect(conn).to receive(:query).with(*query_opts).and_yield(Object.new).exactly(3).times
+    expect(Serfx).to receive(:connect).with(conn_opts).and_yield(conn).exactly(3).times
+    driver.execute(tasks, hosts)
+  end
+  it '#execute serf query based on tags' do
+    conn = double('connection')
+    conn_opts = { host: 'foo', port: 123, authkey: 'xyz'}
+    query_opts = [
+      'test-query',
+      'test-payload',
+      { FilterTags: {"fake-tag"=>"h1"}, :Timeout=>15000000000}
+    ]
+    expect(conn).to receive(:query).with(*query_opts).and_yield(Object.new).exactly(3).times
+    expect(Serfx).to receive(:connect).with(conn_opts).and_yield(conn).exactly(3).times
+    tag_driver.execute(tasks, hosts)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+require 'blender/drivers/serf'
+require 'blender/tasks/serf'
+
+module Helper
+  def create_serf_task(name)
+    Blender::Task::Serf.new(name).tap do |t|
+      t.query 'test-query'
+      t.payload 'test-payload'
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include Helper
+  config.mock_with :rspec do |mocks|
+    mocks.verify_doubled_constant_names = true
+  end
+  config.backtrace_exclusion_patterns = []
+end

metadata

@@ -0,0 +1,188 @@
+--- !ruby/object:Gem::Specification
+name: blender-serf
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ranjib Dey
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pd-blender
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: serfx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Discover and execute tasks using serf
+email:
+- ranjib@pagerduty.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Rakefile
+- blender-serf.gemspec
+- lib/blender/discoveries/serf.rb
+- lib/blender/drivers/serf.rb
+- lib/blender/drivers/serf_async.rb
+- lib/blender/serf.rb
+- lib/blender/serf_dsl.rb
+- lib/blender/tasks/serf.rb
+- spec/blender/discoveries/serf_spec.rb
+- spec/blender/drivers/serf_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/PagerDuty/blender-serf
+licenses:
+- Apache 2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Serf backend for blender
+test_files:
+- spec/blender/discoveries/serf_spec.rb
+- spec/blender/drivers/serf_spec.rb
+- spec/spec_helper.rb
+has_rdoc: