pd-blender 0.0.1

data/blender.gemspec

@@ -0,0 +1,36 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'blender/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'pd-blender'
+  spec.version       = Blender::VERSION
+  spec.authors       = ['Ranjib Dey']
+  spec.email         = ['ranjib@pagerduty.com']
+  spec.summary       = %q{A modular orchestration engine}
+  spec.description   = %q{Discover hosts, run tasks against them and control their execution order}
+  spec.homepage      = 'http://github.com/PagerDuty/blender'
+  spec.license       = 'Apache 2'
+  spec.bindir        = 'bin'
+  spec.executables  = ['blend']
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'highline'
+  spec.add_dependency 'thor'
+  spec.add_dependency 'mixlib-shellout'
+  spec.add_dependency 'mixlib-log'
+  spec.add_dependency 'net-ssh'
+  spec.add_dependency 'rufus-scheduler'
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'yard'
+  spec.add_development_dependency 'pry'
+end

data/lib/blender.rb

@@ -0,0 +1,67 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'blender/version'
+require 'blender/scheduler'
+require 'blender/log'
+require 'blender/drivers/shellout'
+require 'json'
+require 'blender/configuration'
+
+# Top level module that holds all blender related libraries under this namespace
+module Blender
+  # Trigger a blender run. If a block is given, then an object of
+  # Blender::Scheduler is yielded, otherwise the argument is treated as a
+  # command and executed with local shellout driver
+  #
+  # @param name [String] Name of the run
+  #
+  # @return [void]
+  def self.blend(name, config_file = nil)
+    scheduler = Scheduler.new(name)
+    if config_file
+      configure(config_file)
+    end
+    if block_given?
+      yield scheduler
+    else
+      scheduler.task(name)
+    end
+    scheduler.run
+    scheduler
+  end
+
+  def self.configure(file)
+    data = JSON.parse(File.read(file))
+    if data['log_level']
+      Blender::Log.level = data['log_level'].to_sym
+    end
+    if data['log_file']
+      Blender::Log.init(data['log_file'])
+    end
+    if data['load_paths']
+      data['load_paths'].each do |path|
+        $LOAD_PATH.unshift(path)
+      end
+    end
+    if data['scheduler']
+      data['scheduler'].each do |key, value|
+        Blender::Configuration[key.to_sym] = value
+      end
+    end
+  end
+end

data/lib/blender/cli.rb

@@ -0,0 +1,71 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'thor'
+require 'blender'
+require 'blender/timer'
+
+module Blender
+  class CLI < Thor
+
+    def self.exit_on_failure?
+      true
+    end
+
+    default_command :from_file
+    package_name 'Blender'
+
+    desc 'from_file ', 'Run blender job from a file'
+    method_option :file,
+      default: 'Blendfile',
+      type: :string,
+      aliases: '-f'
+
+    method_option :config_file,
+      default: nil,
+      type: :string,
+      aliases: '-c',
+      banner: 'Provide additional configuration via json file'
+
+    method_option :noop,
+      default: false,
+      type: :boolean,
+      aliases: '-n',
+      banner: 'No-op mode, run blender without executing jobs'
+
+    def from_file
+      Configuration[:noop] = options[:noop]
+      des = File.read(options[:file])
+      $LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(options[:file]), 'lib')))
+      Blender.blend(options[:file], options[:config_file]) do |sch|
+        sch.instance_eval(des, __FILE__, __LINE__)
+      end
+    end
+
+    desc 'schedule', 'Run blender in daemon mode, with job scheduled in periodic interval'
+    method_option :schedule,
+      default: 'Schedule_it',
+      type: :string,
+      aliases: '-s'
+    def schedule
+      sched = Blender::Timer.new
+      des = File.read(options[:schedule])
+      sched.instance_eval(des, __FILE__, __LINE__)
+      sched.join
+    end
+  end
+end

data/lib/blender/configuration.rb

@@ -0,0 +1,45 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'singleton'
+require 'thread'
+
+module Blender
+  class Configuration
+    include Singleton
+
+    attr_reader :data, :mutex
+
+    def initialize
+      @data = Hash.new{|h,k| h[k] = Hash.new}
+      @data[:noop] = false
+      @mutex = Mutex.new
+    end
+
+    def self.[]=(key, value)
+      instance.mutex.synchronize do
+        instance.data[key] = value
+      end
+    end
+
+    def self.[](key)
+      instance.mutex.synchronize do
+        instance.data[key]
+      end
+    end
+  end
+end

data/lib/blender/discovery.rb

@@ -0,0 +1,41 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'blender/utils/refinements'
+require 'blender/configuration'
+
+module Blender
+  module Discovery
+    include Blender::Utils::Refinements
+
+    def build_discovery(type, opts = {})
+      disco_klass = Blender::Discovery.const_get(camelcase(type.to_s).to_sym)
+      disco_opts = Blender::Configuration[type].merge(opts)
+      disco_klass.new(symbolize(disco_opts))
+    end
+
+    def search_with_config(type, opts = {})
+      options = opts.dup
+      search_opts = options.delete(:search)
+      build_discovery(type, options).search(search_opts)
+    end
+
+    def search(type, options = nil)
+      search_with_config(type, search: options)
+    end
+  end
+end

data/lib/blender/drivers/base.rb

@@ -0,0 +1,40 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'blender/log'
+
+module Blender
+  module Driver
+    class Base
+      attr_reader :config, :events, :stdout, :stderr
+
+      ExecOutput = Struct.new(:exitstatus, :stdout, :stderr)
+
+      def initialize(config = {})
+        cfg = config.dup
+        @stdout = cfg.delete(:stdout) || File.open(File::NULL, 'w')
+        @stderr = cfg.delete(:stderr) || File.open(File::NULL, 'w')
+        @events = cfg.delete(:events) or fail 'Events needed'
+        @config = cfg
+      end
+
+      def execute(tasks, hosts)
+        raise RuntimeError, 'this method must be overridden'
+      end
+    end
+  end
+end

data/lib/blender/drivers/compound.rb

@@ -0,0 +1,29 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+module Blender
+  module Driver
+    class Compound
+      def execute(tasks, hosts)
+        hosts.each do |host|
+          tasks.each do |task|
+            task.driver.execute([task], [host])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/blender/drivers/ruby.rb

@@ -0,0 +1,55 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'blender/drivers/base'
+
+module Blender
+  module Driver
+    class Ruby < Base
+
+      def execute(tasks, hosts)
+        tasks.each do |task|
+          hosts.each do |host|
+            cmd = run_command(task.command, host)
+            if cmd.exitstatus != 0 and !task.metadata[:ignore_failure]
+              raise ExecutionFailed, cmd.stderr
+            end
+          end
+        end
+      end
+
+      def run_command(command, host)
+        exit_status = 0
+        err = ''
+        current_stdout = STDOUT.clone
+        current_stderr = STDERR.clone
+        begin
+          STDOUT.reopen(stdout)
+          STDERR.reopen(stderr)
+          command.call(host)
+        rescue StandardError => e
+          err = e.message + "\nBacktrace:" + e.backtrace.join("\n")
+          exit_status = -1
+        ensure
+          STDOUT.reopen(current_stdout)
+          STDERR.reopen(current_stderr)
+        end
+        ExecOutput.new(exit_status, '', err)
+      end
+    end
+  end
+end

data/lib/blender/drivers/shellout.rb

@@ -0,0 +1,63 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'mixlib/shellout'
+
+module Blender
+  module Driver
+    class ShellOut < Base
+
+      def initialize(config = {})
+        @options = {}
+        cfg = config.dup
+        [:user, :group, :cwd, :umask, :returns, :environment, :timeout].each do |key|
+          if cfg.key?(key)
+            @options[key] = cfg.delete(key)
+          end
+        end
+        super(cfg)
+      end
+
+      def execute(tasks, hosts)
+        verify_local_host!(hosts)
+        tasks.each do |task|
+          cmd = run_command(task.command)
+          if cmd.exitstatus != 0 and !task.metadata[:ignore_failure]
+            raise ExecutionFailed, cmd.stderr
+          end
+        end
+      end
+
+      def run_command(command)
+        cmd = Mixlib::ShellOut.new(command, @options)
+        begin
+          cmd.live_stream = stdout
+          cmd.run_command
+          ExecOutput.new(cmd.exitstatus, cmd.stdout, cmd.stderr)
+        rescue Errno::ENOENT => e
+          ExecOutput.new(-1, '', e.message)
+        end
+      end
+
+      def verify_local_host!(hosts)
+        unless hosts.all?{|h|h == 'localhost'}
+          raise UnsupportedFeature, 'This driver does not support any host other than localhost'
+        end
+      end
+    end
+  end
+end