pd-blender 0.0.1

data/lib/blender/lock/flock.rb

@@ -0,0 +1,64 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'timeout'
+require 'fcntl'
+
+module Blender
+  module Lock
+    class Flock
+
+      def initialize(name, options)
+        @path = options['path'] || File.join('/tmp', name)
+        @timeout = options[:timeout] || 0
+        @job_name = name
+        @lock_fd = nil
+      end
+
+      def acquire
+        @lock_fd = File.open(@path, File::CREAT|File::RDWR, 0644)
+        @lock_fd.fcntl( Fcntl::F_SETFD, @lock_fd.fcntl(Fcntl::F_GETFD, 0) | Fcntl::FD_CLOEXEC )
+        if @timeout > 0
+          Timeout.timeout(@timeout) do
+            @lock_fd.flock(File::LOCK_EX)
+          end
+        else
+          locked = @lock_fd.flock(File::LOCK_NB | File::LOCK_EX)
+          raise LockAcquisitionError, 'Failed to lock file' if locked == false
+        end
+        @lock_fd.write({job: @job_name, pid: Process.pid }.inspect)
+      end
+
+      def release
+        @lock_fd
+        @lock_fd.flock(File::LOCK_UN)
+        @lock_fd.close
+      end
+
+      def with_lock
+        acquire
+        yield if block_given?
+      rescue Timeout::Error => e
+        raise LockAcquisitionError, 'Timeout while waiting for lock acquisition'
+      rescue LockAcquisitionError => e
+        raise e
+      ensure
+        release
+      end
+    end
+  end
+end

data/lib/blender/log.rb

@@ -0,0 +1,24 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'mixlib/log'
+
+module Blender
+  class Log
+    extend Mixlib::Log
+  end
+end

data/lib/blender/rspec.rb

@@ -0,0 +1,68 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+begin
+  require 'rspec'
+  require 'rspec/mocks'
+rescue LoadError
+  abort 'Blender::RSpec requires RSpec, RSpec::Mocks'
+end
+
+require 'blender'
+require 'blender/rspec/stub_registry'
+
+module Blender
+  module Discovery
+    alias_method :old_search, :search
+    def search(type, options = nil)
+      stub = Blender::RSpec::StubRegistry.instance.data.detect do |st|
+        st.type == type && st.opts == options
+      end
+      if stub
+        stub.return_value
+      else
+        old_search(type, options)
+      end
+    end
+  end
+  class Utils::UI
+    def puts(string)
+    end
+  end
+
+  module RSpec
+    extend self
+    include Blender::Utils::Refinements
+    def stub_search(type, options = nil)
+      StubRegistry.add(type, options)
+    end
+
+    def noop_scheduler_from_file(file)
+      Blender::Configuration[:noop] = true
+      des = File.read(file)
+      $LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(file), 'lib')))
+      Blender.blend(file) do |sch|
+        sch.lock_options(nil)
+        sch.instance_eval(des, __FILE__, __LINE__)
+      end
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include Blender::RSpec
+end

data/lib/blender/rspec/stub_registry.rb

@@ -0,0 +1,45 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'singleton'
+
+module Blender
+  module RSpec
+    class SearchStub
+      attr_reader :type, :opts, :return_value
+      def initialize(type, opts)
+        @type = type
+        @opts = opts
+      end
+      def and_return(value)
+        @return_value = value
+      end
+    end
+    class StubRegistry
+      include Singleton
+      attr_reader :data
+      def initialize
+        @data = []
+      end
+      def self.add(type, opts)
+        obj = SearchStub.new(type, opts)
+        instance.data << obj
+        obj
+      end
+    end
+  end
+end

data/lib/blender/scheduled_job.rb

@@ -0,0 +1,66 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Blender
+  # A scheduled job encapsulates a blender based job to be executed
+  # at certain interval. Job is specified as a file path, where
+  # the file contains job written in blender's DSL. Job interval can be
+  # specified either via +cron+ or  +every+ method
+  #
+  # +Blender::Timer+ object uses +ScheduledJob+ and to execute the job
+  # and Rufus::Scheduler to schedule it
+  class ScheduledJob
+    attr_reader :schedule, :file
+    # create a new instance
+    # @param name [String] name of the job
+    def initialize(name)
+      @name = name
+      @file = name
+    end
+
+    # set the path of the file holding blender job
+    #
+    # @param file [String] path of the blender file
+    def blender_file(file)
+      @file = file
+    end
+
+    # set the job inteval via cron syntax. The value is passed as it  is
+    # to rufus scheduler.
+    #
+    # @param line [String] job interval in cron syntax e.g (*/5 * * * *)
+    def cron(line)
+      @schedule = [ __method__, line]
+    end
+
+    # set the job inteval after every specified seconds
+    # to rufus scheduler.
+    #
+    # @param interval [Fixnum] job interval in seconds
+    def every(interval)
+      @schedule = [ __method__, interval]
+    end
+
+    # invoke a blender run based on the +blender_file+
+    def run
+      des = File.read(file)
+      Blender.blend(file) do |sch|
+        sch.instance_eval(des, __FILE__, __LINE__)
+      end
+    end
+  end
+end

data/lib/blender/scheduler.rb

@@ -0,0 +1,114 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'blender/log'
+require 'blender/configuration'
+require 'blender/utils/thread_pool'
+require 'blender/exceptions'
+require 'blender/scheduling_strategies/default'
+require 'blender/scheduling_strategies/per_host'
+require 'blender/scheduling_strategies/per_task'
+require 'blender/utils/thread_pool'
+require 'blender/scheduler/dsl'
+require 'blender/event_dispatcher'
+require 'blender/handlers/doc'
+require 'blender/tasks/base'
+
+module Blender
+  class Scheduler
+
+    include SchedulerDSL
+    include Lock
+
+    attr_reader :metadata, :name
+    attr_reader :scheduling_strategy
+    attr_reader :events, :tasks
+    attr_reader :lock_properties
+
+    def initialize(name, tasks = [], metadata = {})
+      @name = name
+      @tasks = tasks
+      @metadata = default_metadata.merge(metadata)
+      @events = Blender::EventDispatcher.new
+      events.register(Blender::Handlers::Doc.new)
+      @scheduling_strategy = nil
+      @lock_properties = {driver: 'flock', driver_options: {}}
+    end
+
+    def run
+      @scheduling_strategy ||= SchedulingStrategy::Default.new
+      events.run_started(self)
+      events.job_computation_started(scheduling_strategy)
+      jobs = scheduling_strategy.compute_jobs(@tasks)
+      events.job_computation_finished(self, jobs)
+      lock do
+        if metadata[:concurrency] > 1
+          concurrent_run(jobs)
+        else
+          serial_run(jobs)
+        end
+        events.run_finished(self)
+        jobs
+      end
+    rescue StandardError => e
+      events.run_failed(self, e)
+      raise e
+    end
+
+    def serial_run(jobs)
+      Log.debug('Invoking serial run')
+      jobs.each do |job|
+        run_job(job)
+      end
+    end
+
+    def concurrent_run(jobs)
+      c = metadata[:concurrency]
+      Log.debug("Invoking concurrent run with concurrency:#{c}")
+      pool = Utils::ThreadPool.new(c)
+      jobs.each do |job|
+        pool.add_job do
+          run_job(job)
+        end
+      end
+      pool.run_till_done
+    end
+
+    def run_job(job)
+      events.job_started(job)
+      Log.debug("Running job #{job.name}")
+      job.run
+      events.job_finished(job)
+    rescue StandardError => e
+      events.job_failed(job, e)
+      if metadata[:ignore_failure]
+        Log.warn("Exception: #{e.inspect} was suppressed, ignoring failure")
+      else
+        raise e
+      end
+    end
+
+    def default_metadata
+      {
+       ignore_failure: false,
+       concurrency: 0,
+       handlers: [],
+       members: []
+      }
+    end
+  end
+end

data/lib/blender/scheduler/dsl.rb

@@ -0,0 +1,160 @@
+#
+# Author:: Ranjib Dey (<ranjib@pagerduty.com>)
+# Copyright:: Copyright (c) 2014 PagerDuty, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'blender/exceptions'
+require 'blender/scheduling_strategies/default'
+require 'blender/tasks/base'
+require 'blender/tasks/ruby'
+require 'blender/tasks/ssh'
+require 'blender/tasks/shell_out'
+require 'highline'
+require 'blender/utils/refinements'
+require 'blender/drivers/ssh'
+require 'blender/drivers/ssh_multi'
+require 'blender/drivers/shellout'
+require 'blender/drivers/ruby'
+require 'blender/discovery'
+require 'blender/handlers/base'
+require 'blender/lock/flock'
+
+module Blender
+  module SchedulerDSL
+    include Blender::Utils::Refinements
+    include Blender::Discovery
+
+    def config(type, opts = {})
+      Blender::Configuration[type].merge!(opts)
+    end
+
+    alias :init :config
+
+    def log_level(level)
+      Blender::Log.level = level
+    end
+
+    def ask(msg, echo = false)
+      HighLine.new.ask(msg){|q| q.echo = echo}
+    end
+
+    def driver(type, opts = {})
+      klass_name = camelcase(type.to_s).to_sym
+      config = symbolize(opts.merge(events: events))
+      yield config if block_given?
+      begin
+        Blender::Driver.const_get(klass_name).new(config)
+      rescue NameError => e
+        raise UnknownDriver, e.message
+      end
+    end
+
+    def add_handler(handler)
+      events.register(handler)
+    end
+
+    alias :register_handler :add_handler
+
+    def on(event_type, &block)
+      add_handler(
+        Class.new(Handlers::Base) do
+          define_method(event_type) do |*args|
+            block.call(args)
+          end
+        end.new
+      )
+    end
+
+    def build_task(name, type)
+      task_klass = Blender::Task.const_get(camelcase(type.to_s).to_sym)
+      driver_klass = Blender::Driver.const_get(camelcase(type.to_s).to_sym)
+      task = task_klass.new(name)
+      task.members(metadata[:members]) unless metadata[:members].empty?
+      task
+    end
+
+    def append_task(type, task)
+      Log.debug("Appended task:#{task.name}")
+      klass = Blender::Driver.const_get(camelcase(type.to_s).to_sym)
+      if task.driver.nil?
+        opts = {}
+        opts.merge!(Blender::Configuration[type]) unless Blender::Configuration[type].empty?
+        opts.merge!(task.driver_opts)
+        task.use_driver(driver(type, opts))
+      end
+      @tasks << task
+    end
+
+    def shell_task(name, &block)
+      task = build_task(name, :shell_out)
+      task.members(['localhost'])
+      task.instance_eval(&block) if block_given?
+      append_task(:shell_out, task)
+    end
+
+    def ruby_task(name, &block)
+      task = build_task(name, :ruby)
+      task.instance_eval(&block) if block_given?
+      append_task(:ruby, task)
+    end
+
+    def ssh_task(name, &block)
+      task = build_task(name, :ssh)
+      task.instance_eval(&block) if block_given?
+      append_task(:ssh, task)
+    end
+
+    def strategy(strategy)
+      klass_name = camelcase(strategy.to_s).to_sym
+      begin
+        @scheduling_strategy = Blender::SchedulingStrategy.const_get(klass_name).new
+        @scheduling_strategy.freeze
+      rescue NameError => e
+        raise UnknownSchedulingStrategy, e.message
+      end
+    end
+
+    def concurrency(value)
+      @metadata[:concurrency] = value
+    end
+
+    def ignore_failure(value)
+      @metadata[:ignore_failure] = value
+    end
+
+    def members(hosts)
+      @metadata[:members] = hosts
+    end
+
+    def lock_options(driver, opts = {})
+      @lock_properties[:driver] = driver
+      @lock_properties[:driver_options].merge!(opts.dup)
+    end
+
+    def lock(opts = {})
+      options = lock_properties.dup.merge(opts)
+      if options[:driver]
+        lock_klass = Lock.const_get(camelcase(options[:driver]).to_sym)
+        lock_klass.new(name, options[:driver_options]).with_lock do
+          yield if block_given?
+        end
+      else
+        yield if block_given?
+      end
+    end
+
+    alias_method :task, :shell_task
+  end
+end