kuroko2 0.2.0

data/lib/kuroko2/workflow/assertion_error.rb

@@ -0,0 +1,6 @@
+module Kuroko2
+  module Workflow
+    class AssertionError < EngineError
+    end
+  end
+end

data/lib/kuroko2/workflow/engine.rb

@@ -0,0 +1,141 @@
+module Kuroko2
+  module Workflow
+    class Engine
+      def process_all
+        Token.processable.each do |token|
+          process(token)
+        end
+      end
+
+      def process(token)
+        unless token.working? || token.waiting?
+          Kuroko2.logger.info { "(token #{token.uuid}) Skip since current status marked as '#{token.status_name}'." }
+
+          return
+        end
+
+        token.with_lock { process_with_lock(token) }
+      end
+
+      def retry(token)
+        token.with_lock do
+          node = extract_node(token)
+
+          message = "(token #{token.uuid}) Retry current node: '#{node.type}: #{node.option}'"
+          token.job_instance.update_column(:error_at, nil)
+          token.job_instance.logs.info(message)
+
+          token.mark_as_working
+          token.save!
+
+          Kuroko2.logger.info(message)
+
+          Notifier.notify(:working, token.job_instance)
+        end
+      end
+
+      def skip(token)
+        token.with_lock do
+          node = extract_node(token)
+
+          message = "(token #{token.uuid}) Skip current node: '#{node.type}: #{node.option}'"
+          token.job_instance.update_column(:error_at, nil)
+          token.job_instance.logs.info(message)
+
+          token.mark_as_working
+          process_next(node.next, token)
+
+          token.save! unless token.destroyed?
+
+          Kuroko2.logger.info(message)
+
+          Notifier.notify(:working, token.job_instance)
+        end
+      end
+
+      def failure(token)
+        message = "(token #{token.uuid}) Mark as failure."
+
+        token.job_instance.logs.error(message)
+        token.job_instance.touch(:error_at)
+        token.mark_as_failure
+
+        Kuroko2.logger.info(message)
+
+        Notifier.notify(:failure, token.job_instance)
+
+        if token.context['AUTO_SKIP_ERROR']
+          skip(token)
+        end
+      end
+
+      private
+
+      def execute_task(node, token)
+        result = node.execute(token)
+
+        case result
+        when :next
+          process_next(node.next, token)
+        when :next_sibling
+          process_next(node.next_sibling, token)
+        when :pass
+          # Do nothing
+        when :failure
+          failure(token)
+        end
+      rescue KeyError => e
+        raise EngineError.new(e.message)
+      end
+
+      def process_next(node, token)
+        if node
+          message = "(token #{token.uuid}) Current node is '#{token.path}'."
+
+          token.path = node.path
+          token.job_instance.logs.info(message)
+
+          Kuroko2.logger.info(message)
+        else
+          message = "(token #{token.uuid}) Marked as 'finished'."
+
+          token.job_instance.logs.info(message)
+          token.mark_as_finished
+          unless token.parent
+            token.job_instance.touch(:finished_at)
+            token.destroy!
+          end
+
+          Kuroko2.logger.info(message)
+
+          Notifier.notify(:finished, token.job_instance)
+        end
+      end
+
+      def process_with_lock(token)
+        node = extract_node(token)
+
+        execute_task(node, token)
+      rescue EngineError => e
+        message = "#{e.message}\n" + e.backtrace.map { |trace| "    #{trace}" }.join("\n")
+
+        token.mark_as_critical(e)
+        token.job_instance.logs.error("(token #{token.uuid}) #{message}")
+        token.job_instance.touch(:canceled_at)
+
+        Token.delete_all(job_definition: token.job_definition)
+        token.job_instance.logs.warn("(token #{token.uuid}) This job is canceled.")
+
+        Kuroko2.logger.error(message)
+        Notifier.notify(:critical, token.job_instance)
+      ensure
+        token.save! unless token.destroyed?
+      end
+
+      def extract_node(token)
+        root = ScriptParser.new(token.script).parse(validate: false)
+        root.find(token.path)
+      end
+    end
+  end
+end

data/lib/kuroko2/workflow/engine_error.rb

@@ -0,0 +1,6 @@
+module Kuroko2
+  module Workflow
+    class EngineError < StandardError
+    end
+  end
+end

data/lib/kuroko2/workflow/node.rb

@@ -0,0 +1,124 @@
+module Kuroko2
+  module Workflow
+    class Node
+      PATH_REGEXP = %r(\A(?:/|(?:/\d+-[a-z0-9_]+)+)\z)
+      TASK_REGISTORY = {
+        root:                  Task::Sequence,
+        noop:                  Task::Noop,
+        sequence:              Task::Sequence,
+        auto_skip_error:       Task::AutoSkipError,
+        fork:                  Task::Fork,
+        env:                   Task::Env,
+        execute:               Task::Execute,
+        queue:                 Task::Queue,
+        sub_process:           Task::SubProcess,
+        subprocess:            Task::SubProcess,
+        timeout:               Task::Timeout,
+        expected_time:         Task::ExpectedTime,
+        wait:                  Task::Wait,
+        sleep:                 Task::Sleep,
+        rails_env:             Task::RailsEnv,
+        kuroko_runner:         Task::KurokoRunner,
+      }
+
+      attr_reader :type, :option, :children
+      attr_accessor :parent
+
+      def self.register(key: nil, klass:)
+        key ||= klass.to_s.demodulize.underscore.to_sym
+
+        unless TASK_REGISTORY.has_key?(key)
+          TASK_REGISTORY.store(key, klass)
+        else
+          Kuroko2.logger.warn("Unable to add '#{klass}' to task registory. '#{TASK_REGISTORY[key]}' is already registered.")
+        end
+      end
+
+      def self.deregister(key)
+        TASK_REGISTORY.delete(key)
+      end
+
+      def initialize(type, option = nil)
+        @type       = type.to_sym
+        @task_klass = TASK_REGISTORY.fetch(@type, nil)
+        @option     = option.try(:strip)
+        @parent     = nil
+        @children   = []
+
+        raise AssertionError, "`#{@type}` is not registered in task repository." unless @task_klass
+      end
+
+      def append_child(child)
+        child.parent = self
+        @children << child
+      end
+
+      def execute(token)
+        Kuroko2.logger.debug { "(token #{token.uuid}) Execute #{@type} with option '#{@option}'." }
+        @task_klass.new(self, token).execute.tap do |result|
+          Kuroko2.logger.debug("(token #{token.uuid}) Result is '#{result}'.")
+        end
+      end
+
+      def find(path)
+        raise AssertionError, "path query('#{path}') is invalid." unless PATH_REGEXP === path
+
+        query = path.split('/')
+        query.shift # drop first empty string.
+
+        traverse(query)
+      end
+
+      def next(index = 0)
+        if (child = children[index])
+          child
+        else
+          next_sibling
+        end
+      end
+
+      def next_sibling
+        if parent
+          parent.next(current_index + 1)
+        else
+          nil
+        end
+      end
+
+      def path
+        if parent
+          parent.path + "/#{current_index}-#{type}"
+        else
+          ''
+        end
+      end
+
+      def to_script(indent = 0)
+        "#{'  ' * indent}#{type}: #{option}\n" + children.map { |child| child.to_script(indent + 1) }.join
+      end
+
+      def validate_all
+        @task_klass.new(self, nil).validate
+        @children.each do |child|
+          child.validate_all
+        end
+      end
+
+      protected
+
+      def current_index
+        @_current_index = parent.children.index(self)
+      end
+
+      def traverse(query)
+        return self if query.empty?
+
+        first    = query.shift
+        index, _ = first.split('-')
+
+        @children[index.to_i].traverse(query)
+      end
+
+    end
+  end
+end

data/lib/kuroko2/workflow/notifier/concerns/chat_message_builder.rb

@@ -0,0 +1,39 @@
+module Kuroko2
+  module Workflow
+    module Notifier
+      module Concerns
+        class ChatMessageBuilder
+          def initialize(instance)
+            @instance   = instance
+            @definition = instance.job_definition
+          end
+
+          def failure_text
+            "Failed to execute '#{@definition.name}'"
+          end
+
+          def finished_text
+            "Finished to execute '#{@definition.name}'"
+          end
+
+          def long_elapsed_time_text
+            "The running time is longer than expected '#{@definition.name}'."
+          end
+
+          def additional_text
+            "Failed to execute '#{@definition.name}' #{@definition.hipchat_additional_text}"
+          end
+
+          def job_instance_path
+            Kuroko2::Engine.routes.url_helpers.job_definition_job_instance_url(
+              @definition,
+              @instance,
+              host: Kuroko2.config.url_host,
+              protocol: Kuroko2.config.url_scheme,
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kuroko2/workflow/notifier/hipchat.rb

@@ -0,0 +1,88 @@
+module Kuroko2
+  module Workflow
+    module Notifier
+      class Hipchat
+        attr_reader :hipchat, :message_builder
+
+        USER_NAME = 'kuroko2'
+
+        def initialize(instance)
+          @instance   = instance
+          @definition = instance.job_definition
+          @hipchat    = Kuroko2.config.notifiers.hipchat.api_token
+          @message_builder = Workflow::Notifier::Concerns::ChatMessageBuilder.new(instance)
+        end
+
+        def notify_working
+          # do nothing
+        end
+
+        def notify_cancellation
+          if @definition.notify_cancellation
+            message = build_message(level: 'WARNING', text: message_builder.failure_text)
+            message << "<br>"
+            message << @instance.logs.last(2).first.message
+
+            send_to_hipchat(message, color: 'yellow')
+          end
+        end
+
+        def notify_failure
+          message = build_message(level: 'FAILURE', text: message_builder.failure_text)
+          message << "<br>"
+          message << @instance.logs.last(2).first.message
+
+          send_to_hipchat(message, color: 'red', notify: true)
+          send_additional_text_to_hipchat
+        end
+
+        def notify_critical
+          message = build_message(level: 'CRITICAL', text: message_builder.failure_text)
+          message << "<br>"
+          message << @instance.logs.last(2).first.message
+
+          send_to_hipchat(message, color: 'red', notify: true)
+          send_additional_text_to_hipchat
+        end
+
+        def notify_finished
+          if @definition.hipchat_notify_finished?
+            message = build_message(level: 'SUCCESS', text: message_builder.finished_text)
+            send_to_hipchat(message)
+          end
+        end
+
+        def notify_long_elapsed_time
+          message = build_message(level: 'WARNING', text: message_builder.long_elapsed_time_text)
+          send_to_hipchat(message, color: 'red')
+        end
+
+        private
+
+        def send_to_hipchat(message, color: 'green', notify: false, format: 'html')
+          if notify_hipchat?
+
+            hipchat[@definition.hipchat_room].send(USER_NAME, message, color: color, notify: notify, message_format: format)
+          end
+        end
+
+        def build_message(level: , text:)
+          message = "<b>[#{level}]</b> "
+          message << text
+          message << "(<a href='#{message_builder.job_instance_path}'>Open</a>)"
+        end
+
+        def send_additional_text_to_hipchat
+          if @definition.hipchat_additional_text.present?
+            message = message_builder.additional_text
+            send_to_hipchat(message, color: 'red', notify: true, format: 'text')
+          end
+        end
+
+        def notify_hipchat?
+          @definition.hipchat_room.present?
+        end
+      end
+    end
+  end
+end

data/lib/kuroko2/workflow/notifier/mail.rb

@@ -0,0 +1,44 @@
+module Kuroko2
+  module Workflow
+    module Notifier
+      class Mail
+        def initialize(job_instance)
+          @job_instance = job_instance
+          @definition   = job_instance.job_definition
+        end
+
+        def notify_working
+          # do nothing
+        end
+
+        def notify_cancellation
+          if @definition.notify_cancellation
+            deliver_job_failure
+          end
+        end
+
+        def notify_failure
+          deliver_job_failure
+        end
+
+        def notify_critical
+          deliver_job_failure
+        end
+
+        def notify_finished
+          # do nothing
+        end
+
+        def notify_long_elapsed_time
+          Kuroko2::Notifications.notify_long_elapsed_time(@job_instance).deliver_now
+        end
+
+        private
+
+        def deliver_job_failure
+          Kuroko2::Notifications.job_failure(@job_instance).deliver_now
+        end
+      end
+    end
+  end
+end

data/lib/kuroko2/workflow/notifier/slack.rb

@@ -0,0 +1,121 @@
+module Kuroko2
+  module Workflow
+    module Notifier
+      class Slack
+        attr_reader :message_builder, :webhook_url
+
+        module LevelToColor
+          WARNING  = 'warning'
+          FAILURE  = 'danger'
+          CRITICAL = 'danger'
+          SUCCESS  = 'good'
+        end
+
+        def initialize(instance)
+          @instance   = instance
+          @definition = instance.job_definition
+          @message_builder = Workflow::Notifier::Concerns::ChatMessageBuilder.new(instance)
+          @webhook_url = Kuroko2.config.notifiers.slack.webhook_url
+        end
+
+        def notify_working
+          # do nothing
+        end
+
+        def notify_cancellation
+          if @definition.notify_cancellation
+            send_attachment_message_to_slack(
+              level: 'WARNING',
+              text: message_builder.failure_text,
+              body: @instance.logs.last(2).first.message,
+            )
+          end
+        end
+
+        def notify_failure
+          send_attachment_message_to_slack(
+            level: 'FAILURE',
+            text: message_builder.failure_text,
+            body: @instance.logs.last(2).first.message,
+          )
+
+          send_additional_text_to_slack
+        end
+
+        def notify_critical
+          send_attachment_message_to_slack(
+            level: 'CRITICAL',
+            text: message_builder.failure_text,
+            body: @instance.logs.last(2).first.message,
+          )
+
+          send_additional_text_to_slack
+        end
+
+        def notify_finished
+          if @definition.hipchat_notify_finished?
+            send_attachment_message_to_slack(
+              level: 'SUCCESS',
+              text: message_builder.finished_text,
+            )
+          end
+        end
+
+        def notify_long_elapsed_time
+          send_attachment_message_to_slack(
+            level: 'WARNING',
+            text: message_builder.long_elapsed_time_text,
+          )
+        end
+
+        private
+
+        def send_attachment_message_to_slack(level: , text: , body: nil)
+          return false unless @definition.slack_channel.present?
+
+          send_to_slack(
+            channel: @definition.slack_channel,
+            link_names: 1,
+            attachments: [
+              {
+                pretext: "[#{level}] #{text}",
+                title: "##{@definition.id} #{@definition.name}",
+                title_link: message_builder.job_instance_path,
+                text: body,
+                fallback: "[#{level}] #{text} #{message_builder.job_instance_path}",
+                color: LevelToColor.const_get(level),
+              }
+            ]
+          )
+        end
+
+        def send_to_slack(payload)
+          url = URI.parse(webhook_url)
+          conn = Faraday.new(:url => "#{url.scheme}://#{url.host}") do |faraday|
+            faraday.adapter Faraday.default_adapter
+          end
+
+          response = conn.post do |req|
+            req.url url.path
+            req.headers['Content-Type'] = 'application/json'
+            req.body = payload.to_json
+          end
+
+          unless response.success?
+            Kuroko2.logger.fatal("Failure sending message to Slack: status=#{response.status} body=#{response.body}")
+          end
+        end
+
+        def send_additional_text_to_slack
+          if @definition.slack_channel.present? && @definition.hipchat_additional_text.present?
+            send_to_slack(
+              channel: @definition.slack_channel,
+              text: message_builder.additional_text,
+              link_names: 1
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kuroko2/workflow/notifier.rb

@@ -0,0 +1,31 @@
+module Kuroko2
+  module Workflow
+    module Notifier
+      NOTIFY_IN_THREAD = !Rails.env.test?
+
+      def self.notify(method, job_instance)
+        Kuroko2.config.notifiers.keys.each do |notifier_name|
+          notifier = const_get(notifier_name.camelize, false)
+          if NOTIFY_IN_THREAD
+            Thread.new { notify_with_notifier(job_instance, method, notifier) }
+          else
+            # for test
+            notify_with_notifier(job_instance, method, notifier)
+          end
+        end
+      end
+
+      def self.notify_with_notifier(job_instance, method, notifier)
+        begin
+          ActiveRecord::Base.connection_pool.with_connection do
+            notifier.new(job_instance).send(:"notify_#{method}")
+          end
+        rescue Exception => e
+          Kuroko2.logger.warn("Failure to notify #{method} with #{notifier} for '#{job_instance.job_definition.name}'. #{e.class}: #{e.message}")
+        end
+      end
+
+      private_class_method :notify_with_notifier
+    end
+  end
+end

data/lib/kuroko2/workflow/processor.rb

@@ -0,0 +1,40 @@
+module Kuroko2
+  module Workflow
+    module Processor
+      def initialize
+        @hostname = Socket.gethostname
+
+        @stop       = ServerEngine::BlockingFlag.new
+        @processing = ServerEngine::BlockingFlag.new
+
+        @workflow = Workflow::Engine.new
+      end
+
+      def run
+        Kuroko2.logger = logger
+        Kuroko2.logger.info "[#{@hostname}-#{worker_id}] Start Workflow::Processor"
+
+        until @stop.wait(1.0)
+          unless @processing.set?
+            begin
+              @processing.set!
+              @workflow.process_all
+              @processing.reset!
+            end
+          end
+        end
+      rescue Exception => e
+        Kuroko2.logger.fatal("[#{@hostname}-#{worker_id}] #{e.class}: #{e.message}\n" +
+          e.backtrace.map { |trace| "    #{trace}" }.join("\n"))
+
+        raise e
+      end
+
+      def stop
+        Kuroko2.logger.info "[#{@hostname}-#{worker_id}] Stop Workflow::Processor"
+
+        @stop.set!
+      end
+    end
+  end
+end

data/lib/kuroko2/workflow/scheduler.rb

@@ -0,0 +1,42 @@
+module Kuroko2
+  module Workflow
+    module Scheduler
+      def initialize
+        @hostname = Socket.gethostname
+
+        @stop       = ServerEngine::BlockingFlag.new
+        @processing = ServerEngine::BlockingFlag.new
+      end
+
+      def run
+        Kuroko2.logger = logger
+        Kuroko2.logger.info "[#{@hostname}-#{worker_id}] Start Workflow::Scheduler"
+
+        until @stop.wait(2.0)
+          unless @processing.set?
+            begin
+              @processing.set!
+              JobSchedule.transaction do
+                now = Time.now
+                last_scheduled_time = Tick.fetch_then_update(now)
+                JobSchedule.launch_scheduled_jobs!(last_scheduled_time, now)
+              end
+              @processing.reset!
+            end
+          end
+        end
+      rescue Exception => e
+        Kuroko2.logger.fatal("[#{@hostname}-#{worker_id}] #{e.class}: #{e.message}\n" +
+          e.backtrace.map { |trace| "    #{trace}" }.join("\n"))
+
+        raise e
+      end
+
+      def stop
+        Kuroko2.logger.info "[#{@hostname}-#{worker_id}] Stop Workflow::Scheduler"
+
+        @stop.set!
+      end
+    end
+  end
+end