kuroko2 0.2.0

data/lib/kuroko2/command/monitor.rb

@@ -0,0 +1,109 @@
+require 'open3'
+module Kuroko2
+  module Command
+    class Monitor
+      NUM_FAILURES = 15
+
+      def initialize(hostname:, worker_id:)
+        @hostname  = hostname
+        @worker_id = worker_id
+        @counter   = Hash.new(0)
+        @intervals = {}
+      end
+
+      def execute
+        execution_ids = Worker.on(@hostname).pluck(:execution_id).compact
+        executions = Execution.where(id: execution_ids, mailed_at: nil).started
+        executions.each do |execution|
+          if execution.pid
+            if check_process_absence(execution) && log_memory_consumption?(execution)
+              get_memory_consumption(execution).try do |value|
+                execution.log_memory_consumption(value)
+              end
+            end
+          else
+            check_assignment_delay(execution)
+          end
+        end
+
+        (@counter.keys - executions.map(&:id)).each do |removable_id|
+          @counter.delete(removable_id)
+        end
+      end
+
+      def counter_size
+        @counter.size
+      end
+
+      private
+
+      # @return [Boolean] true means process is exists
+      def check_process_absence(execution)
+        begin
+          process_num = Process.kill(0, execution.pid)
+          @counter.delete(execution.id)
+          !!process_num
+        rescue Errno::EPERM
+          true
+        rescue Errno::ESRCH
+          if Execution.exists?(execution.id)
+            @counter[execution.id] += 1
+
+            message = "(execution.id #{execution.id}) : PID #{execution.pid} not found, increment monitor counter to #{@counter[execution.id]}."
+            Kuroko2.logger.info { message }
+            Kuroko2.logger.info(@counter) # TODO: will remove this logging (for debug only).
+
+            if @counter[execution.id] >= NUM_FAILURES
+              notify_process_absence(execution)
+              @counter.delete(execution.id)
+              @intervals.delete(execution.id)
+            end
+          else
+            @counter.delete(execution.id)
+            @intervals.delete(execution.id)
+          end
+          false
+        end
+      end
+
+      def log_memory_consumption?(execution)
+        if @intervals[execution.id]
+          @intervals[execution.id].reached?(Time.now)
+        else # first time
+          @intervals[execution.id] = MemoryConsumptionLog::Interval.new(Time.now)
+          true
+        end
+      end
+
+      def get_memory_consumption(execution)
+        result = MemorySampler.get_by_pgid(execution.pid)
+        if result
+          @intervals[execution.id] = @intervals[execution.id].next
+          result
+        else
+          nil
+        end
+      end
+
+      def notify_process_absence(execution)
+        message = "(execution #{execution.uuid}) Deliver notification mail: PID #{execution.pid} is not running."
+        Kuroko2.logger.info { message }
+        execution.job_instance.logs.warn(message)
+
+        Kuroko2::Notifications.process_absence(execution, @hostname).deliver_now
+        execution.touch(:mailed_at)
+      end
+
+      def check_assignment_delay(execution)
+        if execution.started_at < 1.minutes.ago
+          message = "(execution #{execution.uuid}) Deliver notification mail: process is not assigned to any job-executor."
+          Kuroko2.logger.info { message }
+          execution.job_instance.logs.warn(message)
+
+          Kuroko2::Notifications.executor_not_assigned(execution, @hostname).deliver_now
+          execution.touch(:mailed_at)
+        end
+      end
+    end
+  end
+end

data/lib/kuroko2/command/shell.rb

@@ -0,0 +1,163 @@
+require 'open3'
+module Kuroko2
+  module Command
+    class Shell
+      MAX_OUTPUT_LENGTH = 60_000
+      MAX_READ_LENGTH   = 1024
+
+      def initialize(hostname:, worker_id: 0, worker:, queue: Execution::DEFAULT_QUEUE)
+        @hostname  = hostname
+        @worker_id = worker_id
+        @worker    = worker
+        @queue     = queue
+      end
+
+      def execute
+        @worker.reload
+        unless @worker.execution_id?
+          if (execution = Execution.poll(@queue))
+            do_execute(execution)
+            execution
+          end
+        end
+      rescue RuntimeError => e
+        Kuroko2.logger.error("[#{@hostname}-#{@worker_id}] #{e.message}\n" +
+          e.backtrace.map { |trace| "[#{@hostname}-#{@worker_id}]    #{trace}" }.join("\n"))
+
+        nil
+      end
+
+      private
+
+      def do_execute(execution)
+        begin
+          @worker.update_column(:execution_id, execution.id)
+
+          invoke(execution)
+        rescue SystemCallError => e
+          message = "[#{@hostname}-#{@worker_id}] (uuid #{execution.uuid}) `#{execution.shell}` failed because #{e.class}: #{e.message}"
+          execution.token.job_instance.logs.warn(message)
+          Kuroko2.logger.warn(message)
+
+          output = truncate_and_escape(e.message)
+          execution.finish(output: output, exit_status: e.errno)
+        ensure
+          @worker.update_column(:execution_id, nil)
+        end
+      end
+
+      def invoke(execution)
+        command = execution.shell
+        env     = execution.context.fetch('ENV', {})
+
+        message = "[#{@hostname}-#{@worker_id}] (uuid #{execution.uuid}) `#{command}` run with env (#{env})"
+        execution.token.job_instance.logs.info(message)
+        Kuroko2.logger.info(message)
+
+        output, status = execute_shell(command, env, execution)
+        output         = truncate_and_escape(output)
+
+        if status.signaled?
+          message = "[#{@hostname}-#{@worker_id}] (uuid #{execution.uuid}) `#{command}` stopped by #{Signal.signame(status.termsig)} signal (pid #{status.pid})"
+          execution.token.job_instance.logs.warn(message)
+          Kuroko2.logger.warn(message)
+
+          execution.finish_by_signal(output: output, term_signal: status.termsig)
+        else
+          message = "[#{@hostname}-#{@worker_id}] (uuid #{execution.uuid}) `#{command}` finished with #{status.exitstatus} (pid #{status.pid})"
+          execution.token.job_instance.logs.info(message)
+          Kuroko2.logger.info(message)
+
+          execution.finish(output: output, exit_status: status.exitstatus)
+        end
+      end
+
+      def execute_shell(command, env, execution)
+        opts = { unsetenv_others: true, pgroup: true }
+        opts[:chdir] = real_path(execution.context['CHDIR']) if execution.context['CHDIR']
+
+        launched_time       = execution.context['meta'].try(:[], 'launched_time').to_s
+        job_definition_id   = execution.context['meta'].try(:[], 'job_definition_id').to_s
+        job_definition_name = execution.context['meta'].try(:[], 'job_definition_name').to_s
+        job_instance_id     = execution.context['meta'].try(:[], 'job_instance_id').to_s
+
+        env.reverse_merge!(
+          'HOME'                        => ENV['HOME'],
+          'PATH'                        => ENV['PATH'],
+          'LANG'                        => ENV['LANG'],
+          'KUROKO2_LAUNCHED_TIME'       => launched_time,
+          'KUROKO2_JOB_DEFINITION_ID'   => job_definition_id,
+          'KUROKO2_JOB_DEFINITION_NAME' => job_definition_name,
+          'KUROKO2_JOB_INSTANCE_ID'     => job_instance_id,
+        )
+
+        execution_logger = ExecutionLogger.get_logger(
+          stream_name: "JOB#{sprintf("%010d", job_definition_id.to_i)}/#{execution.token.job_instance.id}",
+        )
+
+        temporally_path_with(env['PATH']) do
+          Open3.popen2e(env, command, opts) do |stdin, stdout_and_stderr, thread|
+            stdin.close
+
+            pid = thread.pid
+            execution.update_attributes(pid: pid)
+
+            reader = Thread.new do
+              begin
+                output = ''
+                stdout_and_stderr.each do |data|
+                  output << data
+
+                  begin
+                    execution_logger.send_log(
+                      {
+                        uuid: execution.uuid,
+                        pid: pid,
+                        level: 'NOTICE',
+                        message: truncate_and_escape(data.chomp),
+                      }
+                    )
+                  rescue => e
+                    Kuroko2.logger.error(
+                      "[#{@hostname}-#{@worker_id}] #{e.message}\n    " + e.backtrace.join("\n    "))
+                  end
+                end
+              rescue EOFError
+                # do nothing
+              ensure
+                next output
+              end
+            end
+
+            status = thread.value # wait until thread is dead
+            output = reader.value
+
+            [output, status]
+          end
+        end
+      end
+
+      def real_path(path)
+        path = Pathname.new(path.sub(/\/\Z/, ''))
+        Retryable.retryable(tries: 3, sleep: 0.5, on: [Errno::ENOENT]) do
+          path.realpath
+        end
+      end
+
+      def temporally_path_with(path)
+        original_path = ENV['PATH']
+
+        ENV['PATH'] = path
+        yield
+      ensure
+        ENV['PATH'] = original_path
+      end
+
+      def truncate_and_escape(str)
+        str.force_encoding('utf-8')
+        truncated = str.length > MAX_OUTPUT_LENGTH ? str[0...MAX_OUTPUT_LENGTH] : str
+        truncated.scrub.each_char.select{ |c| c.bytes.count < 4 }.join('')
+      end
+    end
+  end
+end

data/lib/kuroko2/configuration.rb

@@ -0,0 +1,19 @@
+require 'hashie'
+
+module Kuroko2
+  class Configuration
+    class << self
+      DEFAULT_CONFIG = { table_name_prefix: 'kuroko2_' }.freeze
+
+      def config
+        @config ||= build_config
+      end
+
+      def build_config
+        filename = Rails.root.join('config', 'kuroko2.yml')
+        yaml = YAML::load(ERB.new(File.read(filename)).result)
+        Hashie::Mash.new(DEFAULT_CONFIG.merge(yaml[Rails.env]))
+      end
+    end
+  end
+end

data/lib/kuroko2/engine.rb

@@ -0,0 +1,59 @@
+module Kuroko2
+  class Engine < ::Rails::Engine
+    isolate_namespace Kuroko2
+
+    config.before_configuration do
+      require 'kaminari'
+      require 'slim'
+      require 'jbuilder'
+      require 'garage'
+      require 'jquery-rails'
+      require 'momentjs-rails'
+      require 'rails_bootstrap_sortable'
+      require 'select2-rails'
+      require 'font-awesome-rails'
+      require 'visjs/rails'
+      require 'dotenv-rails'
+      require 'weak_parameters'
+    end
+
+    config.autoload_paths << root.join('lib')
+
+    initializer "kuroko2.configuration" do |app|
+      URI.parse(Kuroko2.config.url).tap do |url|
+        Kuroko2.config.url_host   = url.host
+        Kuroko2.config.url_scheme = url.scheme
+        Kuroko2.config.url_port   = url.port
+      end
+
+      config.active_record.table_name_prefix = Kuroko2.config.table_name_prefix
+
+      if Kuroko2.config.custom_tasks
+        Kuroko2.config.custom_tasks.each do |key, klass|
+          unless Workflow::Node::TASK_REGISTORY.has_key?(key)
+            Workflow::Node.register(
+              key: key.to_sym,
+              klass: Workflow::Task.const_get(klass, false)
+            )
+          end
+        end
+      end
+
+      config.action_mailer.default_url_options = {
+        host:     Kuroko2.config.url_host,
+        protocol: Kuroko2.config.url_scheme,
+        port:     Kuroko2.config.url_port
+      }
+
+      config.action_mailer.delivery_method = Kuroko2.config.action_mailer.delivery_method.to_sym
+      config.action_mailer.smtp_settings =
+        Kuroko2.config.action_mailer.smtp_settings.to_h.symbolize_keys || {}
+
+      if Kuroko2.config.extentions && Kuroko2.config.extentions.controller
+        Kuroko2.config.extentions.controller.each do |extention|
+          Kuroko2::ApplicationController.include(Module.const_get(extention, false))
+        end
+      end
+    end
+  end
+end

data/lib/kuroko2/execution_logger/cloud_watch_logs.rb

@@ -0,0 +1,92 @@
+module Kuroko2
+  class ExecutionLogger::CloudWatchLogs
+    MAX_RETRY_COUNT = 5
+    RETRY_ERRORS = [
+      Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException,
+      Aws::CloudWatchLogs::Errors::ThrottlingException,
+      Aws::CloudWatchLogs::Errors::ResourceNotFoundException,
+    ]
+
+    attr_reader :client
+
+    def initialize(stream_name:, group_name:, region: 'ap-northeast-1')
+      @client = Aws::CloudWatchLogs::Client.new(region: region)
+
+      @group_name    = group_name
+      @stream_name   = stream_name
+      @put_log_token = nil
+      @get_log_token = nil
+    end
+
+    def send_log(message)
+      put_logs([{ timestamp: timestamp_now, message: message.to_json }])
+    end
+
+    def put_logs(events)
+      exception_cb = lambda do |exception|
+        Kuroko2.logger.warn("#{exception.class} #{exception.message} #{events}")
+
+        case exception
+        when Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException
+          old_token = @put_log_token
+          new_token = exception.message.match(%r{\AThe given sequenceToken is invalid. The next expected sequenceToken is:\s*(\w+)\z})[1]
+          if new_token
+            @put_log_token = new_token
+            Kuroko2.logger.warn("Refreshed sequenceToken from '#{old_token}' to '#{@put_log_token}'")
+          end
+        when Aws::CloudWatchLogs::Errors::ResourceNotFoundException
+          create_log_stream
+        when Aws::CloudWatchLogs::Errors::ThrottlingException
+          sleep(0.5)
+        end
+      end
+
+      retry_options = {
+        exception_cb: exception_cb,
+        on: RETRY_ERRORS,
+        tries: MAX_RETRY_COUNT,
+        sleep: 0,
+      }
+
+      Retryable.retryable(retry_options) do
+        response = client.put_log_events(
+          log_group_name: @group_name,
+          log_stream_name: @stream_name,
+          log_events: events,
+          sequence_token: @put_log_token,
+        )
+        @put_log_token = response.data[:next_sequence_token]
+
+        Kuroko2.logger.debug("Put logs: #{@group_name} #{@stream_name} / #{response.data}")
+        response
+      end
+    end
+
+    def get_logs(token = @get_log_token)
+      response = client.get_log_events({
+        log_group_name: @group_name,
+        log_stream_name: @stream_name,
+        next_token: token,
+        start_from_head: true,
+      })
+
+      @get_log_token = response.next_forward_token
+      response
+    rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
+      raise ExecutionLogger::NotFound
+    end
+
+    private
+
+    def timestamp_now
+      (Time.now.to_f * 1000).to_i # milliseconds
+    end
+
+    def create_log_stream
+      Kuroko2.logger.info("Create log stream: #{@group_name} #{@stream_name}")
+      client.create_log_stream(log_group_name: @group_name, log_stream_name: @stream_name)
+    rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+      warn "Log stream '#{@stream_name}' already exists"
+    end
+  end
+end

data/lib/kuroko2/execution_logger/void.rb

@@ -0,0 +1,13 @@
+module Kuroko2
+  class ExecutionLogger::Void
+    def initialize(option = {})
+    end
+
+    def send_log(message)
+    end
+
+    def get_logs(token = nil)
+      raise ExecutionLogger::NotFound
+    end
+  end
+end

data/lib/kuroko2/execution_logger.rb

@@ -0,0 +1,20 @@
+module Kuroko2
+  module ExecutionLogger
+    class NotFound < StandardError
+    end
+
+    def self.get_logger(option = {})
+      config = Kuroko2.config.execution_logger
+      if config.present? && config.type.present?
+        logger_class = const_get(config.type, false)
+        if config.option.present?
+          logger_class.new(config.option.to_h.merge(option).symbolize_keys)
+        else
+          logger_class.new(option)
+        end
+      else
+        Void.new(option)
+      end
+    end
+  end
+end

data/lib/kuroko2/memory_sampler.rb

@@ -0,0 +1,50 @@
+require 'open3'
+module Kuroko2
+  module MemorySampler
+    extend self
+
+    # @param [Integer] pgid process group id
+    # @return [Integer] sum of memory consumptions of given process group
+    def get_by_pgid(pgid)
+      case platform
+      when /linux/
+        get_by_pgid_linux(pgid)
+      when /darwin/
+        get_by_pgid_osx(pgid)
+      else
+        raise "Unknown platform: #{platform}"
+      end
+    rescue SystemCallError
+      nil
+    end
+
+    private
+
+    # Note:
+    #   taiki-ono@ci-slave-ruby-001:~$ ps -o pgid= -o rss=
+    #   22848   888
+    #   25848  4056
+    def get_by_pgid_linux(pgid)
+      output, _, status = Open3.capture3('ps', '-o', 'pgid=', '-o', 'rss=')
+      if status.success?
+        targets = output.split("\n").select {|line| line.split(' ').first == pgid.to_s }
+        calculate_sum(targets.map {|line| line.split(' ')[1] })
+      else
+        nil
+      end
+    end
+
+    def get_by_pgid_osx(pgid)
+      output, _, status = Open3.capture3('ps', '-o' 'rss=', '-g', pgid.to_s)
+      status.success? ? calculate_sum(output.split("\n")) : nil
+    end
+
+    def calculate_sum(rss_lines)
+      rss_lines.reject(&:blank?).map {|s| s.scan(/\d+/).first }.map(&:to_i).reduce(&:+)
+    end
+
+    def platform
+      RUBY_PLATFORM.downcase
+    end
+  end
+end

data/lib/kuroko2/return_to_validator.rb

@@ -0,0 +1,14 @@
+module Kuroko2
+  module ReturnToValidator
+    def self.valid?(return_to)
+      if return_to.nil?
+        return false
+      end
+
+      uri = Addressable::URI.parse(return_to)
+      !uri.nil? && uri.host.nil? && uri.scheme.nil? && uri.path.start_with?('/')
+    rescue Addressable::URI::InvalidURIError
+      false
+    end
+  end
+end

data/lib/kuroko2/servers/base.rb

@@ -0,0 +1,30 @@
+require 'kuroko2'
+require 'serverengine'
+
+module Kuroko2
+  module Servers
+    class Base
+      if Rails.env.development?
+        ActionMailer::Base.logger = Kuroko2.logger
+      end
+
+      def initialize(options = {})
+        @options = options
+      end
+
+      def run
+        ServerEngine.create(nil, worker, default_options.merge(@options)).run
+      end
+
+      private
+
+      def worker
+        raise NotImplementedError
+      end
+
+      def default_options
+        {}
+      end
+    end
+  end
+end

data/lib/kuroko2/servers/command_executor.rb

@@ -0,0 +1,27 @@
+module Kuroko2
+  module Servers
+    class CommandExecutor < Base
+
+      private
+
+      def worker
+        Kuroko2::Command::Executor
+      end
+
+      def default_options
+        {
+          worker_type: 'process',
+          workers:     Kuroko2::Command::Executor.num_workers,
+          daemonize:   Rails.env.production?,
+          log:         Rails.env.production? ?
+            Rails.root.join("log/command-executor.log").to_s :
+            $stdout,
+          log_level:   Rails.env.production? ? :info : :debug,
+          pid_path:    Rails.root.join('tmp/pids/command-executor.pid').to_s,
+          supervisor:  Rails.env.production?,
+          worker_graceful_kill_timeout: -1,
+        }
+      end
+    end
+  end
+end

data/lib/kuroko2/servers/job_scheduler.rb

@@ -0,0 +1,25 @@
+require 'kuroko2/servers/base'
+
+module Kuroko2
+  module Servers
+    class JobScheduler < Base
+
+      private
+
+      def worker
+        Kuroko2::Workflow::Scheduler
+      end
+
+      def default_options
+        {
+          worker_type: 'embedded',
+          daemonize:   Rails.env.production?,
+          log:         Rails.env.production? ? Rails.root.join('log/job-scheduler.log').to_s : $stdout,
+          log_level:   Rails.env.production? ? :info : :debug,
+          pid_path:    Rails.root.join('tmp/pids/job-scheduler.pid').to_s,
+          supervisor:  Rails.env.production?,
+        }
+      end
+    end
+  end
+end

data/lib/kuroko2/servers/workflow_processor.rb

@@ -0,0 +1,25 @@
+module Kuroko2
+  module Servers
+    class WorkflowProcessor < Base
+
+      private
+
+      def worker
+        Kuroko2::Workflow::Processor
+      end
+
+      def default_options
+        {
+          worker_type: 'embedded',
+          daemonize:   Rails.env.production?,
+          log:         Rails.env.production? ?
+            Rails.root.join("log/workflow-processor.log").to_s :
+            $stdout,
+          log_level:   Rails.env.production? ? :info : :debug,
+          pid_path:    Rails.root.join('tmp/pids/workflow-processor.pid').to_s,
+          supervisor:  Rails.env.production?,
+        }
+      end
+    end
+  end
+end

data/lib/kuroko2/util/logger.rb

@@ -0,0 +1,19 @@
+module Kuroko2
+  module Util
+    class Logger < ::Logger
+      def initialize(*args)
+        super
+
+        @formatter = LoggerFormatter.new
+      end
+
+      class LoggerFormatter < ::Logger::Formatter
+        def call(severity, timestamp, progname, msg)
+          location = caller_locations(4, 1).first
+
+          "%s %-5s - %s: %s\n" % [timestamp.iso8601, severity, location.label, msg]
+        end
+      end
+    end
+  end
+end

data/lib/kuroko2/util/rails_logger_formatter.rb

@@ -0,0 +1,9 @@
+module Kuroko2
+  module Util
+    class RailsLoggerFormatter < ::Logger::Formatter
+      def call(severity, timestamp, _, msg)
+        "%s %-5s: %s\n" % [timestamp.iso8601, severity, msg]
+      end
+    end
+  end
+end

data/lib/kuroko2/version.rb

@@ -0,0 +1,3 @@
+module Kuroko2
+  VERSION = '0.2.0'
+end