kuroko2 0.2.0

data/lib/kuroko2/workflow/task/wait.rb

@@ -0,0 +1,143 @@
+module Kuroko2
+  module Workflow
+    module Task
+      class Wait < Base
+        PERIODS = %w(hourly daily weekly monthly)
+
+        def execute
+          if token.context['WAIT'].present?
+            if token.waiting?
+              process_waiting_job
+            else
+              Kuroko2.logger.info { "(token #{token.uuid}) Skip since current status marked as '#{token.status_name}'." }
+              token.mark_as_waiting
+
+              :pass
+            end
+          else
+            token.context['WAIT'] = parse_option(option, start_at: token.job_instance.created_at)
+
+            Kuroko2.logger.info { "(token #{token.uuid}) waiting jobs: #{token.context['WAIT']["jobs"]}" }
+
+            token.mark_as_waiting
+            message = "(token #{token.uuid}) Marked as 'waiting' #{node.option}."
+            token.job_instance.logs.info(message)
+            Kuroko2.logger.info(message)
+            token.save!
+
+            :pass
+          end
+        end
+
+        def process_waiting_job
+          receive_waiting_job_completion!
+
+          wait_option = token.context['WAIT']
+          if wait_option["jobs"].all? { |wait_job| wait_job["received"] }
+            token.mark_as_working
+            token.context.delete('WAIT')
+            token.save!
+
+            message = "(token #{token.uuid}) All waiting jobs are finished."
+            Kuroko2.logger.info(message)
+            token.job_instance.logs.info(message)
+
+            :next
+          elsif wait_option["timeout"].minutes.since(token.created_at).past?
+            message = "(token #{token.uuid}) waiting jobs `#{node.option}` timeout."
+            Kuroko2.logger.error(message)
+            token.job_instance.logs.error(message)
+
+            :failure
+          else
+            :pass
+          end
+        end
+
+        def validate
+          parse_option(option)
+        end
+
+        private
+
+        # ex. wait: 100/daily 200/daily
+        def parse_option(option, start_at: Time.now)
+          raise_assertion_error unless option
+
+          wait_option = { "jobs" => [], "timeout" => 60.minutes.to_i / 1.minute }
+          scanner = StringScanner.new(option)
+          until scanner.eos?
+            if scanner.scan(%r!(\d+)\s*/\s*(#{PERIODS.join('|')})!)
+              start_from, start_to = period_to_time(scanner[2], at: start_at)
+              wait_option["jobs"] << {
+                "job_definition_id" => parse_definition_id(scanner[1]),
+                "period"            => scanner[2],
+                "start_from"        => start_from.to_s,
+                "start_to"          => start_to.to_s,
+                "received"          => false,
+              }
+            elsif scanner.scan(/timeout=(\d+)h/)
+              wait_option["timeout"] = scanner[1].to_i.hours / 60
+            elsif scanner.scan(/timeout=(\d+)m/) || scanner.scan(/timeout=(\d+)/)
+              wait_option["timeout"] = scanner[1].to_i
+            elsif scanner.scan(/\s+|,/)
+              # do nothing
+            else
+              raise_assertion_error
+            end
+          end
+
+          wait_option
+        end
+
+        def receive_waiting_job_completion!
+          token.context['WAIT']["jobs"].each do |wait_job|
+            next if wait_job["received"] == true
+
+            start_from = Time.zone.parse(wait_job["start_from"])
+            start_to   = Time.zone.parse(wait_job["start_to"])
+
+            received_instance = JobInstance.finished.where(
+              job_definition_id: wait_job["job_definition_id"].to_i,
+              created_at: start_from .. start_to
+            ).first
+
+            if received_instance.present?
+              wait_job["received"] = true
+              token.save!
+
+              message = "(token #{token.uuid}) A waiting job instance##{received_instance.job_definition_id}/#{received_instance.id} is finished."
+              Kuroko2.logger.info(message)
+              token.job_instance.logs.info(message)
+            end
+          end
+        end
+
+        def period_to_time(period, at: Time.now)
+          case period
+          when "hourly"
+            [at.beginning_of_hour, at.end_of_hour]
+          when "daily"
+            [at.beginning_of_day, at.end_of_day]
+          when "weekly"
+            [at.beginning_of_week, at.end_of_week]
+          when "monthly"
+            [at.beginning_of_month, at.end_of_month]
+          else
+            raise_assertion_error
+          end
+        end
+
+        def parse_definition_id(id)
+          JobDefinition.find(id.to_i).id
+        rescue ActiveRecord::RecordNotFound
+          raise Workflow::AssertionError, "Given Job Definition ID not found: #{id}"
+        end
+
+        def raise_assertion_error
+          raise Workflow::AssertionError, "Syntax error option value of wait: #{option}"
+        end
+      end
+    end
+  end
+end

data/lib/kuroko2.rb

@@ -0,0 +1,26 @@
+require 'chrono'
+require 'addressable'
+require 'aws-sdk'
+require 'retryable'
+require 'faraday'
+require 'html/pipeline'
+
+require "kuroko2/engine"
+require "kuroko2/configuration"
+require "kuroko2/util/logger"
+
+module Kuroko2
+  class << self
+    def logger
+      @logger ||= defined?(Rails) && Rails.env.test? ? Rails.logger : Util::Logger.new($stdout)
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+
+    def config
+      Configuration.config
+    end
+  end
+end

data/lib/tasks/kuroko2_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :kuroko2 do
+#   # Task goes here
+# end

data/spec/command/kill_spec.rb

@@ -0,0 +1,20 @@
+require 'rails_helper'
+
+module Kuroko2::Command
+  describe Kill do
+
+    describe '#execute' do
+      subject { Kill.new('test', '1').execute }
+
+      before { Process.detach(pid) }
+
+      let!(:signal) { create(:process_signal, pid: pid, hostname: 'test') }
+      let(:pid) { Process.spawn('sleep 10') }
+
+      it 'terminates spawned process' do
+        is_expected.to eq signal
+        expect { Process.kill(0, pid) }.to raise_error(Errno::ESRCH)
+      end
+    end
+  end
+end

data/spec/command/monitor_spec.rb

@@ -0,0 +1,68 @@
+require 'rails_helper'
+
+
+module Kuroko2::Command
+  describe Monitor do
+    let(:pid) { 32769 } # max int + 1
+    let(:hostname) { 'rspec' }
+
+    let(:definition) { create(:job_definition) }
+    let(:instance) { create(:job_instance, job_definition: definition) }
+    let(:token) { create(:token, job_definition: definition, job_instance: instance) }
+    let(:worker) { create(:worker, hostname: hostname) }
+
+    describe '#execute' do
+      let(:sent_mail) { ActionMailer::Base.deliveries.last }
+
+      context 'not running process' do
+        let!(:execution) { create(:execution, token: token, worker: worker, started_at: Time.now, finished_at: nil, pid: pid) }
+        let(:monitor) { Kuroko2::Command::Monitor.new(hostname: hostname, worker_id: 1) }
+
+        subject! { 15.times { monitor.execute } }
+
+        it 'sends a notification mail' do
+          execution.reload
+
+          expect(monitor.counter_size).to be_zero
+          expect(sent_mail).not_to be_nil
+          expect(sent_mail.subject).to eq '[CRITICAL] Process is not running on kuroko'
+          expect(execution.mailed_at).not_to be_nil
+        end
+      end
+
+      context 'not assigned process' do
+        let!(:execution) { create(:execution, token: token, worker: worker, started_at: 2.minutes.ago, finished_at: nil, pid: nil) }
+        subject! { Kuroko2::Command::Monitor.new(hostname: hostname, worker_id: 1).execute }
+
+        it 'sends a notification mail' do
+          execution.reload
+
+          expect(sent_mail.subject).to eq '[CRITICAL] Process is not assigned to any job-executor'
+          expect(execution.mailed_at).not_to be_nil
+        end
+      end
+    end
+
+    describe 'memory consumption monitoring' do
+      let!(:execution) { create(:execution, token: token, worker: worker, started_at: Time.now, finished_at: nil, pid: pid) }
+      let(:monitor) { Kuroko2::Command::Monitor.new(hostname: hostname, worker_id: 1) }
+      before do
+        allow(monitor).to receive(:check_process_absence).and_return(true)
+        allow(Kuroko2::MemorySampler).to receive(:get_by_pgid).and_return(1)
+      end
+
+      it 'logs memory consumption' do
+        expect {
+          monitor.execute
+        }.to change {
+          execution.job_definition.memory_expectancy.memory_consumption_logs.count
+        }.from(0).to(1)
+
+        log = execution.job_definition.memory_expectancy.memory_consumption_logs.first
+        expect(log.value).to be_kind_of(Integer)
+        expect(log.value).not_to eq(0)
+        expect(log.job_instance_id).to eq(instance.id)
+      end
+    end
+  end
+end

data/spec/command/shell_spec.rb

@@ -0,0 +1,87 @@
+require 'rails_helper'
+
+module Kuroko2::Command
+  describe Shell do
+
+    describe '#execute' do
+      subject { Shell.new(hostname: 'rspec', worker: worker).execute }
+
+      let(:job_definition) { create(:job_definition) }
+      let(:job_instance) { create(:job_instance, job_definition: job_definition) }
+      let(:token) { create(:token, job_instance: job_instance, job_definition: job_definition) }
+      let!(:execution) { create(:execution, shell: shell, context: { 'ENV' => { 'NAME' => "kuroko" } }, started_at: nil, finished_at: nil, token: token) }
+      let(:shell) { 'ruby -e "true"' }
+
+      context 'when available worker exists' do
+        let(:worker) { create(:worker) }
+
+        context 'successfully' do
+          let(:shell) { 'ruby -e "puts ENV[\'NAME\'] + \\"\\\x99\\""' }
+
+          it do
+            is_expected.to eq execution
+
+            expect(subject.started_at).not_to be_nil
+            expect(subject.finished_at).not_to be_nil
+            expect(subject).to be_success
+          end
+        end
+
+        context 'failure' do
+          let(:shell) { 'wrong_script' }
+
+          it do
+            is_expected.to eq execution
+
+            expect(subject.started_at).not_to be_nil
+            expect(subject.finished_at).not_to be_nil
+            expect(subject).not_to be_success
+          end
+        end
+
+        context 'with 4byte utf-8 charactor' do
+          let(:shell) { 'ruby -e "puts ENV[\'NAME\'] + \\"\xF0\x9F\x98\x81\\""' }
+
+          it do
+            is_expected.to eq execution
+
+            expect(subject.started_at).not_to be_nil
+            expect(subject.finished_at).not_to be_nil
+            expect(subject).to be_success
+          end
+        end
+      end
+
+      context 'when all workers are busy' do
+        let(:worker) { create(:worker, execution_id: execution.id) }
+
+        it 'skips execution' do
+          is_expected.to be_nil
+          execution.reload
+          expect(execution.started_at).to be_nil
+        end
+      end
+
+      describe 'memory expectancy calculation' do
+        let(:worker) { create(:worker) }
+        let(:memory_expectancy) { execution.job_definition.memory_expectancy }
+
+        context 'when memory consumption logs exist' do
+          before do
+            (1..10).each {|i| job_instance.create_memory_consumption_log!(value: i) }
+          end
+
+          it 'calculates memory expectancy of related job definition' do
+            expect { subject }.to change { memory_expectancy.reload.expected_value }
+          end
+        end
+
+        context 'when no memory consumption logs exist' do
+          it 'does not calculates memory expectancy of related job definition' do
+            expect { subject }.not_to change { memory_expectancy.reload.expected_value }
+          end
+        end
+      end
+    end
+  end
+end

data/spec/controllers/dashboard_controller_spec.rb

@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe Kuroko2::DashboardController, :type => :controller do
+  routes { Kuroko2::Engine.routes }
+end

data/spec/controllers/executions_controller_spec.rb

@@ -0,0 +1,23 @@
+require 'rails_helper'
+
+describe Kuroko2::ExecutionsController do
+  routes { Kuroko2::Engine.routes }
+
+  before { sign_in }
+
+  describe '#destroy' do
+    let(:definition) { create(:job_definition) }
+    let(:instance) { create(:job_instance, job_definition: definition) }
+    let(:token) { create(:token, job_instance: instance) }
+    let(:execution) { create(:execution, token: token, exit_status: nil, pid: 1) }
+    let!(:worker) { create(:worker, execution: execution) }
+
+    subject! { delete :destroy, params: { job_definition_id: definition.id, job_instance_id: instance.id, id: execution.id } }
+
+    it do
+      expect(response).to redirect_to(job_definition_job_instance_path(job_definition_id: definition.id, id: instance.id))
+      expect(Kuroko2::ProcessSignal.count).to eq 1
+    end
+  end
+
+end

data/spec/controllers/job_definition_stats_controller_spec.rb

@@ -0,0 +1,95 @@
+require 'rails_helper'
+
+describe Kuroko2::JobDefinitionStatsController do
+  routes { Kuroko2::Engine.routes }
+
+  before { sign_in }
+
+  let(:instance_num) { 2 }
+  let(:definition) do
+    create(
+      :job_definition_with_instances,
+      job_instances_count: instance_num,
+      job_instances_token_status: Kuroko2::Token::FINISHED
+    )
+  end
+
+  before do
+    definition.job_instances.first.update!(created_at: 2.months.ago)
+
+    definition.job_instances.each do |instance|
+      Kuroko2::MemoryConsumptionLog.create!(job_instance_id: instance.id, value: 1000 + rand(10))
+    end
+  end
+
+  describe '#index' do
+    before { get :index, params: { job_definition_id: definition.id } }
+
+    it do
+      expect(response).to have_http_status(:ok)
+      expect(response).to render_template('index')
+      expect(assigns(:definition)).to eq definition
+    end
+  end
+
+  describe '#memory' do
+    context 'without parameters' do
+      before { get :memory, params: { job_definition_id: definition.id }, xhr: true }
+
+      it do
+        expect(response).to have_http_status(:ok)
+        expect(assigns(:definition)).to eq definition
+        expect(assigns(:logs).size).to eq 2
+      end
+    end
+
+    context 'with period parameter' do
+      before { get :memory, params: { job_definition_id: definition.id, period: period }, xhr: true }
+
+      context 'with period is 1d' do
+        let(:period) { '1d' }
+        it { expect(assigns(:logs).size).to eq 1 }
+      end
+
+      context 'with period is 2w' do
+        let(:period) { '2w' }
+        it { expect(assigns(:logs).size).to eq 1 }
+      end
+
+      context 'with period is 3m' do
+        let(:period) { '3m' }
+        it { expect(assigns(:logs).size).to eq 2 }
+      end
+    end
+  end
+
+  describe '#execution_time' do
+    context 'without parameters' do
+      before { get :execution_time, params: { job_definition_id: definition.id }, xhr: true }
+      it do
+        expect(response).to have_http_status(:ok)
+        expect(assigns(:definition)).to eq definition
+        expect(assigns(:logs).size).to eq 2
+      end
+    end
+
+    context 'with peripd parameter' do
+      before { get :execution_time, params: { job_definition_id: definition.id, period: period }, xhr: true }
+
+      context 'with period is 1d' do
+        let(:period) { '1d' }
+        it { expect(assigns(:logs).size).to eq 1 }
+      end
+
+      context 'with period is 2w' do
+        let(:period) { '2w' }
+        it { expect(assigns(:logs).size).to eq 1 }
+      end
+
+      context 'with period is 3m' do
+        let(:period) { '3m' }
+        it { expect(assigns(:logs).size).to eq 2 }
+      end
+    end
+  end
+end

data/spec/controllers/job_definitions_controller_spec.rb

@@ -0,0 +1,89 @@
+require 'rails_helper'
+
+describe Kuroko2::JobDefinitionsController do
+  routes { Kuroko2::Engine.routes }
+
+  before { sign_in }
+
+  let(:definition) { create(:job_definition) }
+
+  describe '#index' do
+    subject! { get :index }
+    let!(:definitions) { create_list(:job_definition, 3) }
+
+    it do
+      expect(response).to have_http_status(:ok)
+      expect(response).to render_template('index')
+
+      expect(assigns(:definitions)).to eq definitions
+    end
+  end
+
+  describe '#show' do
+    subject! { get :show, params: { id: definition.id } }
+
+    it do
+      expect(response).to have_http_status(:ok)
+      expect(response).to render_template('show')
+
+      expect(assigns(:definition)).to eq definition
+    end
+  end
+
+  describe '#new' do
+    subject! { get :new }
+    it do
+      expect(response).to have_http_status(:ok)
+      expect(response).to render_template('new')
+
+      expect(assigns(:definition)).to be_new_record
+    end
+  end
+
+  describe '#create' do
+    subject! do
+      post :create, params: { job_definition: { name: 'Job Definition', description: 'This is description', script: "noop:\n" } }
+    end
+
+    it do
+      expect(response).to redirect_to(assigns(:definition))
+
+      expect(assigns(:definition)).not_to be_new_record
+      expect(assigns(:definition).admins).to eq [controller.current_user]
+      expect(assigns(:definition).memory_expectancy).not_to be_nil
+    end
+  end
+
+  describe '#edit' do
+    subject! { get :edit, params: { id: definition.id } }
+
+    it do
+      expect(response).to have_http_status(:ok)
+      expect(assigns(:definition)).to eq definition
+    end
+  end
+
+  describe '#update' do
+    let(:admin) { create(:user) }
+    subject! do
+      patch :update, params: { id: definition.id, job_definition: { name: 'Job Definition', description: 'This is description', script: "noop:\n" }, admin_assignments: { user_id: ["", admin.id] } }
+    end
+
+    it do
+      expect(response).to redirect_to(assigns(:definition))
+
+      expect(assigns(:definition)).not_to be_new_record
+    end
+  end
+
+  describe '#destroy' do
+    subject! { delete :destroy, params: { id: definition.id } }
+
+    it do
+      expect(response).to redirect_to(job_definitions_path)
+
+      expect(assigns(:definition)).to be_destroyed
+    end
+  end
+
+end

data/spec/controllers/job_instances_controller_spec.rb

@@ -0,0 +1,62 @@
+require 'rails_helper'
+
+describe Kuroko2::JobInstancesController do
+  routes { Kuroko2::Engine.routes }
+
+  before { sign_in }
+
+  let(:definition) { create(:job_definition_with_instances, job_instances_count: num_instances) }
+
+  describe '#index' do
+    let(:num_instances) { 2 }
+    before { get :index, params: { job_definition_id: definition.id } }
+
+    it do
+      expect(response).to have_http_status(:ok)
+      expect(response).to render_template('index')
+
+      expect(assigns(:definition)).to eq definition
+      expect(assigns(:instances).size).to eq num_instances
+    end
+  end
+
+  describe '#create' do
+    let(:num_instances) { 2 }
+    before { post :create, params: { job_definition_id: definition.id }, xhr: true }
+
+    it do
+      expect(response).to redirect_to(job_definition_job_instance_path(definition, assigns(:instance)))
+
+      expect(assigns(:definition)).to eq definition
+    end
+
+    context 'with Ad-Hoc `script` parameter' do
+      let(:script) { "execute: echo 1" }
+      before { post :create, params: { job_definition_id: definition.id, job_definition: { script: script } }, xhr: true }
+      it 'creates instance in Ad-Hoc script' do
+        expect(assigns(:instance).script).to eq script
+      end
+    end
+  end
+
+  describe '#destroy' do
+    before do
+      instance.tokens.each do |token|
+        token.update_column(:status, Kuroko2::Token::FAILURE)
+      end
+
+      delete :destroy, params: { job_definition_id: definition, id: instance }
+    end
+
+    let(:num_instances) { 1 }
+    let(:instance) { definition.job_instances.first }
+
+    it do
+      instance.reload
+
+      expect(response).to redirect_to(job_definition_job_instance_path(definition, instance))
+      expect(instance.tokens.size).to eq 0
+    end
+
+  end
+end

data/spec/controllers/job_schedules_controller_spec.rb

@@ -0,0 +1,39 @@
+require 'rails_helper'
+
+describe Kuroko2::JobSchedulesController do
+  routes { Kuroko2::Engine.routes }
+
+  before { sign_in }
+
+  let(:schedules) { create_list(:job_schedule, 1) }
+  let(:definition) { create(:job_definition, job_schedules: schedules) }
+
+  describe '#index' do
+    it do
+      get :index, params: { job_definition_id: definition.id }
+
+      expect(response).to have_http_status(:ok)
+      expect(assigns(:schedule)).to be_new_record
+      expect(assigns(:schedules)).to eq schedules
+    end
+  end
+
+  describe '#create' do
+    it do
+      post :create, params: { job_definition_id: definition.id, job_schedule: { cron: '* * * * *' } }
+
+      expect(response).to have_http_status(:created)
+    end
+  end
+
+  describe '#destroy' do
+
+    it do
+      delete :destroy, params: { job_definition_id: definition.id, id: schedules.first.id }
+      definition.reload
+
+      expect(response).to have_http_status(:ok)
+      expect(definition.job_schedules.size).to be 0
+    end
+  end
+end