kuroko2 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9190d02ed872bc2e4e7c0443fdd1f4c2eb66beda1ddaafd51eb58ed776ec1397
-  data.tar.gz: e92a5acec127505fc4388050f831747101bf1e83f4eb99cad3263ef1a113c854
+  metadata.gz: 0dbca2c6d6e477d98cfb6148007365028b63c7902bfe028968dd0be9a969347c
+  data.tar.gz: 6718f15b2ca4bda2d6527249561cc57357ab225c90f0a881a1c32663af021b82
 SHA512:
-  metadata.gz: 4ee951f567eed8d6ed087b399cf55eb07490c6d5dc7b853f4c80e657d2b47afb206a84b807e331a6df602392bb0439e1bb4c98e4926bd18d14479665ca31a83a
-  data.tar.gz: 45038b996196cf0019cfb9ac4ada24a8274b4a9303a087a60cc25234f7ba4629c0e6aaf7dab3a9f36dc94725c6e11c994eaf8253a43e52e363224e98f23a3ce1
+  metadata.gz: c622f1a547644b2a7bfa0f2d399d9b94a86d8dbf12be28f51a45c4062a4c175c70525aaff62d9ba361a168abef058ba5fc109795d21a5de2d6568b4edaf228a6
+  data.tar.gz: d68b87b94dc13aef1b93b447941a818d7c1f3721691ad430d40434cd267a79304cba3fe90dfb07289d76b5cd5db9e485ae9155c48e3032fd6e9cb3422b166046

data/README.md

@@ -1,5 +1,5 @@
 # Kuroko2
-[![Build Status](https://travis-ci.org/cookpad/kuroko2.svg?branch=master)](https://travis-ci.org/cookpad/kuroko2) [![Gem Version](https://badge.fury.io/rb/kuroko2.svg)](https://badge.fury.io/rb/kuroko2)
+[![Build Status](https://github.com/cookpad/kuroko2/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/cookpad/kuroko2/actions/workflows/ci.yml) [![Gem Version](https://badge.fury.io/rb/kuroko2.svg)](https://badge.fury.io/rb/kuroko2)
 
 ![](app/assets/images/kuroko2/kuroko-logo-horizontal.png)
 
@@ -39,6 +39,8 @@ Documentation is available at [docs/index.md](docs/index.md).
 - yohfee
 - takonomura
 - errm
+- itkq
+
 
 ## License
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/app/assets/config/kuroko2_manifest.js

@@ -0,0 +1,3 @@
+//= link_tree ../images/kuroko2
+//= link_directory ../javascripts/kuroko2 .js
+//= link_directory ../stylesheets/kuroko2 .css

data/app/assets/javascripts/kuroko2/job_timelines.js

@@ -0,0 +1,32 @@
+$(function() {
+  const datasetPath = $('#timeline').data('dataset-path');
+  if (!datasetPath) {
+    return;
+  }
+
+  const container = $('#timeline');
+  const options = {
+    zoomable: false,
+    orientation: 'both',
+    order(a, b) {
+      return a.end - b.end;
+    }
+  };
+  let timeline = null;
+
+  const callback = function(response) {
+    if (timeline === null) {
+      return timeline = new vis.Timeline(
+        container.get(0),
+        response.data,
+        $.extend(options, {start: response.start, end: response.end})
+      );
+    } else {
+      timeline.setItems(response.data);
+      return timeline.setOptions({start: response.start, end: response.end});
+    }
+  };
+
+  $.get(datasetPath, callback);
+  return setInterval(() => $.get(datasetPath, callback), 10000);
+});

data/app/controllers/kuroko2/api/application_controller.rb

@@ -7,15 +7,15 @@ class Kuroko2::Api::ApplicationController < ActionController::Base
     respond_with_error(404, 'record_not_found', exception.message)
   end
 
-  rescue_from HTTP::Forbidden do |exception|
+  rescue_from Http::Forbidden do |exception|
     respond_with_error(403, 'forbidden', exception.message)
   end
 
-  rescue_from HTTP::Unauthorized do |exception|
+  rescue_from Http::Unauthorized do |exception|
     respond_with_error(401, 'unauthorized', exception.message)
   end
 
-  rescue_from HTTP::UnprocessableEntity do |exception|
+  rescue_from Http::UnprocessableEntity do |exception|
     respond_with_error(422, 'unprocessable_entity', exception.message)
   end
 
@@ -43,7 +43,7 @@ class Kuroko2::Api::ApplicationController < ActionController::Base
     end
 
     if service_name.nil?
-      raise HTTP::Unauthorized
+      raise Http::Unauthorized
     end
   end
 

data/app/controllers/kuroko2/api/job_definitions_controller.rb

@@ -4,39 +4,52 @@ class Kuroko2::Api::JobDefinitionsController < Kuroko2::Api::ApplicationControll
   private
 
   def require_resources
+    definitions = Kuroko2::JobDefinition.includes(:tags, :job_schedules)
+    if params[:tags].present?
+      definitions = definitions.tagged_by(params[:tags])
+    end
+    @resources = definitions.ordered.map { |definition| Kuroko2::Api::JobDefinitionResource.new(definition) }
   end
 
   def create_resource
     definition = Kuroko2::JobDefinition.new(definition_params(params))
     user_ids = admin_id_params(params)
     definition.admins = Kuroko2::User.active.with(user_ids)
+    definition.tags = tags(params)
 
     if definition.save_and_record_revision
+      definition.job_schedules = job_schedules(params, definition)
       definition.admins.each do |user|
         user.stars.create(job_definition: definition) if user.google_account?
       end
 
       @resource = Kuroko2::Api::JobDefinitionResource.new(definition)
     else
-      raise HTTP::UnprocessableEntity.new("#{definition.name}: #{definition.errors.full_messages.join()}")
+      raise Http::UnprocessableEntity.new("#{definition.name}: #{definition.errors.full_messages.join}")
     end
   end
 
   def require_resource
-    definition = Kuroko2::JobDefinition.find(params[:id])
+    definition = Kuroko2::JobDefinition.includes(:tags, :job_schedules).find(params[:id])
     @resource = Kuroko2::Api::JobDefinitionResource.new(definition)
   end
 
   def update_resource
     definition = Kuroko2::JobDefinition.find(params[:id])
+    definition.tags = tags(params)
+    definition.job_schedules = job_schedules(params, definition)
 
     if definition.update_and_record_revision(definition_params(params))
       @resource = Kuroko2::Api::JobDefinitionResource.new(definition)
     else
-      raise HTTP::UnprocessableEntity.new("#{definition.name}: #{definition.errors.full_messages.join()}")
+      raise Http::UnprocessableEntity.new("#{definition.name}: #{definition.errors.full_messages.join}")
     end
   end
 
+  def destroy_resource
+    @resource.destroy
+  end
+
   def definition_params(params)
     params.permit(
       :name,
@@ -55,6 +68,22 @@ class Kuroko2::Api::JobDefinitionsController < Kuroko2::Api::ApplicationControll
       :webhook_url)
   end
 
+  def tags(params)
+    tag_strings = params.permit(tags: []).fetch(:tags, [])
+    tag_strings.map do |name|
+      Kuroko2::Tag.find_or_create_by(name: name)
+    end
+  end
+
+  def job_schedules(params, definition)
+    cron_strings = params.permit(cron: []).fetch(:cron, [])
+    cron_strings.map do |cron|
+      schedule = definition.job_schedules.find_or_create_by(cron: cron)
+      raise Http::UnprocessableEntity.new("#{cron}: #{schedule.errors.full_messages.join}") unless schedule.valid?
+      schedule
+    end
+  end
+
   def admin_id_params(params)
     params.permit(user_id: []).
       try!(:[], :user_id).

data/app/controllers/kuroko2/api/job_instances_controller.rb

@@ -16,7 +16,7 @@ class Kuroko2::Api::JobInstancesController < Kuroko2::Api::ApplicationController
   def create_resource
     definition = Kuroko2::JobDefinition.find(params[:job_definition_id])
     unless definition.api_allowed?
-      raise HTTP::Forbidden.new("#{definition.name} is not allowed to be executed via API")
+      raise Http::Forbidden.new("#{definition.name} is not allowed to be executed via API")
     end
 
     instance = definition.create_instance(

data/app/controllers/kuroko2/application_controller.rb

@@ -6,7 +6,7 @@ class Kuroko2::ApplicationController < ActionController::Base
   helper_method :current_user, :signed_in?
   before_action :require_sign_in
 
-  rescue_from HTTP::BadRequest do
+  rescue_from Http::BadRequest do
     respond_to do |format|
       format.html { render 'public/500.html', layout: false, status: :bad_request }
       format.json { render json: { message: 'Bad Request' }, status: :bad_request }

data/app/controllers/kuroko2/stars_controller.rb

@@ -8,7 +8,7 @@ class Kuroko2::StarsController < Kuroko2::ApplicationController
     if (star.save)
       render json: star, status: :created
     else
-      raise HTTP::BadRequest
+      raise Http::BadRequest
     end
   end
 
@@ -18,7 +18,7 @@ class Kuroko2::StarsController < Kuroko2::ApplicationController
     if (star.destroy)
       render json: star, status: :ok
     else
-      raise HTTP::BadRequest
+      raise Http::BadRequest
     end
   end
 

data/app/controllers/kuroko2/tokens_controller.rb

@@ -24,7 +24,7 @@ class Kuroko2::TokensController < Kuroko2::ApplicationController
 
       @engine.retry(@token)
     else
-      raise HTTP::BadRequest
+      raise Http::BadRequest
     end
 
     redirect_to job_definition_job_instance_path(job_definition_id: @token.job_definition_id, id: @token.job_instance.id)

data/app/controllers/kuroko2/workers_controller.rb

@@ -6,7 +6,7 @@ class Kuroko2::WorkersController < Kuroko2::ApplicationController
   end
 
   def update
-    @worker.update_attributes(worker_params)
+    @worker.update(worker_params)
     redirect_to workers_path
   end
 

data/app/errors/http/bad_request.rb

@@ -1,4 +1,4 @@
-module HTTP
+module Http
   class BadRequest < StandardError
   end
 end

data/app/errors/http/forbidden.rb

@@ -1,4 +1,4 @@
-module HTTP
+module Http
   class Forbidden < StandardError
   end
 end

data/app/errors/http/unauthorized.rb

@@ -1,4 +1,4 @@
-module HTTP
+module Http
   class Unauthorized < StandardError
   end
 end

data/app/errors/http/unprocessable_entity.rb

@@ -1,4 +1,4 @@
-module HTTP
+module Http
   class UnprocessableEntity < StandardError
   end
 end

data/app/models/kuroko2/api/job_definition_resource.rb

@@ -7,5 +7,17 @@ class Kuroko2::Api::JobDefinitionResource < Kuroko2::Api::ApplicationResource
 
   property :script
 
-  delegate :id, :name, :description, :script, to: :model
+  property :tags
+
+  property :cron
+
+  def tags
+    model.tags.map(&:name)
+  end
+
+  def cron
+    model.job_schedules.map(&:cron)
+  end
+
+  delegate :id, :name, :description, :script, :destroy, to: :model
 end

data/app/models/kuroko2/job_definition.rb

@@ -35,7 +35,7 @@ class Kuroko2::JobDefinition < Kuroko2::ApplicationRecord
 
   before_destroy :confirm_active_instances
   after_initialize :set_default_values
-  after_save :create_default_memory_expectancy, on: :create
+  after_create :create_default_memory_expectancy
 
   scope :ordered, -> { order(:id) }
   scope :tagged_by, ->(tags) {
@@ -62,14 +62,14 @@ class Kuroko2::JobDefinition < Kuroko2::ApplicationRecord
   }
 
 
-  validates :name, length: { maximum: 40 }, presence: true
+  validates :name, length: { maximum: 180 }, presence: true
   validates :description, presence: true
   validates :script, presence: true
   validate :script_syntax
   validate :validate_number_of_admins
   validates :hipchat_additional_text, length: { maximum: 180 }
   validates :slack_channel,
-    length: { maximum: 22, too_long: ' is too long (maximum is 21 characters without `#` symbol at the head)' },
+    length: { maximum: 81, too_long: ' is too long (maximum is 80 characters without `#` symbol at the head)' },
     format: {
       with: /\A#[^\.\s]+\z/, allow_blank: true,
       message: ' must start with # and must not include any dots or spaces'
@@ -102,14 +102,26 @@ class Kuroko2::JobDefinition < Kuroko2::ApplicationRecord
   end
 
   def save_and_record_revision(edited_user: nil)
-    record_revision(edited_user: edited_user)
-    save
+    transaction do
+      if save
+        record_revision(edited_user: edited_user)
+        true
+      else
+        false
+      end
+    end
   end
 
   def update_and_record_revision(attributes, edited_user: nil)
     assign_attributes(attributes)
-    record_revision(edited_user: edited_user)
-    save
+    transaction do
+      if save
+        record_revision(edited_user: edited_user)
+        true
+      else
+        false
+      end
+    end
   end
 
   private
@@ -142,8 +154,9 @@ class Kuroko2::JobDefinition < Kuroko2::ApplicationRecord
 
   def record_revision(edited_user: nil)
     unless revisions.first.try(:script) == script
-      revisions.new(script: script, user: edited_user, changed_at: Time.current)
+      revisions.create!(script: script, user: edited_user, changed_at: Time.current)
     end
+    nil
   end
 
   def create_default_memory_expectancy

data/app/models/kuroko2/job_schedule.rb

@@ -17,7 +17,7 @@ class Kuroko2::JobSchedule < Kuroko2::ApplicationRecord
 
   CHRONO_SCHEDULE_METHODS = %i[minutes hours days months wdays]
 
-  validates :cron, format: { with: CRON_FORMAT }, uniqueness: { scope: :job_definition_id }
+  validates :cron, format: { with: CRON_FORMAT }, uniqueness: { scope: :job_definition_id, case_sensitive: true }
   validate :validate_cron_schedule
 
   def next(now = Time.current)

data/app/models/kuroko2/job_suspend_schedule.rb

@@ -3,7 +3,7 @@ class Kuroko2::JobSuspendSchedule < Kuroko2::ApplicationRecord
 
   belongs_to :job_definition
 
-  validates :cron, format: { with: Kuroko2::JobSchedule::CRON_FORMAT }, uniqueness: { scope: :job_definition_id }
+  validates :cron, format: { with: Kuroko2::JobSchedule::CRON_FORMAT }, uniqueness: { scope: :job_definition_id, case_sensitive: true }
   validate :validate_cron_schedule
 
   def suspend_times(time_from, time_to)

data/app/views/kuroko2/sessions/new.html.slim

@@ -14,7 +14,7 @@ html.bg-black
       .body
         p.lead Please sign in with Google account.
         .text-center
-          a.btn.btn-social.btn-google-plus href=auth_path("google_oauth2", state: params[:return_to])
+          = button_to auth_path("google_oauth2", state: params[:return_to]), class: 'btn btn-social btn-google-plus' do
             i.fa.fa-google-plus
             span Sign in with Google
       .footer

data/config/initializers/omniauth.rb

@@ -1,4 +1,3 @@
- require 'omniauth-google-oauth2'
 Rails.application.config.middleware.use OmniAuth::Builder do
   config = Kuroko2.config.app_authentication.google_oauth2
   provider :google_oauth2, config.client_id, config.client_secret, config.options.to_h.symbolize_keys

data/config/routes.rb

@@ -48,7 +48,7 @@ Kuroko2::Engine.routes.draw do
   get '/osd' => 'dashboard#osd', as: :osd
 
   scope :v1, module: 'api', as: 'api' do
-    resources :job_definitions, path: 'definitions', only: %w(create show update) do
+    resources :job_definitions, path: 'definitions', except: %w(new edit) do
       resources :job_instances, path: 'instances', only: %w(show create)
     end
 

data/lib/autoload/kuroko2/command/shell.rb

@@ -102,7 +102,7 @@ module Kuroko2
             stdin.close
 
             pid = thread.pid
-            execution.update_attributes(pid: pid)
+            execution.update(pid: pid)
 
             reader = Thread.new do
               begin

data/lib/autoload/kuroko2/workflow/engine.rb

@@ -39,20 +39,7 @@ module Kuroko2
 
       def skip(token)
         token.with_lock do
-          node = extract_node(token)
-
-          message = "(token #{token.uuid}) Skip current node: '#{node.type}: #{node.option}'"
-          token.job_instance.update_column(:error_at, nil)
-          token.job_instance.logs.info(message)
-
-          token.mark_as_working
-          process_next(node.next, token)
-
-          token.save! unless token.destroyed?
-
-          Kuroko2.logger.info(message)
-
-          Notifier.notify(:skipping, token.job_instance)
+          skip_with_lock(token)
         end
       end
 
@@ -68,7 +55,7 @@ module Kuroko2
         Notifier.notify(:failure, token.job_instance)
 
         if token.context['AUTO_SKIP_ERROR']
-          skip(token)
+          skip_with_lock(token)
         end
       end
 
@@ -141,7 +128,7 @@ module Kuroko2
         token.job_instance.logs.error("(token #{token.uuid}) #{message}")
         token.job_instance.touch(:canceled_at)
 
-        Token.where(job_definition: token.job_definition).delete_all
+        Token.where(job_instance: token.job_instance).delete_all
         token.job_instance.logs.warn("(token #{token.uuid}) This job is canceled.")
 
         Kuroko2.logger.error(message)
@@ -150,6 +137,23 @@ module Kuroko2
         token.save! unless token.destroyed?
       end
 
+      def skip_with_lock(token)
+        node = extract_node(token)
+
+        message = "(token #{token.uuid}) Skip current node: '#{node.type}: #{node.option}'"
+        token.job_instance.update_column(:error_at, nil)
+        token.job_instance.logs.info(message)
+
+        token.mark_as_working
+        process_next(node.next, token)
+
+        token.save! unless token.destroyed?
+
+        Kuroko2.logger.info(message)
+
+        Notifier.notify(:skipping, token.job_instance)
+      end
+
       def extract_node(token)
         root = ScriptParser.new(token.script).parse(validate: false)
         root.find(token.path)

data/lib/autoload/kuroko2/workflow/task/fork.rb

@@ -43,9 +43,9 @@ module Kuroko2
         end
 
         def create_child_token(child_node:, env: {})
-          attributes = token.attributes.except('id', 'uuid', 'script', 'path', 'message', 'created_at', 'updated_at')
-          attributes = attributes.merge(uuid: SecureRandom.uuid, parent: token, script: child_node.to_script, path: '/')
-          attributes['context']['ENV'] = (attributes['context']['ENV'] || {}).merge(env)
+          attributes = token.attributes.except('id', 'uuid', 'script', 'path', 'message', 'created_at', 'updated_at', 'context')
+          attributes = attributes.merge(uuid: SecureRandom.uuid, parent: token, script: child_node.to_script, path: '/', context: token.context.deep_dup)
+          attributes[:context]['ENV'] = (attributes[:context]['ENV'] || {}).merge(env)
 
           Token.create!(attributes).tap do |created|
             fork_children_ids << created.id

data/lib/kuroko2/engine.rb

@@ -30,17 +30,6 @@ module Kuroko2
 
       config.active_record.table_name_prefix = Kuroko2.config.table_name_prefix
 
-      if Kuroko2.config.custom_tasks
-        Kuroko2.config.custom_tasks.each do |key, klass|
-          unless Workflow::Node::TASK_REGISTRY.has_key?(key)
-            Workflow::Node.register(
-              key: key.to_sym,
-              klass: Workflow::Task.const_get(klass, false)
-            )
-          end
-        end
-      end
-
       config.action_mailer.default_url_options = {
         host:     Kuroko2.config.url_host,
         protocol: Kuroko2.config.url_scheme,
@@ -52,6 +41,19 @@ module Kuroko2
         Kuroko2.config.action_mailer.smtp_settings.to_h.symbolize_keys || {}
 
       app.config.assets.precompile += %w(kuroko2/kuroko-logo-success.png kuroko2/kuroko-logo-error.png)
+    end
+
+    config.after_initialize do
+      if Kuroko2.config.custom_tasks
+        Kuroko2.config.custom_tasks.each do |key, klass|
+          unless Workflow::Node::TASK_REGISTRY.has_key?(key)
+            Workflow::Node.register(
+              key: key.to_sym,
+              klass: Workflow::Task.const_get(klass, false)
+            )
+          end
+        end
+      end
 
       if Kuroko2.config.extensions && Kuroko2.config.extensions.controller
         Kuroko2.config.extensions.controller.each do |extension|

data/lib/kuroko2/version.rb

@@ -1,3 +1,3 @@
 module Kuroko2
-  VERSION = '0.5.1'
+  VERSION = '0.6.0'
 end

data/lib/kuroko2.rb

@@ -1,10 +1,12 @@
 require 'chrono'
 require 'addressable'
-require 'aws-sdk'
+require 'aws-sdk-cloudwatchlogs'
 require 'retryable'
 require 'faraday'
 require 'html/pipeline'
 require 'hipchat'
+require 'omniauth-google-oauth2'
+require 'omniauth/rails_csrf_protection'
 
 require "kuroko2/engine"
 require "kuroko2/configuration"

data/spec/controllers/users_controller_spec.rb

@@ -12,7 +12,7 @@ describe Kuroko2::UsersController do
 
     context 'with groups parameter' do
       before do
-        users.each {|u| u.update_attributes!(provider: 'group_mail') }
+        users.each {|u| u.update!(provider: 'group_mail') }
       end
       it { expect(assigns(:users).to_a).to eq users }
     end

data/spec/dummy/config/application.rb

@@ -8,6 +8,7 @@ Bundler.require(*Rails.groups)
 
 module Dummy
   class Application < Rails::Application
+    config.load_defaults 6.0
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration should go into files in config/initializers
     # -- all .rb files in that directory are automatically loaded.

data/spec/dummy/config/environments/production.rb

@@ -1,3 +1,5 @@
+require 'uglifier'
+
 Rails.application.configure do
   # Settings specified here will take precedence over those in config/application.rb.
 
@@ -19,7 +21,7 @@ Rails.application.configure do
   config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present?
 
   # Compress JavaScripts and CSS.
-  config.assets.js_compressor = :uglifier
+  config.assets.js_compressor = Uglifier.new(harmony: true)
   # config.assets.css_compressor = :sass
 
   # Do not fallback to assets pipeline if a precompiled asset is missed.

data/spec/dummy/config/kuroko2.yml

@@ -1,5 +1,6 @@
 default: &default
   url: 'http://localhost:3000'
+  table_name_prefix: ''
   action_mailer:
     delivery_method: 'test'
   execution_logger: